From 508fc5b45d4f50c00d6b2495f8fe831ce27f28cc Mon Sep 17 00:00:00 2001 From: Patrick Stephens Date: Sun, 3 May 2026 12:34:36 +0100 Subject: [PATCH 001/213] [upstream] in_forward: tests: Add a regression test case for Upstream-Ref: https://github.com/fluent/fluent-bit/commit/de3dc063cb889a972db6853ff67a1e613a8a27d5 Cherry-picked from Fluent Bit v4.2.4 --- source/tests/runtime/in_forward.c | 113 ++++++++++++++++++++++++++++++ 1 file changed, 113 insertions(+) diff --git a/source/tests/runtime/in_forward.c b/source/tests/runtime/in_forward.c index 359ac240..3f9f6d7d 100644 --- a/source/tests/runtime/in_forward.c +++ b/source/tests/runtime/in_forward.c @@ -1047,6 +1047,118 @@ void flb_test_forward_zstd() test_ctx_destroy(ctx); } +static int cb_count_only(void *record, size_t size, void *data) +{ + int n = get_output_num(); + set_output_num(n + 1); + flb_free(record); + return 0; +} + +void flb_test_threaded_forward_issue_10946() +{ + struct flb_lib_out_cb cb = {0}; + flb_ctx_t *ctx; + int in_ffd, out_ffd, ret; + int out_count; + flb_sockfd_t fd; + char *buf; + size_t size; + int root_type; + struct flb_processor *proc; + struct flb_processor_unit *pu; + struct cfl_variant v_key = { + .type = CFL_VARIANT_STRING, + .data.as_string = "log" + }; + struct cfl_variant v_mode = { + .type = CFL_VARIANT_STRING, + .data.as_string = "partial_message" + }; + char *json = "[\"logs\",1234567890,{\"log\":\"hello\"}]"; + + clear_output_num(); + + cb.cb = cb_count_only; + cb.data = &out_count; + + /* Service */ + ctx = flb_create(); + TEST_CHECK(ctx != NULL); + flb_service_set(ctx, + "Flush", "0.200000000", + "Grace", "1", + "Log_Level", "error", + NULL); + + in_ffd = flb_input(ctx, (char *) "forward", NULL); + TEST_CHECK(in_ffd >= 0); + ret = flb_input_set(ctx, in_ffd, + "tag", "logs", + "threaded", "true", + NULL); + TEST_CHECK(ret == 0); + + /* Attach a logs-processor: multiline (minimal settings). + * This mirrors the YAML: + * processors.logs: + * - name: multiline + * multiline.key_content: log + * mode: partial_message + */ + proc = flb_processor_create(ctx->config, "ut", NULL, 0); + TEST_CHECK(proc != NULL); + + pu = flb_processor_unit_create(proc, FLB_PROCESSOR_LOGS, "multiline"); + TEST_CHECK(pu != NULL); + + ret = flb_processor_unit_set_property(pu, "multiline.key_content", &v_key); + TEST_CHECK(ret == 0); + + ret = flb_processor_unit_set_property(pu, "mode", &v_mode); + TEST_CHECK(ret == 0); + + ret = flb_input_set_processor(ctx, in_ffd, proc); + TEST_CHECK(ret == 0); + + /* Output: lib -> count arrivals of tag 'logs' (after processors) */ + out_ffd = flb_output(ctx, (char *) "lib", (void *) &cb); + TEST_CHECK(out_ffd >= 0); + ret = flb_output_set(ctx, out_ffd, + "match", "logs", + "format", "json", + NULL); + TEST_CHECK(ret == 0); + + /* Start engine */ + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + /* Send a single Forward frame to 'logs' */ + fd = connect_tcp(NULL, -1); + TEST_CHECK(fd >= 0); + + /* ["logs", 1234567890, {"log":"hello"}] */ + ret = flb_pack_json(json, strlen(json), &buf, &size, &root_type, NULL); + TEST_CHECK(ret == 0); + TEST_CHECK(send(fd, buf, size, 0) == (ssize_t) size); + flb_free(buf); + + /* Give it a moment to flush */ + flb_time_msleep(1500); + + /* With the fix, at least one record must arrive */ + out_count = get_output_num(); + TEST_CHECK(out_count > 0); + if (!TEST_CHECK(out_count > 0)) { + TEST_MSG("no outputs with threaded+multiline; emitter RB/collector likely missing"); + } + + /* Cleanup */ + flb_socket_close(fd); + flb_stop(ctx); + flb_destroy(ctx); +} TEST_LIST = { {"forward", flb_test_forward}, @@ -1063,5 +1175,6 @@ TEST_LIST = { {"fw_auth_shared_key_plus_users_start_ok", flb_test_fw_auth_shared_key_plus_users_start_ok}, {"forward_gzip", flb_test_forward_gzip}, {"forward_zstd", flb_test_forward_zstd}, + {"issue_10946", flb_test_threaded_forward_issue_10946}, {NULL, NULL} }; From 6ce3daf818a57c14fe23a84b7f98018ee95986aa Mon Sep 17 00:00:00 2001 From: Patrick Stephens Date: Sun, 3 May 2026 12:34:37 +0100 Subject: [PATCH 002/213] [upstream] in_winevtlog: Try to reconnect on cancellations Upstream-Ref: https://github.com/fluent/fluent-bit/commit/b5e53c0879c3e269b7171c11ac1b98816e05f675 Cherry-picked from Fluent Bit v4.2.4 --- source/plugins/in_winevtlog/winevtlog.c | 153 +++++++++++++++++++++++- source/plugins/in_winevtlog/winevtlog.h | 5 + 2 files changed, 156 insertions(+), 2 deletions(-) diff --git a/source/plugins/in_winevtlog/winevtlog.c b/source/plugins/in_winevtlog/winevtlog.c index 8f46b11d..68514243 100644 --- a/source/plugins/in_winevtlog/winevtlog.c +++ b/source/plugins/in_winevtlog/winevtlog.c @@ -144,6 +144,9 @@ struct winevtlog_channel *winevtlog_subscribe(const char *channel, struct winevt return NULL; } ch->signal_event = signal_event; + ch->cancelled_by_us = FALSE; + ch->reconnect_needed = FALSE; + ch->last_error = 0; if (stored_bookmark) { ch->bookmark = stored_bookmark; @@ -184,9 +187,16 @@ struct winevtlog_channel *winevtlog_subscribe(const char *channel, struct winevt BOOL cancel_subscription(struct winevtlog_channel *ch) { + ch->cancelled_by_us = TRUE; return EvtCancel(ch->subscription); } +void winevtlog_request_cancel(struct winevtlog_channel *ch) +{ + ch->cancelled_by_us = TRUE; + EvtCancel(ch->subscription); +} + static void close_handles(struct winevtlog_channel *ch) { int i; @@ -604,10 +614,21 @@ static int winevtlog_next(struct winevtlog_channel *ch, int hit_threshold) if (!has_next) { status = GetLastError(); - if (ERROR_CANCELLED == status) { + if (status == ERROR_CANCELLED) { + if (ch->cancelled_by_us) { + /* Conmsume this flag and return early */ + ch->cancelled_by_us = FALSE; + return FLB_FALSE; + } + ch->reconnect_needed = TRUE; + ch->last_error = status; + flb_debug("[in_winevtlog] subscription cancelled unexpectedly (err=%lu), will reconnect", status); return FLB_FALSE; } - if (ERROR_NO_MORE_ITEMS != status) { + if (status != ERROR_NO_MORE_ITEMS) { + ch->reconnect_needed = TRUE; + ch->last_error = status; + flb_warn("[in_winevtlog] EvtNext failed (err=%lu), will reconnect", status); return FLB_FALSE; } @@ -627,6 +648,126 @@ static int winevtlog_next(struct winevtlog_channel *ch, int hit_threshold) return FLB_FALSE; } +int winevtlog_reconnect(struct winevtlog_channel *ch, struct winevtlog_config *ctx) +{ + HANDLE new_signal = NULL; + EVT_HANDLE new_remote = NULL; + EVT_HANDLE new_sub = NULL; + DWORD flags = 0; + DWORD err = 0; + PWSTR wide_channel = NULL; + PWSTR wide_query = NULL; + DWORD len; + + len = MultiByteToWideChar(CP_UTF8, 0, ch->name, -1, NULL, 0); + if (len == 0) { + return -1; + } + wide_channel = flb_malloc(sizeof(WCHAR) * len); + if (!wide_channel) { + return -1; + } + MultiByteToWideChar(CP_UTF8, 0, ch->name, -1, wide_channel, len); + + if (ch->query) { + len = MultiByteToWideChar(CP_UTF8, 0, ch->query, -1, NULL, 0); + if (len == 0) { + flb_free(wide_channel); + return -1; + } + wide_query = flb_malloc(sizeof(WCHAR) * len); + if (!wide_query) { + flb_free(wide_channel); + return -1; + } + MultiByteToWideChar(CP_UTF8, 0, ch->query, -1, wide_query, len); + } + + new_signal = CreateEvent(NULL, TRUE, TRUE, NULL); + if (!new_signal) { + flb_free(wide_channel); + if (wide_query) { + flb_free(wide_query); + } + return -1; + } + + if (ch->session) { + new_remote = create_remote_handle(ch->session, &err); + if (err != ERROR_SUCCESS || !new_remote) { + flb_plg_error(ctx->ins, "reconnect: cannot create remote handle '%s' in %ls (err=%lu)", + ch->name, ch->session->server, err); + CloseHandle(new_signal); + flb_free(wide_channel); + if (wide_query) { + flb_free(wide_query); + } + return -1; + } + } + + if (ch->bookmark) { + flags = EvtSubscribeStartAfterBookmark; + } + else if (ctx->read_existing_events) { + flags = EvtSubscribeStartAtOldestRecord; + } + else { + flags = EvtSubscribeToFutureEvents; + } + + new_sub = EvtSubscribe(new_remote, new_signal, wide_channel, wide_query, + ch->bookmark, NULL, NULL, flags); + if (!new_sub) { + DWORD sub_err = GetLastError(); + if (sub_err == ERROR_EVT_QUERY_RESULT_STALE) { + flb_plg_warn(ctx->ins, "reconnect: bookmark stale on '%s' (err=%lu), falling back to latest", + ch->name, sub_err); + flags = ctx->read_existing_events ? EvtSubscribeStartAtOldestRecord + : EvtSubscribeToFutureEvents; + new_sub = EvtSubscribe(new_remote, new_signal, wide_channel, wide_query, + NULL, NULL, NULL, flags); + } + } + + if (!new_sub) { + DWORD sub_err = GetLastError(); + flb_plg_error(ctx->ins, "reconnect: EvtSubscribe failed on '%s' (err=%lu)", ch->name, sub_err); + if (new_remote) EvtClose(new_remote); + CloseHandle(new_signal); + flb_free(wide_channel); + if (wide_query) { + flb_free(wide_query); + } + return -1; + } + + if (ch->subscription) { + EvtClose(ch->subscription); + } + if (ch->remote) { + EvtClose(ch->remote); + } + if (ch->signal_event) { + CloseHandle(ch->signal_event); + } + + ch->subscription = new_sub; + ch->remote = new_remote; + ch->signal_event = new_signal; + ch->reconnect_needed = FALSE; + ch->last_error = 0; + ch->count = 0; + + flb_plg_debug(ctx->ins, "reconnected subscription for '%s'", ch->name); + + flb_free(wide_channel); + if (wide_query) { + flb_free(wide_query); + } + return 0; +} + /* * Read from an open Windows Event Log channel. */ @@ -645,6 +786,7 @@ int winevtlog_read(struct winevtlog_channel *ch, struct winevtlog_config *ctx, PEVT_VARIANT string_inserts = NULL; UINT count_inserts = 0; DWORD i = 0; + int rc = 0; while (winevtlog_next(ch, hit_threshold)) { for (i = 0; i < ch->count; i++) { @@ -699,6 +841,13 @@ int winevtlog_read(struct winevtlog_channel *ch, struct winevtlog_config *ctx, *read = read_size; + if (ch->reconnect_needed) { + rc = winevtlog_reconnect(ch, ctx); + if (rc != 0) { + flb_plg_error(ctx->ins, "reconnect attempt failed for '%s' (last_error=%lu)", + ch->name, ch->last_error); + } + } return 0; } diff --git a/source/plugins/in_winevtlog/winevtlog.h b/source/plugins/in_winevtlog/winevtlog.h index 7b55698a..5e19c5b0 100644 --- a/source/plugins/in_winevtlog/winevtlog.h +++ b/source/plugins/in_winevtlog/winevtlog.h @@ -64,6 +64,11 @@ struct winevtlog_channel { int count; struct winevtlog_session *session; + /* reconnect */ + BOOL cancelled_by_us; + BOOL reconnect_needed; + DWORD last_error; + char *name; char *query; unsigned int time_updated; From fb43aae7adab3abb51d44ed3a6a284f3aefc2dc6 Mon Sep 17 00:00:00 2001 From: Patrick Stephens Date: Sun, 3 May 2026 12:34:37 +0100 Subject: [PATCH 003/213] [upstream] in_winevtlog: Add retry backoff mechanism Upstream-Ref: https://github.com/fluent/fluent-bit/commit/3b917739ee3adc33c16fce783b1f31fbdee9d2eb Cherry-picked from Fluent Bit v4.2.4 --- source/plugins/in_winevtlog/in_winevtlog.c | 69 ++++++++++++++++++ source/plugins/in_winevtlog/winevtlog.c | 85 +++++++++++++++++++--- source/plugins/in_winevtlog/winevtlog.h | 19 +++++ 3 files changed, 164 insertions(+), 9 deletions(-) diff --git a/source/plugins/in_winevtlog/in_winevtlog.c b/source/plugins/in_winevtlog/in_winevtlog.c index 7303ae63..360544e2 100644 --- a/source/plugins/in_winevtlog/in_winevtlog.c +++ b/source/plugins/in_winevtlog/in_winevtlog.c @@ -161,6 +161,8 @@ static int in_winevtlog_init(struct flb_input_instance *in, struct winevtlog_config *ctx; struct winevtlog_session *session; int status = WINEVTLOG_SESSION_CREATE_OK; + double mult = 2.0; + DWORD tmp_ms = 0; /* Initialize context */ ctx = flb_calloc(1, sizeof(struct winevtlog_config)); @@ -187,6 +189,47 @@ static int in_winevtlog_init(struct flb_input_instance *in, return -1; } + if (ctx->backoff_multiplier_str && ctx->backoff_multiplier_str[0] != '\0') { + mult = atof(ctx->backoff_multiplier_str); + if (mult <= 0.0) { + flb_plg_warn(in, "invalid reconnect.multiplier='%s', fallback to 2.0", + ctx->backoff_multiplier_str); + mult = 2.0; + } + } + ctx->backoff.multiplier_x1000 = (DWORD)(mult * 1000.0); + + if (ctx->backoff.base_ms == 0) { + ctx->backoff.base_ms = 500; + } + + if (ctx->backoff.max_ms == 0) { + ctx->backoff.max_ms = 30000; + } + + if (ctx->backoff.jitter_pct == 0) { + ctx->backoff.jitter_pct = 20; + } + + if (ctx->backoff.max_retries == 0) { + ctx->backoff.max_retries = 8; + } + + if (ctx->backoff.max_ms < ctx->backoff.base_ms) { + flb_plg_warn(in, "reconnect.max_ms < reconnect.base_ms, swapping values"); + tmp_ms = ctx->backoff.base_ms; + ctx->backoff.base_ms = ctx->backoff.max_ms; + ctx->backoff.max_ms = tmp_ms; + } + + if (ctx->backoff.multiplier_x1000 < 500) { + ctx->backoff.multiplier_x1000 = 500; + } + + if (ctx->backoff.multiplier_x1000 > 10000) { + ctx->backoff.multiplier_x1000 = 10000; + } + /* Initialize session context */ session = in_winevtlog_session_create(ctx, config, &status); if (status == WINEVTLOG_SESSION_ALLOC_FAILED || @@ -451,6 +494,32 @@ static struct flb_config_map config_map[] = { 0, FLB_TRUE, offsetof(struct winevtlog_config, remote_password), "Specify password of remote access for Windows EventLog" }, + /* ---- reconnect backoff parameters ---- */ + { + FLB_CONFIG_MAP_INT, "reconnect.base_ms", "500", + 0, FLB_TRUE, offsetof(struct winevtlog_config, backoff.base_ms), + "Initial reconnect backoff in milliseconds" + }, + { + FLB_CONFIG_MAP_INT, "reconnect.max_ms", "30000", + 0, FLB_TRUE, offsetof(struct winevtlog_config, backoff.max_ms), + "Maximum reconnect backoff in milliseconds" + }, + { + FLB_CONFIG_MAP_STR, "reconnect.multiplier", "2.0", + 0, FLB_TRUE, offsetof(struct winevtlog_config, backoff_multiplier_str), + "Exponential backoff multiplier (float, e.g. 2.0)" + }, + { + FLB_CONFIG_MAP_INT, "reconnect.jitter_pct", "20", + 0, FLB_TRUE, offsetof(struct winevtlog_config, backoff.jitter_pct), + "Jitter percentage applied to backoff (e.g. 20 means ±20%)" + }, + { + FLB_CONFIG_MAP_INT, "reconnect.max_retries", "8", + 0, FLB_TRUE, offsetof(struct winevtlog_config, backoff.max_retries), + "Max reconnect attempts before giving up" + }, /* EOF */ {0} }; diff --git a/source/plugins/in_winevtlog/winevtlog.c b/source/plugins/in_winevtlog/winevtlog.c index 68514243..036d59af 100644 --- a/source/plugins/in_winevtlog/winevtlog.c +++ b/source/plugins/in_winevtlog/winevtlog.c @@ -147,6 +147,9 @@ struct winevtlog_channel *winevtlog_subscribe(const char *channel, struct winevt ch->cancelled_by_us = FALSE; ch->reconnect_needed = FALSE; ch->last_error = 0; + ch->retry_attempts = 0; + ch->next_retry_deadline = 0; + ch->prng_state = GetTickCount64() ^ (ULONGLONG)(uintptr_t)ch; if (stored_bookmark) { ch->bookmark = stored_bookmark; @@ -648,7 +651,56 @@ static int winevtlog_next(struct winevtlog_channel *ch, int hit_threshold) return FLB_FALSE; } -int winevtlog_reconnect(struct winevtlog_channel *ch, struct winevtlog_config *ctx) +static inline void backoff_defaults(struct winevtlog_backoff *dst, const struct winevtlog_backoff *src) +{ + dst->base_ms = (src && src->base_ms) ? src->base_ms : 500; + dst->max_ms = (src && src->max_ms) ? src->max_ms : 30000; + dst->multiplier_x1000 = (src && src->multiplier_x1000) ? src->multiplier_x1000 : 2000; /* x2.0 */ + dst->jitter_pct = (src && src->jitter_pct) ? src->jitter_pct : 20; + dst->max_retries = (src && src->max_retries) ? src->max_retries : 8; +} + +static inline DWORD prng16(ULONGLONG *state) +{ + *state = (*state * 6364136223846793005ULL + 1ULL); + return (DWORD)((*state >> 33) & 0xFFFF); +} + +static DWORD calc_backoff_ms(struct winevtlog_channel *ch, const struct winevtlog_backoff *cfg, DWORD attempt) +{ + DWORD i = 0; + DWORD ms = 0; + LONG span = 0; + LONG delta = 0; + LONG with_jitter = 0; + double mult = (double)cfg->multiplier_x1000 / 1000.0; + double t = (double)cfg->base_ms; + + for (i = 0; i < attempt; i++) { + t *= mult; + if (t >= (double)cfg->max_ms) { t = (double)cfg->max_ms; break; } + } + ms = (DWORD)((t > (double)cfg->max_ms) ? cfg->max_ms : t); + /* ±jitter% */ + span = (LONG)((ms * cfg->jitter_pct) / 100); + delta = (LONG)(prng16(&ch->prng_state) % (2 * span + 1)) - span; + with_jitter = (LONG)ms + delta; + if (with_jitter < 0) { + with_jitter = 0; + } + return (DWORD)with_jitter; +} + +void winevtlog_schedule_retry(struct winevtlog_channel *ch, struct winevtlog_config *ctx) +{ + struct winevtlog_backoff cfg; + DWORD delay = 0; + backoff_defaults(&cfg, ctx ? &ctx->backoff : NULL); + delay = calc_backoff_ms(ch, &cfg, ch->retry_attempts); + ch->next_retry_deadline = GetTickCount64() + (ULONGLONG)delay; +} + +int winevtlog_try_reconnect(struct winevtlog_channel *ch, struct winevtlog_config *ctx) { HANDLE new_signal = NULL; EVT_HANDLE new_remote = NULL; @@ -755,12 +807,13 @@ int winevtlog_reconnect(struct winevtlog_channel *ch, struct winevtlog_config *c ch->subscription = new_sub; ch->remote = new_remote; ch->signal_event = new_signal; - ch->reconnect_needed = FALSE; - ch->last_error = 0; - ch->count = 0; + ch->reconnect_needed = FALSE; + ch->retry_attempts = 0; + ch->next_retry_deadline = 0; + ch->last_error = 0; + ch->count = 0; flb_plg_debug(ctx->ins, "reconnected subscription for '%s'", ch->name); - flb_free(wide_channel); if (wide_query) { flb_free(wide_query); @@ -842,10 +895,24 @@ int winevtlog_read(struct winevtlog_channel *ch, struct winevtlog_config *ctx, *read = read_size; if (ch->reconnect_needed) { - rc = winevtlog_reconnect(ch, ctx); - if (rc != 0) { - flb_plg_error(ctx->ins, "reconnect attempt failed for '%s' (last_error=%lu)", - ch->name, ch->last_error); + ULONGLONG now = GetTickCount64(); + if (ch->next_retry_deadline == 0 || now >= ch->next_retry_deadline) { + int rc = winevtlog_try_reconnect(ch, ctx); + if (rc != 0) { + struct winevtlog_backoff eff; + backoff_defaults(&eff, ctx ? &ctx->backoff : NULL); + if (ch->retry_attempts < eff.max_retries) { + ch->retry_attempts++; + winevtlog_schedule_retry(ch, ctx); + flb_plg_warn(ctx->ins, "reconnect attempt %lu failed for '%s' (err=%lu), next at +%lums", + (unsigned long)ch->retry_attempts, ch->name, ch->last_error, + (unsigned long)(ch->next_retry_deadline - now)); + } + else { + flb_plg_error(ctx->ins, "reconnect exhausted for '%s' (last err=%lu)", ch->name, ch->last_error); + ch->reconnect_needed = FALSE; + } + } } } return 0; diff --git a/source/plugins/in_winevtlog/winevtlog.h b/source/plugins/in_winevtlog/winevtlog.h index 5e19c5b0..f2ea19f0 100644 --- a/source/plugins/in_winevtlog/winevtlog.h +++ b/source/plugins/in_winevtlog/winevtlog.h @@ -27,6 +27,15 @@ struct winevtlog_session; +/* reconnect backoff */ +struct winevtlog_backoff { + DWORD base_ms; + DWORD max_ms; + DWORD multiplier_x1000; + DWORD jitter_pct; + DWORD max_retries; +}; + struct winevtlog_config { unsigned int interval_sec; unsigned int interval_nsec; @@ -48,6 +57,8 @@ struct winevtlog_config { flb_pipefd_t coll_fd; struct flb_input_instance *ins; struct flb_log_event_encoder *log_encoder; + struct winevtlog_backoff backoff; + flb_sds_t backoff_multiplier_str; }; /* Some channels has very heavy contents for 10 events at same time. @@ -68,6 +79,9 @@ struct winevtlog_channel { BOOL cancelled_by_us; BOOL reconnect_needed; DWORD last_error; + DWORD retry_attempts; + ULONGLONG next_retry_deadline; + ULONGLONG prng_state; char *name; char *query; @@ -129,6 +143,11 @@ void winevtlog_pack_event(PEVT_VARIANT system, WCHAR *message, int winevtlog_sqlite_load(struct winevtlog_channel *ch, struct winevtlog_config *ctx, struct flb_sqldb *db); int winevtlog_sqlite_save(struct winevtlog_channel *ch, struct winevtlog_config *ctx, struct flb_sqldb *db); +/* Non blocking reconnection utilities */ +int winevtlog_try_reconnect(struct winevtlog_channel *ch, struct winevtlog_config *ctx); +void winevtlog_schedule_retry(struct winevtlog_channel *ch, struct winevtlog_config *ctx); +void winevtlog_request_cancel(struct winevtlog_channel *ch); + /* * SQL templates */ From 6249e6cc075d341d346cdf9bffeb44d773d66d3b Mon Sep 17 00:00:00 2001 From: Patrick Stephens Date: Sun, 3 May 2026 12:34:37 +0100 Subject: [PATCH 004/213] [upstream] in_winevtlog: Address coderabbitai comments Upstream-Ref: https://github.com/fluent/fluent-bit/commit/e3735f7a352ce9e878ea335be89157ff78f2e2e9 Cherry-picked from Fluent Bit v4.2.4 --- source/plugins/in_winevtlog/in_winevtlog.c | 15 ++++++++++++--- source/plugins/in_winevtlog/winevtlog.c | 13 ++++++------- 2 files changed, 18 insertions(+), 10 deletions(-) diff --git a/source/plugins/in_winevtlog/in_winevtlog.c b/source/plugins/in_winevtlog/in_winevtlog.c index 360544e2..7aef090f 100644 --- a/source/plugins/in_winevtlog/in_winevtlog.c +++ b/source/plugins/in_winevtlog/in_winevtlog.c @@ -199,22 +199,31 @@ static int in_winevtlog_init(struct flb_input_instance *in, } ctx->backoff.multiplier_x1000 = (DWORD)(mult * 1000.0); + /* normalize base/max/jitter/retries to sane ranges */ if (ctx->backoff.base_ms == 0) { ctx->backoff.base_ms = 500; } - if (ctx->backoff.max_ms == 0) { ctx->backoff.max_ms = 30000; } - if (ctx->backoff.jitter_pct == 0) { ctx->backoff.jitter_pct = 20; } - if (ctx->backoff.max_retries == 0) { ctx->backoff.max_retries = 8; } + /* clamp out-of-range values, protecting against negative INT written into DWORD */ + if (ctx->backoff.base_ms > 3600000U) { /* cap at 1 hour */ + ctx->backoff.base_ms = 3600000U; + } + if (ctx->backoff.max_ms > 86400000U) { /* cap at 24 hours */ + ctx->backoff.max_ms = 86400000U; + } + if (ctx->backoff.jitter_pct > 100U) { /* jitter as percentage */ + ctx->backoff.jitter_pct = 100U; + } + /* ensure ordering */ if (ctx->backoff.max_ms < ctx->backoff.base_ms) { flb_plg_warn(in, "reconnect.max_ms < reconnect.base_ms, swapping values"); tmp_ms = ctx->backoff.base_ms; diff --git a/source/plugins/in_winevtlog/winevtlog.c b/source/plugins/in_winevtlog/winevtlog.c index 036d59af..975ebdea 100644 --- a/source/plugins/in_winevtlog/winevtlog.c +++ b/source/plugins/in_winevtlog/winevtlog.c @@ -619,13 +619,13 @@ static int winevtlog_next(struct winevtlog_channel *ch, int hit_threshold) status = GetLastError(); if (status == ERROR_CANCELLED) { if (ch->cancelled_by_us) { - /* Conmsume this flag and return early */ + /* Consume this flag and return early */ ch->cancelled_by_us = FALSE; return FLB_FALSE; } ch->reconnect_needed = TRUE; ch->last_error = status; - flb_debug("[in_winevtlog] subscription cancelled unexpectedly (err=%lu), will reconnect", status); + flb_warn("[in_winevtlog] subscription cancelled unexpectedly (err=%lu), will reconnect", status); return FLB_FALSE; } if (status != ERROR_NO_MORE_ITEMS) { @@ -673,6 +673,7 @@ static DWORD calc_backoff_ms(struct winevtlog_channel *ch, const struct winevtlo LONG span = 0; LONG delta = 0; LONG with_jitter = 0; + DWORD jitter = 0; double mult = (double)cfg->multiplier_x1000 / 1000.0; double t = (double)cfg->base_ms; @@ -681,13 +682,11 @@ static DWORD calc_backoff_ms(struct winevtlog_channel *ch, const struct winevtlo if (t >= (double)cfg->max_ms) { t = (double)cfg->max_ms; break; } } ms = (DWORD)((t > (double)cfg->max_ms) ? cfg->max_ms : t); - /* ±jitter% */ - span = (LONG)((ms * cfg->jitter_pct) / 100); + /* ±jitter% (clamped 0..100) */ + jitter = cfg->jitter_pct > 100 ? 100 : cfg->jitter_pct; + span = (LONG)((ms * jitter) / 100); delta = (LONG)(prng16(&ch->prng_state) % (2 * span + 1)) - span; with_jitter = (LONG)ms + delta; - if (with_jitter < 0) { - with_jitter = 0; - } return (DWORD)with_jitter; } From a88c6bcb517723577d7311e6497457f6d941a94d Mon Sep 17 00:00:00 2001 From: Patrick Stephens Date: Sun, 3 May 2026 12:34:37 +0100 Subject: [PATCH 005/213] [upstream] in_winevtlog: Fix type glitches Upstream-Ref: https://github.com/fluent/fluent-bit/commit/4510a897642de187f72bbccd2ae08a32429a4e7c Cherry-picked from Fluent Bit v4.2.4 --- source/plugins/in_winevtlog/winevtlog.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/source/plugins/in_winevtlog/winevtlog.c b/source/plugins/in_winevtlog/winevtlog.c index 975ebdea..846ccae0 100644 --- a/source/plugins/in_winevtlog/winevtlog.c +++ b/source/plugins/in_winevtlog/winevtlog.c @@ -88,12 +88,12 @@ struct winevtlog_channel *winevtlog_subscribe(const char *channel, struct winevt // channel : To wide char len = MultiByteToWideChar(CP_UTF8, 0, channel, -1, NULL, 0); - wide_channel = flb_malloc(sizeof(PWSTR) * len); + wide_channel = flb_malloc(sizeof(WCHAR) * len); MultiByteToWideChar(CP_UTF8, 0, channel, -1, wide_channel, len); if (query != NULL) { // query : To wide char len = MultiByteToWideChar(CP_UTF8, 0, query, -1, NULL, 0); - wide_query = flb_malloc(sizeof(PWSTR) * len); + wide_query = flb_malloc(sizeof(WCHAR) * len); MultiByteToWideChar(CP_UTF8, 0, query, -1, wide_query, len); ch->query = flb_strdup(query); } @@ -1013,7 +1013,7 @@ static wchar_t* convert_str(char *str) return NULL; } - buf = flb_malloc(sizeof(PWSTR) * size); + buf = flb_malloc(sizeof(WCHAR) * size); if (buf == NULL) { flb_errno(); return NULL; From 2fc6988c0f78743dc0d06e8136037238bbe7438a Mon Sep 17 00:00:00 2001 From: Patrick Stephens Date: Sun, 3 May 2026 12:34:37 +0100 Subject: [PATCH 006/213] [upstream] in_winevtlog: Handle capped values Upstream-Ref: https://github.com/fluent/fluent-bit/commit/0a95d6277c204b89f40fb7933e297859f1d2f911 Cherry-picked from Fluent Bit v4.2.4 --- source/plugins/in_winevtlog/in_winevtlog.c | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/source/plugins/in_winevtlog/in_winevtlog.c b/source/plugins/in_winevtlog/in_winevtlog.c index 7aef090f..939f7ce0 100644 --- a/source/plugins/in_winevtlog/in_winevtlog.c +++ b/source/plugins/in_winevtlog/in_winevtlog.c @@ -200,15 +200,21 @@ static int in_winevtlog_init(struct flb_input_instance *in, ctx->backoff.multiplier_x1000 = (DWORD)(mult * 1000.0); /* normalize base/max/jitter/retries to sane ranges */ - if (ctx->backoff.base_ms == 0) { + if (ctx->backoff.base_ms <= 0) { ctx->backoff.base_ms = 500; } - if (ctx->backoff.max_ms == 0) { + if (ctx->backoff.max_ms <= 0) { ctx->backoff.max_ms = 30000; } + if (ctx->backoff.jitter_pct < 0) { + ctx->backoff.jitter_pct = 0; + } if (ctx->backoff.jitter_pct == 0) { ctx->backoff.jitter_pct = 20; } + if (ctx->backoff.max_retries < 0) { + ctx->backoff.max_retries = 0; + } if (ctx->backoff.max_retries == 0) { ctx->backoff.max_retries = 8; } @@ -223,6 +229,9 @@ static int in_winevtlog_init(struct flb_input_instance *in, if (ctx->backoff.jitter_pct > 100U) { /* jitter as percentage */ ctx->backoff.jitter_pct = 100U; } + if ((unsigned) ctx->backoff.max_retries > 100U) { /* cap retries */ + ctx->backoff.max_retries = 100; + } /* ensure ordering */ if (ctx->backoff.max_ms < ctx->backoff.base_ms) { flb_plg_warn(in, "reconnect.max_ms < reconnect.base_ms, swapping values"); From 40073beff17d3edc1b2598c2f66671c4eb6e9863 Mon Sep 17 00:00:00 2001 From: Patrick Stephens Date: Sun, 3 May 2026 12:34:37 +0100 Subject: [PATCH 007/213] [upstream] in_winevtlog: Plug NULL pointer deref lines Upstream-Ref: https://github.com/fluent/fluent-bit/commit/1d6d32ccd7eb7679a2069068133a317dc1fdb012 Cherry-picked from Fluent Bit v4.2.4 --- source/plugins/in_winevtlog/winevtlog.c | 50 ++++++++++++++++++++++++- 1 file changed, 48 insertions(+), 2 deletions(-) diff --git a/source/plugins/in_winevtlog/winevtlog.c b/source/plugins/in_winevtlog/winevtlog.c index 846ccae0..fdda6650 100644 --- a/source/plugins/in_winevtlog/winevtlog.c +++ b/source/plugins/in_winevtlog/winevtlog.c @@ -89,12 +89,58 @@ struct winevtlog_channel *winevtlog_subscribe(const char *channel, struct winevt // channel : To wide char len = MultiByteToWideChar(CP_UTF8, 0, channel, -1, NULL, 0); wide_channel = flb_malloc(sizeof(WCHAR) * len); - MultiByteToWideChar(CP_UTF8, 0, channel, -1, wide_channel, len); + if (wide_channel == NULL) { + if (signal_event) { + CloseHandle(signal_event); + } + flb_free(ch->name); + if (ch->query) { + flb_free(ch->query); + } + flb_free(ch); + return NULL; + } + if (0 == MultiByteToWideChar(CP_UTF8, 0, channel, -1, wide_channel, len)) { + if (signal_event) { + CloseHandle(signal_event); + } + flb_free(wide_channel); + flb_free(ch->name); + if (ch->query) { + flb_free(ch->query); + } + flb_free(ch); + return NULL; + } if (query != NULL) { // query : To wide char len = MultiByteToWideChar(CP_UTF8, 0, query, -1, NULL, 0); wide_query = flb_malloc(sizeof(WCHAR) * len); - MultiByteToWideChar(CP_UTF8, 0, query, -1, wide_query, len); + if (wide_query == NULL) { + if (signal_event) { + CloseHandle(signal_event); + } + flb_free(wide_channel); + flb_free(ch->name); + if (ch->query) { + flb_free(ch->query); + } + flb_free(ch); + return NULL; + } + if (0 == MultiByteToWideChar(CP_UTF8, 0, query, -1, wide_query, len)) { + if (signal_event) { + CloseHandle(signal_event); + } + flb_free(wide_query); + flb_free(wide_channel); + flb_free(ch->name); + if (ch->query) { + flb_free(ch->query); + } + flb_free(ch); + return NULL; + } ch->query = flb_strdup(query); } From dfb0700ff60110e9f01c62e434c20f3debb0e1a8 Mon Sep 17 00:00:00 2001 From: Patrick Stephens Date: Sun, 3 May 2026 12:34:37 +0100 Subject: [PATCH 008/213] [upstream] in_winevtlog: Process remote handle to retrive Upstream-Ref: https://github.com/fluent/fluent-bit/commit/84e73c1a7e6f40b3e09ed8b4798828b361ab27a9 Cherry-picked from Fluent Bit v4.2.4 --- source/plugins/in_winevtlog/winevtlog.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/source/plugins/in_winevtlog/winevtlog.c b/source/plugins/in_winevtlog/winevtlog.c index fdda6650..c50e467e 100644 --- a/source/plugins/in_winevtlog/winevtlog.c +++ b/source/plugins/in_winevtlog/winevtlog.c @@ -83,6 +83,7 @@ struct winevtlog_channel *winevtlog_subscribe(const char *channel, struct winevt return NULL; } ch->query = NULL; + ch->remote = NULL; signal_event = CreateEvent(NULL, TRUE, TRUE, NULL); @@ -508,7 +509,7 @@ PWSTR get_message(EVT_HANDLE metadata, EVT_HANDLE handle, unsigned int *message_ return message; } -PWSTR get_description(EVT_HANDLE handle, LANGID langID, unsigned int *message_size) +PWSTR get_description(EVT_HANDLE handle, LANGID langID, unsigned int *message_size, HANDLE remote) { PEVT_VARIANT values = NULL; DWORD buffer_size = 0; @@ -555,7 +556,7 @@ PWSTR get_description(EVT_HANDLE handle, LANGID langID, unsigned int *message_si /* Metadata can be NULL because some of the events do not have an * associated publisher metadata. */ metadata = EvtOpenPublisherMetadata( - NULL, // TODO: Remote handle + remote, values[0].StringVal, NULL, MAKELCID(langID, SORT_DEFAULT), @@ -890,7 +891,7 @@ int winevtlog_read(struct winevtlog_channel *ch, struct winevtlog_config *ctx, for (i = 0; i < ch->count; i++) { if (ctx->render_event_as_xml) { system_xml = render_event(ch->events[i], EvtRenderEventXml, &system_size); - message = get_description(ch->events[i], LANG_NEUTRAL, &message_size); + message = get_description(ch->events[i], LANG_NEUTRAL, &message_size, ch->remote); get_string_inserts(ch->events[i], &string_inserts, &count_inserts, &string_inserts_size); if (system_xml) { /* Caluculate total allocated size: system + message + string_inserts */ @@ -906,7 +907,7 @@ int winevtlog_read(struct winevtlog_channel *ch, struct winevtlog_config *ctx, } else { render_system_event(ch->events[i], &rendered_system, &system_size); - message = get_description(ch->events[i], LANG_NEUTRAL, &message_size); + message = get_description(ch->events[i], LANG_NEUTRAL, &message_size, ch->remote); get_string_inserts(ch->events[i], &string_inserts, &count_inserts, &string_inserts_size); if (rendered_system) { /* Caluculate total allocated size: system + message + string_inserts */ From ea97b1fd75cb4575f1103d53e90fd99c2ab50f56 Mon Sep 17 00:00:00 2001 From: Patrick Stephens Date: Sun, 3 May 2026 12:34:38 +0100 Subject: [PATCH 009/213] [upstream] in_winevtlog: Plug a negative overflow possibility Upstream-Ref: https://github.com/fluent/fluent-bit/commit/20a63b867fe881a09a8d10abe69476af847f7a92 Cherry-picked from Fluent Bit v4.2.4 --- source/plugins/in_winevtlog/winevtlog.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/source/plugins/in_winevtlog/winevtlog.c b/source/plugins/in_winevtlog/winevtlog.c index c50e467e..007e2350 100644 --- a/source/plugins/in_winevtlog/winevtlog.c +++ b/source/plugins/in_winevtlog/winevtlog.c @@ -734,6 +734,9 @@ static DWORD calc_backoff_ms(struct winevtlog_channel *ch, const struct winevtlo span = (LONG)((ms * jitter) / 100); delta = (LONG)(prng16(&ch->prng_state) % (2 * span + 1)) - span; with_jitter = (LONG)ms + delta; + if (with_jitter < 0) { + with_jitter = 0; + } return (DWORD)with_jitter; } From 68019f73c543a16eaeb07cda1f18e8d6347986be Mon Sep 17 00:00:00 2001 From: Patrick Stephens Date: Sun, 3 May 2026 12:34:38 +0100 Subject: [PATCH 010/213] [upstream] in_winevtlog: Address more coderabbitai comments Upstream-Ref: https://github.com/fluent/fluent-bit/commit/088fdfeefe223aadb875f99fe6010760044c411e Cherry-picked from Fluent Bit v4.2.4 --- source/plugins/in_winevtlog/in_winevtlog.c | 6 ------ source/plugins/in_winevtlog/winevtlog.c | 15 ++++++++++++--- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/source/plugins/in_winevtlog/in_winevtlog.c b/source/plugins/in_winevtlog/in_winevtlog.c index 939f7ce0..34a83bc1 100644 --- a/source/plugins/in_winevtlog/in_winevtlog.c +++ b/source/plugins/in_winevtlog/in_winevtlog.c @@ -209,15 +209,9 @@ static int in_winevtlog_init(struct flb_input_instance *in, if (ctx->backoff.jitter_pct < 0) { ctx->backoff.jitter_pct = 0; } - if (ctx->backoff.jitter_pct == 0) { - ctx->backoff.jitter_pct = 20; - } if (ctx->backoff.max_retries < 0) { ctx->backoff.max_retries = 0; } - if (ctx->backoff.max_retries == 0) { - ctx->backoff.max_retries = 8; - } /* clamp out-of-range values, protecting against negative INT written into DWORD */ if (ctx->backoff.base_ms > 3600000U) { /* cap at 1 hour */ diff --git a/source/plugins/in_winevtlog/winevtlog.c b/source/plugins/in_winevtlog/winevtlog.c index 007e2350..627ff34b 100644 --- a/source/plugins/in_winevtlog/winevtlog.c +++ b/source/plugins/in_winevtlog/winevtlog.c @@ -180,13 +180,22 @@ struct winevtlog_channel *winevtlog_subscribe(const char *channel, struct winevt err = GetLastError(); if (!ch->subscription) { flb_plg_error(ctx->ins, "cannot subscribe '%s' (%i)", channel, err); - flb_free(ch->name); - if (ch->query != NULL) { - flb_free(ch->query); + if (signal_event) { + CloseHandle(signal_event); } if (ch->remote) { EvtClose(ch->remote); } + if (wide_channel) { + flb_free(wide_channel); + } + if (wide_query) { + flb_free(wide_query); + } + flb_free(ch->name); + if (ch->query != NULL) { + flb_free(ch->query); + } flb_free(ch); return NULL; } From b3f6ee59032b65be804cee3c796e3bd4390239e9 Mon Sep 17 00:00:00 2001 From: Patrick Stephens Date: Sun, 3 May 2026 12:34:38 +0100 Subject: [PATCH 011/213] [upstream] in_winevtlog: Apply backoff settings effectively Upstream-Ref: https://github.com/fluent/fluent-bit/commit/5fff9442bfa1ea5f3ff637ac122f5a80f093f4d0 Cherry-picked from Fluent Bit v4.2.4 --- source/plugins/in_winevtlog/winevtlog.c | 26 +++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/source/plugins/in_winevtlog/winevtlog.c b/source/plugins/in_winevtlog/winevtlog.c index 627ff34b..df9f4358 100644 --- a/source/plugins/in_winevtlog/winevtlog.c +++ b/source/plugins/in_winevtlog/winevtlog.c @@ -707,13 +707,23 @@ static int winevtlog_next(struct winevtlog_channel *ch, int hit_threshold) return FLB_FALSE; } -static inline void backoff_defaults(struct winevtlog_backoff *dst, const struct winevtlog_backoff *src) +static const struct winevtlog_backoff WINEVTLOG_BACKOFF_DEFAULTS = { + 500, /* base_ms */ + 30000, /* max_ms */ + 2000, /* multiplier_x1000 == x2.0 */ + 20, /* jitter_pct */ + 8 /* max_retries */ +}; + +static inline void backoff_effective(struct winevtlog_backoff *dst, + const struct winevtlog_backoff *src) { - dst->base_ms = (src && src->base_ms) ? src->base_ms : 500; - dst->max_ms = (src && src->max_ms) ? src->max_ms : 30000; - dst->multiplier_x1000 = (src && src->multiplier_x1000) ? src->multiplier_x1000 : 2000; /* x2.0 */ - dst->jitter_pct = (src && src->jitter_pct) ? src->jitter_pct : 20; - dst->max_retries = (src && src->max_retries) ? src->max_retries : 8; + if (src) { + *dst = *src; + } + else { + *dst = WINEVTLOG_BACKOFF_DEFAULTS; + } } static inline DWORD prng16(ULONGLONG *state) @@ -753,7 +763,7 @@ void winevtlog_schedule_retry(struct winevtlog_channel *ch, struct winevtlog_con { struct winevtlog_backoff cfg; DWORD delay = 0; - backoff_defaults(&cfg, ctx ? &ctx->backoff : NULL); + backoff_effective(&cfg, ctx ? &ctx->backoff : NULL); delay = calc_backoff_ms(ch, &cfg, ch->retry_attempts); ch->next_retry_deadline = GetTickCount64() + (ULONGLONG)delay; } @@ -958,7 +968,7 @@ int winevtlog_read(struct winevtlog_channel *ch, struct winevtlog_config *ctx, int rc = winevtlog_try_reconnect(ch, ctx); if (rc != 0) { struct winevtlog_backoff eff; - backoff_defaults(&eff, ctx ? &ctx->backoff : NULL); + backoff_effective(&eff, ctx ? &ctx->backoff : NULL); if (ch->retry_attempts < eff.max_retries) { ch->retry_attempts++; winevtlog_schedule_retry(ch, ctx); From 303b286a268e8e400fb90508bbddaf1bb35d1d64 Mon Sep 17 00:00:00 2001 From: Patrick Stephens Date: Sun, 3 May 2026 12:34:39 +0100 Subject: [PATCH 012/213] [upstream] in_forward: tests: Add fail-close around test cases Upstream-Ref: https://github.com/fluent/fluent-bit/commit/67ae3b047258287234122a0b2344939f8c4e1f38 Cherry-picked from Fluent Bit v4.2.4 --- source/tests/runtime/in_forward.c | 146 ++++++++++++++++++++++++++++++ 1 file changed, 146 insertions(+) diff --git a/source/tests/runtime/in_forward.c b/source/tests/runtime/in_forward.c index 3f9f6d7d..7b36394f 100644 --- a/source/tests/runtime/in_forward.c +++ b/source/tests/runtime/in_forward.c @@ -1160,6 +1160,148 @@ void flb_test_threaded_forward_issue_10946() flb_destroy(ctx); } +static flb_ctx_t *fw_make_ctx_with_forward(int *in_ffd_out, int *out_ffd_out) +{ + struct flb_lib_out_cb cb = {0}; + flb_ctx_t *ctx; + int in_ffd, out_ffd, ret; + + ctx = flb_create(); + TEST_CHECK(ctx != NULL); + if (!ctx) { return NULL; } + + flb_service_set(ctx, + "Flush", "0.200000000", + "Grace", "1", + "Log_Level", "error", + NULL); + + /* forward input */ + in_ffd = flb_input(ctx, (char *) "forward", NULL); + TEST_CHECK(in_ffd >= 0); + if (in_ffd < 0) { flb_destroy(ctx); return NULL; } + + /* lib output: count only (no payload check) */ + cb.cb = cb_count_only; + cb.data = NULL; + out_ffd = flb_output(ctx, (char *) "lib", (void *) &cb); + TEST_CHECK(out_ffd >= 0); + if (out_ffd < 0) { + flb_destroy(ctx); + return NULL; + } + ret = flb_output_set(ctx, out_ffd, + "match", "*", + "format", "json", + NULL); + TEST_CHECK(ret == 0); + + if (in_ffd_out) *in_ffd_out = in_ffd; + if (out_ffd_out) *out_ffd_out = out_ffd; + return ctx; +} + +/* 1) users-only => must fail to start (fail-close) */ +void flb_test_fw_auth_users_only_fail_start() +{ + flb_ctx_t *ctx; + int in_ffd, out_ffd, ret; + + ctx = fw_make_ctx_with_forward(&in_ffd, &out_ffd); + TEST_CHECK(ctx != NULL); + if (!ctx) { + return; + } + + ret = flb_input_set(ctx, in_ffd, + "tag", "test", + "security.users", "alice s3cr3t", + NULL); + TEST_CHECK(ret == 0); + + ret = flb_start(ctx); + TEST_CHECK(ret != 0); + if (ret == 0) { + TEST_MSG("users-only config unexpectedly started; fail-close not enforced"); + flb_stop(ctx); + } + flb_destroy(ctx); +} + +/* 2) empty_shared_key + users => start OK */ +void flb_test_fw_auth_empty_shared_key_plus_users_start_ok() +{ + flb_ctx_t *ctx; + int in_ffd, out_ffd, ret; + + ctx = fw_make_ctx_with_forward(&in_ffd, &out_ffd); + TEST_CHECK(ctx != NULL); + if (!ctx) { return; } + + ret = flb_input_set(ctx, in_ffd, + "tag", "test", + "empty_shared_key", "true", + "security.users", "alice s3cr3t", + NULL); + TEST_CHECK(ret == 0); + + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + if (ret == 0) { + flb_stop(ctx); + } + flb_destroy(ctx); +} + +/* 3) shared_key only => start OK (backward compatible) */ +void flb_test_fw_auth_shared_key_only_start_ok() +{ + flb_ctx_t *ctx; + int in_ffd, out_ffd, ret; + + ctx = fw_make_ctx_with_forward(&in_ffd, &out_ffd); + TEST_CHECK(ctx != NULL); + if (!ctx) { return; } + + ret = flb_input_set(ctx, in_ffd, + "tag", "test", + "shared_key", "k", + NULL); + TEST_CHECK(ret == 0); + + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + if (ret == 0) { + flb_stop(ctx); + } + flb_destroy(ctx); +} + +/* 4) shared_key + users => start OK (both checks) */ +void flb_test_fw_auth_shared_key_plus_users_start_ok() +{ + flb_ctx_t *ctx; + int in_ffd, out_ffd, ret; + + ctx = fw_make_ctx_with_forward(&in_ffd, &out_ffd); + TEST_CHECK(ctx != NULL); + if (!ctx) { return; } + + ret = flb_input_set(ctx, in_ffd, + "tag", "test", + "shared_key", "k", + "security.users", "alice s3cr3t", + NULL); + TEST_CHECK(ret == 0); + + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + if (ret == 0) { + flb_stop(ctx); + } + flb_destroy(ctx); +} + TEST_LIST = { {"forward", flb_test_forward}, {"forward_port", flb_test_forward_port}, @@ -1176,5 +1318,9 @@ TEST_LIST = { {"forward_gzip", flb_test_forward_gzip}, {"forward_zstd", flb_test_forward_zstd}, {"issue_10946", flb_test_threaded_forward_issue_10946}, + {"fw_auth_users_only_fail_start", flb_test_fw_auth_users_only_fail_start}, + {"fw_auth_empty_shared_key_plus_users_start_ok", flb_test_fw_auth_empty_shared_key_plus_users_start_ok}, + {"fw_auth_shared_key_only_start_ok", flb_test_fw_auth_shared_key_only_start_ok}, + {"fw_auth_shared_key_plus_users_start_ok", flb_test_fw_auth_shared_key_plus_users_start_ok}, {NULL, NULL} }; From 2c2d589fb29451312fe7856a56723dee4e19a92b Mon Sep 17 00:00:00 2001 From: Patrick Stephens Date: Sun, 3 May 2026 12:34:40 +0100 Subject: [PATCH 013/213] [upstream] out_es: create space for header before use Upstream-Ref: https://github.com/fluent/fluent-bit/commit/912b7d783a328c09c82aeb16bd26330a468f2005 Cherry-picked from Fluent Bit v4.2.4 --- source/plugins/out_es/es.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/source/plugins/out_es/es.c b/source/plugins/out_es/es.c index f7e7a5de..8320fab6 100644 --- a/source/plugins/out_es/es.c +++ b/source/plugins/out_es/es.c @@ -890,11 +890,13 @@ static void cb_es_flush(struct flb_event_chunk *event_chunk, flb_http_basic_auth(c, ctx->cloud_user, ctx->cloud_passwd); } else if (ctx->http_api_key) { - header_line = flb_sds_printf(NULL, "ApiKey %s", ctx->http_api_key); + /* 7 for ApiKey + space */ + header_line = flb_sds_create_size(strlen(ctx->http_api_key)+7); if (header_line == NULL) { flb_plg_error(ctx->ins, "failed to format API key auth header"); goto retry; } + header_line = flb_sds_printf(&header_line, "ApiKey %s", ctx->http_api_key); if (flb_http_add_header(c, FLB_HTTP_HEADER_AUTH, strlen(FLB_HTTP_HEADER_AUTH), From dd05d8f1d9b49727d94226f3b62fd6fb7b8f825f Mon Sep 17 00:00:00 2001 From: Patrick Stephens Date: Sun, 3 May 2026 12:34:41 +0100 Subject: [PATCH 014/213] [upstream] help: add space for outputs and processors when Upstream-Ref: https://github.com/fluent/fluent-bit/commit/36db973c31a8cf432062e756b28affa8c90b3853 Cherry-picked from Fluent Bit v4.2.4 --- source/src/flb_help.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/source/src/flb_help.c b/source/src/flb_help.c index 234b1218..3dd1cfc6 100644 --- a/source/src/flb_help.c +++ b/source/src/flb_help.c @@ -710,10 +710,11 @@ flb_sds_t flb_help_build_json_schema(struct flb_config *config) * - fluent-bit * - customs * - inputs + * - processors * - filters * - outputs */ - msgpack_pack_map(&mp_pck, 5); + msgpack_pack_map(&mp_pck, 6); /* Fluent Bit */ msgpack_pack_str(&mp_pck, 10); From b953333cc6a9b27b196a6442dfba579936db20a8 Mon Sep 17 00:00:00 2001 From: Patrick Stephens Date: Sun, 3 May 2026 12:34:41 +0100 Subject: [PATCH 015/213] [upstream] opentelemetry: unify trace JSON parser Upstream-Ref: https://github.com/fluent/fluent-bit/commit/03208dbc64a1fe87b7e7d74f45da2f42a6f809c9 Cherry-picked from Fluent Bit v4.2.4 --- source/include/fluent-bit/flb_opentelemetry.h | 3 + .../opentelemetry/flb_opentelemetry_traces.c | 1102 +++++++++++++++++ 2 files changed, 1105 insertions(+) create mode 100644 source/src/opentelemetry/flb_opentelemetry_traces.c diff --git a/source/include/fluent-bit/flb_opentelemetry.h b/source/include/fluent-bit/flb_opentelemetry.h index 8919f0ab..dc137bf9 100644 --- a/source/include/fluent-bit/flb_opentelemetry.h +++ b/source/include/fluent-bit/flb_opentelemetry.h @@ -124,6 +124,9 @@ int flb_opentelemetry_logs_json_to_msgpack(struct flb_log_event_encoder *encoder const char *logs_body_key, int *error_status); +struct ctrace *flb_opentelemetry_json_traces_to_ctrace(const char *body, size_t len, + int *error_status); + /* OpenTelemetry utils */ int flb_otel_utils_find_map_entry_by_key(msgpack_object_map *map, char *key, diff --git a/source/src/opentelemetry/flb_opentelemetry_traces.c b/source/src/opentelemetry/flb_opentelemetry_traces.c new file mode 100644 index 00000000..146cbf5d --- /dev/null +++ b/source/src/opentelemetry/flb_opentelemetry_traces.c @@ -0,0 +1,1102 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2025 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#include + +static int process_attribute(struct ctrace_attributes *attr, + msgpack_object *key, msgpack_object *value, int type) +{ + int ret; + char *key_str; + char *value_str; + int64_t value_int; + double value_double; + int value_bool; + + if (key->type != MSGPACK_OBJECT_STR) { + return -1; + } + + /* temporary buffer for key since it needs to be NULL terminated */ + key_str = flb_sds_create_len(key->via.str.ptr, key->via.str.size); + if (key_str == NULL) { + return -1; + } + + /* + * the value of 'type' is set by the JSON wrapped value, we need to to convert it + * since msgpack_object *value is always a string + */ + switch (type) { + case MSGPACK_OBJECT_STR: + if (value->type != MSGPACK_OBJECT_STR) { + flb_sds_destroy(key_str); + return -1; + } + + value_str = flb_sds_create_len(value->via.str.ptr, value->via.str.size); + if (value_str == NULL) { + flb_sds_destroy(key_str); + return -1; + } + + ret = ctr_attributes_set_string(attr, key_str, value_str); + flb_sds_destroy(value_str); + break; + case MSGPACK_OBJECT_POSITIVE_INTEGER: + case MSGPACK_OBJECT_NEGATIVE_INTEGER: + if (value->type == MSGPACK_OBJECT_STR) { + value_str = flb_sds_create_len(value->via.str.ptr, value->via.str.size); + if (value_str == NULL) { + flb_sds_destroy(key_str); + return -1; + } + value_int = strtoll(value_str, NULL, 10); + flb_sds_destroy(value_str); + } + else if (value->type == MSGPACK_OBJECT_POSITIVE_INTEGER || + value->type == MSGPACK_OBJECT_NEGATIVE_INTEGER) { + value_int = value->via.i64; + } + else { + flb_sds_destroy(key_str); + return -1; + } + + ret = ctr_attributes_set_int64(attr, key_str, value_int); + break; + case MSGPACK_OBJECT_FLOAT32: + case MSGPACK_OBJECT_FLOAT64: + if (value->type == MSGPACK_OBJECT_STR) { + value_str = flb_sds_create_len(value->via.str.ptr, value->via.str.size); + if (value_str == NULL) { + flb_sds_destroy(key_str); + return -1; + } + value_double = strtod(value_str, NULL); + flb_sds_destroy(value_str); + } + else if (value->type == MSGPACK_OBJECT_FLOAT32 || + value->type == MSGPACK_OBJECT_FLOAT64) { + value_double = value->via.f64; + } + else { + flb_sds_destroy(key_str); + return -1; + } + + ret = ctr_attributes_set_double(attr, key_str, value_double); + break; + case MSGPACK_OBJECT_BOOLEAN: + if (value->type != MSGPACK_OBJECT_BOOLEAN) { + flb_sds_destroy(key_str); + return -1; + } + + value_bool = value->via.boolean; + ret = ctr_attributes_set_bool(attr, key_str, value_bool); + break; + case MSGPACK_OBJECT_ARRAY: + /* + * The less fun part (OTel JSON encoding), the value can be an array and + * only allows values such as string, bool, int64, double. I am glad this + * don't support nested arrays or maps. + */ + ret = 0; + break; + default: + flb_sds_destroy(key_str); + return -1; + } + + flb_sds_destroy(key_str); + return ret; +} + +static int process_resource_unwrap_attribute(msgpack_object *attr, + msgpack_object *out_key, + msgpack_object *out_value, int *out_value_type) +{ + int ret; + int type; + msgpack_object key; + msgpack_object val; + msgpack_object *real_value; + + if (attr->type != MSGPACK_OBJECT_MAP) { + return -1; + } + + ret = flb_otel_utils_find_map_entry_by_key(&attr->via.map, "key", 0, FLB_TRUE); + if (ret == -1) { + return -1; + } + + key = attr->via.map.ptr[ret].val; + if (key.type != MSGPACK_OBJECT_STR) { + return -1; + } + + ret = flb_otel_utils_find_map_entry_by_key(&attr->via.map, "value", 0, FLB_TRUE); + if (ret == -1) { + return -1; + } + + val = attr->via.map.ptr[ret].val; + + ret = flb_otel_utils_json_payload_get_wrapped_value(&val, &real_value, &type); + if (ret != 0) { + return -1; + } + + *out_key = key; + *out_value = *real_value; + *out_value_type = type; + + return 0; +} + +/* + * Convert a list of attributes in msgpack format to a cfl attributes by + * unwrapping JSON encoded value types. + */ +static struct ctrace_attributes *convert_attributes(msgpack_object *attributes, + const char *log_context) +{ + int i; + int ret; + int value_type; + msgpack_object key; + msgpack_object value; + struct ctrace_attributes *attr; + + attr = ctr_attributes_create(); + if (attr == NULL) { + return NULL; + } + + for (i = 0; i < attributes->via.array.size; i++) { + ret = process_resource_unwrap_attribute(&attributes->via.array.ptr[i], + &key, &value, &value_type); + if (ret == -1) { + flb_warn("found invalid %s attribute, skipping", log_context); + continue; + } + + /* set attribute */ + ret = process_attribute(attr, &key, &value, value_type); + if (ret == -1) { + flb_warn("failed to set %s attribute, skipping", log_context); + continue; + } + } + + return attr; +} + +static int process_resource_attributes(struct ctrace *ctr, + struct ctrace_resource_span *resource_span, + msgpack_object *attributes) +{ + struct ctrace_resource *resource; + struct ctrace_attributes *attr; + + attr = convert_attributes(attributes, "trace resource"); + if (!attr) { + return -1; + } + + resource = ctr_resource_span_get_resource(resource_span); + ctr_resource_set_attributes(resource, attr); + + return 0; +} + +static int process_scope_attributes(struct ctrace *ctr, + struct ctrace_scope_span *scope_span, + msgpack_object *name, + msgpack_object *version, + msgpack_object *attributes, + msgpack_object *dropped_attributes_count) +{ + int dropped = 0; + cfl_sds_t name_str = NULL; + cfl_sds_t version_str = NULL; + struct ctrace_attributes *attr = NULL; + struct ctrace_instrumentation_scope *ins_scope; + + if (attributes) { + attr = convert_attributes(attributes, "trace scope"); + if (!attr) { + return -1; + } + } + + if (name) { + name_str = cfl_sds_create_len(name->via.str.ptr, name->via.str.size); + } + if (version) { + version_str = cfl_sds_create_len(version->via.str.ptr, version->via.str.size); + } + + if (dropped_attributes_count) { + dropped = dropped_attributes_count->via.u64; + } + + ins_scope = ctr_instrumentation_scope_create(name_str, version_str, dropped, attr); + if (!ins_scope) { + if (name_str) { + cfl_sds_destroy(name_str); + } + if (version_str) { + cfl_sds_destroy(version_str); + } + if (attr) { + ctr_attributes_destroy(attr); + } + return -1; + } + + if (name_str) { + cfl_sds_destroy(name_str); + } + if (version_str) { + cfl_sds_destroy(version_str); + } + + ctr_scope_span_set_instrumentation_scope(scope_span, ins_scope); + return 0; +} + +static int process_events(struct ctrace *ctr, + struct ctrace_span *span, + msgpack_object *events) +{ + int i; + int ret; + int len; + uint64_t ts = 0; + char tmp[64]; + cfl_sds_t name_str = NULL; + msgpack_object event; + msgpack_object *name = NULL; + msgpack_object *attr = NULL; + struct ctrace_span_event *ctr_event = NULL; + struct ctrace_attributes *ctr_attr = NULL; + + for (i = 0; i < events->via.array.size; i++) { + event = events->via.array.ptr[i]; + if (event.type != MSGPACK_OBJECT_MAP) { + flb_error("unexpected event type"); + return -1; + } + + name_str = NULL; + ts = 0; + + /* name */ + ret = flb_otel_utils_find_map_entry_by_key(&event.via.map, "name", 0, FLB_TRUE); + if (ret >= 0 && event.via.map.ptr[ret].val.type == MSGPACK_OBJECT_STR) { + name = &event.via.map.ptr[ret].val; + name_str = cfl_sds_create_len(name->via.str.ptr, name->via.str.size); + if (name_str == NULL) { + return -1; + } + } + + if (!name_str) { + flb_warn("span event name is missing"); + return -1; + } + + /* time_unix_nano */ + ret = flb_otel_utils_find_map_entry_by_key(&event.via.map, "timeUnixNano", 0, FLB_TRUE); + if (ret >= 0 && event.via.map.ptr[ret].val.type == MSGPACK_OBJECT_STR) { + /* convert to uint64_t */ + len = event.via.map.ptr[ret].val.via.str.size; + if (len > sizeof(tmp) - 1) { + len = sizeof(tmp) - 1; + memcpy(tmp, event.via.map.ptr[ret].val.via.str.ptr, len); + tmp[len] = '\0'; + + flb_error("invalid timeUnixNano: '%s'", tmp); + if (name_str) { + cfl_sds_destroy(name_str); + } + return -1; + } + + memcpy(tmp, event.via.map.ptr[ret].val.via.str.ptr, len); + tmp[len] = '\0'; + + ts = strtoull(tmp, NULL, 10); + } + + ctr_event = ctr_span_event_add_ts(span, name_str, ts); + cfl_sds_destroy(name_str); + if (ctr_event == NULL) { + return -1; + } + + /* attributes */ + ret = flb_otel_utils_find_map_entry_by_key(&event.via.map, "attributes", 0, FLB_TRUE); + if (ret >= 0 && event.via.map.ptr[ret].val.type == MSGPACK_OBJECT_ARRAY) { + attr = &event.via.map.ptr[ret].val; + ctr_attr = convert_attributes(attr, "span event"); + if (ctr_attr) { + ctr_span_event_set_attributes(ctr_event, ctr_attr); + } + } + + /* dropped_attributes_count */ + ret = flb_otel_utils_find_map_entry_by_key(&event.via.map, "droppedAttributesCount", 0, FLB_FALSE); + if (ret >= 0 && event.via.map.ptr[ret].val.type == MSGPACK_OBJECT_POSITIVE_INTEGER) { + ctr_span_event_set_dropped_attributes_count(ctr_event, event.via.map.ptr[ret].val.via.u64); + } + } + + return 0; +} + +static int process_links(struct ctrace *ctr, + struct ctrace_span *span, + msgpack_object *links) +{ + int i; + int ret; + int len; + char tmp[64]; + char trace_id_bin[16]; + char span_id_bin[8]; + cfl_sds_t buf; + msgpack_object link; + msgpack_object *trace_id = NULL; + msgpack_object *span_id = NULL; + msgpack_object *trace_state = NULL; + msgpack_object *dropped_attr_count = NULL; + msgpack_object *flags = NULL; + msgpack_object *attr = NULL; + + struct ctrace_link *ctr_link = NULL; + struct ctrace_attributes *ctr_attr = NULL; + + for (i = 0; i < links->via.array.size; i++) { + link = links->via.array.ptr[i]; + if (link.type != MSGPACK_OBJECT_MAP) { + flb_error("unexpected link type"); + return -1; + } + + trace_id = NULL; + span_id = NULL; + trace_state = NULL; + dropped_attr_count = NULL; + flags = NULL; + ctr_attr = NULL; + + /* traceId */ + ret = flb_otel_utils_find_map_entry_by_key(&link.via.map, "traceId", 0, FLB_TRUE); + if (ret >= 0 && link.via.map.ptr[ret].val.type == MSGPACK_OBJECT_STR) { + trace_id = &link.via.map.ptr[ret].val; + + /* trace_id is an hex of 32 bytes */ + if (trace_id->via.str.size != 32) { + len = trace_id->via.str.size; + if (len > sizeof(tmp) - 1) { + len = sizeof(tmp) - 1; + } + memcpy(tmp, trace_id->via.str.ptr, len); + tmp[len] = '\0'; + + flb_error("invalid event traceId: '%s'", tmp); + return -1; + } + + /* decode the hex string (16 bytes) */ + flb_otel_utils_hex_to_id((char *) trace_id->via.str.ptr, trace_id->via.str.size, + (unsigned char *) trace_id_bin, 16); + } + + if (!trace_id) { + flb_error("link traceId is missing"); + return -1; + } + + /* spanId */ + ret = flb_otel_utils_find_map_entry_by_key(&link.via.map, "spanId", 0, FLB_TRUE); + if (ret >= 0 && link.via.map.ptr[ret].val.type == MSGPACK_OBJECT_STR) { + span_id = &link.via.map.ptr[ret].val; + + /* span_id is an hex of 16 bytes */ + if (span_id->via.str.size != 16) { + len = span_id->via.str.size; + if (len > sizeof(tmp) - 1) { + len = sizeof(tmp) - 1; + } + memcpy(tmp, span_id->via.str.ptr, len); + tmp[len] = '\0'; + flb_error("invalid spanId: '%s'", tmp); + return -1; + } + + /* decode the hex string (8 bytes) */ + memset(tmp, '\0', sizeof(tmp)); + flb_otel_utils_hex_to_id((char *) span_id->via.str.ptr, span_id->via.str.size, + (unsigned char *) span_id_bin, 8); + } + + if (!span_id) { + flb_error("link spanId is missing"); + return -1; + } + + /* traceState */ + ret = flb_otel_utils_find_map_entry_by_key(&link.via.map, "traceState", 0, FLB_FALSE); + if (ret >= 0 && link.via.map.ptr[ret].val.type == MSGPACK_OBJECT_STR) { + trace_state = &link.via.map.ptr[ret].val; + } + + /* attributes */ + ret = flb_otel_utils_find_map_entry_by_key(&link.via.map, "attributes", 0, FLB_FALSE); + if (ret >= 0 && link.via.map.ptr[ret].val.type == MSGPACK_OBJECT_ARRAY) { + attr = &link.via.map.ptr[ret].val; + ctr_attr = convert_attributes(attr, "event link"); + } + + /* droped_attributes_count */ + ret = flb_otel_utils_find_map_entry_by_key(&link.via.map, "droppedAttributesCount", 0, FLB_FALSE); + if (ret >= 0 && link.via.map.ptr[ret].val.type == MSGPACK_OBJECT_POSITIVE_INTEGER) { + dropped_attr_count = &link.via.map.ptr[ret].val; + } + + /* flags */ + ret = flb_otel_utils_find_map_entry_by_key(&link.via.map, "flags", 0, FLB_FALSE); + if (ret >= 0 && link.via.map.ptr[ret].val.type == MSGPACK_OBJECT_POSITIVE_INTEGER) { + flags = &link.via.map.ptr[ret].val; + } + + ctr_link = ctr_link_create(span, + trace_id_bin, 16, + span_id_bin, 8); + + if (ctr_link == NULL) { + if (ctr_attr) { + ctr_attributes_destroy(ctr_attr); + } + return -1; + } + + if (trace_state) { + buf = cfl_sds_create_len(trace_state->via.str.ptr, trace_state->via.str.size); + if (buf) { + ctr_link_set_trace_state(ctr_link, buf); + cfl_sds_destroy(buf); + } + } + + if (ctr_attr) { + ctr_link_set_attributes(ctr_link, ctr_attr); + } + + if (dropped_attr_count) { + ctr_link_set_dropped_attr_count(ctr_link, dropped_attr_count->via.u64); + } + + if (flags) { + ctr_link_set_flags(ctr_link, flags->via.u64); + } + } + + return 0; +} + +static int process_span_status(struct ctrace *ctr, + struct ctrace_span *span, + msgpack_object *status) +{ + int ret; + int code = 0; + cfl_sds_t tmp = NULL; + char *message = NULL; + + if (status->type != MSGPACK_OBJECT_MAP) { + flb_error("unexpected status type"); + return -1; + } + + /* code */ + ret = flb_otel_utils_find_map_entry_by_key(&status->via.map, "code", 0, FLB_TRUE); + if (ret >= 0 && status->via.map.ptr[ret].val.type == MSGPACK_OBJECT_STR) { + tmp = cfl_sds_create_len(status->via.map.ptr[ret].val.via.str.ptr, + status->via.map.ptr[ret].val.via.str.size); + if (!tmp) { + return -1; + } + + if (strcasecmp(tmp, "UNSET") == 0) { + code = CTRACE_SPAN_STATUS_CODE_UNSET; + } + else if (strcasecmp(tmp, "OK") == 0) { + code = CTRACE_SPAN_STATUS_CODE_OK; + } + else if (strcasecmp(tmp, "ERROR") == 0) { + code = CTRACE_SPAN_STATUS_CODE_ERROR; + } + else { + flb_error("status code value is invalid: %s", tmp); + cfl_sds_destroy(tmp); + return -1; + } + cfl_sds_destroy(tmp); + } + else { + flb_error("status code is missing"); + return -1; + } + + /* message */ + ret = flb_otel_utils_find_map_entry_by_key(&status->via.map, "message", 0, FLB_FALSE); + if (ret >= 0 && status->via.map.ptr[ret].val.type == MSGPACK_OBJECT_STR) { + message = flb_sds_create_len(status->via.map.ptr[ret].val.via.str.ptr, + status->via.map.ptr[ret].val.via.str.size); + } + + ctr_span_set_status(span, code, message); + if (message) { + flb_sds_destroy(message); + } + + return 0; +} + +static int process_spans(struct ctrace *ctr, + struct ctrace_scope_span *scope_span, + msgpack_object *spans) +{ + int i; + int ret; + int len; + uint64_t val; + char tmp[64]; + cfl_sds_t val_str = NULL; + msgpack_object span; + msgpack_object *name = NULL; + msgpack_object *attr = NULL; + struct ctrace_span *ctr_span = NULL; + struct ctrace_attributes *ctr_attr = NULL; + + for (i = 0; i < spans->via.array.size; i++) { + span = spans->via.array.ptr[i]; + if (span.type != MSGPACK_OBJECT_MAP) { + flb_error("unexpected span type"); + return -1; + } + + val_str = NULL; + + /* name */ + ret = flb_otel_utils_find_map_entry_by_key(&span.via.map, "name", 0, FLB_TRUE); + if (ret >= 0 && span.via.map.ptr[ret].val.type == MSGPACK_OBJECT_STR) { + name = &span.via.map.ptr[ret].val; + val_str = cfl_sds_create_len(name->via.str.ptr, name->via.str.size); + } + + if (!val_str) { + flb_error("span name is missing"); + return -1; + } + + /* create the span */ + ctr_span = ctr_span_create(ctr, scope_span, val_str, NULL); + cfl_sds_destroy(val_str); + + if (ctr_span == NULL) { + return -1; + } + + /* traceId */ + ret = flb_otel_utils_find_map_entry_by_key(&span.via.map, "traceId", 0, FLB_TRUE); + if (ret >= 0 && span.via.map.ptr[ret].val.type == MSGPACK_OBJECT_STR) { + /* trace_id is an hex of 32 bytes */ + if (span.via.map.ptr[ret].val.via.str.size != 32) { + len = span.via.map.ptr[ret].val.via.str.size; + if (len > sizeof(tmp) - 1) { + len = sizeof(tmp) - 1; + } + memcpy(tmp, span.via.map.ptr[ret].val.via.str.ptr, len); + tmp[len] = '\0'; + + flb_error("invalid traceId: '%s'", tmp); + return -1; + } + + /* decode the hex string (16 bytes) */ + memset(tmp, '\0', sizeof(tmp)); + flb_otel_utils_hex_to_id((char *) span.via.map.ptr[ret].val.via.str.ptr, + span.via.map.ptr[ret].val.via.str.size, + (unsigned char *) tmp, 16); + ctr_span_set_trace_id(ctr_span, tmp, 16); + } + + /* spanId */ + ret = flb_otel_utils_find_map_entry_by_key(&span.via.map, "spanId", 0, FLB_TRUE); + if (ret >= 0 && span.via.map.ptr[ret].val.type == MSGPACK_OBJECT_STR) { + /* span_id is an hex of 16 bytes */ + if (span.via.map.ptr[ret].val.via.str.size != 16) { + len = span.via.map.ptr[ret].val.via.str.size; + if (len > sizeof(tmp) - 1) { + len = sizeof(tmp) - 1; + } + memcpy(tmp, span.via.map.ptr[ret].val.via.str.ptr, len); + tmp[len] = '\0'; + flb_error("invalid spanId: '%s'", tmp); + return -1; + } + + /* decode the hex string (8 bytes) */ + memset(tmp, '\0', sizeof(tmp)); + flb_otel_utils_hex_to_id((char *) span.via.map.ptr[ret].val.via.str.ptr, + span.via.map.ptr[ret].val.via.str.size, + (unsigned char *) tmp, 8); + ctr_span_set_span_id(ctr_span, tmp, 8); + } + + /* traceState */ + ret = flb_otel_utils_find_map_entry_by_key(&span.via.map, "traceState", 0, FLB_TRUE); + if (ret >= 0 && span.via.map.ptr[ret].val.type == MSGPACK_OBJECT_STR) { + val_str = cfl_sds_create_len(span.via.map.ptr[ret].val.via.str.ptr, + span.via.map.ptr[ret].val.via.str.size); + ctr_span->trace_state = val_str; + val_str = NULL; + } + + /* flags */ + ret = flb_otel_utils_find_map_entry_by_key(&span.via.map, "flags", 0, FLB_TRUE); + if (ret >= 0 && span.via.map.ptr[ret].val.type == MSGPACK_OBJECT_POSITIVE_INTEGER) { + /* unsupported in CTraces */ + } + + /* parentSpanId */ + ret = flb_otel_utils_find_map_entry_by_key(&span.via.map, "parentSpanId", 0, FLB_TRUE); + if (ret >= 0 && + span.via.map.ptr[ret].val.type == MSGPACK_OBJECT_STR && + span.via.map.ptr[ret].val.via.str.size > 0) { + + /* parent_span_id is an hex of 16 bytes */ + if (span.via.map.ptr[ret].val.via.str.size != 16) { + len = span.via.map.ptr[ret].val.via.str.size; + if (len > sizeof(tmp) - 1) { + len = sizeof(tmp) - 1; + } + memcpy(tmp, span.via.map.ptr[ret].val.via.str.ptr, len); + tmp[len] = '\0'; + flb_error("invalid parentSpanId: '%s'", tmp); + return -1; + } + + /* decode the hex string (8 bytes) */ + memset(tmp, '\0', sizeof(tmp)); + flb_otel_utils_hex_to_id((char *) span.via.map.ptr[ret].val.via.str.ptr, + span.via.map.ptr[ret].val.via.str.size, + (unsigned char *) tmp, 8); + ctr_span_set_parent_span_id(ctr_span, tmp, 8); + } + + /* flags */ + ret = flb_otel_utils_find_map_entry_by_key(&span.via.map, "flags", 0, FLB_TRUE); + if (ret >= 0 && span.via.map.ptr[ret].val.type == MSGPACK_OBJECT_POSITIVE_INTEGER) { + ctr_span_set_flags(ctr_span, span.via.map.ptr[ret].val.via.u64); + } + + /* start_time_unix_nano */ + ret = flb_otel_utils_find_map_entry_by_key(&span.via.map, "startTimeUnixNano", 0, FLB_TRUE); + if (ret >= 0 && span.via.map.ptr[ret].val.type == MSGPACK_OBJECT_STR) { + /* convert string number to integer */ + val = flb_otel_utils_convert_string_number_to_u64((char *) span.via.map.ptr[ret].val.via.str.ptr, + span.via.map.ptr[ret].val.via.str.size); + ctr_span_start_ts(ctr, ctr_span, val); + } + + /* end_time_unix_nano */ + ret = flb_otel_utils_find_map_entry_by_key(&span.via.map, "endTimeUnixNano", 0, FLB_TRUE); + if (ret >= 0 && span.via.map.ptr[ret].val.type == MSGPACK_OBJECT_STR) { + /* convert string number to integer */ + val = flb_otel_utils_convert_string_number_to_u64((char *) span.via.map.ptr[ret].val.via.str.ptr, + span.via.map.ptr[ret].val.via.str.size); + ctr_span_end_ts(ctr, ctr_span, val); + } + + /* kind */ + ret = flb_otel_utils_find_map_entry_by_key(&span.via.map, "kind", 0, FLB_TRUE); + if (ret >= 0 && span.via.map.ptr[ret].val.type == MSGPACK_OBJECT_POSITIVE_INTEGER) { + ctr_span_kind_set(ctr_span, span.via.map.ptr[ret].val.via.u64); + } + + /* attributes */ + ret = flb_otel_utils_find_map_entry_by_key(&span.via.map, "attributes", 0, FLB_TRUE); + if (ret >= 0 && span.via.map.ptr[ret].val.type == MSGPACK_OBJECT_ARRAY) { + attr = &span.via.map.ptr[ret].val; + ctr_attr = convert_attributes(attr, "span"); + if (ctr_attr) { + ctr_span_set_attributes(ctr_span, ctr_attr); + } + } + + /* dropped_attributes_count */ + ret = flb_otel_utils_find_map_entry_by_key(&span.via.map, "droppedAttributesCount", 0, FLB_TRUE); + if (ret >= 0 && span.via.map.ptr[ret].val.type == MSGPACK_OBJECT_POSITIVE_INTEGER) { + ctr_span_set_dropped_attributes_count(ctr_span, span.via.map.ptr[ret].val.via.u64); + } + + /* events */ + ret = flb_otel_utils_find_map_entry_by_key(&span.via.map, "events", 0, FLB_TRUE); + if (ret >= 0 && span.via.map.ptr[ret].val.type == MSGPACK_OBJECT_ARRAY) { + ret = process_events(ctr, ctr_span, &span.via.map.ptr[ret].val); + } + + /* dropped_events_count */ + ret = flb_otel_utils_find_map_entry_by_key(&span.via.map, "droppedEventsCount", 0, FLB_TRUE); + if (ret >= 0 && span.via.map.ptr[ret].val.type == MSGPACK_OBJECT_POSITIVE_INTEGER) { + ctr_span_set_dropped_events_count(ctr_span, span.via.map.ptr[ret].val.via.u64); + } + + /* dropped_links_count */ + ret = flb_otel_utils_find_map_entry_by_key(&span.via.map, "droppedLinksCount", 0, FLB_TRUE); + if (ret >= 0 && span.via.map.ptr[ret].val.type == MSGPACK_OBJECT_POSITIVE_INTEGER) { + ctr_span_set_dropped_links_count(ctr_span, span.via.map.ptr[ret].val.via.u64); + } + + /* links */ + ret = flb_otel_utils_find_map_entry_by_key(&span.via.map, "links", 0, FLB_TRUE); + if (ret >= 0 && span.via.map.ptr[ret].val.type == MSGPACK_OBJECT_ARRAY) { + ret = process_links(ctr, ctr_span, &span.via.map.ptr[ret].val); + } + + /* schema_url */ + ret = flb_otel_utils_find_map_entry_by_key(&span.via.map, "schemaUrl", 0, FLB_TRUE); + if (ret >= 0 && span.via.map.ptr[ret].val.type == MSGPACK_OBJECT_STR) { + val_str = cfl_sds_create_len(span.via.map.ptr[ret].val.via.str.ptr, + span.via.map.ptr[ret].val.via.str.size); + ctr_span_set_schema_url(ctr_span, val_str); + cfl_sds_destroy(val_str); + val_str = NULL; + } + + /* status */ + ret = flb_otel_utils_find_map_entry_by_key(&span.via.map, "status", 0, FLB_TRUE); + if (ret >= 0 && span.via.map.ptr[ret].val.type == MSGPACK_OBJECT_MAP) { + process_span_status(ctr, ctr_span, &span.via.map.ptr[ret].val); + } + } + + return 0; +} + +static int process_scope_span(struct ctrace *ctr, + struct ctrace_resource_span *resource_span, + msgpack_object *scope_spans) +{ + int ret; + msgpack_object scope; + msgpack_object *name; + msgpack_object *attr; + msgpack_object *version; + msgpack_object *schema_url; + msgpack_object *dropped_attr; + msgpack_object *spans; + cfl_sds_t url = NULL; + struct ctrace_scope_span *scope_span; + + /* get 'scope' */ + ret = flb_otel_utils_find_map_entry_by_key(&scope_spans->via.map, "scope", 0, FLB_TRUE); + if (ret >= 0) { + scope = scope_spans->via.map.ptr[ret].val; + if (scope.type != MSGPACK_OBJECT_MAP) { + flb_error("unexpected scope type in scope span"); + return -1; + } + + /* create the scope_span */ + scope_span = ctr_scope_span_create(resource_span); + if (scope_span == NULL) { + return -1; + } + + /* instrumentation scope: name */ + name = NULL; + ret = flb_otel_utils_find_map_entry_by_key(&scope.via.map, "name", 0, FLB_TRUE); + if (ret >= 0 && scope.via.map.ptr[ret].val.type == MSGPACK_OBJECT_STR) { + name = &scope.via.map.ptr[ret].val; + } + + /* instrumentation scope: version */ + version = NULL; + ret = flb_otel_utils_find_map_entry_by_key(&scope.via.map, "version", 0, FLB_TRUE); + if (ret >= 0 && scope.via.map.ptr[ret].val.type == MSGPACK_OBJECT_STR) { + version = &scope.via.map.ptr[ret].val; + } + + /* instrumentation scope: attributes */ + attr = NULL; + ret = flb_otel_utils_find_map_entry_by_key(&scope.via.map, "attributes", 0, FLB_TRUE); + if (ret >= 0 && scope.via.map.ptr[ret].val.type == MSGPACK_OBJECT_ARRAY) { + attr = &scope.via.map.ptr[ret].val; + } + + /* instrumentation scope: dropped_attributes_count */ + dropped_attr = NULL; + ret = flb_otel_utils_find_map_entry_by_key(&scope.via.map, "droppedAttributesCount", 0, FLB_TRUE); + if (ret >= 0 && scope.via.map.ptr[ret].val.type == MSGPACK_OBJECT_POSITIVE_INTEGER) { + dropped_attr = &scope.via.map.ptr[ret].val; + } + + ret = process_scope_attributes(ctr, + scope_span, + name, version, attr, dropped_attr); + if (ret == -1) { + flb_warn("failed to process scope attributes"); + } + } + else { + /* If scope is not defined we still need the scope span container */ + scope_span = ctr_scope_span_create(resource_span); + if (scope_span == NULL) { + return -1; + } + } + + /* schema_url */ + ret = flb_otel_utils_find_map_entry_by_key(&scope_spans->via.map, "schemaUrl", 0, FLB_TRUE); + if (ret >= 0) { + if (scope_spans->via.map.ptr[ret].val.type == MSGPACK_OBJECT_STR) { + schema_url = &scope_spans->via.map.ptr[ret].val; + url = cfl_sds_create_len(schema_url->via.str.ptr, schema_url->via.str.size); + if (url) { + ctr_scope_span_set_schema_url(scope_span, url); + cfl_sds_destroy(url); + url = NULL; + } + } + } + + /* spans */ + spans = NULL; + ret = flb_otel_utils_find_map_entry_by_key(&scope_spans->via.map, "spans", 0, FLB_TRUE); + if (ret >= 0 && scope_spans->via.map.ptr[ret].val.type == MSGPACK_OBJECT_ARRAY) { + spans = &scope_spans->via.map.ptr[ret].val; + } + + if (spans) { + ret = process_spans(ctr, scope_span, spans); + if (ret == -1) { + flb_error("failed to process spans"); + return -1; + } + } + + return 0; +} + +static int process_resource_span(struct ctrace *ctr, + msgpack_object *resource_span) +{ + int ret; + msgpack_object resource; + msgpack_object *attr; + msgpack_object *schema_url; + msgpack_object *scope_spans; + msgpack_object_array *scope_spans_array; + cfl_sds_t url = NULL; + struct ctrace_resource_span *ctr_resource_span; + + if (resource_span->type != MSGPACK_OBJECT_MAP) { + flb_error("unexpected resource span type"); + return -1; + } + + /* create a resource span */ + ctr_resource_span = ctr_resource_span_create(ctr); + if (ctr_resource_span == NULL) { + return -1; + } + + /* process resource data */ + ret = flb_otel_utils_find_map_entry_by_key(&resource_span->via.map, "resource", 0, FLB_TRUE); + if (ret >= 0 && resource_span->via.map.ptr[ret].val.type == MSGPACK_OBJECT_MAP) { + resource = resource_span->via.map.ptr[ret].val; + + /* attributes */ + attr = NULL; + ret = flb_otel_utils_find_map_entry_by_key(&resource.via.map, "attributes", 0, FLB_TRUE); + if (ret >= 0 && resource.via.map.ptr[ret].val.type == MSGPACK_OBJECT_ARRAY) { + attr = &resource.via.map.ptr[ret].val; + ret = process_resource_attributes(ctr, ctr_resource_span, attr); + if (ret == -1) { + flb_warn("failed to process resource attributes"); + } + } + + /* dropped_attributes_count */ + ret = flb_otel_utils_find_map_entry_by_key(&resource.via.map, "droppedAttributesCount", 0, FLB_TRUE); + if (ret >= 0 && resource.via.map.ptr[ret].val.type == MSGPACK_OBJECT_POSITIVE_INTEGER) { + ctr_resource_set_dropped_attr_count(ctr_resource_span->resource, + resource.via.map.ptr[ret].val.via.u64); + } + + /* schema_url */ + ret = flb_otel_utils_find_map_entry_by_key(&resource.via.map, "schemaUrl", 0, FLB_TRUE); + if (ret >= 0 && resource.via.map.ptr[ret].val.type == MSGPACK_OBJECT_STR) { + schema_url = &resource.via.map.ptr[ret].val; + url = cfl_sds_create_len(schema_url->via.str.ptr, schema_url->via.str.size); + if (url) { + ctr_resource_span_set_schema_url(ctr_resource_span, url); + cfl_sds_destroy(url); + url = NULL; + } + } + } + + /* schema_url */ + ret = flb_otel_utils_find_map_entry_by_key(&resource_span->via.map, "schemaUrl", 0, FLB_TRUE); + if (ret >= 0 && resource_span->via.map.ptr[ret].val.type == MSGPACK_OBJECT_STR) { + schema_url = &resource_span->via.map.ptr[ret].val; + url = cfl_sds_create_len(schema_url->via.str.ptr, schema_url->via.str.size); + if (url) { + ctr_resource_span_set_schema_url(ctr_resource_span, url); + cfl_sds_destroy(url); + url = NULL; + } + } + + /* scopeSpans */ + scope_spans = NULL; + ret = flb_otel_utils_find_map_entry_by_key(&resource_span->via.map, "scopeSpans", 0, FLB_TRUE); + if (ret >= 0 && resource_span->via.map.ptr[ret].val.type == MSGPACK_OBJECT_ARRAY) { + scope_spans = &resource_span->via.map.ptr[ret].val; + } + + if (scope_spans) { + scope_spans_array = &scope_spans->via.array; + for (ret = 0; ret < scope_spans_array->size; ret++) { + if (process_scope_span(ctr, + ctr_resource_span, + &scope_spans_array->ptr[ret]) == -1) { + flb_error("failed to process scope span"); + return -1; + } + } + } + + return 0; +} + +static struct ctrace *process_root_msgpack(msgpack_object *obj) +{ + int ret; + msgpack_object *resource_spans; + struct ctrace *ctr; + + if (obj->type != MSGPACK_OBJECT_MAP) { + flb_error("unexpected root object type for traces"); + return NULL; + } + + ret = flb_otel_utils_find_map_entry_by_key(&obj->via.map, "resourceSpans", 0, FLB_TRUE); + if (ret < 0) { + flb_error("resourceSpans entry is missing"); + return NULL; + } + + if (obj->via.map.ptr[ret].val.type != MSGPACK_OBJECT_ARRAY) { + flb_error("resourceSpans entry is invalid"); + return NULL; + } + + resource_spans = &obj->via.map.ptr[ret].val; + ret = 0; + + ctr = ctr_create(NULL); + if (!ctr) { + return NULL; + } + + for (ret = 0; ret < resource_spans->via.array.size; ret++) { + if (process_resource_span(ctr, &resource_spans->via.array.ptr[ret]) == -1) { + flb_warn("failed to process resource span"); + + ctr_destroy(ctr); + return NULL; + } + } + + return ctr; +} + +struct ctrace *flb_opentelemetry_json_traces_to_ctrace(const char *body, size_t len, int *error_status) + +{ + int result; + int root_type; + char *msgpack_body; + size_t msgpack_body_length; + size_t offset = 0; + msgpack_unpacked unpacked_root; + struct ctrace *ctr; + + result = flb_pack_json(body, len, &msgpack_body, &msgpack_body_length, + &root_type, NULL); + + if (result != 0) { + flb_error("invalid JSON trace: msgpack conversion error"); + return NULL; + } + + msgpack_unpacked_init(&unpacked_root); + + result = msgpack_unpack_next(&unpacked_root, + msgpack_body, + msgpack_body_length, + &offset); + + if (result != MSGPACK_UNPACK_SUCCESS) { + msgpack_unpacked_destroy(&unpacked_root); + flb_free(msgpack_body); + return NULL; + } + + /* iterate msgpack and compose a CTraces context */ + ctr = process_root_msgpack(&unpacked_root.data); + + msgpack_unpacked_destroy(&unpacked_root); + flb_free(msgpack_body); + + return ctr; +} From 107465bdf7b459eabe62343bffebe33cdf5bc48f Mon Sep 17 00:00:00 2001 From: Patrick Stephens Date: Sun, 3 May 2026 12:34:42 +0100 Subject: [PATCH 016/213] [upstream] build: register new flb_opentelemetry_traces.c Upstream-Ref: https://github.com/fluent/fluent-bit/commit/67685c0203fffd693b5da1396de50bf9c6fcb241 Cherry-picked from Fluent Bit v4.2.4 --- source/src/CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/source/src/CMakeLists.txt b/source/src/CMakeLists.txt index c40c4a99..e2fddf7a 100644 --- a/source/src/CMakeLists.txt +++ b/source/src/CMakeLists.txt @@ -121,6 +121,7 @@ endif() set(src ${src} opentelemetry/flb_opentelemetry_logs.c + opentelemetry/flb_opentelemetry_traces.c opentelemetry/flb_opentelemetry_utils.c ) From 586f73de66c3580e1a721e7f2ab420e5f9e69dd6 Mon Sep 17 00:00:00 2001 From: Patrick Stephens Date: Sun, 3 May 2026 12:34:42 +0100 Subject: [PATCH 017/213] [upstream] in_opentelemetry: use new otel json trace interface Upstream-Ref: https://github.com/fluent/fluent-bit/commit/602ed5bbf6bdc8bab9b7d665f5907233c691a074 Cherry-picked from Fluent Bit v4.2.4 --- .../in_opentelemetry/opentelemetry_traces.c | 1081 +---------------- 1 file changed, 9 insertions(+), 1072 deletions(-) diff --git a/source/plugins/in_opentelemetry/opentelemetry_traces.c b/source/plugins/in_opentelemetry/opentelemetry_traces.c index 0e8e61b6..75fa1c7b 100644 --- a/source/plugins/in_opentelemetry/opentelemetry_traces.c +++ b/source/plugins/in_opentelemetry/opentelemetry_traces.c @@ -20,13 +20,13 @@ #include #include #include +#include #include #include #include "opentelemetry.h" #include "opentelemetry_traces.h" -#include "opentelemetry_utils.h" int opentelemetry_traces_process_protobuf(struct flb_opentelemetry *ctx, flb_sds_t tag, @@ -51,1092 +51,29 @@ int opentelemetry_traces_process_protobuf(struct flb_opentelemetry *ctx, return result; } -static int process_attribute(struct ctrace_attributes *attr, - msgpack_object *key, msgpack_object *value, int type) -{ - int ret; - char *key_str; - char *value_str; - int64_t value_int; - double value_double; - int value_bool; - - if (key->type != MSGPACK_OBJECT_STR) { - return -1; - } - - /* temporary buffer for key since it needs to be NULL terminated */ - key_str = flb_sds_create_len(key->via.str.ptr, key->via.str.size); - if (key_str == NULL) { - return -1; - } - - /* - * the value of 'type' is set by the JSON wrapped value, we need to to convert it - * since msgpack_object *value is always a string - */ - switch (type) { - case MSGPACK_OBJECT_STR: - if (value->type != MSGPACK_OBJECT_STR) { - flb_sds_destroy(key_str); - return -1; - } - - value_str = flb_sds_create_len(value->via.str.ptr, value->via.str.size); - if (value_str == NULL) { - flb_sds_destroy(key_str); - return -1; - } - - ret = ctr_attributes_set_string(attr, key_str, value_str); - flb_sds_destroy(value_str); - break; - case MSGPACK_OBJECT_POSITIVE_INTEGER: - case MSGPACK_OBJECT_NEGATIVE_INTEGER: - if (value->type == MSGPACK_OBJECT_STR) { - value_str = flb_sds_create_len(value->via.str.ptr, value->via.str.size); - if (value_str == NULL) { - flb_sds_destroy(key_str); - return -1; - } - value_int = strtoll(value_str, NULL, 10); - flb_sds_destroy(value_str); - } - else if (value->type == MSGPACK_OBJECT_POSITIVE_INTEGER || - value->type == MSGPACK_OBJECT_NEGATIVE_INTEGER) { - value_int = value->via.i64; - } - else { - flb_sds_destroy(key_str); - return -1; - } - - ret = ctr_attributes_set_int64(attr, key_str, value_int); - break; - case MSGPACK_OBJECT_FLOAT32: - case MSGPACK_OBJECT_FLOAT64: - if (value->type == MSGPACK_OBJECT_STR) { - value_str = flb_sds_create_len(value->via.str.ptr, value->via.str.size); - if (value_str == NULL) { - flb_sds_destroy(key_str); - return -1; - } - value_double = strtod(value_str, NULL); - flb_sds_destroy(value_str); - } - else if (value->type == MSGPACK_OBJECT_FLOAT32 || - value->type == MSGPACK_OBJECT_FLOAT64) { - value_double = value->via.f64; - } - else { - flb_sds_destroy(key_str); - return -1; - } - - ret = ctr_attributes_set_double(attr, key_str, value_double); - break; - case MSGPACK_OBJECT_BOOLEAN: - if (value->type != MSGPACK_OBJECT_BOOLEAN) { - flb_sds_destroy(key_str); - return -1; - } - - value_bool = value->via.boolean; - ret = ctr_attributes_set_bool(attr, key_str, value_bool); - break; - case MSGPACK_OBJECT_ARRAY: - /* - * The less fun part (OTel JSON encoding), the value can be an array and - * only allows values such as string, bool, int64, double. I am glad this - * don't support nested arrays or maps. - */ - ret = 0; - break; - default: - flb_sds_destroy(key_str); - return -1; - } - - flb_sds_destroy(key_str); - return ret; -} - -static int process_resource_unwrap_attribute(msgpack_object *attr, - msgpack_object *out_key, - msgpack_object *out_value, int *out_value_type) -{ - int ret; - int type; - msgpack_object key; - msgpack_object val; - msgpack_object *real_value; - - if (attr->type != MSGPACK_OBJECT_MAP) { - return -1; - } - - ret = find_map_entry_by_key(&attr->via.map, "key", 0, FLB_TRUE); - if (ret == -1) { - return -1; - } - - key = attr->via.map.ptr[ret].val; - if (key.type != MSGPACK_OBJECT_STR) { - return -1; - } - - ret = find_map_entry_by_key(&attr->via.map, "value", 0, FLB_TRUE); - if (ret == -1) { - return -1; - } - - val = attr->via.map.ptr[ret].val; - - ret = json_payload_get_wrapped_value(&val, &real_value, &type); - if (ret != 0) { - return -1; - } - - *out_key = key; - *out_value = *real_value; - *out_value_type = type; - - return 0; -} - -/* - * Convert a list of attributes in msgpack format to a cfl attributes by - * unwrapping JSON encoded value types. - */ -static struct ctrace_attributes *convert_attributes(struct flb_opentelemetry *ctx, - msgpack_object *attributes, - char *log_context) -{ - int i; - int ret; - int value_type; - msgpack_object key; - msgpack_object value; - struct ctrace_attributes *attr; - - attr = ctr_attributes_create(); - if (attr == NULL) { - return NULL; - } - - for (i = 0; i < attributes->via.array.size; i++) { - ret = process_resource_unwrap_attribute(&attributes->via.array.ptr[i], - &key, &value, &value_type); - if (ret == -1) { - flb_plg_warn(ctx->ins, "found invalid %s attribute, skipping", - log_context); - continue; - } - - /* set attribute */ - ret = process_attribute(attr, &key, &value, value_type); - if (ret == -1) { - flb_plg_warn(ctx->ins, "failed to set %s attribute, skipping", - log_context); - continue; - } - } - - return attr; -} - -static int process_resource_attributes(struct flb_opentelemetry *ctx, - struct ctrace *ctr, - struct ctrace_resource_span *resource_span, - msgpack_object *attributes) - -{ - struct ctrace_resource *resource; - struct ctrace_attributes *attr; - - attr = convert_attributes(ctx, attributes, "trace resource"); - if (!attr) { - return -1; - } - - resource = ctr_resource_span_get_resource(resource_span); - ctr_resource_set_attributes(resource, attr); - - return 0; -} - -static int process_scope_attributes(struct flb_opentelemetry *ctx, - struct ctrace *ctr, - struct ctrace_scope_span *scope_span, - msgpack_object *name, - msgpack_object *version, - msgpack_object *attributes, - msgpack_object *dropped_attributes_count) - -{ - int dropped = 0; - cfl_sds_t name_str = NULL; - cfl_sds_t version_str = NULL; - struct ctrace_attributes *attr = NULL; - struct ctrace_instrumentation_scope *ins_scope; - - if (attributes) { - attr = convert_attributes(ctx, attributes, "trace scope"); - if (!attr) { - return -1; - } - } - - if (name) { - name_str = cfl_sds_create_len(name->via.str.ptr, name->via.str.size); - } - if (version) { - version_str = cfl_sds_create_len(version->via.str.ptr, version->via.str.size); - } - - if (dropped_attributes_count) { - dropped = dropped_attributes_count->via.u64; - } - - ins_scope = ctr_instrumentation_scope_create(name_str, version_str, dropped, attr); - if (!ins_scope) { - if (name_str) { - cfl_sds_destroy(name_str); - } - if (version_str) { - cfl_sds_destroy(version_str); - } - if (attr) { - ctr_attributes_destroy(attr); - } - return -1; - } - - if (name_str) { - cfl_sds_destroy(name_str); - } - if (version_str) { - cfl_sds_destroy(version_str); - } - - ctr_scope_span_set_instrumentation_scope(scope_span, ins_scope); - return 0; -} - -static int process_events(struct flb_opentelemetry *ctx, - struct ctrace *ctr, - struct ctrace_span *span, - msgpack_object *events) -{ - int i; - int ret; - int len; - uint64_t ts = 0; - char tmp[64]; - cfl_sds_t name_str = NULL; - msgpack_object event; - msgpack_object *name = NULL; - msgpack_object *attr = NULL; - struct ctrace_span_event *ctr_event = NULL; - struct ctrace_attributes *ctr_attr = NULL; - - for (i = 0; i < events->via.array.size; i++) { - event = events->via.array.ptr[i]; - if (event.type != MSGPACK_OBJECT_MAP) { - flb_plg_error(ctx->ins, "unexpected event type"); - return -1; - } - - name_str = NULL; - ts = 0; - - /* name */ - ret = find_map_entry_by_key(&event.via.map, "name", 0, FLB_TRUE); - if (ret >= 0 && event.via.map.ptr[ret].val.type == MSGPACK_OBJECT_STR) { - name = &event.via.map.ptr[ret].val; - name_str = cfl_sds_create_len(name->via.str.ptr, name->via.str.size); - if (name_str == NULL) { - return -1; - } - } - - if (!name_str) { - flb_plg_warn(ctx->ins, "span event name is missing"); - return -1; - } - - /* time_unix_nano */ - ret = find_map_entry_by_key(&event.via.map, "timeUnixNano", 0, FLB_TRUE); - if (ret >= 0 && event.via.map.ptr[ret].val.type == MSGPACK_OBJECT_STR) { - /* convert to uint64_t */ - len = event.via.map.ptr[ret].val.via.str.size; - if (len > sizeof(tmp) - 1) { - len = sizeof(tmp) - 1; - memcpy(tmp, event.via.map.ptr[ret].val.via.str.ptr, len); - tmp[len] = '\0'; - - flb_plg_error(ctx->ins, "invalid timeUnixNano: '%s'", tmp); - if (name_str) { - cfl_sds_destroy(name_str); - } - return -1; - } - - memcpy(tmp, event.via.map.ptr[ret].val.via.str.ptr, len); - tmp[len] = '\0'; - - ts = strtoull(tmp, NULL, 10); - } - - ctr_event = ctr_span_event_add_ts(span, name_str, ts); - cfl_sds_destroy(name_str); - if (ctr_event == NULL) { - return -1; - } - - /* attributes */ - ret = find_map_entry_by_key(&event.via.map, "attributes", 0, FLB_TRUE); - if (ret >= 0 && event.via.map.ptr[ret].val.type == MSGPACK_OBJECT_ARRAY) { - attr = &event.via.map.ptr[ret].val; - ctr_attr = convert_attributes(ctx, attr, "span event"); - if (ctr_attr) { - ctr_span_event_set_attributes(ctr_event, ctr_attr); - } - } - - /* dropped_attributes_count */ - ret = find_map_entry_by_key(&event.via.map, "droppedAttributesCount", 0, FLB_FALSE); - if (ret >= 0 && event.via.map.ptr[ret].val.type == MSGPACK_OBJECT_POSITIVE_INTEGER) { - ctr_span_event_set_dropped_attributes_count(ctr_event, event.via.map.ptr[ret].val.via.u64); - } - } - - return 0; -} - -static int process_links(struct flb_opentelemetry *ctx, - struct ctrace *ctr, - struct ctrace_span *span, - msgpack_object *links) -{ - int i; - int ret; - int len; - char tmp[64]; - char trace_id_bin[16]; - char span_id_bin[8]; - cfl_sds_t buf; - msgpack_object link; - msgpack_object *trace_id = NULL; - msgpack_object *span_id = NULL; - msgpack_object *trace_state = NULL; - msgpack_object *dropped_attr_count = NULL; - msgpack_object *flags = NULL; - msgpack_object *attr = NULL; - - struct ctrace_link *ctr_link = NULL; - struct ctrace_attributes *ctr_attr = NULL; - - for (i = 0; i < links->via.array.size; i++) { - link = links->via.array.ptr[i]; - if (link.type != MSGPACK_OBJECT_MAP) { - flb_plg_error(ctx->ins, "unexpected link type"); - return -1; - } - trace_id = NULL; - span_id = NULL; - trace_state = NULL; - dropped_attr_count = NULL; - flags = NULL; - ctr_attr = NULL; - /* traceId */ - ret = find_map_entry_by_key(&link.via.map, "traceId", 0, FLB_TRUE); - if (ret >= 0 && link.via.map.ptr[ret].val.type == MSGPACK_OBJECT_STR) { - trace_id = &link.via.map.ptr[ret].val; - /* trace_id is an hex of 32 bytes */ - if (trace_id->via.str.size != 32) { - len = trace_id->via.str.size; - if (len > sizeof(tmp) - 1) { - len = sizeof(tmp) - 1; - } - memcpy(tmp, trace_id->via.str.ptr, len); - tmp[len] = '\0'; - - flb_plg_error(ctx->ins, "invalid event traceId: '%s'", tmp); - return -1; - } - - /* decode the hex string (16 bytes) */ - hex_to_id((char *) trace_id->via.str.ptr, trace_id->via.str.size, - (unsigned char *) trace_id_bin, 16); - } - - if (!trace_id) { - flb_plg_error(ctx->ins, "link traceId is missing"); - return -1; - } - - /* spanId */ - ret = find_map_entry_by_key(&link.via.map, "spanId", 0, FLB_TRUE); - if (ret >= 0 && link.via.map.ptr[ret].val.type == MSGPACK_OBJECT_STR) { - span_id = &link.via.map.ptr[ret].val; - - /* span_id is an hex of 16 bytes */ - if (span_id->via.str.size != 16) { - len = span_id->via.str.size; - if (len > sizeof(tmp) - 1) { - len = sizeof(tmp) - 1; - } - memcpy(tmp, span_id->via.str.ptr, len); - tmp[len] = '\0'; - flb_plg_error(ctx->ins, "invalid spanId: '%s'", tmp); - return -1; - } - - /* decode the hex string (8 bytes) */ - memset(tmp, '\0', sizeof(tmp)); - hex_to_id((char *) span_id->via.str.ptr, span_id->via.str.size, - (unsigned char *) span_id_bin, 8); - } - - if (!span_id) { - flb_plg_error(ctx->ins, "link spanId is missing"); - return -1; - } - - /* traceState */ - ret = find_map_entry_by_key(&link.via.map, "traceState", 0, FLB_FALSE); - if (ret >= 0 && link.via.map.ptr[ret].val.type == MSGPACK_OBJECT_STR) { - trace_state = &link.via.map.ptr[ret].val; - } - - /* attributes */ - ret = find_map_entry_by_key(&link.via.map, "attributes", 0, FLB_FALSE); - if (ret >= 0 && link.via.map.ptr[ret].val.type == MSGPACK_OBJECT_ARRAY) { - attr = &link.via.map.ptr[ret].val; - ctr_attr = convert_attributes(ctx, attr, "event link"); - } - - /* droped_attributes_count */ - ret = find_map_entry_by_key(&link.via.map, "droppedAttributesCount", 0, FLB_FALSE); - if (ret >= 0 && link.via.map.ptr[ret].val.type == MSGPACK_OBJECT_POSITIVE_INTEGER) { - dropped_attr_count = &link.via.map.ptr[ret].val; - } - - /* flags */ - ret = find_map_entry_by_key(&link.via.map, "flags", 0, FLB_FALSE); - if (ret >= 0 && link.via.map.ptr[ret].val.type == MSGPACK_OBJECT_POSITIVE_INTEGER) { - flags = &link.via.map.ptr[ret].val; - } - - ctr_link = ctr_link_create(span, - trace_id_bin, 16, - span_id_bin, 8); - - if (ctr_link == NULL) { - if (ctr_attr) { - ctr_attributes_destroy(ctr_attr); - } - return -1; - } - - if (trace_state) { - buf = cfl_sds_create_len(trace_state->via.str.ptr, trace_state->via.str.size); - if (buf) { - ctr_link_set_trace_state(ctr_link, buf); - cfl_sds_destroy(buf); - } - } - - if (ctr_attr) { - ctr_link_set_attributes(ctr_link, ctr_attr); - } - - if (dropped_attr_count) { - ctr_link_set_dropped_attr_count(ctr_link, dropped_attr_count->via.u64); - } - - if (flags) { - ctr_link_set_flags(ctr_link, flags->via.u64); - } - } - - return 0; -} - -static int process_span_status(struct flb_opentelemetry *ctx, - struct ctrace *ctr, - struct ctrace_span *span, - msgpack_object *status) -{ - int ret; - int code = 0; - cfl_sds_t tmp = NULL; - char *message = NULL; - - if (status->type != MSGPACK_OBJECT_MAP) { - flb_plg_error(ctx->ins, "unexpected status type"); - return -1; - } - - /* code */ - ret = find_map_entry_by_key(&status->via.map, "code", 0, FLB_TRUE); - if (ret >= 0 && status->via.map.ptr[ret].val.type == MSGPACK_OBJECT_STR) { - tmp = cfl_sds_create_len(status->via.map.ptr[ret].val.via.str.ptr, - status->via.map.ptr[ret].val.via.str.size); - if (!tmp) { - return -1; - } - - if (strcasecmp(tmp, "UNSET") == 0) { - code = CTRACE_SPAN_STATUS_CODE_UNSET; - } - else if (strcasecmp(tmp, "OK") == 0) { - code = CTRACE_SPAN_STATUS_CODE_OK; - } - else if (strcasecmp(tmp, "ERROR") == 0) { - code = CTRACE_SPAN_STATUS_CODE_ERROR; - } - else { - flb_plg_error(ctx->ins, "status code value is invalid: %s", tmp); - cfl_sds_destroy(tmp); - return -1; - } - cfl_sds_destroy(tmp); - } - else { - flb_plg_error(ctx->ins, "status code is missing"); - return -1; - } - - /* message */ - ret = find_map_entry_by_key(&status->via.map, "message", 0, FLB_FALSE); - if (ret >= 0 && status->via.map.ptr[ret].val.type == MSGPACK_OBJECT_STR) { - message = flb_sds_create_len(status->via.map.ptr[ret].val.via.str.ptr, - status->via.map.ptr[ret].val.via.str.size); - } - - ctr_span_set_status(span, code, message); - if (message) { - flb_sds_destroy(message); - } - - return 0; -} - -static int process_spans(struct flb_opentelemetry *ctx, - struct ctrace *ctr, - struct ctrace_scope_span *scope_span, - msgpack_object *spans) -{ - int i; - int ret; - int len; - uint64_t val; - char tmp[64]; - cfl_sds_t val_str = NULL; - msgpack_object span; - msgpack_object *name = NULL; - msgpack_object *attr = NULL; - struct ctrace_span *ctr_span = NULL; - struct ctrace_attributes *ctr_attr = NULL; - - for (i = 0; i < spans->via.array.size; i++) { - span = spans->via.array.ptr[i]; - if (span.type != MSGPACK_OBJECT_MAP) { - flb_plg_error(ctx->ins, "unexpected span type"); - return -1; - } - - val_str = NULL; - - /* name */ - ret = find_map_entry_by_key(&span.via.map, "name", 0, FLB_TRUE); - if (ret >= 0 && span.via.map.ptr[ret].val.type == MSGPACK_OBJECT_STR) { - name = &span.via.map.ptr[ret].val; - val_str = cfl_sds_create_len(name->via.str.ptr, name->via.str.size); - } - - if (!val_str) { - flb_plg_error(ctx->ins, "span name is missing"); - return -1; - } - - /* create the span */ - ctr_span = ctr_span_create(ctr, scope_span, val_str, NULL); - cfl_sds_destroy(val_str); - - if (ctr_span == NULL) { - return -1; - } - - /* traceId */ - ret = find_map_entry_by_key(&span.via.map, "traceId", 0, FLB_TRUE); - if (ret >= 0 && span.via.map.ptr[ret].val.type == MSGPACK_OBJECT_STR) { - /* trace_id is an hex of 32 bytes */ - if (span.via.map.ptr[ret].val.via.str.size != 32) { - len = span.via.map.ptr[ret].val.via.str.size; - if (len > sizeof(tmp) - 1) { - len = sizeof(tmp) - 1; - } - memcpy(tmp, span.via.map.ptr[ret].val.via.str.ptr, len); - tmp[len] = '\0'; - - flb_plg_error(ctx->ins, "invalid traceId: '%s'", tmp); - return -1; - } - - /* decode the hex string (16 bytes) */ - memset(tmp, '\0', sizeof(tmp)); - hex_to_id((char *) span.via.map.ptr[ret].val.via.str.ptr, - span.via.map.ptr[ret].val.via.str.size, - (unsigned char *) tmp, 16); - ctr_span_set_trace_id(ctr_span, tmp, 16); - } - - /* spanId */ - ret = find_map_entry_by_key(&span.via.map, "spanId", 0, FLB_TRUE); - if (ret >= 0 && span.via.map.ptr[ret].val.type == MSGPACK_OBJECT_STR) { - /* span_id is an hex of 16 bytes */ - if (span.via.map.ptr[ret].val.via.str.size != 16) { - len = span.via.map.ptr[ret].val.via.str.size; - if (len > sizeof(tmp) - 1) { - len = sizeof(tmp) - 1; - } - memcpy(tmp, span.via.map.ptr[ret].val.via.str.ptr, len); - tmp[len] = '\0'; - flb_plg_error(ctx->ins, "invalid spanId: '%s'", tmp); - return -1; - } - - /* decode the hex string (8 bytes) */ - memset(tmp, '\0', sizeof(tmp)); - hex_to_id((char *) span.via.map.ptr[ret].val.via.str.ptr, - span.via.map.ptr[ret].val.via.str.size, - (unsigned char *) tmp, 8); - ctr_span_set_span_id(ctr_span, tmp, 8); - } - - /* traceState */ - ret = find_map_entry_by_key(&span.via.map, "traceState", 0, FLB_TRUE); - if (ret >= 0 && span.via.map.ptr[ret].val.type == MSGPACK_OBJECT_STR) { - val_str = cfl_sds_create_len(span.via.map.ptr[ret].val.via.str.ptr, - span.via.map.ptr[ret].val.via.str.size); - ctr_span->trace_state = val_str; - val_str = NULL; - } - - /* flags */ - ret = find_map_entry_by_key(&span.via.map, "flags", 0, FLB_TRUE); - if (ret >= 0 && span.via.map.ptr[ret].val.type == MSGPACK_OBJECT_POSITIVE_INTEGER) { - /* unsupported in CTraces */ - } - - /* parentSpanId */ - ret = find_map_entry_by_key(&span.via.map, "parentSpanId", 0, FLB_TRUE); - if (ret >= 0 && - span.via.map.ptr[ret].val.type == MSGPACK_OBJECT_STR && - span.via.map.ptr[ret].val.via.str.size > 0) { - - /* parent_span_id is an hex of 16 bytes */ - if (span.via.map.ptr[ret].val.via.str.size != 16) { - len = span.via.map.ptr[ret].val.via.str.size; - if (len > sizeof(tmp) - 1) { - len = sizeof(tmp) - 1; - } - memcpy(tmp, span.via.map.ptr[ret].val.via.str.ptr, len); - tmp[len] = '\0'; - flb_plg_error(ctx->ins, "invalid parentSpanId: '%s'", tmp); - return -1; - } - - /* decode the hex string (8 bytes) */ - memset(tmp, '\0', sizeof(tmp)); - hex_to_id((char *) span.via.map.ptr[ret].val.via.str.ptr, - span.via.map.ptr[ret].val.via.str.size, - (unsigned char *) tmp, 8); - ctr_span_set_parent_span_id(ctr_span, tmp, 8); - } - - /* flags */ - ret = find_map_entry_by_key(&span.via.map, "flags", 0, FLB_TRUE); - if (ret >= 0 && span.via.map.ptr[ret].val.type == MSGPACK_OBJECT_POSITIVE_INTEGER) { - ctr_span_set_flags(ctr_span, span.via.map.ptr[ret].val.via.u64); - } - - /* start_time_unix_nano */ - ret = find_map_entry_by_key(&span.via.map, "startTimeUnixNano", 0, FLB_TRUE); - if (ret >= 0 && span.via.map.ptr[ret].val.type == MSGPACK_OBJECT_STR) { - /* convert string number to integer */ - val = convert_string_number_to_u64((char *) span.via.map.ptr[ret].val.via.str.ptr, - span.via.map.ptr[ret].val.via.str.size); - ctr_span_start_ts(ctr, ctr_span, val); - } - - /* end_time_unix_nano */ - ret = find_map_entry_by_key(&span.via.map, "endTimeUnixNano", 0, FLB_TRUE); - if (ret >= 0 && span.via.map.ptr[ret].val.type == MSGPACK_OBJECT_STR) { - /* convert string number to integer */ - val = convert_string_number_to_u64((char *) span.via.map.ptr[ret].val.via.str.ptr, - span.via.map.ptr[ret].val.via.str.size); - ctr_span_end_ts(ctr, ctr_span, val); - } - - /* kind */ - ret = find_map_entry_by_key(&span.via.map, "kind", 0, FLB_TRUE); - if (ret >= 0 && span.via.map.ptr[ret].val.type == MSGPACK_OBJECT_POSITIVE_INTEGER) { - ctr_span_kind_set(ctr_span, span.via.map.ptr[ret].val.via.u64); - } - - /* attributes */ - ret = find_map_entry_by_key(&span.via.map, "attributes", 0, FLB_TRUE); - if (ret >= 0 && span.via.map.ptr[ret].val.type == MSGPACK_OBJECT_ARRAY) { - attr = &span.via.map.ptr[ret].val; - ctr_attr = convert_attributes(ctx, attr, "span"); - if (ctr_attr) { - ctr_span_set_attributes(ctr_span, ctr_attr); - } - } - - /* dropped_attributes_count */ - ret = find_map_entry_by_key(&span.via.map, "droppedAttributesCount", 0, FLB_TRUE); - if (ret >= 0 && span.via.map.ptr[ret].val.type == MSGPACK_OBJECT_POSITIVE_INTEGER) { - ctr_span_set_dropped_attributes_count(ctr_span, span.via.map.ptr[ret].val.via.u64); - } - - /* events */ - ret = find_map_entry_by_key(&span.via.map, "events", 0, FLB_TRUE); - if (ret >= 0 && span.via.map.ptr[ret].val.type == MSGPACK_OBJECT_ARRAY) { - ret = process_events(ctx, ctr, ctr_span, &span.via.map.ptr[ret].val); - } - - /* dropped_events_count */ - ret = find_map_entry_by_key(&span.via.map, "droppedEventsCount", 0, FLB_TRUE); - if (ret >= 0 && span.via.map.ptr[ret].val.type == MSGPACK_OBJECT_POSITIVE_INTEGER) { - ctr_span_set_dropped_events_count(ctr_span, span.via.map.ptr[ret].val.via.u64); - } - - /* dropped_links_count */ - ret = find_map_entry_by_key(&span.via.map, "droppedLinksCount", 0, FLB_TRUE); - if (ret >= 0 && span.via.map.ptr[ret].val.type == MSGPACK_OBJECT_POSITIVE_INTEGER) { - ctr_span_set_dropped_links_count(ctr_span, span.via.map.ptr[ret].val.via.u64); - } - - /* links */ - ret = find_map_entry_by_key(&span.via.map, "links", 0, FLB_TRUE); - if (ret >= 0 && span.via.map.ptr[ret].val.type == MSGPACK_OBJECT_ARRAY) { - ret = process_links(ctx, ctr, ctr_span, &span.via.map.ptr[ret].val); - } - - /* schema_url */ - ret = find_map_entry_by_key(&span.via.map, "schemaUrl", 0, FLB_TRUE); - if (ret >= 0 && span.via.map.ptr[ret].val.type == MSGPACK_OBJECT_STR) { - val_str = cfl_sds_create_len(span.via.map.ptr[ret].val.via.str.ptr, - span.via.map.ptr[ret].val.via.str.size); - ctr_span_set_schema_url(ctr_span, val_str); - cfl_sds_destroy(val_str); - val_str = NULL; - } - - /* status */ - ret = find_map_entry_by_key(&span.via.map, "status", 0, FLB_TRUE); - if (ret >= 0 && span.via.map.ptr[ret].val.type == MSGPACK_OBJECT_MAP) { - process_span_status(ctx, ctr, ctr_span, &span.via.map.ptr[ret].val); - } - } - - return 0; -} - - -static int process_scope_span(struct flb_opentelemetry *ctx, - struct ctrace *ctr, - struct ctrace_resource_span *resource_span, - msgpack_object *scope_spans) -{ - int ret; - msgpack_object scope; - msgpack_object *name; - msgpack_object *attr; - msgpack_object *version; - msgpack_object *schema_url; - msgpack_object *dropped_attr; - msgpack_object *spans; - cfl_sds_t url = NULL; - struct ctrace_scope_span *scope_span; - - /* get 'scope' */ - ret = find_map_entry_by_key(&scope_spans->via.map, "scope", 0, FLB_TRUE); - if (ret >= 0) { - scope = scope_spans->via.map.ptr[ret].val; - if (scope.type != MSGPACK_OBJECT_MAP) { - flb_plg_error(ctx->ins, "unexpected scope type in scope span"); - return -1; - } - - /* create the scope_span */ - scope_span = ctr_scope_span_create(resource_span); - if (scope_span == NULL) { - return -1; - } - - /* instrumentation scope: name */ - name = NULL; - ret = find_map_entry_by_key(&scope.via.map, "name", 0, FLB_TRUE); - if (ret >= 0 && scope.via.map.ptr[ret].val.type == MSGPACK_OBJECT_STR) { - name = &scope.via.map.ptr[ret].val; - } - - /* instrumentation scope: version */ - version = NULL; - ret = find_map_entry_by_key(&scope.via.map, "version", 0, FLB_TRUE); - if (ret >= 0 && scope.via.map.ptr[ret].val.type == MSGPACK_OBJECT_STR) { - version = &scope.via.map.ptr[ret].val; - } - - /* instrumentation scope: attributes */ - attr = NULL; - ret = find_map_entry_by_key(&scope.via.map, "attributes", 0, FLB_TRUE); - if (ret >= 0 && scope.via.map.ptr[ret].val.type == MSGPACK_OBJECT_ARRAY) { - attr = &scope.via.map.ptr[ret].val; - } - - /* instrumentation scope: dropped_attributes_count */ - dropped_attr = NULL; - ret = find_map_entry_by_key(&scope.via.map, "droppedAttributesCount", 0, FLB_TRUE); - if (ret >= 0 && scope.via.map.ptr[ret].val.type == MSGPACK_OBJECT_POSITIVE_INTEGER) { - dropped_attr = &scope.via.map.ptr[ret].val; - } - - ret = process_scope_attributes(ctx, - ctr, - scope_span, - name, version, attr, dropped_attr); - if (ret == -1) { - flb_plg_warn(ctx->ins, "failed to process scope attributes"); - } - } - - /* schema_url */ - ret = find_map_entry_by_key(&scope_spans->via.map, "schemaUrl", 0, FLB_TRUE); - if (ret >= 0) { - schema_url = &scope_spans->via.map.ptr[ret].val; - if (schema_url->type == MSGPACK_OBJECT_STR) { - /* set schema url */ - url = cfl_sds_create_len(schema_url->via.str.ptr, schema_url->via.str.size); - if (url) { - ctr_scope_span_set_schema_url(scope_span, url); - cfl_sds_destroy(url); - } - } - } - - /* process the scope spans[] */ - ret = find_map_entry_by_key(&scope_spans->via.map, "spans", 0, FLB_TRUE); - if (ret >= 0 && scope_spans->via.map.ptr[ret].val.type == MSGPACK_OBJECT_ARRAY) { - spans = &scope_spans->via.map.ptr[ret].val; - ret = process_spans(ctx, ctr, scope_span, spans); - if (ret == -1) { - flb_plg_warn(ctx->ins, "failed to process spans"); - } - } - - return 0; -} - -static int process_resource_span(struct flb_opentelemetry *ctx, - struct ctrace *ctr, - msgpack_object *resource_spans) -{ - int i; - int ret; - cfl_sds_t url; - struct ctrace_resource *ctr_resource; - struct ctrace_resource_span *resource_span; - msgpack_object resource; - msgpack_object attr; - msgpack_object scope_spans; - msgpack_object schema_url; - - if (resource_spans->type != MSGPACK_OBJECT_MAP) { - return -1; - } - - /* get the resource */ - ret = find_map_entry_by_key(&resource_spans->via.map, "resource", 0, FLB_TRUE); - if (ret == -1) { - flb_plg_error(ctx->ins, "resource missing"); - return -1; - } - - resource = resource_spans->via.map.ptr[ret].val; - if (resource.type != MSGPACK_OBJECT_MAP) { - flb_plg_error(ctx->ins, "unexpected resource type in resource span"); - return -1; - } - - - resource_span = ctr_resource_span_create(ctr); - if (resource_span == NULL) { - return -1; - } - - ctr_resource = ctr_resource_span_get_resource(resource_span); - - /* droppedAttributesCount */ - ret = find_map_entry_by_key(&resource.via.map, "droppedAttributesCount", 0, FLB_FALSE); - if (ret >= 0 && resource.via.map.ptr[ret].val.type == MSGPACK_OBJECT_POSITIVE_INTEGER) { - ctr_resource_set_dropped_attr_count(ctr_resource, resource.via.map.ptr[ret].val.via.u64); - } - - - /* Get resource attributes */ - ret = find_map_entry_by_key(&resource.via.map, "attributes", 0, FLB_TRUE); - if (ret >= 0) { - attr = resource.via.map.ptr[ret].val; - if (attr.type == MSGPACK_OBJECT_ARRAY) { - /* iterate and register attributes */ - ret = process_resource_attributes(ctx, - ctr, - resource_span, - &attr); - if (ret == -1) { - flb_plg_warn(ctx->ins, "failed to process resource attributes"); - } - } - } - - /* scopeSpans */ - ret = find_map_entry_by_key(&resource_spans->via.map, "scopeSpans", 0, FLB_TRUE); - if (ret == -1) { - flb_plg_error(ctx->ins, "scopeSpans missing"); - return -1; - } - scope_spans = resource_spans->via.map.ptr[ret].val; - - if (scope_spans.type != MSGPACK_OBJECT_ARRAY) { - flb_plg_error(ctx->ins, "unexpected scopeSpans type"); - ctr_destroy(ctr); - return -1; - } - - for (i = 0; i < scope_spans.via.array.size; i++) { - ret = process_scope_span(ctx, - ctr, - resource_span, - &scope_spans.via.array.ptr[i]); - if (ret == -1) { - flb_plg_warn(ctx->ins, "failed to process scope span"); - } - } - - /* schema_url */ - ret = find_map_entry_by_key(&resource.via.map, "schemaUrl", 0, FLB_TRUE); - if (ret >= 0) { - schema_url = resource.via.map.ptr[ret].val; - if (schema_url.type == MSGPACK_OBJECT_STR) { - url = cfl_sds_create_len(schema_url.via.str.ptr, schema_url.via.str.size); - if (url) { - ctr_resource_span_set_schema_url(resource_span, url); - cfl_sds_destroy(url); - } - } - } - - return 0; -} - -static struct ctrace *process_root_msgpack(struct flb_opentelemetry *ctx, msgpack_object *obj) -{ - int i; - int ret; - struct ctrace *ctr; - msgpack_object_array *resource_spans; - - if (obj->type != MSGPACK_OBJECT_MAP) { - return NULL; - } - - ret = find_map_entry_by_key(&obj->via.map, "resourceSpans", 0, FLB_TRUE); - if (ret == -1) { - ret = find_map_entry_by_key(&obj->via.map, "resource_spans", 0, FLB_TRUE); - if (ret == -1) { - flb_plg_error(ctx->ins, "resourceSpans missing"); - return NULL; - } - } - - if (obj->via.map.ptr[ret].val.type != MSGPACK_OBJECT_ARRAY) { - flb_plg_error(ctx->ins, "unexpected resourceSpans type"); - return NULL; - } - - resource_spans = &obj->via.map.ptr[ret].val.via.array; - ret = 0; - - ctr = ctr_create(NULL); - if (!ctr) { - return NULL; - } - - for (i = 0; i < resource_spans->size; i++) { - ret = process_resource_span(ctx, ctr, &resource_spans->ptr[i]); - if (ret == -1) { - flb_plg_warn(ctx->ins, "failed to process resource span"); - - ctr_destroy(ctr); - return NULL; - } - } - - return ctr; -} static int process_json(struct flb_opentelemetry *ctx, char *tag, size_t tag_len, const char *body, size_t len) { - int result = -1; - int root_type; - char *msgpack_body; - size_t msgpack_body_length; - size_t offset = 0; - msgpack_unpacked unpacked_root; - struct ctrace *ctr; - - result = flb_pack_json(body, len, &msgpack_body, &msgpack_body_length, - &root_type, NULL); - - if (result != 0) { - flb_plg_error(ctx->ins, "invalid JSON trace: msgpack conversion error"); - return -1; - } - - msgpack_unpacked_init(&unpacked_root); - - result = msgpack_unpack_next(&unpacked_root, - msgpack_body, - msgpack_body_length, - &offset); - - if (result != MSGPACK_UNPACK_SUCCESS) { - return -1; - } + int result = -1; + int error_status = 0; + struct ctrace *ctr; - /* iterate msgpack and comppose a CTraces context */ - ctr = process_root_msgpack(ctx, &unpacked_root.data); + /* Use the new centralized API for JSON to ctrace conversion */ + ctr = flb_opentelemetry_json_traces_to_ctrace(body, len, &error_status); if (ctr) { result = flb_input_trace_append(ctx->ins, tag, tag_len, ctr); if (result == -1) { ctr_destroy(ctr); } } - - msgpack_unpacked_destroy(&unpacked_root); - flb_free(msgpack_body); + else { + flb_plg_error(ctx->ins, "invalid JSON trace: conversion error (status: %d)", error_status); + } return result; } From 1ceb31108c4d2170e125483b65f1ca6baf2f236e Mon Sep 17 00:00:00 2001 From: Patrick Stephens Date: Sun, 3 May 2026 12:34:42 +0100 Subject: [PATCH 018/213] [upstream] opentelemetry: traces: propagate error status Upstream-Ref: https://github.com/fluent/fluent-bit/commit/4fbd6a0ff2068538e15033952e64742ab853cb0c Cherry-picked from Fluent Bit v4.2.4 --- source/include/fluent-bit/flb_opentelemetry.h | 53 +++- .../opentelemetry/flb_opentelemetry_traces.c | 258 +++++++++++++----- 2 files changed, 248 insertions(+), 63 deletions(-) diff --git a/source/include/fluent-bit/flb_opentelemetry.h b/source/include/fluent-bit/flb_opentelemetry.h index dc137bf9..cc521862 100644 --- a/source/include/fluent-bit/flb_opentelemetry.h +++ b/source/include/fluent-bit/flb_opentelemetry.h @@ -24,8 +24,9 @@ #include #include -/* Error code values from flb_opentelemetry_logs.c */ +/* Error code values from flb_opentelemetry logs/traces helpers */ #define FLB_OTEL_LOGS_ERR_GENERIC_ERROR -1 +#define FLB_OTEL_TRACES_ERR_GENERIC_ERROR -1 enum { @@ -50,7 +51,31 @@ enum { FLB_OTEL_LOGS_ERR_ENCODER_FAILURE, FLB_OTEL_LOGS_ERR_APPEND_BODY_FAILURE, FLB_OTEL_LOGS_ERR_INVALID_TRACE_ID, - FLB_OTEL_LOGS_ERR_INVALID_SPAN_ID + FLB_OTEL_LOGS_ERR_INVALID_SPAN_ID, + + /* trace specific errors */ + FLB_OTEL_TRACES_ERR_UNEXPECTED_ROOT_OBJECT_TYPE, + FLB_OTEL_TRACES_ERR_INVALID_JSON, + FLB_OTEL_TRACES_ERR_RESOURCE_SPANS_MISSING, + FLB_OTEL_TRACES_ERR_UNEXPECTED_RESOURCE_SPANS_TYPE, + FLB_OTEL_TRACES_ERR_UNEXPECTED_RESOURCE_SPANS_ENTRY_TYPE, + FLB_OTEL_TRACES_ERR_SCOPE_SPANS_MISSING, + FLB_OTEL_TRACES_ERR_UNEXPECTED_SCOPE_SPANS_TYPE, + FLB_OTEL_TRACES_ERR_UNEXPECTED_SCOPE_SPANS_ENTRY_TYPE, + FLB_OTEL_TRACES_ERR_SPANS_MISSING, + FLB_OTEL_TRACES_ERR_UNEXPECTED_SPANS_TYPE, + FLB_OTEL_TRACES_ERR_UNEXPECTED_SPAN_ENTRY_TYPE, + FLB_OTEL_TRACES_ERR_SPAN_NAME_MISSING, + FLB_OTEL_TRACES_ERR_INVALID_ATTRIBUTES, + FLB_OTEL_TRACES_ERR_INVALID_TRACE_ID, + FLB_OTEL_TRACES_ERR_INVALID_SPAN_ID, + FLB_OTEL_TRACES_ERR_INVALID_PARENT_SPAN_ID, + FLB_OTEL_TRACES_ERR_INVALID_EVENT_ENTRY, + FLB_OTEL_TRACES_ERR_INVALID_EVENT_TIMESTAMP, + FLB_OTEL_TRACES_ERR_INVALID_LINK_ENTRY, + FLB_OTEL_TRACES_ERR_INVALID_LINK_TRACE_ID, + FLB_OTEL_TRACES_ERR_INVALID_LINK_SPAN_ID, + FLB_OTEL_TRACES_ERR_STATUS_FAILURE }; @@ -88,7 +113,31 @@ static struct flb_otel_error_map otel_error_map[] = { {"FLB_OTEL_LOGS_ERR_APPEND_BODY_FAILURE", FLB_OTEL_LOGS_ERR_APPEND_BODY_FAILURE}, {"FLB_OTEL_LOGS_ERR_INVALID_TRACE_ID", FLB_OTEL_LOGS_ERR_INVALID_TRACE_ID}, {"FLB_OTEL_LOGS_ERR_INVALID_SPAN_ID", FLB_OTEL_LOGS_ERR_INVALID_SPAN_ID}, + + {"FLB_OTEL_TRACES_ERR_UNEXPECTED_ROOT_OBJECT_TYPE", FLB_OTEL_TRACES_ERR_UNEXPECTED_ROOT_OBJECT_TYPE}, + {"FLB_OTEL_TRACES_ERR_INVALID_JSON", FLB_OTEL_TRACES_ERR_INVALID_JSON}, + {"FLB_OTEL_TRACES_ERR_RESOURCE_SPANS_MISSING", FLB_OTEL_TRACES_ERR_RESOURCE_SPANS_MISSING}, + {"FLB_OTEL_TRACES_ERR_UNEXPECTED_RESOURCE_SPANS_TYPE", FLB_OTEL_TRACES_ERR_UNEXPECTED_RESOURCE_SPANS_TYPE}, + {"FLB_OTEL_TRACES_ERR_UNEXPECTED_RESOURCE_SPANS_ENTRY_TYPE", FLB_OTEL_TRACES_ERR_UNEXPECTED_RESOURCE_SPANS_ENTRY_TYPE}, + {"FLB_OTEL_TRACES_ERR_SCOPE_SPANS_MISSING", FLB_OTEL_TRACES_ERR_SCOPE_SPANS_MISSING}, + {"FLB_OTEL_TRACES_ERR_UNEXPECTED_SCOPE_SPANS_TYPE", FLB_OTEL_TRACES_ERR_UNEXPECTED_SCOPE_SPANS_TYPE}, + {"FLB_OTEL_TRACES_ERR_UNEXPECTED_SCOPE_SPANS_ENTRY_TYPE",FLB_OTEL_TRACES_ERR_UNEXPECTED_SCOPE_SPANS_ENTRY_TYPE}, + {"FLB_OTEL_TRACES_ERR_SPANS_MISSING", FLB_OTEL_TRACES_ERR_SPANS_MISSING}, + {"FLB_OTEL_TRACES_ERR_UNEXPECTED_SPANS_TYPE", FLB_OTEL_TRACES_ERR_UNEXPECTED_SPANS_TYPE}, + {"FLB_OTEL_TRACES_ERR_UNEXPECTED_SPAN_ENTRY_TYPE", FLB_OTEL_TRACES_ERR_UNEXPECTED_SPAN_ENTRY_TYPE}, + {"FLB_OTEL_TRACES_ERR_SPAN_NAME_MISSING", FLB_OTEL_TRACES_ERR_SPAN_NAME_MISSING}, + {"FLB_OTEL_TRACES_ERR_INVALID_ATTRIBUTES", FLB_OTEL_TRACES_ERR_INVALID_ATTRIBUTES}, + {"FLB_OTEL_TRACES_ERR_INVALID_TRACE_ID", FLB_OTEL_TRACES_ERR_INVALID_TRACE_ID}, + {"FLB_OTEL_TRACES_ERR_INVALID_SPAN_ID", FLB_OTEL_TRACES_ERR_INVALID_SPAN_ID}, + {"FLB_OTEL_TRACES_ERR_INVALID_PARENT_SPAN_ID", FLB_OTEL_TRACES_ERR_INVALID_PARENT_SPAN_ID}, + {"FLB_OTEL_TRACES_ERR_INVALID_EVENT_ENTRY", FLB_OTEL_TRACES_ERR_INVALID_EVENT_ENTRY}, + {"FLB_OTEL_TRACES_ERR_INVALID_EVENT_TIMESTAMP", FLB_OTEL_TRACES_ERR_INVALID_EVENT_TIMESTAMP}, + {"FLB_OTEL_TRACES_ERR_INVALID_LINK_ENTRY", FLB_OTEL_TRACES_ERR_INVALID_LINK_ENTRY}, + {"FLB_OTEL_TRACES_ERR_INVALID_LINK_TRACE_ID", FLB_OTEL_TRACES_ERR_INVALID_LINK_TRACE_ID}, + {"FLB_OTEL_TRACES_ERR_INVALID_LINK_SPAN_ID", FLB_OTEL_TRACES_ERR_INVALID_LINK_SPAN_ID}, + {"FLB_OTEL_TRACES_ERR_STATUS_FAILURE", FLB_OTEL_TRACES_ERR_STATUS_FAILURE}, {"GENERIC_ERROR", FLB_OTEL_LOGS_ERR_GENERIC_ERROR}, + {"FLB_OTEL_TRACES_ERR_GENERIC_ERROR", FLB_OTEL_TRACES_ERR_GENERIC_ERROR}, /* ---- */ {"FLB_OTEL_LOGS_ERR_EMPTY_PAYLOAD", FLB_OTEL_LOGS_ERR_EMPTY_PAYLOAD}, diff --git a/source/src/opentelemetry/flb_opentelemetry_traces.c b/source/src/opentelemetry/flb_opentelemetry_traces.c index 146cbf5d..b9473dad 100644 --- a/source/src/opentelemetry/flb_opentelemetry_traces.c +++ b/source/src/opentelemetry/flb_opentelemetry_traces.c @@ -17,11 +17,6 @@ * limitations under the License. */ -#include -#include -#include -#include - #include #include #include @@ -224,13 +219,17 @@ static struct ctrace_attributes *convert_attributes(msgpack_object *attributes, static int process_resource_attributes(struct ctrace *ctr, struct ctrace_resource_span *resource_span, - msgpack_object *attributes) + msgpack_object *attributes, + int *error_status) { struct ctrace_resource *resource; struct ctrace_attributes *attr; attr = convert_attributes(attributes, "trace resource"); if (!attr) { + if (error_status) { + *error_status = FLB_OTEL_TRACES_ERR_INVALID_ATTRIBUTES; + } return -1; } @@ -245,7 +244,8 @@ static int process_scope_attributes(struct ctrace *ctr, msgpack_object *name, msgpack_object *version, msgpack_object *attributes, - msgpack_object *dropped_attributes_count) + msgpack_object *dropped_attributes_count, + int *error_status) { int dropped = 0; cfl_sds_t name_str = NULL; @@ -256,6 +256,9 @@ static int process_scope_attributes(struct ctrace *ctr, if (attributes) { attr = convert_attributes(attributes, "trace scope"); if (!attr) { + if (error_status) { + *error_status = FLB_OTEL_TRACES_ERR_INVALID_ATTRIBUTES; + } return -1; } } @@ -282,6 +285,9 @@ static int process_scope_attributes(struct ctrace *ctr, if (attr) { ctr_attributes_destroy(attr); } + if (error_status) { + *error_status = FLB_OTEL_TRACES_ERR_GENERIC_ERROR; + } return -1; } @@ -298,7 +304,8 @@ static int process_scope_attributes(struct ctrace *ctr, static int process_events(struct ctrace *ctr, struct ctrace_span *span, - msgpack_object *events) + msgpack_object *events, + int *error_status) { int i; int ret; @@ -315,7 +322,9 @@ static int process_events(struct ctrace *ctr, for (i = 0; i < events->via.array.size; i++) { event = events->via.array.ptr[i]; if (event.type != MSGPACK_OBJECT_MAP) { - flb_error("unexpected event type"); + if (error_status) { + *error_status = FLB_OTEL_TRACES_ERR_INVALID_EVENT_ENTRY; + } return -1; } @@ -328,12 +337,17 @@ static int process_events(struct ctrace *ctr, name = &event.via.map.ptr[ret].val; name_str = cfl_sds_create_len(name->via.str.ptr, name->via.str.size); if (name_str == NULL) { + if (error_status) { + *error_status = FLB_OTEL_TRACES_ERR_INVALID_EVENT_ENTRY; + } return -1; } } if (!name_str) { - flb_warn("span event name is missing"); + if (error_status) { + *error_status = FLB_OTEL_TRACES_ERR_INVALID_EVENT_ENTRY; + } return -1; } @@ -347,10 +361,12 @@ static int process_events(struct ctrace *ctr, memcpy(tmp, event.via.map.ptr[ret].val.via.str.ptr, len); tmp[len] = '\0'; - flb_error("invalid timeUnixNano: '%s'", tmp); if (name_str) { cfl_sds_destroy(name_str); } + if (error_status) { + *error_status = FLB_OTEL_TRACES_ERR_INVALID_EVENT_TIMESTAMP; + } return -1; } @@ -363,6 +379,9 @@ static int process_events(struct ctrace *ctr, ctr_event = ctr_span_event_add_ts(span, name_str, ts); cfl_sds_destroy(name_str); if (ctr_event == NULL) { + if (error_status) { + *error_status = FLB_OTEL_TRACES_ERR_INVALID_EVENT_ENTRY; + } return -1; } @@ -388,7 +407,8 @@ static int process_events(struct ctrace *ctr, static int process_links(struct ctrace *ctr, struct ctrace_span *span, - msgpack_object *links) + msgpack_object *links, + int *error_status) { int i; int ret; @@ -411,7 +431,9 @@ static int process_links(struct ctrace *ctr, for (i = 0; i < links->via.array.size; i++) { link = links->via.array.ptr[i]; if (link.type != MSGPACK_OBJECT_MAP) { - flb_error("unexpected link type"); + if (error_status) { + *error_status = FLB_OTEL_TRACES_ERR_INVALID_LINK_ENTRY; + } return -1; } @@ -436,7 +458,9 @@ static int process_links(struct ctrace *ctr, memcpy(tmp, trace_id->via.str.ptr, len); tmp[len] = '\0'; - flb_error("invalid event traceId: '%s'", tmp); + if (error_status) { + *error_status = FLB_OTEL_TRACES_ERR_INVALID_LINK_TRACE_ID; + } return -1; } @@ -446,7 +470,9 @@ static int process_links(struct ctrace *ctr, } if (!trace_id) { - flb_error("link traceId is missing"); + if (error_status) { + *error_status = FLB_OTEL_TRACES_ERR_INVALID_LINK_TRACE_ID; + } return -1; } @@ -463,7 +489,9 @@ static int process_links(struct ctrace *ctr, } memcpy(tmp, span_id->via.str.ptr, len); tmp[len] = '\0'; - flb_error("invalid spanId: '%s'", tmp); + if (error_status) { + *error_status = FLB_OTEL_TRACES_ERR_INVALID_LINK_SPAN_ID; + } return -1; } @@ -474,7 +502,9 @@ static int process_links(struct ctrace *ctr, } if (!span_id) { - flb_error("link spanId is missing"); + if (error_status) { + *error_status = FLB_OTEL_TRACES_ERR_INVALID_LINK_SPAN_ID; + } return -1; } @@ -511,6 +541,9 @@ static int process_links(struct ctrace *ctr, if (ctr_attr) { ctr_attributes_destroy(ctr_attr); } + if (error_status) { + *error_status = FLB_OTEL_TRACES_ERR_INVALID_LINK_ENTRY; + } return -1; } @@ -540,7 +573,8 @@ static int process_links(struct ctrace *ctr, static int process_span_status(struct ctrace *ctr, struct ctrace_span *span, - msgpack_object *status) + msgpack_object *status, + int *error_status) { int ret; int code = 0; @@ -548,7 +582,7 @@ static int process_span_status(struct ctrace *ctr, char *message = NULL; if (status->type != MSGPACK_OBJECT_MAP) { - flb_error("unexpected status type"); + flb_error("unexpected type for status"); return -1; } @@ -571,14 +605,18 @@ static int process_span_status(struct ctrace *ctr, code = CTRACE_SPAN_STATUS_CODE_ERROR; } else { - flb_error("status code value is invalid: %s", tmp); cfl_sds_destroy(tmp); + if (error_status) { + *error_status = FLB_OTEL_TRACES_ERR_STATUS_FAILURE; + } return -1; } cfl_sds_destroy(tmp); } else { - flb_error("status code is missing"); + if (error_status) { + *error_status = FLB_OTEL_TRACES_ERR_STATUS_FAILURE; + } return -1; } @@ -599,7 +637,8 @@ static int process_span_status(struct ctrace *ctr, static int process_spans(struct ctrace *ctr, struct ctrace_scope_span *scope_span, - msgpack_object *spans) + msgpack_object *spans, + int *error_status) { int i; int ret; @@ -616,7 +655,9 @@ static int process_spans(struct ctrace *ctr, for (i = 0; i < spans->via.array.size; i++) { span = spans->via.array.ptr[i]; if (span.type != MSGPACK_OBJECT_MAP) { - flb_error("unexpected span type"); + if (error_status) { + *error_status = FLB_OTEL_TRACES_ERR_UNEXPECTED_SPAN_ENTRY_TYPE; + } return -1; } @@ -630,7 +671,9 @@ static int process_spans(struct ctrace *ctr, } if (!val_str) { - flb_error("span name is missing"); + if (error_status) { + *error_status = FLB_OTEL_TRACES_ERR_SPAN_NAME_MISSING; + } return -1; } @@ -639,6 +682,9 @@ static int process_spans(struct ctrace *ctr, cfl_sds_destroy(val_str); if (ctr_span == NULL) { + if (error_status) { + *error_status = FLB_OTEL_TRACES_ERR_GENERIC_ERROR; + } return -1; } @@ -654,7 +700,9 @@ static int process_spans(struct ctrace *ctr, memcpy(tmp, span.via.map.ptr[ret].val.via.str.ptr, len); tmp[len] = '\0'; - flb_error("invalid traceId: '%s'", tmp); + if (error_status) { + *error_status = FLB_OTEL_TRACES_ERR_INVALID_TRACE_ID; + } return -1; } @@ -677,7 +725,9 @@ static int process_spans(struct ctrace *ctr, } memcpy(tmp, span.via.map.ptr[ret].val.via.str.ptr, len); tmp[len] = '\0'; - flb_error("invalid spanId: '%s'", tmp); + if (error_status) { + *error_status = FLB_OTEL_TRACES_ERR_INVALID_SPAN_ID; + } return -1; } @@ -718,7 +768,9 @@ static int process_spans(struct ctrace *ctr, } memcpy(tmp, span.via.map.ptr[ret].val.via.str.ptr, len); tmp[len] = '\0'; - flb_error("invalid parentSpanId: '%s'", tmp); + if (error_status) { + *error_status = FLB_OTEL_TRACES_ERR_INVALID_PARENT_SPAN_ID; + } return -1; } @@ -779,7 +831,10 @@ static int process_spans(struct ctrace *ctr, /* events */ ret = flb_otel_utils_find_map_entry_by_key(&span.via.map, "events", 0, FLB_TRUE); if (ret >= 0 && span.via.map.ptr[ret].val.type == MSGPACK_OBJECT_ARRAY) { - ret = process_events(ctr, ctr_span, &span.via.map.ptr[ret].val); + ret = process_events(ctr, ctr_span, &span.via.map.ptr[ret].val, error_status); + if (ret == -1) { + return -1; + } } /* dropped_events_count */ @@ -797,7 +852,10 @@ static int process_spans(struct ctrace *ctr, /* links */ ret = flb_otel_utils_find_map_entry_by_key(&span.via.map, "links", 0, FLB_TRUE); if (ret >= 0 && span.via.map.ptr[ret].val.type == MSGPACK_OBJECT_ARRAY) { - ret = process_links(ctr, ctr_span, &span.via.map.ptr[ret].val); + ret = process_links(ctr, ctr_span, &span.via.map.ptr[ret].val, error_status); + if (ret == -1) { + return -1; + } } /* schema_url */ @@ -813,7 +871,10 @@ static int process_spans(struct ctrace *ctr, /* status */ ret = flb_otel_utils_find_map_entry_by_key(&span.via.map, "status", 0, FLB_TRUE); if (ret >= 0 && span.via.map.ptr[ret].val.type == MSGPACK_OBJECT_MAP) { - process_span_status(ctr, ctr_span, &span.via.map.ptr[ret].val); + ret = process_span_status(ctr, ctr_span, &span.via.map.ptr[ret].val, error_status); + if (ret == -1) { + return -1; + } } } @@ -822,7 +883,8 @@ static int process_spans(struct ctrace *ctr, static int process_scope_span(struct ctrace *ctr, struct ctrace_resource_span *resource_span, - msgpack_object *scope_spans) + msgpack_object *scope_spans, + int *error_status) { int ret; msgpack_object scope; @@ -841,12 +903,18 @@ static int process_scope_span(struct ctrace *ctr, scope = scope_spans->via.map.ptr[ret].val; if (scope.type != MSGPACK_OBJECT_MAP) { flb_error("unexpected scope type in scope span"); + if (error_status) { + *error_status = FLB_OTEL_TRACES_ERR_UNEXPECTED_SCOPE_SPANS_ENTRY_TYPE; + } return -1; } /* create the scope_span */ scope_span = ctr_scope_span_create(resource_span); if (scope_span == NULL) { + if (error_status) { + *error_status = FLB_OTEL_TRACES_ERR_GENERIC_ERROR; + } return -1; } @@ -880,15 +948,23 @@ static int process_scope_span(struct ctrace *ctr, ret = process_scope_attributes(ctr, scope_span, - name, version, attr, dropped_attr); + name, version, attr, dropped_attr, + error_status); if (ret == -1) { flb_warn("failed to process scope attributes"); + if (error_status && *error_status == 0) { + *error_status = FLB_OTEL_TRACES_ERR_INVALID_ATTRIBUTES; + } + return -1; } } else { /* If scope is not defined we still need the scope span container */ scope_span = ctr_scope_span_create(resource_span); if (scope_span == NULL) { + if (error_status) { + *error_status = FLB_OTEL_TRACES_ERR_GENERIC_ERROR; + } return -1; } } @@ -910,23 +986,34 @@ static int process_scope_span(struct ctrace *ctr, /* spans */ spans = NULL; ret = flb_otel_utils_find_map_entry_by_key(&scope_spans->via.map, "spans", 0, FLB_TRUE); - if (ret >= 0 && scope_spans->via.map.ptr[ret].val.type == MSGPACK_OBJECT_ARRAY) { - spans = &scope_spans->via.map.ptr[ret].val; + if (ret < 0) { + if (error_status) { + *error_status = FLB_OTEL_TRACES_ERR_SPANS_MISSING; + } + return -1; } - if (spans) { - ret = process_spans(ctr, scope_span, spans); - if (ret == -1) { - flb_error("failed to process spans"); - return -1; + if (scope_spans->via.map.ptr[ret].val.type == MSGPACK_OBJECT_ARRAY) { + spans = &scope_spans->via.map.ptr[ret].val; + } + else { + if (error_status) { + *error_status = FLB_OTEL_TRACES_ERR_UNEXPECTED_SPANS_TYPE; } + return -1; + } + + ret = process_spans(ctr, scope_span, spans, error_status); + if (ret == -1) { + return -1; } return 0; } static int process_resource_span(struct ctrace *ctr, - msgpack_object *resource_span) + msgpack_object *resource_span, + int *error_status) { int ret; msgpack_object resource; @@ -938,13 +1025,18 @@ static int process_resource_span(struct ctrace *ctr, struct ctrace_resource_span *ctr_resource_span; if (resource_span->type != MSGPACK_OBJECT_MAP) { - flb_error("unexpected resource span type"); + if (error_status) { + *error_status = FLB_OTEL_TRACES_ERR_UNEXPECTED_RESOURCE_SPANS_ENTRY_TYPE; + } return -1; } /* create a resource span */ ctr_resource_span = ctr_resource_span_create(ctr); if (ctr_resource_span == NULL) { + if (error_status) { + *error_status = FLB_OTEL_TRACES_ERR_GENERIC_ERROR; + } return -1; } @@ -958,9 +1050,12 @@ static int process_resource_span(struct ctrace *ctr, ret = flb_otel_utils_find_map_entry_by_key(&resource.via.map, "attributes", 0, FLB_TRUE); if (ret >= 0 && resource.via.map.ptr[ret].val.type == MSGPACK_OBJECT_ARRAY) { attr = &resource.via.map.ptr[ret].val; - ret = process_resource_attributes(ctr, ctr_resource_span, attr); + ret = process_resource_attributes(ctr, ctr_resource_span, attr, error_status); if (ret == -1) { - flb_warn("failed to process resource attributes"); + if (error_status && *error_status == 0) { + *error_status = FLB_OTEL_TRACES_ERR_INVALID_ATTRIBUTES; + } + return -1; } } @@ -999,44 +1094,69 @@ static int process_resource_span(struct ctrace *ctr, /* scopeSpans */ scope_spans = NULL; ret = flb_otel_utils_find_map_entry_by_key(&resource_span->via.map, "scopeSpans", 0, FLB_TRUE); - if (ret >= 0 && resource_span->via.map.ptr[ret].val.type == MSGPACK_OBJECT_ARRAY) { + if (ret < 0) { + if (error_status) { + *error_status = FLB_OTEL_TRACES_ERR_SCOPE_SPANS_MISSING; + } + return -1; + } + + if (resource_span->via.map.ptr[ret].val.type == MSGPACK_OBJECT_ARRAY) { scope_spans = &resource_span->via.map.ptr[ret].val; } + else { + if (error_status) { + *error_status = FLB_OTEL_TRACES_ERR_UNEXPECTED_SCOPE_SPANS_TYPE; + } + return -1; + } - if (scope_spans) { - scope_spans_array = &scope_spans->via.array; - for (ret = 0; ret < scope_spans_array->size; ret++) { - if (process_scope_span(ctr, - ctr_resource_span, - &scope_spans_array->ptr[ret]) == -1) { - flb_error("failed to process scope span"); - return -1; + scope_spans_array = &scope_spans->via.array; + for (ret = 0; ret < scope_spans_array->size; ret++) { + if (scope_spans_array->ptr[ret].type != MSGPACK_OBJECT_MAP) { + if (error_status) { + *error_status = FLB_OTEL_TRACES_ERR_UNEXPECTED_SCOPE_SPANS_ENTRY_TYPE; } + return -1; + } + + if (process_scope_span(ctr, + ctr_resource_span, + &scope_spans_array->ptr[ret], + error_status) == -1) { + return -1; } } return 0; } -static struct ctrace *process_root_msgpack(msgpack_object *obj) +static struct ctrace *process_root_msgpack(msgpack_object *obj, + int *error_status) { int ret; msgpack_object *resource_spans; struct ctrace *ctr; if (obj->type != MSGPACK_OBJECT_MAP) { - flb_error("unexpected root object type for traces"); + if (error_status) { + *error_status = FLB_OTEL_TRACES_ERR_UNEXPECTED_ROOT_OBJECT_TYPE; + } return NULL; } ret = flb_otel_utils_find_map_entry_by_key(&obj->via.map, "resourceSpans", 0, FLB_TRUE); if (ret < 0) { - flb_error("resourceSpans entry is missing"); + if (error_status) { + *error_status = FLB_OTEL_TRACES_ERR_RESOURCE_SPANS_MISSING; + } return NULL; } if (obj->via.map.ptr[ret].val.type != MSGPACK_OBJECT_ARRAY) { - flb_error("resourceSpans entry is invalid"); + if (error_status) { + *error_status = FLB_OTEL_TRACES_ERR_UNEXPECTED_RESOURCE_SPANS_TYPE; + } return NULL; } @@ -1045,13 +1165,16 @@ static struct ctrace *process_root_msgpack(msgpack_object *obj) ctr = ctr_create(NULL); if (!ctr) { + if (error_status) { + *error_status = FLB_OTEL_TRACES_ERR_GENERIC_ERROR; + } return NULL; } for (ret = 0; ret < resource_spans->via.array.size; ret++) { - if (process_resource_span(ctr, &resource_spans->via.array.ptr[ret]) == -1) { - flb_warn("failed to process resource span"); - + if (process_resource_span(ctr, + &resource_spans->via.array.ptr[ret], + error_status) == -1) { ctr_destroy(ctr); return NULL; } @@ -1071,11 +1194,17 @@ struct ctrace *flb_opentelemetry_json_traces_to_ctrace(const char *body, size_t msgpack_unpacked unpacked_root; struct ctrace *ctr; + if (error_status) { + *error_status = 0; + } + result = flb_pack_json(body, len, &msgpack_body, &msgpack_body_length, &root_type, NULL); if (result != 0) { - flb_error("invalid JSON trace: msgpack conversion error"); + if (error_status) { + *error_status = FLB_OTEL_TRACES_ERR_INVALID_JSON; + } return NULL; } @@ -1089,14 +1218,21 @@ struct ctrace *flb_opentelemetry_json_traces_to_ctrace(const char *body, size_t if (result != MSGPACK_UNPACK_SUCCESS) { msgpack_unpacked_destroy(&unpacked_root); flb_free(msgpack_body); + if (error_status) { + *error_status = FLB_OTEL_TRACES_ERR_UNEXPECTED_ROOT_OBJECT_TYPE; + } return NULL; } /* iterate msgpack and compose a CTraces context */ - ctr = process_root_msgpack(&unpacked_root.data); + ctr = process_root_msgpack(&unpacked_root.data, error_status); msgpack_unpacked_destroy(&unpacked_root); flb_free(msgpack_body); + if (!ctr && error_status && *error_status == 0) { + *error_status = FLB_OTEL_TRACES_ERR_GENERIC_ERROR; + } + return ctr; } From 3f05f20bb8449c0ef9dd48b6d4c6488761f27e7a Mon Sep 17 00:00:00 2001 From: Patrick Stephens Date: Sun, 3 May 2026 12:34:42 +0100 Subject: [PATCH 019/213] [upstream] opentelemetry: logs: strict check when decoding IDs Upstream-Ref: https://github.com/fluent/fluent-bit/commit/c576b1e35ba071f006585e3d2422be1a053b7e15 Cherry-picked from Fluent Bit v4.2.4 --- source/src/opentelemetry/flb_opentelemetry_logs.c | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/source/src/opentelemetry/flb_opentelemetry_logs.c b/source/src/opentelemetry/flb_opentelemetry_logs.c index 85894225..700f8c6e 100644 --- a/source/src/opentelemetry/flb_opentelemetry_logs.c +++ b/source/src/opentelemetry/flb_opentelemetry_logs.c @@ -302,14 +302,24 @@ static int process_json_payload_log_records_entry( } if (trace_id != NULL && trace_id->type == MSGPACK_OBJECT_STR && trace_id->via.str.size == 32) { - flb_otel_utils_hex_to_id(trace_id->via.str.ptr, trace_id->via.str.size, tmp_id, 16); + if (flb_otel_utils_hex_to_id(trace_id->via.str.ptr, trace_id->via.str.size, tmp_id, 16) != 0) { + if (error_status) { + *error_status = FLB_OTEL_LOGS_ERR_INVALID_TRACE_ID; + } + return -FLB_OTEL_LOGS_ERR_INVALID_TRACE_ID; + } flb_log_event_encoder_append_metadata_values(encoder, FLB_LOG_EVENT_STRING_VALUE("trace_id", 8), FLB_LOG_EVENT_BINARY_VALUE(tmp_id, 16)); } if (span_id != NULL && span_id->type == MSGPACK_OBJECT_STR && span_id->via.str.size == 16) { - flb_otel_utils_hex_to_id(span_id->via.str.ptr, span_id->via.str.size, tmp_id, 8); + if (flb_otel_utils_hex_to_id(span_id->via.str.ptr, span_id->via.str.size, tmp_id, 8) != 0) { + if (error_status) { + *error_status = FLB_OTEL_LOGS_ERR_INVALID_SPAN_ID; + } + return -FLB_OTEL_LOGS_ERR_INVALID_SPAN_ID; + } flb_log_event_encoder_append_metadata_values(encoder, FLB_LOG_EVENT_STRING_VALUE("span_id", 7), FLB_LOG_EVENT_BINARY_VALUE(tmp_id, 8)); From 43f4aa3d8df39489f1700f4d986e254026e39f1e Mon Sep 17 00:00:00 2001 From: Patrick Stephens Date: Sun, 3 May 2026 12:34:42 +0100 Subject: [PATCH 020/213] [upstream] opentelemetry: traces: strict check when decoding IDs Upstream-Ref: https://github.com/fluent/fluent-bit/commit/f2244cde0c58af3b9a0c6caad2afb2983426287d Cherry-picked from Fluent Bit v4.2.4 --- .../opentelemetry/flb_opentelemetry_traces.c | 51 ++++++++++++++----- 1 file changed, 38 insertions(+), 13 deletions(-) diff --git a/source/src/opentelemetry/flb_opentelemetry_traces.c b/source/src/opentelemetry/flb_opentelemetry_traces.c index b9473dad..4d61f323 100644 --- a/source/src/opentelemetry/flb_opentelemetry_traces.c +++ b/source/src/opentelemetry/flb_opentelemetry_traces.c @@ -465,8 +465,13 @@ static int process_links(struct ctrace *ctr, } /* decode the hex string (16 bytes) */ - flb_otel_utils_hex_to_id((char *) trace_id->via.str.ptr, trace_id->via.str.size, - (unsigned char *) trace_id_bin, 16); + if (flb_otel_utils_hex_to_id((char *) trace_id->via.str.ptr, trace_id->via.str.size, + (unsigned char *) trace_id_bin, 16) != 0) { + if (error_status) { + *error_status = FLB_OTEL_TRACES_ERR_INVALID_LINK_TRACE_ID; + } + return -1; + } } if (!trace_id) { @@ -497,8 +502,13 @@ static int process_links(struct ctrace *ctr, /* decode the hex string (8 bytes) */ memset(tmp, '\0', sizeof(tmp)); - flb_otel_utils_hex_to_id((char *) span_id->via.str.ptr, span_id->via.str.size, - (unsigned char *) span_id_bin, 8); + if (flb_otel_utils_hex_to_id((char *) span_id->via.str.ptr, span_id->via.str.size, + (unsigned char *) span_id_bin, 8) != 0) { + if (error_status) { + *error_status = FLB_OTEL_TRACES_ERR_INVALID_LINK_SPAN_ID; + } + return -1; + } } if (!span_id) { @@ -708,9 +718,14 @@ static int process_spans(struct ctrace *ctr, /* decode the hex string (16 bytes) */ memset(tmp, '\0', sizeof(tmp)); - flb_otel_utils_hex_to_id((char *) span.via.map.ptr[ret].val.via.str.ptr, - span.via.map.ptr[ret].val.via.str.size, - (unsigned char *) tmp, 16); + if (flb_otel_utils_hex_to_id((char *) span.via.map.ptr[ret].val.via.str.ptr, + span.via.map.ptr[ret].val.via.str.size, + (unsigned char *) tmp, 16) != 0) { + if (error_status) { + *error_status = FLB_OTEL_TRACES_ERR_INVALID_TRACE_ID; + } + return -1; + } ctr_span_set_trace_id(ctr_span, tmp, 16); } @@ -733,9 +748,14 @@ static int process_spans(struct ctrace *ctr, /* decode the hex string (8 bytes) */ memset(tmp, '\0', sizeof(tmp)); - flb_otel_utils_hex_to_id((char *) span.via.map.ptr[ret].val.via.str.ptr, - span.via.map.ptr[ret].val.via.str.size, - (unsigned char *) tmp, 8); + if (flb_otel_utils_hex_to_id((char *) span.via.map.ptr[ret].val.via.str.ptr, + span.via.map.ptr[ret].val.via.str.size, + (unsigned char *) tmp, 8) != 0) { + if (error_status) { + *error_status = FLB_OTEL_TRACES_ERR_INVALID_SPAN_ID; + } + return -1; + } ctr_span_set_span_id(ctr_span, tmp, 8); } @@ -776,9 +796,14 @@ static int process_spans(struct ctrace *ctr, /* decode the hex string (8 bytes) */ memset(tmp, '\0', sizeof(tmp)); - flb_otel_utils_hex_to_id((char *) span.via.map.ptr[ret].val.via.str.ptr, - span.via.map.ptr[ret].val.via.str.size, - (unsigned char *) tmp, 8); + if (flb_otel_utils_hex_to_id((char *) span.via.map.ptr[ret].val.via.str.ptr, + span.via.map.ptr[ret].val.via.str.size, + (unsigned char *) tmp, 8) != 0) { + if (error_status) { + *error_status = FLB_OTEL_TRACES_ERR_INVALID_PARENT_SPAN_ID; + } + return -1; + } ctr_span_set_parent_span_id(ctr_span, tmp, 8); } From f812ce81f99fc2302d01ad12e9c964313696787d Mon Sep 17 00:00:00 2001 From: Patrick Stephens Date: Sun, 3 May 2026 12:34:43 +0100 Subject: [PATCH 021/213] [upstream] config_format: add new function to add property Upstream-Ref: https://github.com/fluent/fluent-bit/commit/03f3803997af19ec7265678d26dbc124c92b577d Cherry-picked from Fluent Bit v4.2.4 --- .../include/fluent-bit/config_format/flb_cf.h | 5 +++ source/src/config_format/flb_config_format.c | 38 +++++++++++++++++++ 2 files changed, 43 insertions(+) diff --git a/source/include/fluent-bit/config_format/flb_cf.h b/source/include/fluent-bit/config_format/flb_cf.h index b5ae20fa..6c6233c2 100644 --- a/source/include/fluent-bit/config_format/flb_cf.h +++ b/source/include/fluent-bit/config_format/flb_cf.h @@ -174,6 +174,11 @@ struct cfl_variant *flb_cf_section_property_add(struct flb_cf *cf, char *k_buf, size_t k_len, char *v_buf, size_t v_len); +struct cfl_variant *flb_cf_section_property_add_variant(struct flb_cf *cf, + struct cfl_kvlist *kv_list, + char *k_buf, size_t k_len, + struct cfl_variant *variant); + struct cfl_array *flb_cf_section_property_add_list(struct flb_cf *cf, struct cfl_kvlist *kv_list, char *k_buf, size_t k_len); diff --git a/source/src/config_format/flb_config_format.c b/source/src/config_format/flb_config_format.c index 28486d6a..e6d55bf2 100644 --- a/source/src/config_format/flb_config_format.c +++ b/source/src/config_format/flb_config_format.c @@ -325,6 +325,44 @@ struct cfl_variant *flb_cf_section_property_add(struct flb_cf *cf, return NULL; } +struct cfl_variant *flb_cf_section_property_add_variant(struct flb_cf *cf, + struct cfl_kvlist *kv_list, + char *k_buf, size_t k_len, + struct cfl_variant *variant) +{ + int rc; + flb_sds_t key; + + if (variant == NULL) { + return NULL; + } + + if (k_len == 0) { + k_len = strlen(k_buf); + } + + key = flb_cf_key_translate(cf, k_buf, k_len); + if (key == NULL) { + return NULL; + } + + rc = flb_sds_trim(key); + if (rc == -1) { + flb_cf_error_set(cf, FLB_CF_ERROR_KV_INVALID_KEY); + flb_sds_destroy(key); + return NULL; + } + + rc = cfl_kvlist_insert(kv_list, key, variant); + if (rc < 0) { + flb_sds_destroy(key); + return NULL; + } + + flb_sds_destroy(key); + return variant; +} + struct cfl_array *flb_cf_section_property_add_list(struct flb_cf *cf, struct cfl_kvlist *kv_list, char *k_buf, size_t k_len) From d65ff798d086c44582272d624428be5cfab9e34c Mon Sep 17 00:00:00 2001 From: Patrick Stephens Date: Sun, 3 May 2026 12:34:43 +0100 Subject: [PATCH 022/213] [upstream] router_config: new interface to parse new input Upstream-Ref: https://github.com/fluent/fluent-bit/commit/cbd9b13f5a595731e0afece255aa72f30bfaed59 Cherry-picked from Fluent Bit v4.2.4 --- source/src/flb_router_config.c | 1225 ++++++++++++++++++++++++++++++++ 1 file changed, 1225 insertions(+) create mode 100644 source/src/flb_router_config.c diff --git a/source/src/flb_router_config.c b/source/src/flb_router_config.c new file mode 100644 index 00000000..f059ac3f --- /dev/null +++ b/source/src/flb_router_config.c @@ -0,0 +1,1225 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2024 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#ifndef _WIN32 +#include +#endif + +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +static flb_sds_t copy_from_cfl_sds(cfl_sds_t value) +{ + if (!value) { + return NULL; + } + + return flb_sds_create_len(value, cfl_sds_len(value)); +} + +static flb_sds_t variant_to_sds(struct cfl_variant *var) +{ + char tmp[64]; + int len; + + if (!var) { + return NULL; + } + + switch (var->type) { + case CFL_VARIANT_STRING: + return copy_from_cfl_sds(var->data.as_string); + case CFL_VARIANT_INT: + len = snprintf(tmp, sizeof(tmp), "%" PRId64, var->data.as_int64); + if (len < 0) { + return NULL; + } + return flb_sds_create_len(tmp, len); + case CFL_VARIANT_UINT: + len = snprintf(tmp, sizeof(tmp), "%" PRIu64, var->data.as_uint64); + if (len < 0) { + return NULL; + } + return flb_sds_create_len(tmp, len); + case CFL_VARIANT_DOUBLE: + len = snprintf(tmp, sizeof(tmp), "%.*g", 17, var->data.as_double); + if (len < 0) { + return NULL; + } + return flb_sds_create_len(tmp, len); + case CFL_VARIANT_BOOL: + return flb_sds_create(var->data.as_bool ? "true" : "false"); + default: + break; + } + + return NULL; +} + +static int variant_to_bool(struct cfl_variant *var, int *out) +{ + const char *val; + + if (!var || !out) { + return -1; + } + + if (var->type == CFL_VARIANT_BOOL) { + *out = var->data.as_bool != 0; + return 0; + } + else if (var->type == CFL_VARIANT_STRING && var->data.as_string) { + val = var->data.as_string; + + if (strcasecmp(val, "true") == 0) { + *out = FLB_TRUE; + return 0; + } + else if (strcasecmp(val, "false") == 0) { + *out = FLB_FALSE; + return 0; + } + } + + return -1; +} + +static int field_allowed_for_logs(const char *field) +{ + if (!field) { + return FLB_FALSE; + } + + if (strncmp(field, "$metric.", 8) == 0) { + return FLB_FALSE; + } + if (strncmp(field, "$span.", 6) == 0) { + return FLB_FALSE; + } + if (strncmp(field, "$scope.", 7) == 0) { + return FLB_FALSE; + } + + return FLB_TRUE; +} + +static int field_allowed_for_metrics(const char *field) +{ + if (!field) { + return FLB_FALSE; + } + + if (strncmp(field, "$metric.", 8) == 0) { + return FLB_TRUE; + } + if (strncmp(field, "$resource[", 10) == 0) { + return FLB_TRUE; + } + if (strncmp(field, "$attributes[", 12) == 0) { + return FLB_TRUE; + } + + return FLB_FALSE; +} + +static int field_allowed_for_traces(const char *field) +{ + if (!field) { + return FLB_FALSE; + } + + if (strncmp(field, "$span.", 6) == 0) { + return FLB_TRUE; + } + if (strncmp(field, "$resource[", 10) == 0) { + return FLB_TRUE; + } + if (strncmp(field, "$scope[", 7) == 0) { + return FLB_TRUE; + } + + return FLB_FALSE; +} + +static int validate_rule_field(const char *field, uint32_t signals) +{ + int ok = FLB_FALSE; + + if (!field) { + return FLB_FALSE; + } + + if (signals == FLB_ROUTER_SIGNAL_ANY) { + signals = FLB_ROUTER_SIGNAL_LOGS | + FLB_ROUTER_SIGNAL_METRICS | + FLB_ROUTER_SIGNAL_TRACES; + } + + if ((signals & FLB_ROUTER_SIGNAL_LOGS) && field_allowed_for_logs(field)) { + ok = FLB_TRUE; + } + + if ((signals & FLB_ROUTER_SIGNAL_METRICS) && + field_allowed_for_metrics(field)) { + ok = FLB_TRUE; + } + + if ((signals & FLB_ROUTER_SIGNAL_TRACES) && + field_allowed_for_traces(field)) { + ok = FLB_TRUE; + } + + return ok; +} + +static uint32_t parse_signal_key(const char *key) +{ + const char *cursor; + const char *start; + size_t len; + uint32_t mask = 0; + + if (!key) { + return 0; + } + + cursor = key; + while (*cursor) { + while (*cursor && (isspace((unsigned char) *cursor) || + *cursor == ',' || *cursor == '|' || *cursor == '+')) { + cursor++; + } + + if (*cursor == '\0') { + break; + } + + start = cursor; + while (*cursor && !isspace((unsigned char) *cursor) && + *cursor != ',' && *cursor != '|' && *cursor != '+') { + cursor++; + } + + len = cursor - start; + if (len == 0) { + continue; + } + + if (len == 4 && strncasecmp(start, "logs", len) == 0) { + mask |= FLB_ROUTER_SIGNAL_LOGS; + } + else if (len == 7 && strncasecmp(start, "metrics", len) == 0) { + mask |= FLB_ROUTER_SIGNAL_METRICS; + } + else if (len == 6 && strncasecmp(start, "traces", len) == 0) { + mask |= FLB_ROUTER_SIGNAL_TRACES; + } + else if (len == 3 && strncasecmp(start, "any", len) == 0) { + mask |= FLB_ROUTER_SIGNAL_ANY; + } + else { + return 0; + } + } + + return mask; +} + +static void route_condition_destroy(struct flb_route_condition *condition) +{ + struct cfl_list *tmp; + struct cfl_list *head; + struct flb_route_condition_rule *rule; + + if (!condition) { + return; + } + + cfl_list_foreach_safe(head, tmp, &condition->rules) { + rule = cfl_list_entry(head, struct flb_route_condition_rule, _head); + cfl_list_del(&rule->_head); + + if (rule->field) { + flb_sds_destroy(rule->field); + } + if (rule->op) { + flb_sds_destroy(rule->op); + } + if (rule->value) { + flb_sds_destroy(rule->value); + } + + flb_free(rule); + } + + flb_free(condition); +} + +static void route_outputs_destroy(struct flb_route *route) +{ + struct cfl_list *tmp; + struct cfl_list *head; + struct flb_route_output *output; + + cfl_list_foreach_safe(head, tmp, &route->outputs) { + output = cfl_list_entry(head, struct flb_route_output, _head); + cfl_list_del(&output->_head); + + if (output->name) { + flb_sds_destroy(output->name); + } + if (output->fallback) { + flb_sds_destroy(output->fallback); + } + flb_free(output); + } +} + +static void route_processors_destroy(struct cfl_list *processors) +{ + struct cfl_list *tmp; + struct cfl_list *head; + struct cfl_list *p_head; + struct cfl_list *p_tmp; + struct flb_route_processor *processor; + struct flb_route_processor_property *prop; + + if (!processors) { + return; + } + + cfl_list_foreach_safe(head, tmp, processors) { + processor = cfl_list_entry(head, struct flb_route_processor, _head); + cfl_list_del(&processor->_head); + + cfl_list_foreach_safe(p_head, p_tmp, &processor->properties) { + prop = cfl_list_entry(p_head, struct flb_route_processor_property, _head); + cfl_list_del(&prop->_head); + + if (prop->key) { + flb_sds_destroy(prop->key); + } + if (prop->value) { + flb_sds_destroy(prop->value); + } + flb_free(prop); + } + + if (processor->name) { + flb_sds_destroy(processor->name); + } + flb_free(processor); + } +} + +static void input_routes_destroy(struct flb_input_routes *input) +{ + struct cfl_list *r_head; + struct cfl_list *r_tmp; + struct flb_route *route; + + if (!input) { + return; + } + + route_processors_destroy(&input->processors); + + cfl_list_foreach_safe(r_head, r_tmp, &input->routes) { + route = cfl_list_entry(r_head, struct flb_route, _head); + cfl_list_del(&route->_head); + + if (route->condition) { + route_condition_destroy(route->condition); + } + + route_outputs_destroy(route); + route_processors_destroy(&route->processors); + + if (route->name) { + flb_sds_destroy(route->name); + } + flb_free(route); + } + + if (input->input_name) { + flb_sds_destroy(input->input_name); + } + + flb_free(input); +} + +void flb_router_routes_destroy(struct cfl_list *input_routes) +{ + struct cfl_list *head; + struct cfl_list *tmp; + struct flb_input_routes *routes; + + if (!input_routes) { + return; + } + + cfl_list_foreach_safe(head, tmp, input_routes) { + routes = cfl_list_entry(head, struct flb_input_routes, _head); + cfl_list_del(&routes->_head); + input_routes_destroy(routes); + } +} + +static int add_processor_properties(struct flb_route_processor *processor, + struct cfl_kvlist *kvlist) +{ + struct cfl_list *head; + struct cfl_kvpair *pair; + struct flb_route_processor_property *prop; + + if (!processor || !kvlist) { + return -1; + } + + cfl_list_foreach(head, &kvlist->list) { + pair = cfl_list_entry(head, struct cfl_kvpair, _head); + if (strcmp(pair->key, "name") == 0) { + continue; + } + + prop = flb_calloc(1, sizeof(struct flb_route_processor_property)); + if (!prop) { + flb_errno(); + return -1; + } + + cfl_list_init(&prop->_head); + + prop->key = flb_sds_create_len(pair->key, cfl_sds_len(pair->key)); + if (!prop->key) { + flb_free(prop); + return -1; + } + + prop->value = variant_to_sds(pair->val); + if (!prop->value) { + flb_sds_destroy(prop->key); + flb_free(prop); + return -1; + } + + cfl_list_add(&prop->_head, &processor->properties); + } + + return 0; +} + +static int parse_processors(struct cfl_variant *variant, + struct cfl_list *out_list, + struct flb_config *config) +{ + size_t idx; + struct cfl_array *array; + struct cfl_variant *entry; + struct cfl_kvlist *kvlist; + struct cfl_variant *name_var; + struct flb_route_processor *processor; + + (void) config; + + if (!variant || !out_list) { + return -1; + } + + if (variant->type != CFL_VARIANT_ARRAY) { + return -1; + } + + array = variant->data.as_array; + for (idx = 0; idx < cfl_array_size(array); idx++) { + entry = cfl_array_fetch_by_index(array, idx); + if (!entry || entry->type != CFL_VARIANT_KVLIST) { + return -1; + } + + kvlist = entry->data.as_kvlist; + name_var = cfl_kvlist_fetch(kvlist, "name"); + if (!name_var || name_var->type != CFL_VARIANT_STRING) { + return -1; + } + + processor = flb_calloc(1, sizeof(struct flb_route_processor)); + if (!processor) { + flb_errno(); + return -1; + } + cfl_list_init(&processor->_head); + cfl_list_init(&processor->properties); + + processor->name = copy_from_cfl_sds(name_var->data.as_string); + if (!processor->name) { + flb_free(processor); + return -1; + } + + if (add_processor_properties(processor, kvlist) != 0) { + route_processors_destroy(&processor->properties); + if (processor->name) { + flb_sds_destroy(processor->name); + } + flb_free(processor); + return -1; + } + + cfl_list_add(&processor->_head, out_list); + } + + return 0; +} + +static struct flb_route_condition_rule *parse_condition_rule(struct cfl_variant *variant) +{ + struct flb_route_condition_rule *rule; + struct cfl_kvlist *kvlist; + struct cfl_variant *field_var; + struct cfl_variant *op_var; + struct cfl_variant *value_var; + + if (!variant || variant->type != CFL_VARIANT_KVLIST) { + return NULL; + } + + kvlist = variant->data.as_kvlist; + field_var = cfl_kvlist_fetch(kvlist, "field"); + op_var = cfl_kvlist_fetch(kvlist, "op"); + value_var = cfl_kvlist_fetch(kvlist, "value"); + + if (!field_var || field_var->type != CFL_VARIANT_STRING) { + return NULL; + } + if (!op_var || op_var->type != CFL_VARIANT_STRING) { + return NULL; + } + + rule = flb_calloc(1, sizeof(struct flb_route_condition_rule)); + if (!rule) { + flb_errno(); + return NULL; + } + cfl_list_init(&rule->_head); + + rule->field = copy_from_cfl_sds(field_var->data.as_string); + if (!rule->field) { + flb_free(rule); + return NULL; + } + + rule->op = copy_from_cfl_sds(op_var->data.as_string); + if (!rule->op) { + flb_sds_destroy(rule->field); + flb_free(rule); + return NULL; + } + + if (value_var) { + rule->value = variant_to_sds(value_var); + if (!rule->value && strcmp(rule->op, "exists") != 0) { + flb_sds_destroy(rule->op); + flb_sds_destroy(rule->field); + flb_free(rule); + return NULL; + } + } + + return rule; +} + +static struct flb_route_condition *parse_condition(struct cfl_variant *variant, + uint32_t signals) +{ + struct flb_route_condition *condition; + struct cfl_variant *rules_var; + struct cfl_variant *default_var; + struct cfl_array *rules_array; + struct cfl_variant *entry; + struct flb_route_condition_rule *rule; + int val; + size_t idx; + + if (!variant || variant->type != CFL_VARIANT_KVLIST) { + return NULL; + } + + condition = flb_calloc(1, sizeof(struct flb_route_condition)); + if (!condition) { + flb_errno(); + return NULL; + } + cfl_list_init(&condition->rules); + + rules_var = cfl_kvlist_fetch(variant->data.as_kvlist, "rules"); + default_var = cfl_kvlist_fetch(variant->data.as_kvlist, "default"); + + if (default_var) { + if (variant_to_bool(default_var, &val) != 0) { + route_condition_destroy(condition); + return NULL; + } + condition->is_default = val; + } + + if (rules_var) { + if (rules_var->type != CFL_VARIANT_ARRAY) { + route_condition_destroy(condition); + return NULL; + } + + rules_array = rules_var->data.as_array; + for (idx = 0; idx < cfl_array_size(rules_array); idx++) { + entry = cfl_array_fetch_by_index(rules_array, idx); + rule = parse_condition_rule(entry); + if (!rule) { + route_condition_destroy(condition); + return NULL; + } + + if (!validate_rule_field(rule->field, signals)) { + route_condition_destroy(condition); + return NULL; + } + + cfl_list_add(&rule->_head, &condition->rules); + } + } + + if (!condition->is_default && cfl_list_is_empty(&condition->rules) == 1) { + route_condition_destroy(condition); + return NULL; + } + + return condition; +} + +static int add_output_from_variant(struct flb_route *route, + struct cfl_variant *variant) +{ + struct flb_route_output *output; + struct cfl_variant *name_var; + struct cfl_variant *fallback_var; + + if (!route || !variant) { + return -1; + } + + output = flb_calloc(1, sizeof(struct flb_route_output)); + if (!output) { + flb_errno(); + return -1; + } + cfl_list_init(&output->_head); + + if (variant->type == CFL_VARIANT_STRING) { + output->name = copy_from_cfl_sds(variant->data.as_string); + if (!output->name) { + flb_free(output); + return -1; + } + } + else if (variant->type == CFL_VARIANT_KVLIST) { + name_var = cfl_kvlist_fetch(variant->data.as_kvlist, "name"); + if (!name_var || name_var->type != CFL_VARIANT_STRING) { + flb_free(output); + return -1; + } + output->name = copy_from_cfl_sds(name_var->data.as_string); + if (!output->name) { + flb_free(output); + return -1; + } + + fallback_var = cfl_kvlist_fetch(variant->data.as_kvlist, "fallback"); + if (fallback_var) { + if (fallback_var->type != CFL_VARIANT_STRING) { + flb_sds_destroy(output->name); + flb_free(output); + return -1; + } + output->fallback = copy_from_cfl_sds(fallback_var->data.as_string); + if (!output->fallback) { + flb_sds_destroy(output->name); + flb_free(output); + return -1; + } + } + } + else { + flb_free(output); + return -1; + } + + cfl_list_add(&output->_head, &route->outputs); + return 0; +} + +static int parse_outputs(struct cfl_variant *variant, struct flb_route *route) +{ + size_t idx; + struct cfl_array *array; + struct cfl_variant *entry; + + if (!variant || !route) { + return -1; + } + + if (variant->type == CFL_VARIANT_ARRAY) { + array = variant->data.as_array; + + if (cfl_array_size(array) == 0) { + return -1; + } + + for (idx = 0; idx < cfl_array_size(array); idx++) { + entry = cfl_array_fetch_by_index(array, idx); + if (!entry) { + return -1; + } + + if (add_output_from_variant(route, entry) != 0) { + return -1; + } + } + + return 0; + } + + if (add_output_from_variant(route, variant) != 0) { + return -1; + } + + return 0; +} + +static int route_name_exists(struct cfl_list *routes, flb_sds_t name) +{ + struct cfl_list *head; + struct flb_route *route; + + if (!routes || !name) { + return FLB_FALSE; + } + + cfl_list_foreach(head, routes) { + route = cfl_list_entry(head, struct flb_route, _head); + if (route->name && strcmp(route->name, name) == 0) { + return FLB_TRUE; + } + } + + return FLB_FALSE; +} + +static int parse_route(struct cfl_variant *variant, + struct flb_input_routes *input, + struct flb_config *config, + uint32_t signals) +{ + struct flb_route *route; + struct cfl_kvlist *kvlist; + struct cfl_variant *name_var; + struct cfl_variant *condition_var; + struct cfl_variant *processors_var; + struct cfl_variant *to_var; + struct cfl_variant *outputs_var; + + (void) config; + + if (!variant || variant->type != CFL_VARIANT_KVLIST) { + return -1; + } + + if (signals == 0) { + return -1; + } + + kvlist = variant->data.as_kvlist; + + name_var = cfl_kvlist_fetch(kvlist, "name"); + if (!name_var || name_var->type != CFL_VARIANT_STRING) { + return -1; + } + + route = flb_calloc(1, sizeof(struct flb_route)); + if (!route) { + flb_errno(); + return -1; + } + cfl_list_init(&route->_head); + cfl_list_init(&route->outputs); + cfl_list_init(&route->processors); + route->signals = signals; + + route->name = copy_from_cfl_sds(name_var->data.as_string); + if (!route->name) { + flb_free(route); + return -1; + } + + if (route_name_exists(&input->routes, route->name)) { + flb_sds_destroy(route->name); + flb_free(route); + return -1; + } + + + condition_var = cfl_kvlist_fetch(kvlist, "condition"); + if (condition_var) { + route->condition = parse_condition(condition_var, route->signals); + if (!route->condition) { + flb_sds_destroy(route->name); + flb_free(route); + return -1; + } + } + + processors_var = cfl_kvlist_fetch(kvlist, "processors"); + if (processors_var) { + if (parse_processors(processors_var, &route->processors, config) != 0) { + if (route->condition) { + route_condition_destroy(route->condition); + } + flb_sds_destroy(route->name); + flb_free(route); + return -1; + } + } + + to_var = cfl_kvlist_fetch(kvlist, "to"); + if (!to_var) { + if (route->condition) { + route_condition_destroy(route->condition); + } + route_processors_destroy(&route->processors); + flb_sds_destroy(route->name); + flb_free(route); + return -1; + } + + if (to_var->type == CFL_VARIANT_KVLIST) { + outputs_var = cfl_kvlist_fetch(to_var->data.as_kvlist, "outputs"); + } + else { + outputs_var = to_var; + } + + if (!outputs_var || parse_outputs(outputs_var, route) != 0 || + cfl_list_is_empty(&route->outputs) == 1) { + if (route->condition) { + route_condition_destroy(route->condition); + } + route_processors_destroy(&route->processors); + route_outputs_destroy(route); + flb_sds_destroy(route->name); + flb_free(route); + return -1; + } + + cfl_list_add(&route->_head, &input->routes); + return 0; +} + +static int parse_routes_block(struct cfl_variant *variant, + struct flb_input_routes *input, + struct flb_config *config, + uint32_t signals) +{ + struct cfl_array *array; + struct cfl_variant *entry; + size_t idx; + + if (!variant) { + return -1; + } + + if (variant->type == CFL_VARIANT_ARRAY) { + array = variant->data.as_array; + if (cfl_array_size(array) == 0) { + return -1; + } + + for (idx = 0; idx < cfl_array_size(array); idx++) { + entry = cfl_array_fetch_by_index(array, idx); + if (!entry) { + return -1; + } + + if (parse_route(entry, input, config, signals) != 0) { + return -1; + } + } + + return 0; + } + + if (variant->type == CFL_VARIANT_KVLIST) { + if (parse_route(variant, input, config, signals) != 0) { + return -1; + } + return 0; + } + + return -1; +} + +static int parse_input_section(struct flb_cf_section *section, + struct cfl_list *input_routes, + struct flb_config *config) +{ + struct flb_input_routes *input; + struct cfl_kvlist *kvlist; + struct cfl_variant *name_var; + struct cfl_variant *processors_var; + struct cfl_variant *routes_var; + struct cfl_kvlist *routes_kvlist; + struct cfl_list *head; + struct cfl_kvpair *pair; + uint32_t mask; + size_t before_count; + + if (!section || !input_routes) { + return -1; + } + + kvlist = section->properties; + if (!kvlist) { + return -1; + } + + name_var = cfl_kvlist_fetch(kvlist, "name"); + if (!name_var || name_var->type != CFL_VARIANT_STRING) { + return -1; + } + + input = flb_calloc(1, sizeof(struct flb_input_routes)); + if (!input) { + flb_errno(); + return -1; + } + + cfl_list_init(&input->_head); + cfl_list_init(&input->processors); + cfl_list_init(&input->routes); + + input->input_name = copy_from_cfl_sds(name_var->data.as_string); + if (!input->input_name) { + flb_free(input); + return -1; + } + + processors_var = cfl_kvlist_fetch(kvlist, "processors"); + if (processors_var) { + if (parse_processors(processors_var, &input->processors, config) != 0) { + goto error; + } + } + + routes_var = cfl_kvlist_fetch(kvlist, "routes"); + if (!routes_var || routes_var->type != CFL_VARIANT_KVLIST) { + goto error; + } + + routes_kvlist = routes_var->data.as_kvlist; + if (cfl_list_is_empty(&routes_kvlist->list) == 1) { + goto error; + } + + cfl_list_foreach(head, &routes_kvlist->list) { + pair = cfl_list_entry(head, struct cfl_kvpair, _head); + if (!pair || !pair->key) { + goto error; + } + + mask = parse_signal_key(pair->key); + if (mask == 0) { + goto error; + } + + if (!pair->val) { + goto error; + } + + before_count = cfl_list_size(&input->routes); + if (parse_routes_block(pair->val, input, config, mask) != 0 || + cfl_list_size(&input->routes) == before_count) { + goto error; + } + } + + if (cfl_list_is_empty(&input->routes) == 1) { + goto error; + } + + cfl_list_add(&input->_head, input_routes); + return 0; + +error: + input_routes_destroy(input); + return -1; +} + +int flb_router_config_parse(struct flb_cf *cf, + struct cfl_list *input_routes, + struct flb_config *config) +{ + struct mk_list *head; + struct flb_cf_section *section; + + if (!cf || !input_routes) { + return -1; + } + + cfl_list_init(input_routes); + + mk_list_foreach(head, &cf->inputs) { + section = mk_list_entry(head, struct flb_cf_section, _head_section); + if (parse_input_section(section, input_routes, config) != 0) { + flb_router_routes_destroy(input_routes); + cfl_list_init(input_routes); + return -1; + } + } + + if (cfl_list_is_empty(input_routes) == 1) { + flb_router_routes_destroy(input_routes); + cfl_list_init(input_routes); + return -1; + } + + return 0; +} + +/* Apply parsed router configuration to actual input/output instances */ +static struct flb_input_instance *find_input_instance(struct flb_config *config, + flb_sds_t name) +{ + struct mk_list *head; + struct flb_input_instance *ins; + + if (!config || !name) { + return NULL; + } + + mk_list_foreach(head, &config->inputs) { + ins = mk_list_entry(head, struct flb_input_instance, _head); + + if (!ins->p) { + continue; + } + + if (ins->alias && strcmp(ins->alias, name) == 0) { + return ins; + } + + if (strcmp(ins->name, name) == 0) { + return ins; + } + + if (ins->p->name && strcmp(ins->p->name, name) == 0) { + return ins; + } + } + + return NULL; +} + +static struct flb_output_instance *find_output_instance(struct flb_config *config, + flb_sds_t name) +{ + struct mk_list *head; + struct flb_output_instance *ins; + + if (!config || !name) { + return NULL; + } + + mk_list_foreach(head, &config->outputs) { + ins = mk_list_entry(head, struct flb_output_instance, _head); + + if (!ins->p) { + continue; + } + + if (ins->alias && strcmp(ins->alias, name) == 0) { + return ins; + } + + if (strcmp(ins->name, name) == 0) { + return ins; + } + + if (ins->p->name && strcmp(ins->p->name, name) == 0) { + return ins; + } + } + + return NULL; +} + +static int input_has_direct_route(struct flb_input_instance *in, + struct flb_output_instance *out) +{ + struct mk_list *head; + struct flb_router_path *path; + + if (!in || !out) { + return FLB_FALSE; + } + + mk_list_foreach(head, &in->routes_direct) { + path = mk_list_entry(head, struct flb_router_path, _head); + if (path->ins == out) { + return FLB_TRUE; + } + } + + return FLB_FALSE; +} + +static int output_supports_signals(struct flb_output_instance *out, uint32_t signals) +{ + if (!out) { + return FLB_FALSE; + } + + if (signals == 0 || signals == FLB_ROUTER_SIGNAL_ANY) { + return FLB_TRUE; + } + + if ((signals & FLB_ROUTER_SIGNAL_LOGS) && (out->event_type & FLB_OUTPUT_LOGS)) { + return FLB_TRUE; + } + + if ((signals & FLB_ROUTER_SIGNAL_METRICS) && (out->event_type & FLB_OUTPUT_METRICS)) { + return FLB_TRUE; + } + + if ((signals & FLB_ROUTER_SIGNAL_TRACES) && (out->event_type & FLB_OUTPUT_TRACES)) { + return FLB_TRUE; + } + + return FLB_FALSE; +} + +int flb_router_apply_config(struct flb_config *config) +{ + struct cfl_list *input_head; + struct cfl_list *route_head; + struct cfl_list *output_head; + struct flb_input_routes *input_routes; + struct flb_route *route; + struct flb_route_output *route_output; + struct flb_input_instance *input_ins; + struct flb_output_instance *output_ins; + struct flb_output_instance *fallback_ins; + int created; + + if (!config) { + return 0; + } + + flb_debug("[router] applying router configuration"); + created = 0; + + cfl_list_foreach(input_head, &config->input_routes) { + input_routes = cfl_list_entry(input_head, struct flb_input_routes, _head); + + input_ins = find_input_instance(config, input_routes->input_name); + if (!input_ins) { + flb_warn("[router] could not find input instance '%s' for routes", + input_routes->input_name ? input_routes->input_name : "(null)"); + continue; + } + + cfl_list_foreach(route_head, &input_routes->routes) { + route = cfl_list_entry(route_head, struct flb_route, _head); + + cfl_list_foreach(output_head, &route->outputs) { + route_output = cfl_list_entry(output_head, struct flb_route_output, _head); + + output_ins = find_output_instance(config, route_output->name); + fallback_ins = NULL; + + if (!output_ins && route_output->fallback) { + fallback_ins = find_output_instance(config, route_output->fallback); + if (fallback_ins) { + flb_warn("[router] output '%s' not found, using fallback '%s'", + route_output->name, route_output->fallback); + output_ins = fallback_ins; + } + } + + if (!output_ins) { + flb_warn("[router] could not find output instance '%s' for route '%s/%s'", + route_output->name ? route_output->name : "(null)", + input_routes->input_name ? input_routes->input_name : "(null)", + route->name ? route->name : "(unnamed)"); + continue; + } + + if (!output_supports_signals(output_ins, route->signals)) { + flb_warn("[router] output '%s' does not support required signals for route '%s/%s'", + flb_output_name(output_ins), + input_routes->input_name ? input_routes->input_name : "(null)", + route->name ? route->name : "(unnamed)"); + continue; + } + + if (input_has_direct_route(input_ins, output_ins)) { + continue; + } + + if (flb_router_connect_direct(input_ins, output_ins) == 0) { + created++; + flb_debug("[router] connected input '%s' route '%s' to output '%s'", + flb_input_name(input_ins), + route->name ? route->name : "(unnamed)", + flb_output_name(output_ins)); + } + else { + flb_error("[router] failed to connect input '%s' to output '%s'", + flb_input_name(input_ins), + flb_output_name(output_ins)); + } + } + } + } + + if (created == 0) { + flb_debug("[router] no direct routes were connected from configuration"); + } + + return 0; +} + From 1f43da5a21176534919ffff1a1f0f8d3df066887 Mon Sep 17 00:00:00 2001 From: Patrick Stephens Date: Sun, 3 May 2026 12:34:43 +0100 Subject: [PATCH 023/213] [upstream] router_condition: add new conditional template Upstream-Ref: https://github.com/fluent/fluent-bit/commit/a2cd096372f5e328cb217ca4ddf49725b6d7d860 Cherry-picked from Fluent Bit v4.2.4 --- source/include/fluent-bit/flb_router.h | 83 ++++++++++++++++- source/src/flb_router_condition.c | 123 +++++++++++++++++++++++++ 2 files changed, 205 insertions(+), 1 deletion(-) create mode 100644 source/src/flb_router_condition.c diff --git a/source/include/fluent-bit/flb_router.h b/source/include/fluent-bit/flb_router.h index 81528248..a8e4f2dd 100644 --- a/source/include/fluent-bit/flb_router.h +++ b/source/include/fluent-bit/flb_router.h @@ -2,7 +2,7 @@ /* Fluent Bit * ========== - * Copyright (C) 2015-2024 The Fluent Bit Authors + * Copyright (C) 2015-2025 The Fluent Bit Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -23,6 +23,11 @@ #include #include #include +#include +#include +#include +#include +#include struct flb_router_path { struct flb_output_instance *ins; @@ -56,6 +61,61 @@ static inline int flb_router_match_type(int in_event_type, return FLB_TRUE; } +enum flb_router_signal { + FLB_ROUTER_SIGNAL_LOGS = (1U << 0), + FLB_ROUTER_SIGNAL_METRICS = (1U << 1), + FLB_ROUTER_SIGNAL_TRACES = (1U << 2), + FLB_ROUTER_SIGNAL_ANY = (FLB_ROUTER_SIGNAL_LOGS | + FLB_ROUTER_SIGNAL_METRICS | + FLB_ROUTER_SIGNAL_TRACES) +}; + +struct flb_route_condition_rule { + flb_sds_t field; + flb_sds_t op; + flb_sds_t value; + struct cfl_list _head; +}; + +struct flb_route_condition { + struct cfl_list rules; + int is_default; +}; + +struct flb_route_output { + flb_sds_t name; + flb_sds_t fallback; + struct cfl_list _head; +}; + +struct flb_route_processor_property { + flb_sds_t key; + flb_sds_t value; + struct cfl_list _head; +}; + +struct flb_route_processor { + flb_sds_t name; + struct cfl_list properties; + struct cfl_list _head; +}; + +struct flb_route { + flb_sds_t name; + uint32_t signals; + struct flb_route_condition *condition; + struct cfl_list outputs; + struct cfl_list processors; + struct cfl_list _head; +}; + +struct flb_input_routes { + flb_sds_t input_name; + struct cfl_list processors; + struct cfl_list routes; + struct cfl_list _head; +}; + int flb_router_connect(struct flb_input_instance *in, struct flb_output_instance *out); int flb_router_connect_direct(struct flb_input_instance *in, @@ -65,4 +125,25 @@ int flb_router_match(const char *tag, int tag_len, const char *match, void *match_regex); int flb_router_io_set(struct flb_config *config); void flb_router_exit(struct flb_config *config); + +uint32_t flb_router_signal_from_chunk(struct flb_event_chunk *chunk); + +int flb_route_condition_eval(struct flb_event_chunk *chunk, + struct flb_route *route); +int flb_condition_eval_logs(struct flb_event_chunk *chunk, + struct flb_route *route); +int flb_condition_eval_metrics(struct flb_event_chunk *chunk, + struct flb_route *route); +int flb_condition_eval_traces(struct flb_event_chunk *chunk, + struct flb_route *route); + +struct flb_cf; + +int flb_router_config_parse(struct flb_cf *cf, + struct cfl_list *input_routes, + struct flb_config *config); +void flb_router_routes_destroy(struct cfl_list *input_routes); +int flb_router_apply_config(struct flb_config *config); + #endif + diff --git a/source/src/flb_router_condition.c b/source/src/flb_router_condition.c new file mode 100644 index 00000000..318227a0 --- /dev/null +++ b/source/src/flb_router_condition.c @@ -0,0 +1,123 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2024 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include + +uint32_t flb_router_signal_from_chunk(struct flb_event_chunk *chunk) +{ + if (!chunk) { + return 0; + } + + switch (chunk->type) { + case FLB_EVENT_TYPE_LOGS: + return FLB_ROUTER_SIGNAL_LOGS; + case FLB_EVENT_TYPE_METRICS: + return FLB_ROUTER_SIGNAL_METRICS; + case FLB_EVENT_TYPE_TRACES: + return FLB_ROUTER_SIGNAL_TRACES; + default: + break; + } + + return 0; +} + +int flb_condition_eval_logs(struct flb_event_chunk *chunk, + struct flb_route *route) +{ + (void) chunk; + (void) route; + + /* + * The full condition evaluation engine requires field resolvers that map + * record accessors to the different telemetry payload shapes. The wiring + * of those resolvers is part of a bigger effort and will be implemented in + * follow-up changes. For the time being we simply report that the + * condition failed so that the runtime can rely on explicit default + * routes. + */ + return FLB_FALSE; +} + +int flb_condition_eval_metrics(struct flb_event_chunk *chunk, + struct flb_route *route) +{ + (void) chunk; + (void) route; + + return FLB_FALSE; +} + +int flb_condition_eval_traces(struct flb_event_chunk *chunk, + struct flb_route *route) +{ + (void) chunk; + (void) route; + + return FLB_FALSE; +} + +int flb_route_condition_eval(struct flb_event_chunk *chunk, + struct flb_route *route) +{ + uint32_t signal; + + if (!route) { + return FLB_FALSE; + } + + if (!route->condition) { + return FLB_TRUE; + } + + signal = flb_router_signal_from_chunk(chunk); + if (signal == 0) { + return FLB_FALSE; + } + + if ((route->signals != 0) && (route->signals != FLB_ROUTER_SIGNAL_ANY) && + ((route->signals & signal) == 0)) { + return FLB_FALSE; + } + + if (route->condition->is_default) { + return FLB_TRUE; + } + + if (cfl_list_is_empty(&route->condition->rules) == 0) { + return FLB_TRUE; + } + + switch (signal) { + case FLB_ROUTER_SIGNAL_LOGS: + return flb_condition_eval_logs(chunk, route); + case FLB_ROUTER_SIGNAL_METRICS: + return flb_condition_eval_metrics(chunk, route); + case FLB_ROUTER_SIGNAL_TRACES: + return flb_condition_eval_traces(chunk, route); + default: + break; + } + + return FLB_FALSE; +} + From 870f027a6bd9f768f2e49702375c83d08b053547 Mon Sep 17 00:00:00 2001 From: Patrick Stephens Date: Sun, 3 May 2026 12:34:43 +0100 Subject: [PATCH 024/213] [upstream] router: check for new input routes Upstream-Ref: https://github.com/fluent/fluent-bit/commit/7dc9a918573bd283a10810ac2a49e9520157ead3 Cherry-picked from Fluent Bit v4.2.4 --- source/src/flb_router.c | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/source/src/flb_router.c b/source/src/flb_router.c index f6bf2477..9312f5ff 100644 --- a/source/src/flb_router.c +++ b/source/src/flb_router.c @@ -256,6 +256,15 @@ int flb_router_io_set(struct flb_config *config) } } + /* Apply new router configuration if available */ + if (!cfl_list_is_empty(&config->input_routes)) { + flb_debug("[router] new router configuration found, applying..."); + if (flb_router_apply_config(config) == -1) { + flb_error("[router] failed to apply new router configuration"); + return -1; + } + } + return 0; } From 0e28d64fcdef5d9b5aadb963c905ed2c2d6719cd Mon Sep 17 00:00:00 2001 From: Patrick Stephens Date: Sun, 3 May 2026 12:34:43 +0100 Subject: [PATCH 025/213] [upstream] config: add list for input routes Upstream-Ref: https://github.com/fluent/fluent-bit/commit/6b33e3d2c199d6cf88f25e3e27d84225b1b78c5b Cherry-picked from Fluent Bit v4.2.4 --- source/include/fluent-bit/flb_config.h | 4 ++++ source/src/flb_config.c | 16 ++++++++++++++++ 2 files changed, 20 insertions(+) diff --git a/source/include/fluent-bit/flb_config.h b/source/include/fluent-bit/flb_config.h index e1e8f2f0..10fe72a4 100644 --- a/source/include/fluent-bit/flb_config.h +++ b/source/include/fluent-bit/flb_config.h @@ -27,6 +27,7 @@ #include #include #include +#include #include @@ -325,6 +326,9 @@ struct flb_config { int json_escape_unicode; int dry_run; + + /* New Router Configuration */ + struct cfl_list input_routes; }; #define FLB_CONFIG_LOG_LEVEL(c) (c->log->level) diff --git a/source/src/flb_config.c b/source/src/flb_config.c index 6c45541b..aefc04f9 100644 --- a/source/src/flb_config.c +++ b/source/src/flb_config.c @@ -44,6 +44,7 @@ #include #include #include +#include const char *FLB_CONF_ENV_LOGLEVEL = "FLB_LOG_LEVEL"; @@ -362,6 +363,7 @@ struct flb_config *flb_config_init() mk_list_init(&config->filters); mk_list_init(&config->outputs); mk_list_init(&config->proxies); + cfl_list_init(&config->input_routes); mk_list_init(&config->workers); mk_list_init(&config->upstreams); mk_list_init(&config->downstreams); @@ -608,6 +610,9 @@ void flb_config_exit(struct flb_config *config) flb_config_task_map_resize(config, 0); flb_routes_empty_mask_destroy(config); + /* Clean up router input routes */ + flb_router_routes_destroy(&config->input_routes); + flb_free(config); } @@ -851,6 +856,9 @@ static int configure_plugins_type(struct flb_config *config, struct flb_cf *cf, if (strcasecmp(kv->key, "name") == 0) { continue; } + if (strcasecmp(kv->key, "routes") == 0) { + continue; + } /* set ret to -1 to ensure that we treat any unhandled plugin or * value types as errors. @@ -972,6 +980,7 @@ static int configure_plugins_type(struct flb_config *config, struct flb_cf *cf, int flb_config_load_config_format(struct flb_config *config, struct flb_cf *cf) { int ret; + flb_debug("[config] starting configuration loading"); struct flb_kv *kv; struct mk_list *head; struct cfl_kvpair *ckv; @@ -1073,6 +1082,13 @@ int flb_config_load_config_format(struct flb_config *config, struct flb_cf *cf) return -1; } + /* Parse new router configuration */ + ret = flb_router_config_parse(cf, &config->input_routes, config); + if (ret == -1) { + flb_debug("[router] router configuration parsing failed"); + return -1; + } + return 0; } From 25ced17fbacc6c481f5222ea98271f17e86f5dc4 Mon Sep 17 00:00:00 2001 From: Patrick Stephens Date: Sun, 3 May 2026 12:34:43 +0100 Subject: [PATCH 026/213] [upstream] build: register new router interface files Upstream-Ref: https://github.com/fluent/fluent-bit/commit/48e773347beddebdd93cb9141652a6e5775efaa5 Cherry-picked from Fluent Bit v4.2.4 --- source/src/CMakeLists.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/source/src/CMakeLists.txt b/source/src/CMakeLists.txt index e2fddf7a..0ada411b 100644 --- a/source/src/CMakeLists.txt +++ b/source/src/CMakeLists.txt @@ -54,6 +54,8 @@ set(src flb_upstream_ha.c flb_upstream_node.c flb_router.c + flb_router_condition.c + flb_router_config.c flb_worker.c flb_coro.c flb_time.c From 8101a9da7bc14a2974ce1187058a261ee90fe7d2 Mon Sep 17 00:00:00 2001 From: Patrick Stephens Date: Sun, 3 May 2026 12:34:44 +0100 Subject: [PATCH 027/213] [upstream] config_format: yaml: fix processors recognition Upstream-Ref: https://github.com/fluent/fluent-bit/commit/24f8677596af3f7fb04418cbe6e43fab28d72b9f Cherry-picked from Fluent Bit v4.2.4 --- source/src/config_format/flb_cf_yaml.c | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/source/src/config_format/flb_cf_yaml.c b/source/src/config_format/flb_cf_yaml.c index aae6d31d..11fb8a79 100644 --- a/source/src/config_format/flb_cf_yaml.c +++ b/source/src/config_format/flb_cf_yaml.c @@ -2085,6 +2085,29 @@ static int consume_event(struct flb_cf *conf, struct local_ctx *ctx, case YAML_MAPPING_END_EVENT: print_current_properties(state); + if (strcmp(state->key, "processors") == 0) { + struct flb_cf_group *group; + + group = flb_cf_group_create(conf, state->cf_section, + state->key, + strlen(state->key)); + + if (group == NULL) { + flb_error("unable to create processors group"); + return YAML_FAILURE; + } + + state->cf_group = group; + state = state_push(ctx, STATE_INPUT_PROCESSORS); + + if (state == NULL) { + flb_error("unable to allocate state"); + return YAML_FAILURE; + } + + break; + } + if (state->section == SECTION_PROCESSOR) { status = state_move_into_config_group(state, state->cf_group); From afd632ecc7599a89133a103dcf2d6694e9053740 Mon Sep 17 00:00:00 2001 From: Patrick Stephens Date: Sun, 3 May 2026 12:34:44 +0100 Subject: [PATCH 028/213] [upstream] router_config: allow not having routes (optional) Upstream-Ref: https://github.com/fluent/fluent-bit/commit/6c63f07e1b743a6bd418122c1f8aca6977214884 Cherry-picked from Fluent Bit v4.2.4 --- source/src/flb_router_config.c | 51 +++++++++++++++++++++++----------- 1 file changed, 35 insertions(+), 16 deletions(-) diff --git a/source/src/flb_router_config.c b/source/src/flb_router_config.c index f059ac3f..ac14c068 100644 --- a/source/src/flb_router_config.c +++ b/source/src/flb_router_config.c @@ -917,6 +917,22 @@ static int parse_input_section(struct flb_cf_section *section, return -1; } + routes_var = cfl_kvlist_fetch(kvlist, "routes"); + if (!routes_var) { + /* No router configuration for this input section */ + return 0; + } + + if (routes_var->type != CFL_VARIANT_KVLIST) { + return -1; + } + + routes_kvlist = routes_var->data.as_kvlist; + if (cfl_list_is_empty(&routes_kvlist->list) == 1) { + /* routes field present but empty, nothing to configure */ + return 0; + } + name_var = cfl_kvlist_fetch(kvlist, "name"); if (!name_var || name_var->type != CFL_VARIANT_STRING) { return -1; @@ -945,16 +961,6 @@ static int parse_input_section(struct flb_cf_section *section, } } - routes_var = cfl_kvlist_fetch(kvlist, "routes"); - if (!routes_var || routes_var->type != CFL_VARIANT_KVLIST) { - goto error; - } - - routes_kvlist = routes_var->data.as_kvlist; - if (cfl_list_is_empty(&routes_kvlist->list) == 1) { - goto error; - } - cfl_list_foreach(head, &routes_kvlist->list) { pair = cfl_list_entry(head, struct cfl_kvpair, _head); if (!pair || !pair->key) { @@ -978,15 +984,19 @@ static int parse_input_section(struct flb_cf_section *section, } if (cfl_list_is_empty(&input->routes) == 1) { - goto error; + goto skip; } cfl_list_add(&input->_head, input_routes); - return 0; + return 1; error: input_routes_destroy(input); return -1; + +skip: + input_routes_destroy(input); + return 0; } int flb_router_config_parse(struct flb_cf *cf, @@ -995,6 +1005,8 @@ int flb_router_config_parse(struct flb_cf *cf, { struct mk_list *head; struct flb_cf_section *section; + int routes_found = FLB_FALSE; + int ret; if (!cf || !input_routes) { return -1; @@ -1004,17 +1016,24 @@ int flb_router_config_parse(struct flb_cf *cf, mk_list_foreach(head, &cf->inputs) { section = mk_list_entry(head, struct flb_cf_section, _head_section); - if (parse_input_section(section, input_routes, config) != 0) { + ret = parse_input_section(section, input_routes, config); + if (ret == -1) { flb_router_routes_destroy(input_routes); cfl_list_init(input_routes); return -1; } + else if (ret == 1) { + routes_found = FLB_TRUE; + } } if (cfl_list_is_empty(input_routes) == 1) { - flb_router_routes_destroy(input_routes); - cfl_list_init(input_routes); - return -1; + if (routes_found == FLB_TRUE) { + flb_router_routes_destroy(input_routes); + cfl_list_init(input_routes); + return -1; + } + return 0; } return 0; From 8ac5de0e99f335404043bb9bec66213ec37d9474 Mon Sep 17 00:00:00 2001 From: Patrick Stephens Date: Sun, 3 May 2026 12:34:44 +0100 Subject: [PATCH 029/213] [upstream] router_config: fix signal compatibility check logic Upstream-Ref: https://github.com/fluent/fluent-bit/commit/1c7c4e1afd60155b4486e33fc4b2e982daeefc0d Cherry-picked from Fluent Bit v4.2.4 --- source/src/flb_router_config.c | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/source/src/flb_router_config.c b/source/src/flb_router_config.c index ac14c068..0513a11a 100644 --- a/source/src/flb_router_config.c +++ b/source/src/flb_router_config.c @@ -1132,23 +1132,23 @@ static int output_supports_signals(struct flb_output_instance *out, uint32_t sig return FLB_FALSE; } - if (signals == 0 || signals == FLB_ROUTER_SIGNAL_ANY) { + /* Handle ANY signal - if ANY is present in the bitmask, allow all signals */ + if (signals == 0 || (signals & FLB_ROUTER_SIGNAL_ANY)) { return FLB_TRUE; } - if ((signals & FLB_ROUTER_SIGNAL_LOGS) && (out->event_type & FLB_OUTPUT_LOGS)) { - return FLB_TRUE; + /* Require support for all requested signal bits */ + if ((signals & FLB_ROUTER_SIGNAL_LOGS) && !(out->event_type & FLB_OUTPUT_LOGS)) { + return FLB_FALSE; } - - if ((signals & FLB_ROUTER_SIGNAL_METRICS) && (out->event_type & FLB_OUTPUT_METRICS)) { - return FLB_TRUE; + if ((signals & FLB_ROUTER_SIGNAL_METRICS) && !(out->event_type & FLB_OUTPUT_METRICS)) { + return FLB_FALSE; } - - if ((signals & FLB_ROUTER_SIGNAL_TRACES) && (out->event_type & FLB_OUTPUT_TRACES)) { - return FLB_TRUE; + if ((signals & FLB_ROUTER_SIGNAL_TRACES) && !(out->event_type & FLB_OUTPUT_TRACES)) { + return FLB_FALSE; } - return FLB_FALSE; + return FLB_TRUE; } int flb_router_apply_config(struct flb_config *config) From 7950d0851dcdde061d8be7ca2d11c4fc4078e90c Mon Sep 17 00:00:00 2001 From: Patrick Stephens Date: Sun, 3 May 2026 12:34:44 +0100 Subject: [PATCH 030/213] [upstream] mp: extend cobj API to expose group metadata and Upstream-Ref: https://github.com/fluent/fluent-bit/commit/78b14aa111671083e1aae7b369e105a034f2402c Cherry-picked from Fluent Bit v4.2.4 --- source/include/fluent-bit/flb_mp_chunk.h | 7 ++ source/src/flb_mp.c | 133 ++++++++++++++++++++++- 2 files changed, 139 insertions(+), 1 deletion(-) diff --git a/source/include/fluent-bit/flb_mp_chunk.h b/source/include/fluent-bit/flb_mp_chunk.h index 050eb352..f925336b 100644 --- a/source/include/fluent-bit/flb_mp_chunk.h +++ b/source/include/fluent-bit/flb_mp_chunk.h @@ -36,6 +36,10 @@ struct flb_mp_chunk_record { struct flb_log_event event; struct cfl_object *cobj_metadata; struct cfl_object *cobj_record; + struct cfl_object *cobj_group_metadata; + struct cfl_object *cobj_group_attributes; + int owns_group_metadata; + int owns_group_attributes; struct cfl_list _head; }; @@ -47,6 +51,9 @@ struct flb_mp_chunk_cobj { struct flb_mp_chunk_record *record_pos; struct cfl_list records; + struct cfl_object *active_group_metadata; + struct cfl_object *active_group_attributes; + /* Condition for filtering records during processing */ struct flb_condition *condition; }; diff --git a/source/src/flb_mp.c b/source/src/flb_mp.c index ec182511..fe3735ba 100644 --- a/source/src/flb_mp.c +++ b/source/src/flb_mp.c @@ -1054,6 +1054,10 @@ struct flb_mp_chunk_record *flb_mp_chunk_record_create(struct flb_mp_chunk_cobj return NULL; } record->modified = FLB_FALSE; + record->cobj_group_metadata = NULL; + record->cobj_group_attributes = NULL; + record->owns_group_metadata = FLB_FALSE; + record->owns_group_attributes = FLB_FALSE; return record; } @@ -1076,6 +1080,8 @@ struct flb_mp_chunk_cobj *flb_mp_chunk_cobj_create(struct flb_log_event_encoder chunk_cobj->log_encoder = log_encoder; chunk_cobj->log_decoder = log_decoder; chunk_cobj->condition = NULL; + chunk_cobj->active_group_metadata = NULL; + chunk_cobj->active_group_attributes = NULL; return chunk_cobj; } @@ -1100,6 +1106,7 @@ static int generate_empty_msgpack_map(char **out_buf, size_t *out_size) int flb_mp_chunk_cobj_encode(struct flb_mp_chunk_cobj *chunk_cobj, char **out_buf, size_t *out_size) { int ret; + int record_type; char *mp_buf; size_t mp_size; struct cfl_list *head; @@ -1123,6 +1130,21 @@ int flb_mp_chunk_cobj_encode(struct flb_mp_chunk_cobj *chunk_cobj, char **out_bu return -1; } + /* Determine record type from timestamp */ + if (record->event.timestamp.tm.tv_sec >= 0) { + record_type = FLB_LOG_EVENT_NORMAL; + } + else if (record->event.timestamp.tm.tv_sec == FLB_LOG_EVENT_GROUP_START) { + record_type = FLB_LOG_EVENT_GROUP_START; + } + else if (record->event.timestamp.tm.tv_sec == FLB_LOG_EVENT_GROUP_END) { + record_type = FLB_LOG_EVENT_GROUP_END; + } + else { + record_type = FLB_LOG_EVENT_NORMAL; + } + + if (record->cobj_metadata) { ret = flb_mp_cfl_to_msgpack(record->cobj_metadata, &mp_buf, &mp_size); if (ret == -1) { @@ -1143,7 +1165,14 @@ int flb_mp_chunk_cobj_encode(struct flb_mp_chunk_cobj *chunk_cobj, char **out_bu } flb_free(mp_buf); - if (record->cobj_record) { + /* For group start records, use group attributes as body if available */ + if (record_type == FLB_LOG_EVENT_GROUP_START && record->cobj_group_attributes) { + ret = flb_mp_cfl_to_msgpack(record->cobj_group_attributes, &mp_buf, &mp_size); + if (ret == -1) { + return -1; + } + } + else if (record->cobj_record) { ret = flb_mp_cfl_to_msgpack(record->cobj_record, &mp_buf, &mp_size); if (ret == -1) { return -1; @@ -1195,6 +1224,14 @@ int flb_mp_chunk_cobj_destroy(struct flb_mp_chunk_cobj *chunk_cobj) if (record->cobj_record) { cfl_object_destroy(record->cobj_record); } + if (record->owns_group_metadata && record->cobj_group_metadata && + record->cobj_group_metadata != record->cobj_metadata) { + cfl_object_destroy(record->cobj_group_metadata); + } + if (record->owns_group_attributes && record->cobj_group_attributes && + record->cobj_group_attributes != record->cobj_record) { + cfl_object_destroy(record->cobj_group_attributes); + } cfl_list_del(&record->_head); flb_free(record); } @@ -1208,6 +1245,7 @@ int flb_mp_chunk_cobj_record_next(struct flb_mp_chunk_cobj *chunk_cobj, { int ret = FLB_MP_CHUNK_RECORD_EOF; size_t bytes; + int record_type = FLB_LOG_EVENT_NORMAL; struct flb_mp_chunk_record *record = NULL; struct flb_condition *condition = NULL; @@ -1246,6 +1284,82 @@ int flb_mp_chunk_cobj_record_next(struct flb_mp_chunk_cobj *chunk_cobj, return -1; } + ret = flb_log_event_decoder_get_record_type(&record->event, &record_type); + if (ret != FLB_EVENT_DECODER_SUCCESS) { + cfl_object_destroy(record->cobj_record); + cfl_object_destroy(record->cobj_metadata); + flb_free(record); + return FLB_MP_CHUNK_RECORD_ERROR; + } + + record->owns_group_metadata = FLB_FALSE; + record->owns_group_attributes = FLB_FALSE; + + if (record_type == FLB_LOG_EVENT_GROUP_START) { + if (record->cobj_metadata) { + record->cobj_group_metadata = record->cobj_metadata; + record->owns_group_metadata = FLB_TRUE; + } + if (record->cobj_record) { + record->cobj_group_attributes = record->cobj_record; + record->owns_group_attributes = FLB_TRUE; + } + + chunk_cobj->active_group_metadata = record->cobj_group_metadata; + chunk_cobj->active_group_attributes = record->cobj_group_attributes; + } + else if (record_type == FLB_LOG_EVENT_GROUP_END) { + record->cobj_group_metadata = chunk_cobj->active_group_metadata; + record->cobj_group_attributes = chunk_cobj->active_group_attributes; + + chunk_cobj->active_group_metadata = NULL; + chunk_cobj->active_group_attributes = NULL; + } + else { + record->cobj_group_metadata = chunk_cobj->active_group_metadata; + record->cobj_group_attributes = chunk_cobj->active_group_attributes; + } + + if (!record->cobj_group_metadata && + record->event.group_metadata && + (record->event.group_metadata->type == MSGPACK_OBJECT_MAP || + record->event.group_metadata->type == MSGPACK_OBJECT_ARRAY)) { + record->cobj_group_metadata = flb_mp_object_to_cfl(record->event.group_metadata); + if (!record->cobj_group_metadata) { + if (record->owns_group_attributes && record->cobj_group_attributes) { + cfl_object_destroy(record->cobj_group_attributes); + } + cfl_object_destroy(record->cobj_record); + cfl_object_destroy(record->cobj_metadata); + flb_free(record); + return FLB_MP_CHUNK_RECORD_ERROR; + } + record->owns_group_metadata = FLB_TRUE; + if (!chunk_cobj->active_group_metadata) { + chunk_cobj->active_group_metadata = record->cobj_group_metadata; + } + } + + if (!record->cobj_group_attributes && + record->event.group_attributes && + (record->event.group_attributes->type == MSGPACK_OBJECT_MAP || + record->event.group_attributes->type == MSGPACK_OBJECT_ARRAY)) { + record->cobj_group_attributes = flb_mp_object_to_cfl(record->event.group_attributes); + if (!record->cobj_group_attributes) { + if (record->owns_group_metadata && record->cobj_group_metadata) { + cfl_object_destroy(record->cobj_group_metadata); + } + cfl_object_destroy(record->cobj_record); + cfl_object_destroy(record->cobj_metadata); + flb_free(record); + return FLB_MP_CHUNK_RECORD_ERROR; + } + record->owns_group_attributes = FLB_TRUE; + if (!chunk_cobj->active_group_attributes) { + chunk_cobj->active_group_attributes = record->cobj_group_attributes; + } + } + cfl_list_add(&record->_head, &chunk_cobj->records); /* If there's a condition, check if the record matches */ @@ -1339,12 +1453,29 @@ int flb_mp_chunk_cobj_record_destroy(struct flb_mp_chunk_cobj *chunk_cobj, } } + if (chunk_cobj && record->owns_group_metadata && + chunk_cobj->active_group_metadata == record->cobj_group_metadata) { + chunk_cobj->active_group_metadata = NULL; + } + if (chunk_cobj && record->owns_group_attributes && + chunk_cobj->active_group_attributes == record->cobj_group_attributes) { + chunk_cobj->active_group_attributes = NULL; + } + if (record->cobj_metadata) { cfl_object_destroy(record->cobj_metadata); } if (record->cobj_record) { cfl_object_destroy(record->cobj_record); } + if (record->owns_group_metadata && record->cobj_group_metadata && + record->cobj_group_metadata != record->cobj_metadata) { + cfl_object_destroy(record->cobj_group_metadata); + } + if (record->owns_group_attributes && record->cobj_group_attributes && + record->cobj_group_attributes != record->cobj_record) { + cfl_object_destroy(record->cobj_group_attributes); + } cfl_list_del(&record->_head); flb_free(record); From 13cd05d2408a18ec496f760c9536b0554281466b Mon Sep 17 00:00:00 2001 From: Patrick Stephens Date: Sun, 3 May 2026 12:34:44 +0100 Subject: [PATCH 031/213] [upstream] tests: internal: new mp_chunk_cobj Upstream-Ref: https://github.com/fluent/fluent-bit/commit/d87ec0dd206fa95c71e96151714d4cd7df8f49a5 Cherry-picked from Fluent Bit v4.2.4 --- source/tests/internal/CMakeLists.txt | 1 + source/tests/internal/log_event_decoder.c | 1 + source/tests/internal/mp_chunk_cobj.c | 419 ++++++++++++++++++++++ 3 files changed, 421 insertions(+) create mode 100644 source/tests/internal/mp_chunk_cobj.c diff --git a/source/tests/internal/CMakeLists.txt b/source/tests/internal/CMakeLists.txt index f78b692b..3c4296cf 100644 --- a/source/tests/internal/CMakeLists.txt +++ b/source/tests/internal/CMakeLists.txt @@ -23,6 +23,7 @@ set(UNIT_TESTS_FILES random.c config_map.c mp.c + mp_chunk_cobj.c input_chunk.c flb_time.c file.c diff --git a/source/tests/internal/log_event_decoder.c b/source/tests/internal/log_event_decoder.c index cc1509c7..8661e4fb 100644 --- a/source/tests/internal/log_event_decoder.c +++ b/source/tests/internal/log_event_decoder.c @@ -21,6 +21,7 @@ #include #include #include +#include #include #include diff --git a/source/tests/internal/mp_chunk_cobj.c b/source/tests/internal/mp_chunk_cobj.c new file mode 100644 index 00000000..2be5b853 --- /dev/null +++ b/source/tests/internal/mp_chunk_cobj.c @@ -0,0 +1,419 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2024 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include +#include + +#include + +#include + +#include "flb_tests_internal.h" + +void decoder_groups_cobj() +{ + struct flb_log_event_encoder *builder = NULL; + struct flb_log_event_encoder *chunk_encoder = NULL; + struct flb_log_event_decoder decoder; + struct flb_log_event_decoder verify_decoder; + struct flb_mp_chunk_cobj *chunk = NULL; + struct flb_mp_chunk_record *record = NULL; + struct flb_log_event verify_event; + struct cfl_kvlist *kvlist; + struct cfl_variant *variant; + struct cfl_object *metadata_obj = NULL; + struct cfl_object *body_obj = NULL; + char *encoded_buf = NULL; + struct flb_time ts; + int decoder_ready = FLB_FALSE; + int verify_decoder_ready = FLB_FALSE; + int ret; + int record_type; + int group_index; + size_t encoded_size = 0; + + memset(&verify_decoder, 0, sizeof(verify_decoder)); + memset(&verify_event, 0, sizeof(verify_event)); + + builder = flb_log_event_encoder_create(FLB_LOG_EVENT_FORMAT_DEFAULT); + if (!TEST_CHECK(builder != NULL)) { + return; + } + + ret = flb_log_event_encoder_group_init(builder); + if (!TEST_CHECK(ret == 0)) { + goto cleanup; + } + + ret = flb_log_event_encoder_append_metadata_values(builder, + FLB_LOG_EVENT_STRING_VALUE("group_id", 8), + FLB_LOG_EVENT_INT64_VALUE(42)); + if (!TEST_CHECK(ret == FLB_EVENT_ENCODER_SUCCESS)) { + goto cleanup; + } + + ret = flb_log_event_encoder_append_body_values(builder, + FLB_LOG_EVENT_STRING_VALUE("resource_type", 13), + FLB_LOG_EVENT_CSTRING_VALUE("demo")); + if (!TEST_CHECK(ret == FLB_EVENT_ENCODER_SUCCESS)) { + goto cleanup; + } + + ret = flb_log_event_encoder_group_header_end(builder); + if (!TEST_CHECK(ret == 0)) { + goto cleanup; + } + + ret = flb_log_event_encoder_begin_record(builder); + if (!TEST_CHECK(ret == FLB_EVENT_ENCODER_SUCCESS)) { + goto cleanup; + } + + flb_time_set(&ts, 1700000000, 0); + ret = flb_log_event_encoder_set_timestamp(builder, &ts); + if (!TEST_CHECK(ret == FLB_EVENT_ENCODER_SUCCESS)) { + goto cleanup; + } + + ret = flb_log_event_encoder_append_body_values(builder, + FLB_LOG_EVENT_STRING_VALUE("message", 7), + FLB_LOG_EVENT_CSTRING_VALUE("hello")); + if (!TEST_CHECK(ret == FLB_EVENT_ENCODER_SUCCESS)) { + goto cleanup; + } + + ret = flb_log_event_encoder_commit_record(builder); + if (!TEST_CHECK(ret == FLB_EVENT_ENCODER_SUCCESS)) { + goto cleanup; + } + + ret = flb_log_event_encoder_group_end(builder); + if (!TEST_CHECK(ret == 0)) { + goto cleanup; + } + + ret = flb_log_event_encoder_group_init(builder); + if (!TEST_CHECK(ret == 0)) { + goto cleanup; + } + + ret = flb_log_event_encoder_append_metadata_values(builder, + FLB_LOG_EVENT_STRING_VALUE("group_id", 8), + FLB_LOG_EVENT_INT64_VALUE(100)); + if (!TEST_CHECK(ret == FLB_EVENT_ENCODER_SUCCESS)) { + goto cleanup; + } + + ret = flb_log_event_encoder_append_body_values(builder, + FLB_LOG_EVENT_STRING_VALUE("resource_type", 13), + FLB_LOG_EVENT_CSTRING_VALUE("prod")); + if (!TEST_CHECK(ret == FLB_EVENT_ENCODER_SUCCESS)) { + goto cleanup; + } + + ret = flb_log_event_encoder_group_header_end(builder); + if (!TEST_CHECK(ret == 0)) { + goto cleanup; + } + + ret = flb_log_event_encoder_begin_record(builder); + if (!TEST_CHECK(ret == FLB_EVENT_ENCODER_SUCCESS)) { + goto cleanup; + } + + flb_time_set(&ts, 1700000001, 0); + ret = flb_log_event_encoder_set_timestamp(builder, &ts); + if (!TEST_CHECK(ret == FLB_EVENT_ENCODER_SUCCESS)) { + goto cleanup; + } + + ret = flb_log_event_encoder_append_body_values(builder, + FLB_LOG_EVENT_STRING_VALUE("message", 7), + FLB_LOG_EVENT_CSTRING_VALUE("world")); + if (!TEST_CHECK(ret == FLB_EVENT_ENCODER_SUCCESS)) { + goto cleanup; + } + + ret = flb_log_event_encoder_commit_record(builder); + if (!TEST_CHECK(ret == FLB_EVENT_ENCODER_SUCCESS)) { + goto cleanup; + } + + ret = flb_log_event_encoder_group_end(builder); + if (!TEST_CHECK(ret == 0)) { + goto cleanup; + } + + ret = flb_log_event_decoder_init(&decoder, + builder->output_buffer, + builder->output_length); + if (!TEST_CHECK(ret == FLB_EVENT_DECODER_SUCCESS)) { + goto cleanup; + } + flb_log_event_decoder_read_groups(&decoder, FLB_TRUE); + decoder_ready = FLB_TRUE; + + chunk_encoder = flb_log_event_encoder_create(FLB_LOG_EVENT_FORMAT_DEFAULT); + if (!TEST_CHECK(chunk_encoder != NULL)) { + goto cleanup; + } + + chunk = flb_mp_chunk_cobj_create(chunk_encoder, &decoder); + if (!TEST_CHECK(chunk != NULL)) { + goto cleanup; + } + + group_index = 0; + while ((ret = flb_mp_chunk_cobj_record_next(chunk, &record)) == FLB_MP_CHUNK_RECORD_OK) { + ret = flb_log_event_decoder_get_record_type(&record->event, &record_type); + if (!TEST_CHECK(ret == 0)) { + goto cleanup; + } + + if (record_type == FLB_LOG_EVENT_GROUP_START) { + group_index++; + + if (!TEST_CHECK(record->cobj_group_metadata != NULL && + record->cobj_group_metadata->variant != NULL)) { + goto cleanup; + } + + kvlist = record->cobj_group_metadata->variant->data.as_kvlist; + variant = cfl_kvlist_fetch(kvlist, "group_id"); + if (group_index == 1) { + if (!TEST_CHECK(variant != NULL && + ((variant->type == CFL_VARIANT_INT && + variant->data.as_int64 == 42) || + (variant->type == CFL_VARIANT_UINT && + variant->data.as_uint64 == 42)))) { + goto cleanup; + } + } + else if (group_index == 2) { + if (!TEST_CHECK(variant != NULL && + ((variant->type == CFL_VARIANT_INT && + variant->data.as_int64 == 100) || + (variant->type == CFL_VARIANT_UINT && + variant->data.as_uint64 == 100)))) { + goto cleanup; + } + } + + if (!TEST_CHECK(record->cobj_group_attributes != NULL && + record->cobj_group_attributes->variant != NULL)) { + goto cleanup; + } + + kvlist = record->cobj_group_attributes->variant->data.as_kvlist; + variant = cfl_kvlist_fetch(kvlist, "resource_type"); + if (group_index == 1) { + if (!TEST_CHECK(variant != NULL && + variant->type == CFL_VARIANT_STRING && + variant->size == 4 && + strncmp(variant->data.as_string, "demo", 4) == 0)) { + goto cleanup; + } + } + else if (group_index == 2) { + if (!TEST_CHECK(variant != NULL && + variant->type == CFL_VARIANT_STRING && + variant->size == 4 && + strncmp(variant->data.as_string, "prod", 4) == 0)) { + goto cleanup; + } + } + } + else if (record_type == FLB_LOG_EVENT_NORMAL) { + if (!TEST_CHECK(record->cobj_group_metadata != NULL && + record->cobj_group_metadata->variant != NULL)) { + goto cleanup; + } + + kvlist = record->cobj_group_metadata->variant->data.as_kvlist; + variant = cfl_kvlist_fetch(kvlist, "group_id"); + if (!TEST_CHECK(variant != NULL)) { + goto cleanup; + } + + if (group_index == 1) { + if (variant->type == CFL_VARIANT_INT) { + variant->data.as_int64 = 4242; + } + else if (variant->type == CFL_VARIANT_UINT) { + variant->data.as_uint64 = 4242; + } + else { + TEST_CHECK(0); + goto cleanup; + } + + if (!TEST_CHECK(record->cobj_group_attributes != NULL && + record->cobj_group_attributes->variant != NULL)) { + goto cleanup; + } + + kvlist = record->cobj_group_attributes->variant->data.as_kvlist; + ret = cfl_kvlist_insert_int64(kvlist, "new_attribute", 1); + if (!TEST_CHECK(ret == 0)) { + goto cleanup; + } + } + else if (group_index == 2) { + if (!TEST_CHECK(((variant->type == CFL_VARIANT_INT && + variant->data.as_int64 == 100) || + (variant->type == CFL_VARIANT_UINT && + variant->data.as_uint64 == 100)))) { + goto cleanup; + } + } + } + } + + if (!TEST_CHECK(ret == FLB_MP_CHUNK_RECORD_EOF)) { + goto cleanup; + } + + ret = flb_mp_chunk_cobj_encode(chunk, &encoded_buf, &encoded_size); + if (!TEST_CHECK(ret == 0)) { + goto cleanup; + } + + ret = flb_log_event_decoder_init(&verify_decoder, encoded_buf, encoded_size); + if (!TEST_CHECK(ret == FLB_EVENT_DECODER_SUCCESS)) { + goto cleanup; + } + flb_log_event_decoder_read_groups(&verify_decoder, FLB_TRUE); + verify_decoder_ready = FLB_TRUE; + + group_index = 0; + while ((ret = flb_log_event_decoder_next(&verify_decoder, &verify_event)) == FLB_EVENT_DECODER_SUCCESS) { + ret = flb_log_event_decoder_get_record_type(&verify_event, &record_type); + if (!TEST_CHECK(ret == 0)) { + goto cleanup; + } + + if (record_type == FLB_LOG_EVENT_GROUP_START) { + group_index++; + if (!TEST_CHECK(verify_event.metadata != NULL && + verify_event.metadata->type == MSGPACK_OBJECT_MAP)) { + goto cleanup; + } + + metadata_obj = flb_mp_object_to_cfl(verify_event.metadata); + if (!TEST_CHECK(metadata_obj != NULL && metadata_obj->variant != NULL)) { + goto cleanup; + } + + kvlist = metadata_obj->variant->data.as_kvlist; + variant = cfl_kvlist_fetch(kvlist, "group_id"); + if (!TEST_CHECK(variant != NULL)) { + goto cleanup; + } + + if (group_index == 1) { + if (!TEST_CHECK(((variant->type == CFL_VARIANT_INT && + variant->data.as_int64 == 4242) || + (variant->type == CFL_VARIANT_UINT && + variant->data.as_uint64 == 4242)))) { + goto cleanup; + } + + if (!TEST_CHECK(verify_event.body != NULL && + verify_event.body->type == MSGPACK_OBJECT_MAP)) { + goto cleanup; + } + + body_obj = flb_mp_object_to_cfl(verify_event.body); + if (!TEST_CHECK(body_obj != NULL && body_obj->variant != NULL)) { + goto cleanup; + } + + kvlist = body_obj->variant->data.as_kvlist; + variant = cfl_kvlist_fetch(kvlist, "new_attribute"); + if (!TEST_CHECK(variant != NULL && + (variant->type == CFL_VARIANT_INT || variant->type == CFL_VARIANT_UINT) && + ((variant->type == CFL_VARIANT_INT && variant->data.as_int64 == 1) || + (variant->type == CFL_VARIANT_UINT && variant->data.as_uint64 == 1)))) { + goto cleanup; + } + } + else if (group_index == 2) { + if (!TEST_CHECK(((variant->type == CFL_VARIANT_INT && + variant->data.as_int64 == 100) || + (variant->type == CFL_VARIANT_UINT && + variant->data.as_uint64 == 100)))) { + goto cleanup; + } + } + + if (metadata_obj) { + cfl_object_destroy(metadata_obj); + metadata_obj = NULL; + } + if (body_obj) { + cfl_object_destroy(body_obj); + body_obj = NULL; + } + } + } + + ret = flb_log_event_decoder_get_last_result(&verify_decoder); + + if (!TEST_CHECK(ret == FLB_EVENT_DECODER_SUCCESS)) { + goto cleanup; + } + +cleanup: + if (metadata_obj) { + cfl_object_destroy(metadata_obj); + } + if (body_obj) { + cfl_object_destroy(body_obj); + } + if (encoded_buf) { + flb_free(encoded_buf); + } + if (chunk) { + flb_mp_chunk_cobj_destroy(chunk); + } + if (chunk_encoder) { + flb_log_event_encoder_destroy(chunk_encoder); + } + if (decoder_ready == FLB_TRUE) { + flb_log_event_decoder_destroy(&decoder); + } + if (builder) { + flb_log_event_encoder_destroy(builder); + } + + if (verify_decoder_ready == FLB_TRUE) { + flb_log_event_decoder_destroy(&verify_decoder); + } +} + + +TEST_LIST = { + { "decoder_groups_cobj", decoder_groups_cobj }, + { 0 } +}; + From fd44ab09e01782a1024b1da82ed4d08a61ffb39c Mon Sep 17 00:00:00 2001 From: Patrick Stephens Date: Sun, 3 May 2026 12:34:45 +0100 Subject: [PATCH 032/213] [upstream] in_node_exporter_metrics: add netstat linux collector Upstream-Ref: https://github.com/fluent/fluent-bit/commit/243d431b88d49873c66d33d2a0f1eb6684d7e426 Cherry-picked from Fluent Bit v4.2.4 --- .../in_node_exporter_metrics/CMakeLists.txt | 1 + source/plugins/in_node_exporter_metrics/ne.c | 8 + source/plugins/in_node_exporter_metrics/ne.h | 12 +- .../in_node_exporter_metrics/ne_netstat.c | 33 ++ .../in_node_exporter_metrics/ne_netstat.h | 27 ++ .../ne_netstat_linux.c | 282 ++++++++++++++++++ 6 files changed, 362 insertions(+), 1 deletion(-) create mode 100644 source/plugins/in_node_exporter_metrics/ne_netstat.c create mode 100644 source/plugins/in_node_exporter_metrics/ne_netstat.h create mode 100644 source/plugins/in_node_exporter_metrics/ne_netstat_linux.c diff --git a/source/plugins/in_node_exporter_metrics/CMakeLists.txt b/source/plugins/in_node_exporter_metrics/CMakeLists.txt index 1b92684c..55e9a860 100644 --- a/source/plugins/in_node_exporter_metrics/CMakeLists.txt +++ b/source/plugins/in_node_exporter_metrics/CMakeLists.txt @@ -7,6 +7,7 @@ set(src ne_stat.c ne_vmstat.c ne_netdev.c + ne_netstat.c ne_sockstat.c ne_time.c ne_loadavg.c diff --git a/source/plugins/in_node_exporter_metrics/ne.c b/source/plugins/in_node_exporter_metrics/ne.c index 4afb098f..d98b8b3d 100644 --- a/source/plugins/in_node_exporter_metrics/ne.c +++ b/source/plugins/in_node_exporter_metrics/ne.c @@ -39,6 +39,7 @@ #include "ne_loadavg.h" #include "ne_vmstat.h" #include "ne_netdev.h" +#include "ne_netstat.h" #include "ne_sockstat.h" #include "ne_textfile.h" #include "ne_systemd.h" @@ -194,6 +195,7 @@ static int in_ne_init(struct flb_input_instance *in, mk_list_add(&loadavg_collector._head, &ctx->collectors); mk_list_add(&vmstat_collector._head, &ctx->collectors); mk_list_add(&netdev_collector._head, &ctx->collectors); + mk_list_add(&netstat_collector._head, &ctx->collectors); mk_list_add(&sockstat_collector._head, &ctx->collectors); mk_list_add(&filefd_collector._head, &ctx->collectors); mk_list_add(&textfile_collector._head, &ctx->collectors); @@ -389,6 +391,12 @@ static struct flb_config_map config_map[] = { "scrape interval to collect netdev metrics from the node." }, + { + FLB_CONFIG_MAP_TIME, "collector.netstat.scrape_interval", "0", + 0, FLB_FALSE, 0, + "scrape interval to collect netstat metrics from the node." + }, + { FLB_CONFIG_MAP_TIME, "collector.sockstat.scrape_interval", "0", 0, FLB_FALSE, 0, diff --git a/source/plugins/in_node_exporter_metrics/ne.h b/source/plugins/in_node_exporter_metrics/ne.h index 6a696882..9bcc215b 100644 --- a/source/plugins/in_node_exporter_metrics/ne.h +++ b/source/plugins/in_node_exporter_metrics/ne.h @@ -33,7 +33,7 @@ /* Default enabled metrics */ #ifdef __linux__ -#define NE_DEFAULT_ENABLED_METRICS "cpu,cpufreq,meminfo,diskstats,filesystem,uname,stat,time,loadavg,vmstat,netdev,sockstat,filefd,systemd,nvme,thermal_zone,hwmon" +#define NE_DEFAULT_ENABLED_METRICS "cpu,cpufreq,meminfo,diskstats,filesystem,uname,stat,time,loadavg,vmstat,netdev,netstat,sockstat,filefd,systemd,nvme,thermal_zone,hwmon" #elif __APPLE__ #define NE_DEFAULT_ENABLED_METRICS "cpu,loadavg,meminfo,diskstats,uname,netdev" #endif @@ -163,6 +163,16 @@ struct flb_ne { struct cmt_gauge *sockstat_FRAG6_inuse; struct cmt_gauge *sockstat_FRAG6_memory; + /* netstat_linux */ + struct cmt_gauge *netstat_Tcp_CurrEstab; + struct cmt_counter *netstat_Tcp_ActiveOpens; + struct cmt_counter *netstat_Tcp_PassiveOpens; + struct cmt_counter *netstat_Tcp_RetransSegs; + struct cmt_counter *netstat_Udp_InDatagrams; + struct cmt_counter *netstat_Udp_InErrors; + struct cmt_counter *netstat_Udp_OutDatagrams; + struct cmt_counter *netstat_Udp_NoPorts; + /* time */ struct cmt_gauge *time; diff --git a/source/plugins/in_node_exporter_metrics/ne_netstat.c b/source/plugins/in_node_exporter_metrics/ne_netstat.c new file mode 100644 index 00000000..5762b56e --- /dev/null +++ b/source/plugins/in_node_exporter_metrics/ne_netstat.c @@ -0,0 +1,33 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2025 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifdef __linux__ +#include "ne_netstat_linux.c" +#else + +#include "ne.h" + +struct flb_ne_collector netstat_collector = { + .name = "netstat", + .cb_init = NULL, + .cb_update = NULL, + .cb_exit = NULL +}; + +#endif diff --git a/source/plugins/in_node_exporter_metrics/ne_netstat.h b/source/plugins/in_node_exporter_metrics/ne_netstat.h new file mode 100644 index 00000000..a05757f6 --- /dev/null +++ b/source/plugins/in_node_exporter_metrics/ne_netstat.h @@ -0,0 +1,27 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2025 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_IN_NE_NETSTAT_H +#define FLB_IN_NE_NETSTAT_H + +#include "ne.h" + +extern struct flb_ne_collector netstat_collector; + +#endif diff --git a/source/plugins/in_node_exporter_metrics/ne_netstat_linux.c b/source/plugins/in_node_exporter_metrics/ne_netstat_linux.c new file mode 100644 index 00000000..7d3820d4 --- /dev/null +++ b/source/plugins/in_node_exporter_metrics/ne_netstat_linux.c @@ -0,0 +1,282 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2025 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#define _GNU_SOURCE + +#include + +#include +#include + +#include "ne.h" +#include "ne_utils.h" + +#define NETSTAT_PROTO_NONE 0 +#define NETSTAT_PROTO_TCP 1 +#define NETSTAT_PROTO_UDP 2 + +static int netstat_configure(struct flb_ne *ctx) +{ + ctx->netstat_Tcp_CurrEstab = + cmt_gauge_create(ctx->cmt, "node", "netstat", "Tcp_CurrEstab", + "Number of TCP connections in ESTABLISHED or CLOSE-WAIT state.", + 0, NULL); + if (!ctx->netstat_Tcp_CurrEstab) { + return -1; + } + + ctx->netstat_Tcp_ActiveOpens = + cmt_counter_create(ctx->cmt, "node", "netstat", "Tcp_ActiveOpens", + "Total number of TCP connections that have made a direct transition to SYN-SENT state.", + 0, NULL); + if (!ctx->netstat_Tcp_ActiveOpens) { + return -1; + } + + ctx->netstat_Tcp_PassiveOpens = + cmt_counter_create(ctx->cmt, "node", "netstat", "Tcp_PassiveOpens", + "Total number of TCP connections made in response to incoming SYN requests.", + 0, NULL); + if (!ctx->netstat_Tcp_PassiveOpens) { + return -1; + } + + ctx->netstat_Tcp_RetransSegs = + cmt_counter_create(ctx->cmt, "node", "netstat", "Tcp_RetransSegs", + "Total number of TCP segments retransmitted.", + 0, NULL); + if (!ctx->netstat_Tcp_RetransSegs) { + return -1; + } + + ctx->netstat_Udp_InDatagrams = + cmt_counter_create(ctx->cmt, "node", "netstat", "Udp_InDatagrams", + "Total number of received UDP datagrams delivered to UDP users.", + 0, NULL); + if (!ctx->netstat_Udp_InDatagrams) { + return -1; + } + + ctx->netstat_Udp_InErrors = + cmt_counter_create(ctx->cmt, "node", "netstat", "Udp_InErrors", + "Total number of UDP datagrams that could not be delivered.", + 0, NULL); + if (!ctx->netstat_Udp_InErrors) { + return -1; + } + + ctx->netstat_Udp_OutDatagrams = + cmt_counter_create(ctx->cmt, "node", "netstat", "Udp_OutDatagrams", + "Total number of UDP datagrams sent from this host.", + 0, NULL); + if (!ctx->netstat_Udp_OutDatagrams) { + return -1; + } + + ctx->netstat_Udp_NoPorts = + cmt_counter_create(ctx->cmt, "node", "netstat", "Udp_NoPorts", + "Total number of received UDP datagrams for which there was no application at the destination port.", + 0, NULL); + if (!ctx->netstat_Udp_NoPorts) { + return -1; + } + + return 0; +} + +static void netstat_process_tcp(struct flb_ne *ctx, + struct mk_list *headers, int headers_count, + struct mk_list *values, int values_count, + uint64_t ts) +{ + int idx; + double d_val; + struct flb_slist_entry *key; + struct flb_slist_entry *val; + + for (idx = 1; idx < headers_count && idx < values_count; idx++) { + key = flb_slist_entry_get(headers, idx); + val = flb_slist_entry_get(values, idx); + + if (!key || !val) { + continue; + } + + if (ne_utils_str_to_double(val->str, &d_val) != 0) { + continue; + } + + if (strcmp(key->str, "CurrEstab") == 0 && ctx->netstat_Tcp_CurrEstab) { + cmt_gauge_set(ctx->netstat_Tcp_CurrEstab, ts, d_val, 0, NULL); + } + else if (strcmp(key->str, "ActiveOpens") == 0 && ctx->netstat_Tcp_ActiveOpens) { + cmt_counter_set(ctx->netstat_Tcp_ActiveOpens, ts, d_val, 0, NULL); + } + else if (strcmp(key->str, "PassiveOpens") == 0 && ctx->netstat_Tcp_PassiveOpens) { + cmt_counter_set(ctx->netstat_Tcp_PassiveOpens, ts, d_val, 0, NULL); + } + else if (strcmp(key->str, "RetransSegs") == 0 && ctx->netstat_Tcp_RetransSegs) { + cmt_counter_set(ctx->netstat_Tcp_RetransSegs, ts, d_val, 0, NULL); + } + } +} + +static void netstat_process_udp(struct flb_ne *ctx, + struct mk_list *headers, int headers_count, + struct mk_list *values, int values_count, + uint64_t ts) +{ + int idx; + double d_val; + struct flb_slist_entry *key; + struct flb_slist_entry *val; + + for (idx = 1; idx < headers_count && idx < values_count; idx++) { + key = flb_slist_entry_get(headers, idx); + val = flb_slist_entry_get(values, idx); + + if (!key || !val) { + continue; + } + + if (ne_utils_str_to_double(val->str, &d_val) != 0) { + continue; + } + + if (strcmp(key->str, "InDatagrams") == 0 && ctx->netstat_Udp_InDatagrams) { + cmt_counter_set(ctx->netstat_Udp_InDatagrams, ts, d_val, 0, NULL); + } + else if (strcmp(key->str, "NoPorts") == 0 && ctx->netstat_Udp_NoPorts) { + cmt_counter_set(ctx->netstat_Udp_NoPorts, ts, d_val, 0, NULL); + } + else if (strcmp(key->str, "InErrors") == 0 && ctx->netstat_Udp_InErrors) { + cmt_counter_set(ctx->netstat_Udp_InErrors, ts, d_val, 0, NULL); + } + else if (strcmp(key->str, "OutDatagrams") == 0 && ctx->netstat_Udp_OutDatagrams) { + cmt_counter_set(ctx->netstat_Udp_OutDatagrams, ts, d_val, 0, NULL); + } + } +} + +static void netstat_process_pair(struct flb_ne *ctx, + const char *header_line, + const char *value_line, + int proto, + uint64_t ts) +{ + int headers_count; + int values_count; + struct mk_list headers; + struct mk_list values; + + mk_list_init(&headers); + mk_list_init(&values); + + headers_count = flb_slist_split_string(&headers, header_line, ' ', -1); + values_count = flb_slist_split_string(&values, value_line, ' ', -1); + + if (headers_count > 1 && values_count > 1) { + if (proto == NETSTAT_PROTO_TCP) { + netstat_process_tcp(ctx, &headers, headers_count, &values, values_count, ts); + } + else if (proto == NETSTAT_PROTO_UDP) { + netstat_process_udp(ctx, &headers, headers_count, &values, values_count, ts); + } + } + + flb_slist_destroy(&headers); + flb_slist_destroy(&values); +} + +static int netstat_update(struct flb_ne *ctx) +{ + int ret; + uint64_t ts; + struct mk_list list; + struct mk_list *head; + struct flb_slist_entry *line; + const char *prev_line; + int prev_proto; + + mk_list_init(&list); + ret = ne_utils_file_read_lines(ctx->path_procfs, "/net/snmp", &list); + if (ret == -1) { + return -1; + } + + ts = cfl_time_now(); + prev_line = NULL; + prev_proto = NETSTAT_PROTO_NONE; + + mk_list_foreach(head, &list) { + line = mk_list_entry(head, struct flb_slist_entry, _head); + + if (prev_proto != NETSTAT_PROTO_NONE) { + if (prev_proto == NETSTAT_PROTO_TCP && strncmp(line->str, "Tcp:", 4) == 0) { + netstat_process_pair(ctx, prev_line, line->str, NETSTAT_PROTO_TCP, ts); + prev_proto = NETSTAT_PROTO_NONE; + prev_line = NULL; + continue; + } + else if (prev_proto == NETSTAT_PROTO_UDP && strncmp(line->str, "Udp:", 4) == 0) { + netstat_process_pair(ctx, prev_line, line->str, NETSTAT_PROTO_UDP, ts); + prev_proto = NETSTAT_PROTO_NONE; + prev_line = NULL; + continue; + } + + prev_proto = NETSTAT_PROTO_NONE; + prev_line = NULL; + } + + if (strncmp(line->str, "Tcp:", 4) == 0) { + prev_line = line->str; + prev_proto = NETSTAT_PROTO_TCP; + } + else if (strncmp(line->str, "Udp:", 4) == 0) { + prev_line = line->str; + prev_proto = NETSTAT_PROTO_UDP; + } + } + + flb_slist_destroy(&list); + return 0; +} + +static int ne_netstat_init(struct flb_ne *ctx) +{ + return netstat_configure(ctx); +} + +static int ne_netstat_update(struct flb_input_instance *ins, + struct flb_config *config, void *in_context) +{ + struct flb_ne *ctx = (struct flb_ne *) in_context; + + netstat_update(ctx); + return 0; +} + +struct flb_ne_collector netstat_collector = { + .name = "netstat", + .cb_init = ne_netstat_init, + .cb_update = ne_netstat_update, + .cb_exit = NULL +}; + From dc41a6e4bb14ae8ac99082bf66d64fe51ba027d5 Mon Sep 17 00:00:00 2001 From: Patrick Stephens Date: Sun, 3 May 2026 12:34:45 +0100 Subject: [PATCH 033/213] [upstream] fstore: safe checks for active chunks on deletion Upstream-Ref: https://github.com/fluent/fluent-bit/commit/e7320a6cbb8e2191bea74ec3c920f2df2ea9ae2c Cherry-picked from Fluent Bit v4.2.4 --- source/src/flb_fstore.c | 29 +++++++++++++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-) diff --git a/source/src/flb_fstore.c b/source/src/flb_fstore.c index e02dbcd1..84b46fc1 100644 --- a/source/src/flb_fstore.c +++ b/source/src/flb_fstore.c @@ -215,12 +215,33 @@ struct flb_fstore_file *flb_fstore_file_get(struct flb_fstore *fs, * Set a file to inactive mode. Inactive means just to remove the reference * from the list. */ +static int chunk_is_linked_to_stream(struct flb_fstore_file *fsf) +{ + struct mk_list *head; + struct cio_chunk *chunk; + + if (fsf == NULL || fsf->chunk == NULL || fsf->stream == NULL) { + return FLB_FALSE; + } + + mk_list_foreach(head, &fsf->stream->chunks) { + chunk = mk_list_entry(head, struct cio_chunk, _head); + + if (chunk == fsf->chunk) { + return FLB_TRUE; + } + } + + return FLB_FALSE; +} + int flb_fstore_file_inactive(struct flb_fstore *fs, struct flb_fstore_file *fsf) { /* close the Chunk I/O reference, but don't delete the real file */ - if (fsf->chunk) { + if (chunk_is_linked_to_stream(fsf) == FLB_TRUE) { cio_chunk_close(fsf->chunk, CIO_FALSE); + fsf->chunk = NULL; } /* release */ @@ -239,7 +260,10 @@ int flb_fstore_file_delete(struct flb_fstore *fs, struct flb_fstore_file *fsf) { /* close the Chunk I/O reference, but don't delete it the real file */ - cio_chunk_close(fsf->chunk, CIO_TRUE); + if (chunk_is_linked_to_stream(fsf) == FLB_TRUE) { + cio_chunk_close(fsf->chunk, CIO_TRUE); + fsf->chunk = NULL; + } /* release */ mk_list_del(&fsf->_head); @@ -415,6 +439,7 @@ static int map_chunks(struct flb_fstore *ctx, struct flb_fstore_stream *fs_strea return -1; } + fsf->stream = stream; fsf->chunk = chunk; /* load metadata */ From 68400c306136e4ccc1e1ac1c6c41042d4fa00f4d Mon Sep 17 00:00:00 2001 From: Patrick Stephens Date: Sun, 3 May 2026 12:34:45 +0100 Subject: [PATCH 034/213] [upstream] tests: internal: fstore: add regression test for Upstream-Ref: https://github.com/fluent/fluent-bit/commit/65722e88460949d58eba030b0ceec3dee7824ef3 Cherry-picked from Fluent Bit v4.2.4 --- source/tests/internal/fstore.c | 51 ++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/source/tests/internal/fstore.c b/source/tests/internal/fstore.c index 1f4de61b..8979378b 100644 --- a/source/tests/internal/fstore.c +++ b/source/tests/internal/fstore.c @@ -30,6 +30,7 @@ #include #include +#include #ifdef FLB_SYSTEM_WINDOWS /* Not yet implemented! */ @@ -78,7 +79,57 @@ void cb_all() flb_fstore_destroy(fs); } +void cb_delete_after_external_close() +{ + int ret; + struct stat st_data; + struct flb_fstore *fs; + struct flb_fstore_stream *st; + struct flb_fstore_file *fsf; + struct cio_chunk *chunk; + + cio_utils_recursive_delete(FSF_STORE_PATH); + + fs = flb_fstore_create(FSF_STORE_PATH, FLB_FSTORE_FS); + TEST_CHECK(fs != NULL); + if (!fs) { + return; + } + + st = flb_fstore_stream_create(fs, "abc"); + TEST_CHECK(st != NULL); + if (!st) { + flb_fstore_destroy(fs); + return; + } + + fsf = flb_fstore_file_create(fs, st, "example.txt", 100); + TEST_CHECK(fsf != NULL); + if (!fsf) { + flb_fstore_destroy(fs); + return; + } + + chunk = fsf->chunk; + TEST_CHECK(chunk != NULL); + if (!chunk) { + flb_fstore_destroy(fs); + return; + } + + cio_chunk_close(chunk, CIO_TRUE); + + ret = stat(FSF_STORE_PATH "/abc/example.txt", &st_data); + TEST_CHECK(ret == -1 && errno == ENOENT); + + ret = flb_fstore_file_delete(fs, fsf); + TEST_CHECK(ret == 0); + + flb_fstore_destroy(fs); +} + TEST_LIST = { { "all" , cb_all}, + { "delete_after_external_close", cb_delete_after_external_close}, { NULL } }; From 68df462bba9afd0c6516a4bb4fdc2be2c0348036 Mon Sep 17 00:00:00 2001 From: Patrick Stephens Date: Sun, 3 May 2026 12:34:45 +0100 Subject: [PATCH 035/213] [upstream] router: implement new router paths and conditions Upstream-Ref: https://github.com/fluent/fluent-bit/commit/d87d0f910c5e20c8f305565c1387defa46eb1d9b Cherry-picked from Fluent Bit v4.2.4 --- source/include/fluent-bit/flb_router.h | 9 +++++++++ source/src/flb_router.c | 2 ++ 2 files changed, 11 insertions(+) diff --git a/source/include/fluent-bit/flb_router.h b/source/include/fluent-bit/flb_router.h index a8e4f2dd..e7c0ba92 100644 --- a/source/include/fluent-bit/flb_router.h +++ b/source/include/fluent-bit/flb_router.h @@ -26,11 +26,13 @@ #include #include #include +#include #include #include struct flb_router_path { struct flb_output_instance *ins; + struct flb_route *route; struct mk_list _head; }; @@ -74,12 +76,17 @@ struct flb_route_condition_rule { flb_sds_t field; flb_sds_t op; flb_sds_t value; + flb_sds_t *values; + size_t values_count; struct cfl_list _head; }; struct flb_route_condition { struct cfl_list rules; int is_default; + enum flb_condition_operator op; + struct flb_condition *compiled; + int compiled_status; }; struct flb_route_output { @@ -136,6 +143,8 @@ int flb_condition_eval_metrics(struct flb_event_chunk *chunk, struct flb_route *route); int flb_condition_eval_traces(struct flb_event_chunk *chunk, struct flb_route *route); +int flb_router_path_should_route(struct flb_event_chunk *chunk, + struct flb_router_path *path); struct flb_cf; diff --git a/source/src/flb_router.c b/source/src/flb_router.c index 9312f5ff..0c8f749d 100644 --- a/source/src/flb_router.c +++ b/source/src/flb_router.c @@ -155,6 +155,7 @@ int flb_router_connect(struct flb_input_instance *in, } p->ins = out; + p->route = NULL; mk_list_add(&p->_head, &in->routes); return 0; @@ -172,6 +173,7 @@ int flb_router_connect_direct(struct flb_input_instance *in, } p->ins = out; + p->route = NULL; mk_list_add(&p->_head, &in->routes_direct); return 0; From bd1e83d8b3691d146327439f2bfa6d22a48f04a9 Mon Sep 17 00:00:00 2001 From: Patrick Stephens Date: Sun, 3 May 2026 12:34:45 +0100 Subject: [PATCH 036/213] [upstream] router_config: add support for rules Upstream-Ref: https://github.com/fluent/fluent-bit/commit/e75e256356cc00d57ad431cc1f2cf8e861fd2686 Cherry-picked from Fluent Bit v4.2.4 --- source/src/flb_router_config.c | 96 +++++++++++++++++++++++++++++++++- 1 file changed, 94 insertions(+), 2 deletions(-) diff --git a/source/src/flb_router_config.c b/source/src/flb_router_config.c index 0513a11a..2a3a5077 100644 --- a/source/src/flb_router_config.c +++ b/source/src/flb_router_config.c @@ -29,6 +29,7 @@ #include #include #include +#include #include #include @@ -276,10 +277,24 @@ static void route_condition_destroy(struct flb_route_condition *condition) if (rule->value) { flb_sds_destroy(rule->value); } + if (rule->values) { + size_t idx; + + for (idx = 0; idx < rule->values_count; idx++) { + if (rule->values[idx]) { + flb_sds_destroy(rule->values[idx]); + } + } + flb_free(rule->values); + } flb_free(rule); } + if (condition->compiled) { + flb_condition_destroy(condition->compiled); + } + flb_free(condition); } @@ -530,6 +545,8 @@ static struct flb_route_condition_rule *parse_condition_rule(struct cfl_variant return NULL; } cfl_list_init(&rule->_head); + rule->values = NULL; + rule->values_count = 0; rule->field = copy_from_cfl_sds(field_var->data.as_string); if (!rule->field) { @@ -544,9 +561,56 @@ static struct flb_route_condition_rule *parse_condition_rule(struct cfl_variant return NULL; } - if (value_var) { + if (!value_var) { + flb_sds_destroy(rule->op); + flb_sds_destroy(rule->field); + flb_free(rule); + return NULL; + } + + if (value_var->type == CFL_VARIANT_ARRAY) { + struct cfl_array *array; + struct cfl_variant *entry; + size_t idx; + + array = value_var->data.as_array; + if (!array || cfl_array_size(array) == 0) { + flb_sds_destroy(rule->op); + flb_sds_destroy(rule->field); + flb_free(rule); + return NULL; + } + + rule->values = flb_calloc(cfl_array_size(array), sizeof(flb_sds_t)); + if (!rule->values) { + flb_errno(); + flb_sds_destroy(rule->op); + flb_sds_destroy(rule->field); + flb_free(rule); + return NULL; + } + + for (idx = 0; idx < cfl_array_size(array); idx++) { + entry = cfl_array_fetch_by_index(array, idx); + rule->values[idx] = variant_to_sds(entry); + if (!rule->values[idx]) { + size_t j; + + for (j = 0; j < idx; j++) { + flb_sds_destroy(rule->values[j]); + } + flb_free(rule->values); + flb_sds_destroy(rule->op); + flb_sds_destroy(rule->field); + flb_free(rule); + return NULL; + } + } + rule->values_count = cfl_array_size(array); + } + else { rule->value = variant_to_sds(value_var); - if (!rule->value && strcmp(rule->op, "exists") != 0) { + if (!rule->value) { flb_sds_destroy(rule->op); flb_sds_destroy(rule->field); flb_free(rule); @@ -563,6 +627,7 @@ static struct flb_route_condition *parse_condition(struct cfl_variant *variant, struct flb_route_condition *condition; struct cfl_variant *rules_var; struct cfl_variant *default_var; + struct cfl_variant *op_var; struct cfl_array *rules_array; struct cfl_variant *entry; struct flb_route_condition_rule *rule; @@ -579,9 +644,31 @@ static struct flb_route_condition *parse_condition(struct cfl_variant *variant, return NULL; } cfl_list_init(&condition->rules); + condition->op = FLB_COND_OP_AND; + condition->compiled = NULL; + condition->compiled_status = 0; rules_var = cfl_kvlist_fetch(variant->data.as_kvlist, "rules"); default_var = cfl_kvlist_fetch(variant->data.as_kvlist, "default"); + op_var = cfl_kvlist_fetch(variant->data.as_kvlist, "op"); + + if (op_var) { + if (op_var->type != CFL_VARIANT_STRING) { + route_condition_destroy(condition); + return NULL; + } + + if (strcasecmp(op_var->data.as_string, "and") == 0) { + condition->op = FLB_COND_OP_AND; + } + else if (strcasecmp(op_var->data.as_string, "or") == 0) { + condition->op = FLB_COND_OP_OR; + } + else { + route_condition_destroy(condition); + return NULL; + } + } if (default_var) { if (variant_to_bool(default_var, &val) != 0) { @@ -1220,6 +1307,11 @@ int flb_router_apply_config(struct flb_config *config) } if (flb_router_connect_direct(input_ins, output_ins) == 0) { + struct flb_router_path *path; + + path = mk_list_entry_last(&input_ins->routes_direct, + struct flb_router_path, _head); + path->route = route; created++; flb_debug("[router] connected input '%s' route '%s' to output '%s'", flb_input_name(input_ins), From b1679f4039c2476740d559442fdb39e518cbb9fb Mon Sep 17 00:00:00 2001 From: Patrick Stephens Date: Sun, 3 May 2026 12:34:45 +0100 Subject: [PATCH 037/213] [upstream] router_condition: add conditional logic for logs Upstream-Ref: https://github.com/fluent/fluent-bit/commit/559b8509a972b61a74be4e4b94197129aec95dd2 Cherry-picked from Fluent Bit v4.2.4 --- source/src/flb_router_condition.c | 287 ++++++++++++++++++++++++++++-- 1 file changed, 270 insertions(+), 17 deletions(-) diff --git a/source/src/flb_router_condition.c b/source/src/flb_router_condition.c index 318227a0..fa1ad515 100644 --- a/source/src/flb_router_condition.c +++ b/source/src/flb_router_condition.c @@ -20,6 +20,16 @@ #include #include #include +#include +#include +#include +#include + +#define FLB_ROUTE_CONDITION_COMPILED_SUCCESS 1 +#define FLB_ROUTE_CONDITION_COMPILED_FAILURE -1 + +static struct flb_condition *route_condition_get_compiled(struct flb_route_condition *condition); +static void route_condition_record_destroy(struct flb_mp_chunk_record *record); uint32_t flb_router_signal_from_chunk(struct flb_event_chunk *chunk) { @@ -44,18 +54,68 @@ uint32_t flb_router_signal_from_chunk(struct flb_event_chunk *chunk) int flb_condition_eval_logs(struct flb_event_chunk *chunk, struct flb_route *route) { - (void) chunk; - (void) route; + int ret; + int result = FLB_FALSE; + struct flb_route_condition *condition; + struct flb_condition *compiled; + struct flb_log_event_decoder decoder; + struct flb_log_event event; + struct flb_mp_chunk_record record; - /* - * The full condition evaluation engine requires field resolvers that map - * record accessors to the different telemetry payload shapes. The wiring - * of those resolvers is part of a bigger effort and will be implemented in - * follow-up changes. For the time being we simply report that the - * condition failed so that the runtime can rely on explicit default - * routes. - */ - return FLB_FALSE; + if (!chunk || !route || !route->condition) { + return FLB_FALSE; + } + + if (!chunk->data || chunk->size == 0) { + return FLB_FALSE; + } + + condition = route->condition; + + compiled = route_condition_get_compiled(condition); + if (!compiled) { + return FLB_FALSE; + } + + ret = flb_log_event_decoder_init(&decoder, chunk->data, chunk->size); + if (ret != FLB_EVENT_DECODER_SUCCESS) { + return FLB_FALSE; + } + + flb_log_event_decoder_read_groups(&decoder, FLB_TRUE); + + while ((ret = flb_log_event_decoder_next(&decoder, &event)) == FLB_EVENT_DECODER_SUCCESS) { + memset(&record, 0, sizeof(record)); + record.event = event; + + if (event.metadata) { + record.cobj_metadata = flb_mp_object_to_cfl(event.metadata); + if (!record.cobj_metadata) { + route_condition_record_destroy(&record); + break; + } + } + + if (event.body) { + record.cobj_record = flb_mp_object_to_cfl(event.body); + if (!record.cobj_record) { + route_condition_record_destroy(&record); + break; + } + } + + if (flb_condition_evaluate(compiled, &record) == FLB_TRUE) { + result = FLB_TRUE; + route_condition_record_destroy(&record); + break; + } + + route_condition_record_destroy(&record); + } + + flb_log_event_decoder_destroy(&decoder); + + return result; } int flb_condition_eval_metrics(struct flb_event_chunk *chunk, @@ -94,8 +154,7 @@ int flb_route_condition_eval(struct flb_event_chunk *chunk, return FLB_FALSE; } - if ((route->signals != 0) && (route->signals != FLB_ROUTER_SIGNAL_ANY) && - ((route->signals & signal) == 0)) { + if ((route->signals != 0) && (route->signals != FLB_ROUTER_SIGNAL_ANY) && ((route->signals & signal) == 0)) { return FLB_FALSE; } @@ -103,10 +162,6 @@ int flb_route_condition_eval(struct flb_event_chunk *chunk, return FLB_TRUE; } - if (cfl_list_is_empty(&route->condition->rules) == 0) { - return FLB_TRUE; - } - switch (signal) { case FLB_ROUTER_SIGNAL_LOGS: return flb_condition_eval_logs(chunk, route); @@ -121,3 +176,201 @@ int flb_route_condition_eval(struct flb_event_chunk *chunk, return FLB_FALSE; } +int flb_router_path_should_route(struct flb_event_chunk *chunk, + struct flb_router_path *path) +{ + if (!path) { + return FLB_FALSE; + } + + if (!path->route) { + return FLB_TRUE; + } + + return flb_route_condition_eval(chunk, path->route); +} + +static int parse_rule_operator(const flb_sds_t op_str, + enum flb_rule_operator *out) +{ + if (!op_str || !out) { + return -1; + } + + if (strcasecmp(op_str, "eq") == 0) { + *out = FLB_RULE_OP_EQ; + } + else if (strcasecmp(op_str, "neq") == 0) { + *out = FLB_RULE_OP_NEQ; + } + else if (strcasecmp(op_str, "gt") == 0) { + *out = FLB_RULE_OP_GT; + } + else if (strcasecmp(op_str, "lt") == 0) { + *out = FLB_RULE_OP_LT; + } + else if (strcasecmp(op_str, "gte") == 0) { + *out = FLB_RULE_OP_GTE; + } + else if (strcasecmp(op_str, "lte") == 0) { + *out = FLB_RULE_OP_LTE; + } + else if (strcasecmp(op_str, "regex") == 0) { + *out = FLB_RULE_OP_REGEX; + } + else if (strcasecmp(op_str, "not_regex") == 0) { + *out = FLB_RULE_OP_NOT_REGEX; + } + else if (strcasecmp(op_str, "in") == 0) { + *out = FLB_RULE_OP_IN; + } + else if (strcasecmp(op_str, "not_in") == 0) { + *out = FLB_RULE_OP_NOT_IN; + } + else { + return -1; + } + + return 0; +} + +static int parse_numeric_value(flb_sds_t value, double *out) +{ + char *endptr = NULL; + double result; + + if (!value || !out) { + return -1; + } + + errno = 0; + result = strtod(value, &endptr); + if (errno == ERANGE || endptr == value || (endptr && *endptr != '\0')) { + return -1; + } + + *out = result; + return 0; +} + +static struct flb_condition *route_condition_compile(struct flb_route_condition *condition) +{ + int ret; + double numeric_value; + enum flb_rule_operator op; + struct cfl_list *head; + struct flb_condition *compiled; + struct flb_route_condition_rule *rule; + + compiled = flb_condition_create(condition->op); + if (!compiled) { + return NULL; + } + + cfl_list_foreach(head, &condition->rules) { + rule = cfl_list_entry(head, struct flb_route_condition_rule, _head); + + if (!rule->field || !rule->op) { + flb_condition_destroy(compiled); + return NULL; + } + + if (parse_rule_operator(rule->op, &op) != 0) { + flb_condition_destroy(compiled); + return NULL; + } + + switch (op) { + case FLB_RULE_OP_EQ: + case FLB_RULE_OP_NEQ: + case FLB_RULE_OP_REGEX: + case FLB_RULE_OP_NOT_REGEX: + if (!rule->value) { + flb_condition_destroy(compiled); + return NULL; + } + ret = flb_condition_add_rule(compiled, rule->field, op, + rule->value, 1, RECORD_CONTEXT_BODY); + break; + case FLB_RULE_OP_GT: + case FLB_RULE_OP_LT: + case FLB_RULE_OP_GTE: + case FLB_RULE_OP_LTE: + if (!rule->value) { + flb_condition_destroy(compiled); + return NULL; + } + if (parse_numeric_value(rule->value, &numeric_value) != 0) { + flb_condition_destroy(compiled); + return NULL; + } + ret = flb_condition_add_rule(compiled, rule->field, op, + &numeric_value, 1, RECORD_CONTEXT_BODY); + break; + case FLB_RULE_OP_IN: + case FLB_RULE_OP_NOT_IN: + if (!rule->values || rule->values_count == 0) { + flb_condition_destroy(compiled); + return NULL; + } + ret = flb_condition_add_rule(compiled, rule->field, op, + rule->values, + (int) rule->values_count, + RECORD_CONTEXT_BODY); + break; + default: + flb_condition_destroy(compiled); + return NULL; + } + + if (ret != FLB_TRUE) { + flb_condition_destroy(compiled); + return NULL; + } + } + + return compiled; +} + +static struct flb_condition *route_condition_get_compiled(struct flb_route_condition *condition) +{ + if (!condition) { + return NULL; + } + + if (condition->compiled_status == FLB_ROUTE_CONDITION_COMPILED_FAILURE) { + return NULL; + } + + if (condition->compiled_status == FLB_ROUTE_CONDITION_COMPILED_SUCCESS && + condition->compiled) { + return condition->compiled; + } + + condition->compiled = route_condition_compile(condition); + if (!condition->compiled) { + condition->compiled_status = FLB_ROUTE_CONDITION_COMPILED_FAILURE; + return NULL; + } + + condition->compiled_status = FLB_ROUTE_CONDITION_COMPILED_SUCCESS; + return condition->compiled; +} + +static void route_condition_record_destroy(struct flb_mp_chunk_record *record) +{ + if (!record) { + return; + } + + if (record->cobj_record) { + cfl_object_destroy(record->cobj_record); + record->cobj_record = NULL; + } + + if (record->cobj_metadata) { + cfl_object_destroy(record->cobj_metadata); + record->cobj_metadata = NULL; + } +} + From e23b90cd04a8de3e4c9122aa53462907db0cccb9 Mon Sep 17 00:00:00 2001 From: Patrick Stephens Date: Sun, 3 May 2026 12:34:45 +0100 Subject: [PATCH 038/213] [upstream] task: add handling for direct route Upstream-Ref: https://github.com/fluent/fluent-bit/commit/9ac9bf9aa2f7e746d483527af6544dd70fb71d7e Cherry-picked from Fluent Bit v4.2.4 --- source/src/flb_task.c | 26 +++++++++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/source/src/flb_task.c b/source/src/flb_task.c index c104102b..5d0a8da0 100644 --- a/source/src/flb_task.c +++ b/source/src/flb_task.c @@ -360,6 +360,7 @@ struct flb_task *flb_task_create(uint64_t ref_id, { int count = 0; int total_events = 0; + int direct_count = 0; struct flb_task *task; struct flb_event_chunk *evc; struct flb_task_route *route; @@ -409,11 +410,18 @@ struct flb_task *flb_task_create(uint64_t ref_id, /* Direct connects betweek input <> outputs (API based) */ if (mk_list_size(&i_ins->routes_direct) > 0) { + direct_count = 0; + mk_list_foreach(i_head, &i_ins->routes_direct) { route_path = mk_list_entry(i_head, struct flb_router_path, _head); + + if (flb_router_path_should_route(task->event_chunk, route_path) == FLB_FALSE) { + continue; + } + o_ins = route_path->ins; - route = flb_malloc(sizeof(struct flb_task_route)); + route = flb_calloc(1, sizeof(struct flb_task_route)); if (!route) { flb_errno(); task->event_chunk->data = NULL; @@ -421,10 +429,22 @@ struct flb_task *flb_task_create(uint64_t ref_id, return NULL; } + route->status = FLB_TASK_ROUTE_INACTIVE; route->out = o_ins; mk_list_add(&route->_head, &task->routes); + direct_count++; + } + + if (direct_count == 0) { + flb_debug("[task] dropping direct task=%p id=%i without matching routes", + task, task->id); + task->event_chunk->data = NULL; + flb_task_destroy(task, FLB_TRUE); + return NULL; } - flb_debug("[task] created direct task=%p id=%i OK", task, task->id); + + flb_debug("[task] created direct task=%p id=%i with %i route(s)", + task, task->id, direct_count); return task; } @@ -438,7 +458,7 @@ struct flb_task *flb_task_create(uint64_t ref_id, continue; } - if (flb_routes_mask_get_bit(task_ic->routes_mask, + if (flb_routes_mask_get_bit(task_ic->routes_mask, o_ins->id, o_ins->config) != 0) { route = flb_calloc(1, sizeof(struct flb_task_route)); From 136f1afb1eb43fee1ccf3158075b9a8c83c4c0cd Mon Sep 17 00:00:00 2001 From: Patrick Stephens Date: Sun, 3 May 2026 12:34:46 +0100 Subject: [PATCH 039/213] [upstream] router: use cfl_list for router path Upstream-Ref: https://github.com/fluent/fluent-bit/commit/7b6e7880297454c745010522069598ff5398321a Cherry-picked from Fluent Bit v4.2.4 --- source/include/fluent-bit/flb_router.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/include/fluent-bit/flb_router.h b/source/include/fluent-bit/flb_router.h index e7c0ba92..78824b8d 100644 --- a/source/include/fluent-bit/flb_router.h +++ b/source/include/fluent-bit/flb_router.h @@ -33,7 +33,7 @@ struct flb_router_path { struct flb_output_instance *ins; struct flb_route *route; - struct mk_list _head; + struct cfl_list _head; }; static inline int flb_router_match_type(int in_event_type, From d0af07921a3a3f797d325b21d4dbebf16e979b97 Mon Sep 17 00:00:00 2001 From: Patrick Stephens Date: Sun, 3 May 2026 12:34:46 +0100 Subject: [PATCH 040/213] [upstream] input: fix data type for routes_direct Upstream-Ref: https://github.com/fluent/fluent-bit/commit/68b7efbb0d277f7943ca931870ddcc8d2ce3d1bc Cherry-picked from Fluent Bit v4.2.4 --- source/include/fluent-bit/flb_input.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/source/include/fluent-bit/flb_input.h b/source/include/fluent-bit/flb_input.h index 9374af3c..df1f5130 100644 --- a/source/include/fluent-bit/flb_input.h +++ b/source/include/fluent-bit/flb_input.h @@ -335,10 +335,10 @@ struct flb_input_instance { struct mk_list _head; /* link to config->inputs */ - struct mk_list routes_direct; /* direct routes set by API */ - struct mk_list routes; /* flb_router_path's list */ - struct mk_list properties; /* properties / configuration */ - struct mk_list collectors; /* collectors */ + struct cfl_list routes_direct; /* direct routes set by API */ + struct mk_list routes; /* flb_router_path's list */ + struct mk_list properties; /* properties / configuration */ + struct mk_list collectors; /* collectors */ /* Storage Chunks */ struct mk_list chunks; /* linked list of all chunks */ From 22e8b40cc572791534a9ee064d8e2b317d181eb2 Mon Sep 17 00:00:00 2001 From: Patrick Stephens Date: Sun, 3 May 2026 12:34:47 +0100 Subject: [PATCH 041/213] [upstream] router_config: fix data type for linked list Upstream-Ref: https://github.com/fluent/fluent-bit/commit/fa8e8b37e0c51ffd45e39db311ee697bce1f82c1 Cherry-picked from Fluent Bit v4.2.4 --- source/src/flb_router_config.c | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/source/src/flb_router_config.c b/source/src/flb_router_config.c index 2a3a5077..d5a6687e 100644 --- a/source/src/flb_router_config.c +++ b/source/src/flb_router_config.c @@ -1196,15 +1196,15 @@ static struct flb_output_instance *find_output_instance(struct flb_config *confi static int input_has_direct_route(struct flb_input_instance *in, struct flb_output_instance *out) { - struct mk_list *head; + struct cfl_list *head; struct flb_router_path *path; if (!in || !out) { return FLB_FALSE; } - mk_list_foreach(head, &in->routes_direct) { - path = mk_list_entry(head, struct flb_router_path, _head); + cfl_list_foreach(head, &in->routes_direct) { + path = cfl_list_entry(head, struct flb_router_path, _head); if (path->ins == out) { return FLB_TRUE; } @@ -1240,6 +1240,7 @@ static int output_supports_signals(struct flb_output_instance *out, uint32_t sig int flb_router_apply_config(struct flb_config *config) { + int created = 0; struct cfl_list *input_head; struct cfl_list *route_head; struct cfl_list *output_head; @@ -1249,15 +1250,12 @@ int flb_router_apply_config(struct flb_config *config) struct flb_input_instance *input_ins; struct flb_output_instance *output_ins; struct flb_output_instance *fallback_ins; - int created; + struct flb_router_path *path; if (!config) { return 0; } - flb_debug("[router] applying router configuration"); - created = 0; - cfl_list_foreach(input_head, &config->input_routes) { input_routes = cfl_list_entry(input_head, struct flb_input_routes, _head); @@ -1307,10 +1305,7 @@ int flb_router_apply_config(struct flb_config *config) } if (flb_router_connect_direct(input_ins, output_ins) == 0) { - struct flb_router_path *path; - - path = mk_list_entry_last(&input_ins->routes_direct, - struct flb_router_path, _head); + path = cfl_list_entry_last(&input_ins->routes_direct, struct flb_router_path, _head); path->route = route; created++; flb_debug("[router] connected input '%s' route '%s' to output '%s'", From 6994bf45418a70793474404ddab9df0b3f571847 Mon Sep 17 00:00:00 2001 From: Patrick Stephens Date: Sun, 3 May 2026 12:34:47 +0100 Subject: [PATCH 042/213] [upstream] task: fix data type for linked list Upstream-Ref: https://github.com/fluent/fluent-bit/commit/9aa07ace15c3b87c8b7eeacfeb3bb2fee8af95dc Cherry-picked from Fluent Bit v4.2.4 --- source/src/flb_task.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/source/src/flb_task.c b/source/src/flb_task.c index 5d0a8da0..64bafc23 100644 --- a/source/src/flb_task.c +++ b/source/src/flb_task.c @@ -367,7 +367,7 @@ struct flb_task *flb_task_create(uint64_t ref_id, struct flb_router_path *route_path; struct flb_output_instance *o_ins; struct flb_input_chunk *task_ic; - struct mk_list *i_head; + struct cfl_list *i_head; struct mk_list *o_head; /* No error status */ @@ -409,11 +409,11 @@ struct flb_task *flb_task_create(uint64_t ref_id, #endif /* Direct connects betweek input <> outputs (API based) */ - if (mk_list_size(&i_ins->routes_direct) > 0) { + if (cfl_list_size(&i_ins->routes_direct) > 0) { direct_count = 0; - mk_list_foreach(i_head, &i_ins->routes_direct) { - route_path = mk_list_entry(i_head, struct flb_router_path, _head); + cfl_list_foreach(i_head, &i_ins->routes_direct) { + route_path = cfl_list_entry(i_head, struct flb_router_path, _head); if (flb_router_path_should_route(task->event_chunk, route_path) == FLB_FALSE) { continue; From 4756eac9468dddb18436f7c23f3dd56b05d731ee Mon Sep 17 00:00:00 2001 From: Patrick Stephens Date: Sun, 3 May 2026 12:34:47 +0100 Subject: [PATCH 043/213] [upstream] input: include cfl header Upstream-Ref: https://github.com/fluent/fluent-bit/commit/380c159d0bbc5723953b3e4ababf182d3ef128da Cherry-picked from Fluent Bit v4.2.4 --- source/include/fluent-bit/flb_input.h | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/source/include/fluent-bit/flb_input.h b/source/include/fluent-bit/flb_input.h index df1f5130..b58c5acf 100644 --- a/source/include/fluent-bit/flb_input.h +++ b/source/include/fluent-bit/flb_input.h @@ -52,6 +52,8 @@ #include #include +#include + #include #include @@ -336,7 +338,7 @@ struct flb_input_instance { struct mk_list _head; /* link to config->inputs */ struct cfl_list routes_direct; /* direct routes set by API */ - struct mk_list routes; /* flb_router_path's list */ + struct cfl_list routes; /* flb_router_path's list */ struct mk_list properties; /* properties / configuration */ struct mk_list collectors; /* collectors */ From 9665c14820bff2d3e858aa253e0bd703b84672b0 Mon Sep 17 00:00:00 2001 From: Patrick Stephens Date: Sun, 3 May 2026 12:34:47 +0100 Subject: [PATCH 044/213] [upstream] router: use cfl_list intead of mk_list Upstream-Ref: https://github.com/fluent/fluent-bit/commit/e3eeb0bf13430ae185d55c577392c2655640b64d Cherry-picked from Fluent Bit v4.2.4 --- source/src/flb_router.c | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/source/src/flb_router.c b/source/src/flb_router.c index 0c8f749d..47251d52 100644 --- a/source/src/flb_router.c +++ b/source/src/flb_router.c @@ -156,7 +156,7 @@ int flb_router_connect(struct flb_input_instance *in, p->ins = out; p->route = NULL; - mk_list_add(&p->_head, &in->routes); + cfl_list_add(&p->_head, &in->routes); return 0; } @@ -174,7 +174,7 @@ int flb_router_connect_direct(struct flb_input_instance *in, p->ins = out; p->route = NULL; - mk_list_add(&p->_head, &in->routes_direct); + cfl_list_add(&p->_head, &in->routes_direct); return 0; } @@ -273,9 +273,9 @@ int flb_router_io_set(struct flb_config *config) void flb_router_exit(struct flb_config *config) { struct mk_list *tmp; - struct mk_list *r_tmp; + struct cfl_list *r_tmp; struct mk_list *head; - struct mk_list *r_head; + struct cfl_list *r_head; struct flb_input_instance *in; struct flb_router_path *r; @@ -284,16 +284,16 @@ void flb_router_exit(struct flb_config *config) in = mk_list_entry(head, struct flb_input_instance, _head); /* Iterate instance routes */ - mk_list_foreach_safe(r_head, r_tmp, &in->routes) { - r = mk_list_entry(r_head, struct flb_router_path, _head); - mk_list_del(&r->_head); + cfl_list_foreach_safe(r_head, r_tmp, &in->routes) { + r = cfl_list_entry(r_head, struct flb_router_path, _head); + cfl_list_del(&r->_head); flb_free(r); } /* Iterate instance routes direct */ - mk_list_foreach_safe(r_head, r_tmp, &in->routes_direct) { - r = mk_list_entry(r_head, struct flb_router_path, _head); - mk_list_del(&r->_head); + cfl_list_foreach_safe(r_head, r_tmp, &in->routes_direct) { + r = cfl_list_entry(r_head, struct flb_router_path, _head); + cfl_list_del(&r->_head); flb_free(r); } } From eb4da064545d768efa5d1df7572d1a8790d54481 Mon Sep 17 00:00:00 2001 From: Patrick Stephens Date: Sun, 3 May 2026 12:34:47 +0100 Subject: [PATCH 045/213] [upstream] input: fix llist initialization Upstream-Ref: https://github.com/fluent/fluent-bit/commit/995dd1261f92e6ea9bfc0a6a88b86d1885c37980 Cherry-picked from Fluent Bit v4.2.4 --- source/src/flb_input.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/src/flb_input.c b/source/src/flb_input.c index 0abdd834..0e0bb267 100644 --- a/source/src/flb_input.c +++ b/source/src/flb_input.c @@ -329,8 +329,8 @@ struct flb_input_instance *flb_input_new(struct flb_config *config, instance->host.ipv6 = FLB_FALSE; /* Initialize list heads */ - mk_list_init(&instance->routes_direct); - mk_list_init(&instance->routes); + cfl_list_init(&instance->routes_direct); + cfl_list_init(&instance->routes); mk_list_init(&instance->tasks); mk_list_init(&instance->chunks); mk_list_init(&instance->collectors); From a7c56ea2ffbfe6e73ccb68239da0fd2531225e56 Mon Sep 17 00:00:00 2001 From: Patrick Stephens Date: Sun, 3 May 2026 12:34:48 +0100 Subject: [PATCH 046/213] [upstream] sosreport: fix list iterator api Upstream-Ref: https://github.com/fluent/fluent-bit/commit/6bab25521afdcffc29d2b94a988bed8c54ce2e6d Cherry-picked from Fluent Bit v4.2.4 --- source/src/flb_sosreport.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/source/src/flb_sosreport.c b/source/src/flb_sosreport.c index 3db87bd1..5074aa5a 100644 --- a/source/src/flb_sosreport.c +++ b/source/src/flb_sosreport.c @@ -167,7 +167,7 @@ int flb_sosreport(struct flb_config *config) { char tmp[32]; struct mk_list *head; - struct mk_list *head_r; + struct cfl_list *head_r; struct flb_input_plugin *in; struct flb_filter_plugin *filter; struct flb_output_plugin *out; @@ -266,10 +266,10 @@ int flb_sosreport(struct flb_config *config) print_properties(&ins_in->properties); /* Fixed Routes */ - if (mk_list_is_empty(&ins_in->routes) != 0) { + if (!cfl_list_is_empty(&ins_in->routes)) { printf(" Routes\t\t"); - mk_list_foreach(head_r, &ins_in->routes) { - route = mk_list_entry(head_r, struct flb_router_path, _head); + cfl_list_foreach(head_r, &ins_in->routes) { + route = cfl_list_entry(head_r, struct flb_router_path, _head); printf("%s ", route->ins->name); } printf("\n"); From 90342d64529b053de80a05ea38ec29205c54c162 Mon Sep 17 00:00:00 2001 From: Patrick Stephens Date: Sun, 3 May 2026 12:34:48 +0100 Subject: [PATCH 047/213] [upstream] input_chunk: expose flb_input_chunk_get_real_size() Upstream-Ref: https://github.com/fluent/fluent-bit/commit/98009ae64b75d7e679e97109362ba790df58c94e Cherry-picked from Fluent Bit v4.2.4 --- source/include/fluent-bit/flb_input_chunk.h | 1 + source/src/flb_input_chunk.c | 3 +-- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/source/include/fluent-bit/flb_input_chunk.h b/source/include/fluent-bit/flb_input_chunk.h index 696e40ac..b9b55abe 100644 --- a/source/include/fluent-bit/flb_input_chunk.h +++ b/source/include/fluent-bit/flb_input_chunk.h @@ -113,6 +113,7 @@ int flb_input_chunk_get_tag(struct flb_input_chunk *ic, void flb_input_chunk_ring_buffer_cleanup(struct flb_input_instance *ins); void flb_input_chunk_ring_buffer_collector(struct flb_config *ctx, void *data); ssize_t flb_input_chunk_get_size(struct flb_input_chunk *ic); +ssize_t flb_input_chunk_get_real_size(struct flb_input_chunk *ic); size_t flb_input_chunk_set_limits(struct flb_input_instance *in); size_t flb_input_chunk_total_size(struct flb_input_instance *in); struct flb_input_chunk *flb_input_chunk_map(struct flb_input_instance *in, diff --git a/source/src/flb_input_chunk.c b/source/src/flb_input_chunk.c index 3d94b9cc..6458cb6d 100644 --- a/source/src/flb_input_chunk.c +++ b/source/src/flb_input_chunk.c @@ -93,7 +93,6 @@ static int flb_input_chunk_drop_task_route( struct flb_output_instance *o_ins, ssize_t *dropped_record_count); -static ssize_t flb_input_chunk_get_real_size(struct flb_input_chunk *ic); static ssize_t get_input_chunk_record_count(struct flb_input_chunk *input_chunk) { @@ -280,7 +279,7 @@ ssize_t flb_input_chunk_get_size(struct flb_input_chunk *ic) * is used to track the size of chunks in filesystem so we need to call * cio_chunk_get_real_size to return the original size in the file system */ -static ssize_t flb_input_chunk_get_real_size(struct flb_input_chunk *ic) +ssize_t flb_input_chunk_get_real_size(struct flb_input_chunk *ic) { ssize_t meta_size; ssize_t size; From 28059290cbdebc8efb80980fc06033ff53413b75 Mon Sep 17 00:00:00 2001 From: Patrick Stephens Date: Sun, 3 May 2026 12:34:48 +0100 Subject: [PATCH 048/213] [upstream] tests: internal: input_chunk: remove unused code Upstream-Ref: https://github.com/fluent/fluent-bit/commit/32ebcda1a6d18d292ce96d8aecde2c2fa08e000f Cherry-picked from Fluent Bit v4.2.4 --- source/tests/internal/input_chunk.c | 36 ----------------------------- 1 file changed, 36 deletions(-) diff --git a/source/tests/internal/input_chunk.c b/source/tests/internal/input_chunk.c index 945ddbbf..773eb03c 100644 --- a/source/tests/internal/input_chunk.c +++ b/source/tests/internal/input_chunk.c @@ -309,42 +309,6 @@ void flb_test_input_chunk_dropping_chunks() flb_destroy(ctx); } -/* - * When chunk is set to DOWN from memory, data_size is set to 0 and - * cio_chunk_get_content_size(1) returns the data_size. fs_chunks_size - * is used to track the size of chunks in filesystem so we need to call - * cio_chunk_get_real_size to return the original size in the file system - */ -static ssize_t flb_input_chunk_get_real_size(struct flb_input_chunk *ic) -{ - ssize_t meta_size; - ssize_t size; - - size = cio_chunk_get_real_size(ic->chunk); - - if (size != 0) { - return size; - } - - // Real size is not synced to chunk yet - size = flb_input_chunk_get_size(ic); - if (size == 0) { - flb_debug("[input chunk] no data in the chunk %s", - flb_input_chunk_get_name(ic)); - return -1; - } - - meta_size = cio_meta_size(ic->chunk); - size += meta_size - /* See https://github.com/edsiper/chunkio#file-layout for more details */ - + 2 /* HEADER BYTES */ - + 4 /* CRC32 */ - + 16 /* PADDING */ - + 2; /* METADATA LENGTH BYTES */ - - return size; -} - static int gen_buf(msgpack_sbuffer *mp_sbuf, char *buf, size_t buf_size) { msgpack_unpacked result; From 2f3cbe748e2f22b4960748d2457f339f8877dcd0 Mon Sep 17 00:00:00 2001 From: Patrick Stephens Date: Sun, 3 May 2026 12:34:48 +0100 Subject: [PATCH 049/213] [upstream] routes_mask: correct memcmp byte count in Upstream-Ref: https://github.com/fluent/fluent-bit/commit/83662c4097f5b25f71a3706544656fb7094d6393 Cherry-picked from Fluent Bit v4.2.4 --- source/src/flb_routes_mask.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/src/flb_routes_mask.c b/source/src/flb_routes_mask.c index e10ad9a4..4ff29587 100644 --- a/source/src/flb_routes_mask.c +++ b/source/src/flb_routes_mask.c @@ -144,7 +144,7 @@ int flb_routes_mask_is_empty(flb_route_mask_element *routes_mask, { return memcmp(routes_mask, config->route_empty_mask, - config->route_mask_size) == 0; + config->route_mask_size * sizeof(flb_route_mask_element)) == 0; } int flb_routes_empty_mask_create(struct flb_config *config) From 6d497b8668005b5a9aec7a8076e3c4c3babc805c Mon Sep 17 00:00:00 2001 From: Patrick Stephens Date: Sun, 3 May 2026 12:34:49 +0100 Subject: [PATCH 050/213] [upstream] log_event_decoder: add context extraction helpers Upstream-Ref: https://github.com/fluent/fluent-bit/commit/f1767017f88e738fa6b1a3342f7f6861053fb3d3 Cherry-picked from Fluent Bit v4.2.4 --- source/src/flb_log_event_decoder.c | 49 +++++++++++++++++++----------- 1 file changed, 32 insertions(+), 17 deletions(-) diff --git a/source/src/flb_log_event_decoder.c b/source/src/flb_log_event_decoder.c index f500d17a..52a39e14 100644 --- a/source/src/flb_log_event_decoder.c +++ b/source/src/flb_log_event_decoder.c @@ -150,11 +150,6 @@ void flb_log_event_decoder_destroy(struct flb_log_event_decoder *context) if (context != NULL) { if (context->initialized) { - if (context->unpacked_group_record.zone == - context->unpacked_event.zone) { - msgpack_unpacked_init(&context->unpacked_event); - } - msgpack_unpacked_destroy(&context->unpacked_group_record); msgpack_unpacked_destroy(&context->unpacked_empty_map); msgpack_unpacked_destroy(&context->unpacked_event); @@ -331,11 +326,6 @@ int flb_log_event_decoder_next(struct flb_log_event_decoder *context, return context->last_result; } - if (context->unpacked_group_record.zone == - context->unpacked_event.zone) { - msgpack_unpacked_init(&context->unpacked_event); - } - previous_offset = context->offset; result = msgpack_unpack_next(&context->unpacked_event, context->buffer, @@ -378,12 +368,38 @@ int flb_log_event_decoder_next(struct flb_log_event_decoder *context, msgpack_unpacked_destroy(&context->unpacked_group_record); if (record_type == FLB_LOG_EVENT_GROUP_START) { - memcpy(&context->unpacked_group_record, - &context->unpacked_event, - sizeof(msgpack_unpacked)); - - context->current_group_metadata = event->metadata; - context->current_group_attributes = event->body; + /* + * Transfer zone ownership from unpacked_event to unpacked_group_record + * instead of using memcpy. This prevents double-free issues when + * both structures would otherwise reference the same zone. + */ + context->unpacked_group_record.zone = msgpack_unpacked_release_zone(&context->unpacked_event); + context->unpacked_group_record.data = context->unpacked_event.data; + + /* + * Extract pointers from the transferred data structure to ensure they + * remain valid. The pointers must come from unpacked_group_record.data + * since that's where the zone (and the data) now reside. + */ + if (context->unpacked_group_record.data.type == MSGPACK_OBJECT_ARRAY && + context->unpacked_group_record.data.via.array.size == 2) { + msgpack_object *header = &context->unpacked_group_record.data.via.array.ptr[0]; + msgpack_object *root_body = &context->unpacked_group_record.data.via.array.ptr[1]; + + if (header->type == MSGPACK_OBJECT_ARRAY && + header->via.array.size == 2) { + context->current_group_metadata = &header->via.array.ptr[1]; + } + else { + context->current_group_metadata = context->empty_map; + } + context->current_group_attributes = root_body; + } + else { + /* Fallback to using event pointers if structure is unexpected */ + context->current_group_metadata = event->metadata; + context->current_group_attributes = event->body; + } } else { context->current_group_metadata = NULL; @@ -392,7 +408,6 @@ int flb_log_event_decoder_next(struct flb_log_event_decoder *context, if (context->read_groups != FLB_TRUE) { memset(event, 0, sizeof(struct flb_log_event)); - return flb_log_event_decoder_next(context, event); } } From 32b86cd907963404b5bf8b236aeefebe3dc952df Mon Sep 17 00:00:00 2001 From: Patrick Stephens Date: Sun, 3 May 2026 12:34:50 +0100 Subject: [PATCH 051/213] [upstream] task: pass context to router condition evaluation Upstream-Ref: https://github.com/fluent/fluent-bit/commit/a7d0510853db3ce0c2741d56782a0bd81fff130e Cherry-picked from Fluent Bit v4.2.4 --- source/src/flb_task.c | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/source/src/flb_task.c b/source/src/flb_task.c index 64bafc23..74b9223d 100644 --- a/source/src/flb_task.c +++ b/source/src/flb_task.c @@ -421,6 +421,15 @@ struct flb_task *flb_task_create(uint64_t ref_id, o_ins = route_path->ins; + /* For conditional routing, also check the route mask */ + if (task_ic->routes_mask) { + if (flb_routes_mask_get_bit(task_ic->routes_mask, + o_ins->id, + o_ins->config) == 0) { + continue; + } + } + route = flb_calloc(1, sizeof(struct flb_task_route)); if (!route) { flb_errno(); From b198a0df067b0b03d23607be4b22b95486c19021 Mon Sep 17 00:00:00 2001 From: Patrick Stephens Date: Sun, 3 May 2026 12:34:50 +0100 Subject: [PATCH 052/213] [upstream] processor_content_modifier: skip GROUP_START markers Upstream-Ref: https://github.com/fluent/fluent-bit/commit/03d3c2e299198bd65c2060b17dc7d3dd984206b4 Cherry-picked from Fluent Bit v4.2.4 --- source/plugins/processor_content_modifier/cm_logs.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/source/plugins/processor_content_modifier/cm_logs.c b/source/plugins/processor_content_modifier/cm_logs.c index 49e6e2fe..6af448cd 100644 --- a/source/plugins/processor_content_modifier/cm_logs.c +++ b/source/plugins/processor_content_modifier/cm_logs.c @@ -316,6 +316,12 @@ int cm_logs_process(struct flb_processor_instance *ins, continue; } + if (record_type == FLB_LOG_EVENT_GROUP_START && + (ctx->context_type == CM_CONTEXT_LOG_METADATA || + ctx->context_type == CM_CONTEXT_LOG_BODY)) { + continue; + } + /* retrieve the target cfl object */ if (ctx->context_type == CM_CONTEXT_LOG_METADATA) { obj = record->cobj_metadata; From 8353bc568a1076ce53b715d6bfb05f057835f61a Mon Sep 17 00:00:00 2001 From: Patrick Stephens Date: Sun, 3 May 2026 12:34:50 +0100 Subject: [PATCH 053/213] [upstream] lib: add context support API for routing Upstream-Ref: https://github.com/fluent/fluent-bit/commit/6b8fbdde9ef4182c32a290de2f589e8c3a662435 Cherry-picked from Fluent Bit v4.2.4 --- source/src/flb_lib.c | 86 +++++++++++++++++++++++++++++++++++++++----- 1 file changed, 78 insertions(+), 8 deletions(-) diff --git a/source/src/flb_lib.c b/source/src/flb_lib.c index 30d4d99d..f679be04 100644 --- a/source/src/flb_lib.c +++ b/source/src/flb_lib.c @@ -34,9 +34,14 @@ #include #include #include +#include #include #include +#include +#include +#include +#include #ifdef FLB_HAVE_MTRACE #include @@ -675,17 +680,81 @@ int flb_service_set(flb_ctx_t *ctx, ...) /* Load a configuration file that may be used by the input or output plugin */ int flb_lib_config_file(struct flb_lib_ctx *ctx, const char *path) { - if (access(path, R_OK) != 0) { + struct flb_cf *cf; + int ret; + char tmp[PATH_MAX + 1]; + char *cfg = NULL; + char *end; + char *real_path; + struct stat st; + + /* Check if file exists and resolve path */ + ret = stat(path, &st); + if (ret == -1 && errno == ENOENT) { + /* Try to resolve the real path (if exists) */ + if (path[0] == '/') { + fprintf(stderr, "Error: configuration file not found: %s\n", path); + return -1; + } + + if (ctx->config->conf_path) { + snprintf(tmp, PATH_MAX, "%s%s", ctx->config->conf_path, path); + cfg = tmp; + } + else { + cfg = (char *) path; + } + } + else { + cfg = (char *) path; + } + + if (access(cfg, R_OK) != 0) { perror("access"); + fprintf(stderr, "Error: cannot read configuration file: %s\n", cfg); return -1; } - ctx->config->file = mk_rconf_open(path); - if (!ctx->config->file) { - fprintf(stderr, "Error reading configuration file: %s\n", path); + /* Use modern config format API that supports both .conf and .yaml/.yml */ + cf = flb_cf_create_from_file(NULL, cfg); + if (!cf) { + fprintf(stderr, "Error reading configuration file: %s\n", cfg); return -1; } + /* Set configuration root path */ + if (cfg) { + real_path = realpath(cfg, NULL); + if (real_path) { + end = strrchr(real_path, FLB_DIRCHAR); + if (end) { + end++; + *end = '\0'; + if (ctx->config->conf_path) { + flb_free(ctx->config->conf_path); + } + ctx->config->conf_path = flb_strdup(real_path); + } + free(real_path); + } + } + + /* Load the configuration format into the config */ + ret = flb_config_load_config_format(ctx->config, cf); + if (ret != 0) { + flb_cf_destroy(cf); + fprintf(stderr, "Error loading configuration from file: %s\n", cfg); + return -1; + } + + /* Destroy old cf_main if it exists (created by flb_config_init) */ + if (ctx->config->cf_main) { + flb_cf_destroy(ctx->config->cf_main); + } + + /* Store the config format object */ + ctx->config->cf_main = cf; + return 0; } @@ -803,7 +872,7 @@ int flb_lib_push(flb_ctx_t *ctx, int ffd, const void *data, size_t len) /* Emulate some data from the response */ int flb_lib_response(flb_ctx_t *ctx, int ffd, int status, const void *data, size_t len) { - int ret; + int ret = -1; struct flb_output_instance *o_ins; if (ctx->status == FLB_LIB_NONE || ctx->status == FLB_LIB_ERROR) { @@ -816,7 +885,7 @@ int flb_lib_response(flb_ctx_t *ctx, int ffd, int status, const void *data, size return -1; } - /* If input's test_formatter is registered, priorize to run it. */ + /* If output's test_response callback is registered, prioritize to run it. */ if (o_ins->test_response.callback != NULL) { ret = flb_output_run_response(ctx, o_ins, status, data, len); } @@ -965,8 +1034,9 @@ int flb_stop(flb_ctx_t *ctx) return 0; } - if (ctx->config->file) { - mk_rconf_free(ctx->config->file); + if (ctx->config->cf_main) { + flb_cf_destroy(ctx->config->cf_main); + ctx->config->cf_main = NULL; } flb_debug("[lib] sending STOP signal to the engine"); From 9a070bdbfd14238520aac9d9c40a34e93f940c69 Mon Sep 17 00:00:00 2001 From: Patrick Stephens Date: Sun, 3 May 2026 12:34:50 +0100 Subject: [PATCH 054/213] [upstream] tests: runtime: in_opentelemetry_routing: add context Upstream-Ref: https://github.com/fluent/fluent-bit/commit/2d07323f5f6fb5b2a54001f08222b0ae9de0535d Cherry-picked from Fluent Bit v4.2.4 --- source/tests/runtime/CMakeLists.txt | 1 + .../data/opentelemetry/routing_logs.json | 203 +++++++++ .../tests/runtime/in_opentelemetry_routing.c | 409 ++++++++++++++++++ 3 files changed, 613 insertions(+) create mode 100644 source/tests/runtime/data/opentelemetry/routing_logs.json create mode 100644 source/tests/runtime/in_opentelemetry_routing.c diff --git a/source/tests/runtime/CMakeLists.txt b/source/tests/runtime/CMakeLists.txt index b209a467..275edf23 100644 --- a/source/tests/runtime/CMakeLists.txt +++ b/source/tests/runtime/CMakeLists.txt @@ -60,6 +60,7 @@ if(FLB_OUT_LIB) FLB_RT_TEST(FLB_IN_FLUENTBIT_METRICS "in_fluentbit_metrics.c") FLB_RT_TEST(FLB_IN_PROMETHEUS_TEXTFILE "in_prometheus_textfile.c") FLB_RT_TEST(FLB_IN_KUBERNETES_EVENTS "in_kubernetes_events.c") + FLB_RT_TEST(FLB_IN_OPENTELEMETRY "in_opentelemetry_routing.c") if (FLB_IN_SYSTEMD) FLB_RT_TEST(FLB_IN_SYSTEMD "in_systemd.c") endif () diff --git a/source/tests/runtime/data/opentelemetry/routing_logs.json b/source/tests/runtime/data/opentelemetry/routing_logs.json new file mode 100644 index 00000000..ce39690c --- /dev/null +++ b/source/tests/runtime/data/opentelemetry/routing_logs.json @@ -0,0 +1,203 @@ +{ + "resourceLogs": [ + { + "resource": { + "attributes": [ + { + "key": "service_name", + "value": { + "stringValue": "service-a" + } + }, + { + "key": "service_version", + "value": { + "stringValue": "1.0.0" + } + }, + { + "key": "environment", + "value": { + "stringValue": "production" + } + } + ], + "droppedAttributesCount": 0 + }, + "scopeLogs": [ + { + "scope": { + "name": "scope-a", + "version": "v1.0", + "attributes": [ + { + "key": "component", + "value": { + "stringValue": "backend" + } + } + ], + "droppedAttributesCount": 0 + }, + "logRecords": [ + { + "timeUnixNano": "1640995200000000000", + "body": { + "stringValue": "record from service-a scope-a" + }, + "attributes": [ + { + "key": "level", + "value": { + "stringValue": "info" + } + } + ] + }, + { + "timeUnixNano": "1640995201000000000", + "body": { + "stringValue": "error record from service-a" + }, + "attributes": [ + { + "key": "level", + "value": { + "stringValue": "error" + } + } + ] + } + ] + } + ] + }, + { + "resource": { + "attributes": [ + { + "key": "service_name", + "value": { + "stringValue": "service-b" + } + }, + { + "key": "service_version", + "value": { + "stringValue": "2.0.0" + } + }, + { + "key": "environment", + "value": { + "stringValue": "staging" + } + } + ], + "droppedAttributesCount": 0 + }, + "scopeLogs": [ + { + "scope": { + "name": "scope-b", + "version": "v2.0", + "attributes": [ + { + "key": "component", + "value": { + "stringValue": "frontend" + } + } + ], + "droppedAttributesCount": 0 + }, + "logRecords": [ + { + "timeUnixNano": "1640995202000000000", + "body": { + "stringValue": "record from service-b scope-b" + }, + "attributes": [ + { + "key": "level", + "value": { + "stringValue": "debug" + } + } + ] + } + ] + }, + { + "scope": { + "name": "scope-c", + "version": "v1.5", + "attributes": [ + { + "key": "component", + "value": { + "stringValue": "database" + } + } + ], + "droppedAttributesCount": 0 + }, + "logRecords": [ + { + "timeUnixNano": "1640995203000000000", + "body": { + "stringValue": "query log from service-b" + }, + "attributes": [ + { + "key": "operation", + "value": { + "stringValue": "SELECT" + } + } + ] + } + ] + } + ] + }, + { + "resource": { + "attributes": [ + { + "key": "service_name", + "value": { + "stringValue": "service-c" + } + }, + { + "key": "service_version", + "value": { + "stringValue": "3.0.0" + } + } + ], + "droppedAttributesCount": 0 + }, + "scopeLogs": [ + { + "scope": { + "name": "scope-d", + "version": "v3.0", + "attributes": [], + "droppedAttributesCount": 0 + }, + "logRecords": [ + { + "timeUnixNano": "1640995204000000000", + "body": { + "stringValue": "unmatched record should go to default" + } + } + ] + } + ] + } + ] +} + diff --git a/source/tests/runtime/in_opentelemetry_routing.c b/source/tests/runtime/in_opentelemetry_routing.c new file mode 100644 index 00000000..0bae5de5 --- /dev/null +++ b/source/tests/runtime/in_opentelemetry_routing.c @@ -0,0 +1,409 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2025 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "flb_tests_runtime.h" +#include "../../plugins/in_opentelemetry/opentelemetry.h" +#include "../../plugins/in_opentelemetry/opentelemetry_logs.h" + +#define JSON_CONTENT_TYPE "application/json" +#define PORT_OTEL 4318 +#define V1_ENDPOINT_LOGS "/v1/logs" +#define MAX_ROUTES 32 + +/* Route expectation: file path and expected count */ +struct route_expectation { + const char *route_name; + const char *output_file; + int expected_count; +}; + +struct test_ctx { + flb_ctx_t *flb; + int i_ffd; + char output_dir[PATH_MAX]; +}; + +#define TEST_OUTPUT_DIR "otlp_routing_test_output" + +/* Construct path to config file in source directory */ +static char *get_config_path(const char *config_file) +{ + char path[PATH_MAX]; + char *resolved; + char *real_resolved; + char cwd[PATH_MAX]; + + /* Try FLB_TESTS_DATA_PATH first (tests/runtime directory) */ + snprintf(path, sizeof(path), "%s/%s", FLB_TESTS_DATA_PATH, config_file); + if (access(path, R_OK) == 0) { + resolved = flb_strdup(path); + return resolved; + } + + /* Try source root (go up from tests/runtime) */ + snprintf(path, sizeof(path), "%s/../../%s", FLB_TESTS_DATA_PATH, config_file); + if (access(path, R_OK) == 0) { + real_resolved = realpath(path, NULL); + if (real_resolved) { + resolved = flb_strdup(real_resolved); + free(real_resolved); + return resolved; + } + resolved = flb_strdup(path); + return resolved; + } + + /* Try current working directory */ + if (getcwd(cwd, sizeof(cwd)) != NULL) { + snprintf(path, sizeof(path), "%s/%s", cwd, config_file); + if (access(path, R_OK) == 0) { + resolved = flb_strdup(path); + return resolved; + } + } + + /* Return original path as fallback */ + return flb_strdup(config_file); +} + +/* Get opentelemetry input instance */ +static struct flb_input_instance *get_opentelemetry_instance(flb_ctx_t *flb_ctx) +{ + struct mk_list *head; + struct flb_input_instance *ins; + + mk_list_foreach(head, &flb_ctx->config->inputs) { + ins = mk_list_entry(head, struct flb_input_instance, _head); + if (ins->p && strcmp(ins->p->name, "opentelemetry") == 0) { + return ins; + } + } + return NULL; +} + +/* Directly inject JSON payload into opentelemetry plugin */ +static int inject_otlp_json(flb_ctx_t *flb_ctx, const char *json_data, size_t json_size) +{ + struct flb_input_instance *ins; + struct flb_opentelemetry *otel_ctx; + flb_sds_t content_type; + flb_sds_t tag; + int ret; + + /* Get opentelemetry input instance */ + ins = get_opentelemetry_instance(flb_ctx); + if (!ins || !ins->context) { + return -1; + } + + otel_ctx = (struct flb_opentelemetry *)ins->context; + + /* Use default tag if not set */ + if (ins->tag && ins->tag_len > 0) { + tag = flb_sds_create_len(ins->tag, ins->tag_len); + } + else { + tag = flb_sds_create("opentelemetry.0"); + } + + /* Set content type */ + content_type = flb_sds_create("application/json"); + + /* Process logs directly */ + ret = opentelemetry_process_logs(otel_ctx, content_type, tag, flb_sds_len(tag), + (void *)json_data, json_size); + + flb_sds_destroy(content_type); + flb_sds_destroy(tag); + + return ret; +} + +/* Create test context from YAML config file */ +static struct test_ctx *test_ctx_create(const char *config_file) +{ + struct test_ctx *ctx; + char *config_path; + int ret; + char cwd[PATH_MAX]; + + ctx = flb_calloc(1, sizeof(struct test_ctx)); + if (!TEST_CHECK(ctx != NULL)) { + return NULL; + } + + ctx->flb = flb_create(); + TEST_CHECK(ctx->flb != NULL); + + /* Create output directory */ + if (getcwd(cwd, sizeof(cwd)) != NULL) { + snprintf(ctx->output_dir, sizeof(ctx->output_dir), "%s/%s", cwd, TEST_OUTPUT_DIR); + } + else { + snprintf(ctx->output_dir, sizeof(ctx->output_dir), "./%s", TEST_OUTPUT_DIR); + } + + /* Create directory if it doesn't exist */ + ret = mkdir(ctx->output_dir, 0755); + if (ret != 0 && errno != EEXIST) { + flb_error("[test] Failed to create output directory: %s", ctx->output_dir); + flb_destroy(ctx->flb); + flb_free(ctx); + return NULL; + } + + + /* Resolve config file path */ + config_path = get_config_path(config_file); + TEST_CHECK(config_path != NULL); + if (!config_path) { + flb_destroy(ctx->flb); + flb_free(ctx); + return NULL; + } + + /* Load config from YAML file */ + ret = flb_lib_config_file(ctx->flb, config_path); + flb_free(config_path); + if (!TEST_CHECK(ret == 0)) { + flb_destroy(ctx->flb); + flb_free(ctx); + return NULL; + } + + return ctx; +} + +static void test_ctx_destroy(struct test_ctx *ctx) +{ + if (!ctx) { + return; + } + + sleep(1); + flb_stop(ctx->flb); + flb_destroy(ctx->flb); + + /* Cleanup output directory (optional - keep for debugging) */ + /* Can uncomment to clean up: + char cmd[PATH_MAX * 2]; + snprintf(cmd, sizeof(cmd), "rm -rf %s", ctx->output_dir); + system(cmd); + */ + + flb_free(ctx); +} + +/* Read file content and count JSON records */ +static int count_records_in_file(const char *filepath) +{ + FILE *fp; + char line[8192]; + int count = 0; + char *trimmed; + + fp = fopen(filepath, "r"); + if (!fp) { + return -1; + } + + while (fgets(line, sizeof(line), fp)) { + /* Trim whitespace */ + trimmed = line; + while (*trimmed == ' ' || *trimmed == '\t' || *trimmed == '\n' || *trimmed == '\r') { + trimmed++; + } + + /* Skip empty lines */ + if (*trimmed == '\0') { + continue; + } + + /* Count lines that contain a timestamp pattern (tag: [timestamp) or JSON object/array */ + /* File output in JSON format writes: "tag: [timestamp, {...}]" */ + /* File output in plain format writes: "{...}" */ + if (strstr(trimmed, ": [") != NULL || + *trimmed == '{' || + *trimmed == '[') { + count++; + } + } + + fclose(fp); + return count; +} + +/* Remove existing output files before test */ +static void cleanup_output_files(struct test_ctx *ctx, struct route_expectation *expectations, int count) +{ + int i; + char filepath[PATH_MAX]; + + for (i = 0; i < count; i++) { + snprintf(filepath, sizeof(filepath), "%s/%s", ctx->output_dir, expectations[i].output_file); + unlink(filepath); + } +} + +/* Verify all expectations by reading output files */ +static int verify_expectations(struct route_expectation *expectations, int count, struct test_ctx *ctx) +{ + int i; + int all_passed = 1; + char filepath[PATH_MAX]; + int actual_count; + + for (i = 0; i < count; i++) { + struct route_expectation *exp = &expectations[i]; + + snprintf(filepath, sizeof(filepath), "%s/%s", ctx->output_dir, exp->output_file); + actual_count = count_records_in_file(filepath); + + if (actual_count < 0) { + flb_error("[test] Route '%s': failed to read output file: %s", + exp->route_name, filepath); + all_passed = 0; + } + else if (actual_count != exp->expected_count) { + flb_error("[test] Route '%s': expected %d records, got %d (file: %s)", + exp->route_name, exp->expected_count, actual_count, filepath); + all_passed = 0; + } + else { + flb_info("[test] Route '%s': ✓ %d records (file: %s)", + exp->route_name, actual_count, filepath); + } + } + + return all_passed; +} + +/* Load JSON test data from file */ +static flb_sds_t load_json_test_data(const char *filename) +{ + char path[PATH_MAX]; + flb_sds_t content; + + /* Try FLB_TESTS_DATA_PATH first */ + snprintf(path, sizeof(path), "%s/data/opentelemetry/%s", FLB_TESTS_DATA_PATH, filename); + content = flb_file_read(path); + + /* Try relative to current directory if not found */ + if (!content) { + snprintf(path, sizeof(path), "data/opentelemetry/%s", filename); + content = flb_file_read(path); + } + + return content; +} + +/* Main test function */ +static void flb_test_otlp_routing(const char *config_file, + const char *json_file, + struct route_expectation *expectations, + int exp_count) +{ + struct test_ctx *ctx; + flb_sds_t json_content; + int ret; + + /* Load JSON test data */ + json_content = load_json_test_data(json_file); + TEST_CHECK(json_content != NULL); + if (!json_content) { + return; + } + + /* Create test context */ + ctx = test_ctx_create(config_file); + TEST_CHECK(ctx != NULL); + if (!ctx) { + flb_sds_destroy(json_content); + return; + } + + /* Clean up any existing output files */ + cleanup_output_files(ctx, expectations, exp_count); + + /* Start Fluent Bit */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + /* Directly inject JSON payload into opentelemetry plugin */ + ret = inject_otlp_json(ctx->flb, json_content, flb_sds_len(json_content)); + TEST_CHECK(ret == 0); + + /* Wait for records to be processed and flushed to files */ + flb_time_msleep(3000); + + /* Verify expectations by reading output files */ + ret = verify_expectations(expectations, exp_count, ctx); + TEST_CHECK(ret == 1); + + /* Cleanup */ + test_ctx_destroy(ctx); + flb_sds_destroy(json_content); +} + +/* Test case: Comprehensive routing with multiple routes */ +void flb_test_otlp_comprehensive_routing() +{ + struct route_expectation expectations[] = { + {"service_a_logs", "service_a_logs.out", 2}, /* Record 1, Record 2 */ + {"version_2_logs", "version_2_logs.out", 2}, /* Record 3, Record 4 */ + {"production_logs", "production_logs.out", 2}, /* Record 1, Record 2 */ + {"scope_a_logs", "scope_a_logs.out", 2}, /* Record 1, Record 2 */ + {"scope_v2_logs", "scope_v2_logs.out", 1}, /* Record 3 */ + {"backend_component_logs", "backend_component_logs.out", 2}, /* Record 1, Record 2 */ + {"error_body_logs", "error_body_logs.out", 1}, /* Record 2 */ + {"info_level_logs", "info_level_logs.out", 1}, /* Record 1 */ + {"select_operation_logs", "select_operation_logs.out", 1}, /* Record 4 */ + {"default_logs", "default_logs.out", 1}, /* Record 5 */ + }; + + /* Config file should be in the same directory as the test */ + flb_test_otlp_routing( + "otlp_comprehensive_routing_test.yaml", + "routing_logs.json", + expectations, + sizeof(expectations) / sizeof(expectations[0]) + ); +} + +TEST_LIST = { + {"otlp_comprehensive_routing", flb_test_otlp_comprehensive_routing}, + {NULL, NULL} +}; + From 92c9767aa561c7c9ea0d641c6e35421c13252bca Mon Sep 17 00:00:00 2001 From: Patrick Stephens Date: Sun, 3 May 2026 12:34:50 +0100 Subject: [PATCH 055/213] [upstream] tests: internal: conditional_routing: add context Upstream-Ref: https://github.com/fluent/fluent-bit/commit/34c08fee7a71fefaa0105a9e90d85dcbbba4cf9b Cherry-picked from Fluent Bit v4.2.4 --- source/tests/internal/conditional_routing.c | 983 ++++++++++++++++++++ 1 file changed, 983 insertions(+) create mode 100644 source/tests/internal/conditional_routing.c diff --git a/source/tests/internal/conditional_routing.c b/source/tests/internal/conditional_routing.c new file mode 100644 index 00000000..430580e6 --- /dev/null +++ b/source/tests/internal/conditional_routing.c @@ -0,0 +1,983 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include "flb_tests_internal.h" + +/* Test data structures */ +struct test_log_record { + const char *level; + const char *service; + const char *message; + const char *expected_route; +}; + +static const struct test_log_record test_records[] = { + {"info", "web-server", "Application started successfully", "info_logs"}, + {"error", "database", "Database connection failed", "error_logs"}, + {"undef", "logger", "Unknown log level detected", "default_logs"}, + {"info", "auth", "User authentication successful", "info_logs"}, + {"error", "file-service", "File not found", "error_logs"}, + {"undef", "parser", "Invalid log format", "default_logs"}, + {"info", "cache", "Cache updated successfully", "info_logs"}, + {"error", "memory-manager", "Memory allocation failed", "error_logs"}, + {"undef", "event-processor", "Unrecognized event type", "default_logs"}, + {"info", "test", "Test log entry", "info_logs"} +}; + +static const size_t test_records_count = sizeof(test_records) / sizeof(test_records[0]); + +/* Helper function to create a test log chunk */ +static int create_test_log_chunk(const char *level, + const char *service, + const char *message, + struct flb_log_event_encoder *encoder, + struct flb_event_chunk *chunk) +{ + int ret; + + if (!level || !service || !message || !encoder || !chunk) { + return -1; + } + + ret = flb_log_event_encoder_init(encoder, FLB_LOG_EVENT_FORMAT_DEFAULT); + TEST_CHECK(ret == FLB_EVENT_ENCODER_SUCCESS); + if (ret != FLB_EVENT_ENCODER_SUCCESS) { + return -1; + } + + ret = flb_log_event_encoder_begin_record(encoder); + TEST_CHECK(ret == FLB_EVENT_ENCODER_SUCCESS); + if (ret != FLB_EVENT_ENCODER_SUCCESS) { + flb_log_event_encoder_destroy(encoder); + return -1; + } + + ret = flb_log_event_encoder_set_current_timestamp(encoder); + TEST_CHECK(ret == FLB_EVENT_ENCODER_SUCCESS); + if (ret != FLB_EVENT_ENCODER_SUCCESS) { + flb_log_event_encoder_destroy(encoder); + return -1; + } + + ret = flb_log_event_encoder_append_body_values( + encoder, + FLB_LOG_EVENT_STRING_VALUE("level", 5), + FLB_LOG_EVENT_CSTRING_VALUE(level), + FLB_LOG_EVENT_STRING_VALUE("service", 7), + FLB_LOG_EVENT_CSTRING_VALUE(service), + FLB_LOG_EVENT_STRING_VALUE("message", 7), + FLB_LOG_EVENT_CSTRING_VALUE(message)); + TEST_CHECK(ret == FLB_EVENT_ENCODER_SUCCESS); + if (ret != FLB_EVENT_ENCODER_SUCCESS) { + flb_log_event_encoder_destroy(encoder); + return -1; + } + + ret = flb_log_event_encoder_commit_record(encoder); + TEST_CHECK(ret == FLB_EVENT_ENCODER_SUCCESS); + if (ret != FLB_EVENT_ENCODER_SUCCESS) { + flb_log_event_encoder_destroy(encoder); + return -1; + } + + memset(chunk, 0, sizeof(*chunk)); + chunk->type = FLB_EVENT_TYPE_LOGS; + chunk->data = encoder->output_buffer; + chunk->size = encoder->output_length; + chunk->total_events = 1; + + return 0; +} + +/* Test conditional routing configuration parsing */ +void test_conditional_routing_config_parse() +{ + struct cfl_list routes; + struct cfl_variant *inputs; + struct flb_cf *cf; + struct flb_input_routes *input_routes; + struct flb_route *route; + struct cfl_list *head; + struct cfl_list *route_head; + int ret; + int seen_info = 0; + int seen_error = 0; + int seen_default = 0; + + cfl_list_init(&routes); + + /* Create test configuration */ + inputs = create_conditional_routing_inputs(); + TEST_CHECK(inputs != NULL); + if (!inputs) { + return; + } + + cf = cf_from_inputs_variant(inputs); + TEST_CHECK(cf != NULL); + if (!cf) { + cfl_variant_destroy(inputs); + return; + } + + ret = flb_router_config_parse(cf, &routes, NULL); + TEST_CHECK(ret == 0); + if (ret == 0) { + TEST_CHECK(cfl_list_size(&routes) == 1); + head = routes.next; + input_routes = cfl_list_entry(head, struct flb_input_routes, _head); + TEST_CHECK(strcmp(input_routes->input_name, "tail") == 0); + TEST_CHECK(cfl_list_size(&input_routes->routes) == 3); + + cfl_list_foreach(route_head, &input_routes->routes) { + route = cfl_list_entry(route_head, struct flb_route, _head); + if (strcmp(route->name, "info_logs") == 0) { + seen_info = 1; + TEST_CHECK(route->per_record_routing == FLB_TRUE); + TEST_CHECK(route->condition != NULL); + TEST_CHECK(route->condition->is_default == FLB_FALSE); + } + else if (strcmp(route->name, "error_logs") == 0) { + seen_error = 1; + TEST_CHECK(route->per_record_routing == FLB_TRUE); + TEST_CHECK(route->condition != NULL); + TEST_CHECK(route->condition->is_default == FLB_FALSE); + } + else if (strcmp(route->name, "default_logs") == 0) { + seen_default = 1; + TEST_CHECK(route->per_record_routing == FLB_TRUE); + TEST_CHECK(route->condition != NULL); + TEST_CHECK(route->condition->is_default == FLB_TRUE); + } + } + + TEST_CHECK(seen_info == 1); + TEST_CHECK(seen_error == 1); + TEST_CHECK(seen_default == 1); + + flb_router_routes_destroy(&routes); + } + + flb_cf_destroy(cf); + cfl_variant_destroy(inputs); +} + +/* Test condition evaluation for individual records */ +void test_conditional_routing_condition_eval() +{ + struct flb_route route; + struct flb_route_condition *condition; + struct flb_route_condition_rule *rule; + struct flb_log_event_encoder encoder; + struct flb_event_chunk chunk; + struct flb_router_chunk_context context; + int ret; + size_t i; + + memset(&route, 0, sizeof(route)); + cfl_list_init(&route.outputs); + cfl_list_init(&route.processors); + + /* Create condition: level == "info" */ + condition = flb_calloc(1, sizeof(struct flb_route_condition)); + TEST_CHECK(condition != NULL); + if (!condition) { + return; + } + + cfl_list_init(&condition->rules); + condition->op = FLB_COND_OP_AND; + condition->compiled_status = 0; + condition->compiled = NULL; + condition->is_default = FLB_FALSE; + + rule = flb_calloc(1, sizeof(struct flb_route_condition_rule)); + TEST_CHECK(rule != NULL); + if (!rule) { + flb_free(condition); + return; + } + + cfl_list_init(&rule->_head); + rule->field = flb_sds_create("$level"); + rule->op = flb_sds_create("eq"); + rule->value = flb_sds_create("info"); + TEST_CHECK(rule->field != NULL && rule->op != NULL && rule->value != NULL); + + cfl_list_add(&rule->_head, &condition->rules); + route.condition = condition; + route.signals = FLB_ROUTER_SIGNAL_LOGS; + route.per_record_routing = FLB_TRUE; + + flb_router_chunk_context_init(&context); + + /* Test each record */ + for (i = 0; i < test_records_count; i++) { + const struct test_log_record *record = &test_records[i]; + int expected_result = (strcmp(record->level, "info") == 0) ? FLB_TRUE : FLB_FALSE; + + ret = create_test_log_chunk(record->level, record->service, record->message, + &encoder, &chunk); + TEST_CHECK(ret == 0); + if (ret == 0) { + int result = flb_condition_eval_logs(&chunk, &context, &route); + TEST_CHECK(result == expected_result); + if (result != expected_result) { + fprintf(stderr, "Condition evaluation failed for record %zu: level=%s, expected=%d, got=%d\n", + i, record->level, expected_result, result); + } + } + flb_router_chunk_context_reset(&context); + flb_log_event_encoder_destroy(&encoder); + } + + flb_router_chunk_context_destroy(&context); + flb_free(condition); + flb_sds_destroy(rule->field); + flb_sds_destroy(rule->op); + flb_sds_destroy(rule->value); + flb_free(rule); +} + +/* Test per-record routing functionality */ +void test_conditional_routing_per_record() +{ + struct flb_config config; + struct flb_input_instance input; + struct flb_output_instance output1, output2, output3; + struct flb_input_plugin input_plugin; + struct flb_output_plugin output_plugin; + struct flb_input_routes input_routes; + struct flb_route route1, route2, route3; + struct flb_route_output route_output1, route_output2, route_output3; + struct flb_router_path *path; + struct flb_log_event_encoder encoder; + struct flb_event_chunk chunk; + int ret; + size_t i; + + /* Setup test instances */ + setup_conditional_routing_instances(&config, &input, &input_plugin, + &output1, &output2, &output3, &output_plugin); + + /* Setup routes */ + setup_conditional_routes(&input_routes, &route1, &route2, &route3, + &route_output1, &route_output2, &route_output3, + &output1, &output2, &output3); + + /* Apply configuration */ + ret = flb_router_apply_config(&config); + TEST_CHECK(ret == 0); + TEST_CHECK(cfl_list_size(&input.routes_direct) == 3); + + /* Test per-record routing for each test record */ + for (i = 0; i < test_records_count; i++) { + const struct test_log_record *record = &test_records[i]; + + ret = create_test_log_chunk(record->level, record->service, record->message, + &encoder, &chunk); + TEST_CHECK(ret == 0); + if (ret == 0) { + /* Test that the record routes to the expected output */ + int routed_correctly = test_record_routing(&input, &chunk, record->expected_route); + TEST_CHECK(routed_correctly == 1); + if (!routed_correctly) { + fprintf(stderr, "Record %zu did not route correctly: level=%s, expected=%s\n", + i, record->level, record->expected_route); + } + } + flb_log_event_encoder_destroy(&encoder); + } + + /* Cleanup */ + flb_router_exit(&config); + cleanup_conditional_routing_instances(&input, &output1, &output2, &output3, + &input_routes, &route1, &route2, &route3, + &route_output1, &route_output2, &route_output3); +} + +/* Test default route handling */ +void test_conditional_routing_default_route() +{ + struct flb_config config; + struct flb_input_instance input; + struct flb_output_instance output1, output2, output3; + struct flb_input_plugin input_plugin; + struct flb_output_plugin output_plugin; + struct flb_input_routes input_routes; + struct flb_route route1, route2, route3; + struct flb_route_output route_output1, route_output2, route_output3; + struct flb_log_event_encoder encoder; + struct flb_event_chunk chunk; + int ret; + size_t i; + int default_route_count = 0; + + /* Setup test instances */ + setup_conditional_routing_instances(&config, &input, &input_plugin, + &output1, &output2, &output3, &output_plugin); + + /* Setup routes */ + setup_conditional_routes(&input_routes, &route1, &route2, &route3, + &route_output1, &route_output2, &route_output3, + &output1, &output2, &output3); + + /* Apply configuration */ + ret = flb_router_apply_config(&config); + TEST_CHECK(ret == 0); + + /* Test that records with "undef" level go to default route */ + for (i = 0; i < test_records_count; i++) { + const struct test_log_record *record = &test_records[i]; + + if (strcmp(record->level, "undef") == 0) { + ret = create_test_log_chunk(record->level, record->service, record->message, + &encoder, &chunk); + TEST_CHECK(ret == 0); + if (ret == 0) { + int routed_to_default = test_record_routing(&input, &chunk, "default_logs"); + TEST_CHECK(routed_to_default == 1); + if (routed_to_default) { + default_route_count++; + } + } + flb_log_event_encoder_destroy(&encoder); + } + } + + /* Verify that all "undef" records went to default route */ + TEST_CHECK(default_route_count == 3); /* Should have 3 "undef" records */ + + /* Cleanup */ + flb_router_exit(&config); + cleanup_conditional_routing_instances(&input, &output1, &output2, &output3, + &input_routes, &route1, &route2, &route3, + &route_output1, &route_output2, &route_output3); +} + +/* Test route mask functionality */ +void test_conditional_routing_route_mask() +{ + struct flb_config config; + struct flb_input_instance input; + struct flb_output_instance output1, output2, output3; + struct flb_input_plugin input_plugin; + struct flb_output_plugin output_plugin; + struct flb_input_routes input_routes; + struct flb_route route1, route2, route3; + struct flb_route_output route_output1, route_output2, route_output3; + struct flb_input_chunk *chunk; + flb_route_mask_element *routes_mask; + int ret; + size_t i; + + /* Setup test instances */ + setup_conditional_routing_instances(&config, &input, &input_plugin, + &output1, &output2, &output3, &output_plugin); + + /* Setup routes */ + setup_conditional_routes(&input_routes, &route1, &route2, &route3, + &route_output1, &route_output2, &route_output3, + &output1, &output2, &output3); + + /* Apply configuration */ + ret = flb_router_apply_config(&config); + TEST_CHECK(ret == 0); + + /* Test route mask for info records */ + for (i = 0; i < test_records_count; i++) { + const struct test_log_record *record = &test_records[i]; + + if (strcmp(record->level, "info") == 0) { + /* Create a test chunk */ + chunk = flb_input_chunk_create(&input, "test_tag", 8, NULL, 0); + TEST_CHECK(chunk != NULL); + if (chunk) { + /* Set route mask for info output only */ + routes_mask = chunk->routes_mask; + flb_routes_mask_set_bit(routes_mask, output1.id, &config); + + /* Verify route mask is set correctly */ + TEST_CHECK(flb_routes_mask_get_bit(routes_mask, output1.id, &config) == 1); + TEST_CHECK(flb_routes_mask_get_bit(routes_mask, output2.id, &config) == 0); + TEST_CHECK(flb_routes_mask_get_bit(routes_mask, output3.id, &config) == 0); + + flb_input_chunk_destroy(chunk); + } + } + } + + /* Cleanup */ + flb_router_exit(&config); + cleanup_conditional_routing_instances(&input, &output1, &output2, &output3, + &input_routes, &route1, &route2, &route3, + &route_output1, &route_output2, &route_output3); +} + +/* Test no duplicate routing */ +void test_conditional_routing_no_duplicates() +{ + struct flb_config config; + struct flb_input_instance input; + struct flb_output_instance output1, output2, output3; + struct flb_input_plugin input_plugin; + struct flb_output_plugin output_plugin; + struct flb_input_routes input_routes; + struct flb_route route1, route2, route3; + struct flb_route_output route_output1, route_output2, route_output3; + struct flb_log_event_encoder encoder; + struct flb_event_chunk chunk; + int ret; + size_t i; + int total_routed = 0; + int info_routed = 0; + int error_routed = 0; + int default_routed = 0; + + /* Setup test instances */ + setup_conditional_routing_instances(&config, &input, &input_plugin, + &output1, &output2, &output3, &output_plugin); + + /* Setup routes */ + setup_conditional_routes(&input_routes, &route1, &route2, &route3, + &route_output1, &route_output2, &route_output3, + &output1, &output2, &output3); + + /* Apply configuration */ + ret = flb_router_apply_config(&config); + TEST_CHECK(ret == 0); + + /* Test all records and count routing */ + for (i = 0; i < test_records_count; i++) { + const struct test_log_record *record = &test_records[i]; + + ret = create_test_log_chunk(record->level, record->service, record->message, + &encoder, &chunk); + TEST_CHECK(ret == 0); + if (ret == 0) { + int routed = test_record_routing(&input, &chunk, record->expected_route); + TEST_CHECK(routed == 1); + if (routed == 1) { + total_routed++; + if (strcmp(record->expected_route, "info_logs") == 0) { + info_routed++; + } else if (strcmp(record->expected_route, "error_logs") == 0) { + error_routed++; + } else if (strcmp(record->expected_route, "default_logs") == 0) { + default_routed++; + } + } + } + flb_log_event_encoder_destroy(&encoder); + } + + /* Verify no duplicates - each record should be routed exactly once */ + TEST_CHECK(total_routed == test_records_count); + TEST_CHECK(info_routed == 4); /* 4 info records */ + TEST_CHECK(error_routed == 3); /* 3 error records */ + TEST_CHECK(default_routed == 3); /* 3 default records */ + + /* Cleanup */ + flb_router_exit(&config); + cleanup_conditional_routing_instances(&input, &output1, &output2, &output3, + &input_routes, &route1, &route2, &route3, + &route_output1, &route_output2, &route_output3); +} + +/* Helper functions for test setup */ + +static struct cfl_variant *create_conditional_routing_inputs() +{ + struct cfl_array *inputs; + struct cfl_kvlist *input; + struct cfl_kvlist *routes; + struct cfl_array *log_routes; + struct cfl_kvlist *route; + struct cfl_kvlist *condition; + struct cfl_array *rules; + struct cfl_kvlist *rule_kv; + struct cfl_variant *rule_variant; + struct cfl_array *outputs; + struct cfl_kvlist *to; + struct cfl_variant *inputs_variant; + + inputs = cfl_array_create(1); + TEST_CHECK(inputs != NULL); + if (!inputs) { + return NULL; + } + + input = cfl_kvlist_create(); + TEST_CHECK(input != NULL); + if (!input) { + cfl_array_destroy(inputs); + return NULL; + } + + TEST_CHECK(cfl_kvlist_insert_string(input, "name", "tail") == 0); + + routes = cfl_kvlist_create(); + TEST_CHECK(routes != NULL); + log_routes = cfl_array_create(3); + TEST_CHECK(log_routes != NULL); + + /* info_logs route */ + route = cfl_kvlist_create(); + TEST_CHECK(route != NULL); + TEST_CHECK(cfl_kvlist_insert_string(route, "name", "info_logs") == 0); + TEST_CHECK(cfl_kvlist_insert_bool(route, "per_record_routing", 1) == 0); + + condition = cfl_kvlist_create(); + TEST_CHECK(condition != NULL); + rules = cfl_array_create(1); + TEST_CHECK(rules != NULL); + + rule_kv = cfl_kvlist_create(); + TEST_CHECK(rule_kv != NULL); + TEST_CHECK(cfl_kvlist_insert_string(rule_kv, "field", "$level") == 0); + TEST_CHECK(cfl_kvlist_insert_string(rule_kv, "op", "eq") == 0); + TEST_CHECK(cfl_kvlist_insert_string(rule_kv, "value", "info") == 0); + rule_variant = cfl_variant_create_from_kvlist(rule_kv); + TEST_CHECK(rule_variant != NULL); + TEST_CHECK(cfl_array_append(rules, rule_variant) == 0); + + TEST_CHECK(cfl_kvlist_insert_array(condition, "rules", rules) == 0); + TEST_CHECK(cfl_kvlist_insert_kvlist(route, "condition", condition) == 0); + + outputs = cfl_array_create(1); + TEST_CHECK(outputs != NULL); + TEST_CHECK(cfl_array_append_string(outputs, "info_destination") == 0); + to = cfl_kvlist_create(); + TEST_CHECK(to != NULL); + TEST_CHECK(cfl_kvlist_insert_array(to, "outputs", outputs) == 0); + TEST_CHECK(cfl_kvlist_insert_kvlist(route, "to", to) == 0); + + TEST_CHECK(cfl_array_append(log_routes, cfl_variant_create_from_kvlist(route)) == 0); + + /* error_logs route */ + route = cfl_kvlist_create(); + TEST_CHECK(route != NULL); + TEST_CHECK(cfl_kvlist_insert_string(route, "name", "error_logs") == 0); + TEST_CHECK(cfl_kvlist_insert_bool(route, "per_record_routing", 1) == 0); + + condition = cfl_kvlist_create(); + TEST_CHECK(condition != NULL); + rules = cfl_array_create(1); + TEST_CHECK(rules != NULL); + + rule_kv = cfl_kvlist_create(); + TEST_CHECK(rule_kv != NULL); + TEST_CHECK(cfl_kvlist_insert_string(rule_kv, "field", "$level") == 0); + TEST_CHECK(cfl_kvlist_insert_string(rule_kv, "op", "eq") == 0); + TEST_CHECK(cfl_kvlist_insert_string(rule_kv, "value", "error") == 0); + rule_variant = cfl_variant_create_from_kvlist(rule_kv); + TEST_CHECK(rule_variant != NULL); + TEST_CHECK(cfl_array_append(rules, rule_variant) == 0); + + TEST_CHECK(cfl_kvlist_insert_array(condition, "rules", rules) == 0); + TEST_CHECK(cfl_kvlist_insert_kvlist(route, "condition", condition) == 0); + + outputs = cfl_array_create(1); + TEST_CHECK(outputs != NULL); + TEST_CHECK(cfl_array_append_string(outputs, "error_destination") == 0); + to = cfl_kvlist_create(); + TEST_CHECK(to != NULL); + TEST_CHECK(cfl_kvlist_insert_array(to, "outputs", outputs) == 0); + TEST_CHECK(cfl_kvlist_insert_kvlist(route, "to", to) == 0); + + TEST_CHECK(cfl_array_append(log_routes, cfl_variant_create_from_kvlist(route)) == 0); + + /* default_logs route */ + route = cfl_kvlist_create(); + TEST_CHECK(route != NULL); + TEST_CHECK(cfl_kvlist_insert_string(route, "name", "default_logs") == 0); + TEST_CHECK(cfl_kvlist_insert_bool(route, "per_record_routing", 1) == 0); + + condition = cfl_kvlist_create(); + TEST_CHECK(condition != NULL); + TEST_CHECK(cfl_kvlist_insert_bool(condition, "default", 1) == 0); + TEST_CHECK(cfl_kvlist_insert_kvlist(route, "condition", condition) == 0); + + outputs = cfl_array_create(1); + TEST_CHECK(outputs != NULL); + TEST_CHECK(cfl_array_append_string(outputs, "default_destination") == 0); + to = cfl_kvlist_create(); + TEST_CHECK(to != NULL); + TEST_CHECK(cfl_kvlist_insert_array(to, "outputs", outputs) == 0); + TEST_CHECK(cfl_kvlist_insert_kvlist(route, "to", to) == 0); + + TEST_CHECK(cfl_array_append(log_routes, cfl_variant_create_from_kvlist(route)) == 0); + + TEST_CHECK(cfl_kvlist_insert_array(routes, "logs", log_routes) == 0); + TEST_CHECK(cfl_kvlist_insert_kvlist(input, "routes", routes) == 0); + TEST_CHECK(cfl_array_append(inputs, cfl_variant_create_from_kvlist(input)) == 0); + + inputs_variant = cfl_variant_create_from_array(inputs); + TEST_CHECK(inputs_variant != NULL); + + return inputs_variant; +} + +static struct flb_cf *cf_from_inputs_variant(struct cfl_variant *inputs) +{ + struct flb_cf *cf; + struct cfl_array *array; + size_t idx; + + if (!inputs || inputs->type != CFL_VARIANT_ARRAY) { + return NULL; + } + + cf = flb_cf_create(); + if (!cf) { + return NULL; + } + + array = inputs->data.as_array; + for (idx = 0; idx < cfl_array_size(array); idx++) { + struct cfl_variant *entry; + struct cfl_kvlist *copy; + struct flb_cf_section *section; + + entry = cfl_array_fetch_by_index(array, idx); + if (!entry || entry->type != CFL_VARIANT_KVLIST) { + flb_cf_destroy(cf); + return NULL; + } + + copy = clone_kvlist(entry->data.as_kvlist); + if (!copy) { + flb_cf_destroy(cf); + return NULL; + } + + section = flb_cf_section_create(cf, "input", 5); + if (!section) { + cfl_kvlist_destroy(copy); + flb_cf_destroy(cf); + return NULL; + } + + cfl_kvlist_destroy(section->properties); + section->properties = copy; + } + + return cf; +} + +static struct cfl_kvlist *clone_kvlist(struct cfl_kvlist *kvlist) +{ + struct cfl_kvlist *copy; + struct cfl_list *head; + struct cfl_kvpair *pair; + struct cfl_variant *value_copy; + + if (!kvlist) { + return NULL; + } + + copy = cfl_kvlist_create(); + if (!copy) { + return NULL; + } + + cfl_list_foreach(head, &kvlist->list) { + pair = cfl_list_entry(head, struct cfl_kvpair, _head); + value_copy = clone_variant(pair->val); + if (!value_copy) { + cfl_kvlist_destroy(copy); + return NULL; + } + + if (cfl_kvlist_insert_s(copy, + pair->key, + cfl_sds_len(pair->key), + value_copy) != 0) { + cfl_variant_destroy(value_copy); + cfl_kvlist_destroy(copy); + return NULL; + } + } + + return copy; +} + +static struct cfl_variant *clone_variant(struct cfl_variant *var) +{ + struct cfl_array *array_copy; + struct cfl_kvlist *kvlist_copy; + int referenced; + + array_copy = NULL; + kvlist_copy = NULL; + referenced = CFL_FALSE; + + if (!var) { + return NULL; + } + + switch (var->type) { + case CFL_VARIANT_STRING: + referenced = (var->referenced == CFL_TRUE) ? CFL_TRUE : CFL_FALSE; + return cfl_variant_create_from_string_s(var->data.as_string, + cfl_sds_len(var->data.as_string), + referenced); + case CFL_VARIANT_BOOL: + return cfl_variant_create_from_bool(var->data.as_bool); + case CFL_VARIANT_ARRAY: + array_copy = clone_array(var->data.as_array); + if (!array_copy) { + return NULL; + } + return cfl_variant_create_from_array(array_copy); + case CFL_VARIANT_KVLIST: + kvlist_copy = clone_kvlist(var->data.as_kvlist); + if (!kvlist_copy) { + return NULL; + } + return cfl_variant_create_from_kvlist(kvlist_copy); + default: + break; + } + + return NULL; +} + +static struct cfl_array *clone_array(struct cfl_array *array) +{ + struct cfl_array *copy; + struct cfl_variant *entry; + struct cfl_variant *entry_copy; + size_t idx; + + if (!array) { + return NULL; + } + + copy = cfl_array_create(cfl_array_size(array)); + if (!copy) { + return NULL; + } + + for (idx = 0; idx < cfl_array_size(array); idx++) { + entry = cfl_array_fetch_by_index(array, idx); + entry_copy = clone_variant(entry); + if (!entry_copy) { + cfl_array_destroy(copy); + return NULL; + } + + if (cfl_array_append(copy, entry_copy) != 0) { + cfl_variant_destroy(entry_copy); + cfl_array_destroy(copy); + return NULL; + } + } + + return copy; +} + +static void setup_conditional_routing_instances(struct flb_config *config, + struct flb_input_instance *input, + struct flb_input_plugin *input_plugin, + struct flb_output_instance *output1, + struct flb_output_instance *output2, + struct flb_output_instance *output3, + struct flb_output_plugin *output_plugin) +{ + memset(config, 0, sizeof(struct flb_config)); + mk_list_init(&config->inputs); + mk_list_init(&config->outputs); + cfl_list_init(&config->input_routes); + + memset(input, 0, sizeof(struct flb_input_instance)); + mk_list_init(&input->_head); + cfl_list_init(&input->routes_direct); + cfl_list_init(&input->routes); + mk_list_init(&input->tasks); + mk_list_init(&input->chunks); + mk_list_init(&input->collectors); + snprintf(input->name, sizeof(input->name), "tail.0"); + input->alias = flb_sds_create("test_input"); + input_plugin->name = "tail"; + input->p = input_plugin; + mk_list_add(&input->_head, &config->inputs); + + memset(output1, 0, sizeof(struct flb_output_instance)); + mk_list_init(&output1->_head); + mk_list_init(&output1->properties); + mk_list_init(&output1->net_properties); + snprintf(output1->name, sizeof(output1->name), "stdout.0"); + output1->alias = flb_sds_create("info_destination"); + output1->event_type = FLB_OUTPUT_LOGS; + output1->id = 1; + output_plugin->name = "stdout"; + output1->p = output_plugin; + mk_list_add(&output1->_head, &config->outputs); + + memset(output2, 0, sizeof(struct flb_output_instance)); + mk_list_init(&output2->_head); + mk_list_init(&output2->properties); + mk_list_init(&output2->net_properties); + snprintf(output2->name, sizeof(output2->name), "stdout.1"); + output2->alias = flb_sds_create("error_destination"); + output2->event_type = FLB_OUTPUT_LOGS; + output2->id = 2; + output2->p = output_plugin; + mk_list_add(&output2->_head, &config->outputs); + + memset(output3, 0, sizeof(struct flb_output_instance)); + mk_list_init(&output3->_head); + mk_list_init(&output3->properties); + mk_list_init(&output3->net_properties); + snprintf(output3->name, sizeof(output3->name), "stdout.2"); + output3->alias = flb_sds_create("default_destination"); + output3->event_type = FLB_OUTPUT_LOGS; + output3->id = 3; + output3->p = output_plugin; + mk_list_add(&output3->_head, &config->outputs); +} + +static void setup_conditional_routes(struct flb_input_routes *input_routes, + struct flb_route *route1, + struct flb_route *route2, + struct flb_route *route3, + struct flb_route_output *route_output1, + struct flb_route_output *route_output2, + struct flb_route_output *route_output3, + struct flb_output_instance *output1, + struct flb_output_instance *output2, + struct flb_output_instance *output3) +{ + memset(input_routes, 0, sizeof(struct flb_input_routes)); + cfl_list_init(&input_routes->_head); + cfl_list_init(&input_routes->routes); + input_routes->input_name = flb_sds_create("tail"); + + /* Route 1: info_logs */ + memset(route1, 0, sizeof(struct flb_route)); + cfl_list_init(&route1->_head); + cfl_list_init(&route1->outputs); + route1->name = flb_sds_create("info_logs"); + route1->signals = FLB_ROUTER_SIGNAL_LOGS; + route1->per_record_routing = FLB_TRUE; + cfl_list_add(&route1->_head, &input_routes->routes); + + memset(route_output1, 0, sizeof(struct flb_route_output)); + cfl_list_init(&route_output1->_head); + route_output1->name = flb_sds_create("info_destination"); + cfl_list_add(&route_output1->_head, &route1->outputs); + + /* Route 2: error_logs */ + memset(route2, 0, sizeof(struct flb_route)); + cfl_list_init(&route2->_head); + cfl_list_init(&route2->outputs); + route2->name = flb_sds_create("error_logs"); + route2->signals = FLB_ROUTER_SIGNAL_LOGS; + route2->per_record_routing = FLB_TRUE; + cfl_list_add(&route2->_head, &input_routes->routes); + + memset(route_output2, 0, sizeof(struct flb_route_output)); + cfl_list_init(&route_output2->_head); + route_output2->name = flb_sds_create("error_destination"); + cfl_list_add(&route_output2->_head, &route2->outputs); + + /* Route 3: default_logs */ + memset(route3, 0, sizeof(struct flb_route)); + cfl_list_init(&route3->_head); + cfl_list_init(&route3->outputs); + route3->name = flb_sds_create("default_logs"); + route3->signals = FLB_ROUTER_SIGNAL_LOGS; + route3->per_record_routing = FLB_TRUE; + cfl_list_add(&route3->_head, &input_routes->routes); + + memset(route_output3, 0, sizeof(struct flb_route_output)); + cfl_list_init(&route_output3->_head); + route_output3->name = flb_sds_create("default_destination"); + cfl_list_add(&route_output3->_head, &route3->outputs); +} + +static int test_record_routing(struct flb_input_instance *input, + struct flb_event_chunk *chunk, + const char *expected_route) +{ + struct mk_list *head; + struct flb_router_path *path; + struct flb_router_chunk_context context; + int found = 0; + + flb_router_chunk_context_init(&context); + + cfl_list_foreach(head, &input->routes_direct) { + path = cfl_list_entry(head, struct flb_router_path, _head); + + if (path->route && strcmp(path->route->name, expected_route) == 0) { + if (flb_router_path_should_route(chunk, &context, path) == FLB_TRUE) { + found = 1; + break; + } + } + } + + flb_router_chunk_context_destroy(&context); + return found; +} + +static void cleanup_conditional_routing_instances(struct flb_input_instance *input, + struct flb_output_instance *output1, + struct flb_output_instance *output2, + struct flb_output_instance *output3, + struct flb_input_routes *input_routes, + struct flb_route *route1, + struct flb_route *route2, + struct flb_route *route3, + struct flb_route_output *route_output1, + struct flb_route_output *route_output2, + struct flb_route_output *route_output3) +{ + flb_sds_destroy(input->alias); + flb_sds_destroy(output1->alias); + flb_sds_destroy(output2->alias); + flb_sds_destroy(output3->alias); + flb_sds_destroy(input_routes->input_name); + flb_sds_destroy(route1->name); + flb_sds_destroy(route2->name); + flb_sds_destroy(route3->name); + flb_sds_destroy(route_output1->name); + flb_sds_destroy(route_output2->name); + flb_sds_destroy(route_output3->name); +} + +TEST_LIST = { + { "config_parse", test_conditional_routing_config_parse }, + { "condition_eval", test_conditional_routing_condition_eval }, + { "per_record", test_conditional_routing_per_record }, + { "default_route", test_conditional_routing_default_route }, + { "route_mask", test_conditional_routing_route_mask }, + { "no_duplicates", test_conditional_routing_no_duplicates }, + { 0 } +}; From e6c8345b16fae1dd8abe26caf00b354563a949a5 Mon Sep 17 00:00:00 2001 From: Patrick Stephens Date: Sun, 3 May 2026 12:34:51 +0100 Subject: [PATCH 056/213] [upstream] tests: runtime: in_opentelemetry_routing: enhance Upstream-Ref: https://github.com/fluent/fluent-bit/commit/6a80c787813bf5830aaf38ec60f4f17c3ef54690 Cherry-picked from Fluent Bit v4.2.4 --- .../tests/runtime/in_opentelemetry_routing.c | 105 +++++++++++++++--- 1 file changed, 90 insertions(+), 15 deletions(-) diff --git a/source/tests/runtime/in_opentelemetry_routing.c b/source/tests/runtime/in_opentelemetry_routing.c index 0bae5de5..a63ddbff 100644 --- a/source/tests/runtime/in_opentelemetry_routing.c +++ b/source/tests/runtime/in_opentelemetry_routing.c @@ -57,17 +57,39 @@ struct test_ctx { #define TEST_OUTPUT_DIR "otlp_routing_test_output" -/* Construct path to config file in source directory */ +/* Construct absolute path to config file */ static char *get_config_path(const char *config_file) { char path[PATH_MAX]; char *resolved; char *real_resolved; char cwd[PATH_MAX]; + size_t cwd_len; + size_t config_file_len; + int ret; - /* Try FLB_TESTS_DATA_PATH first (tests/runtime directory) */ + /* Try FLB_TESTS_DATA_PATH/data/routing/ first */ + snprintf(path, sizeof(path), "%s/data/routing/%s", FLB_TESTS_DATA_PATH, config_file); + if (access(path, R_OK) == 0) { + real_resolved = realpath(path, NULL); + if (real_resolved) { + resolved = flb_strdup(real_resolved); + free(real_resolved); + return resolved; + } + resolved = flb_strdup(path); + return resolved; + } + + /* Try FLB_TESTS_DATA_PATH (tests/runtime directory) */ snprintf(path, sizeof(path), "%s/%s", FLB_TESTS_DATA_PATH, config_file); if (access(path, R_OK) == 0) { + real_resolved = realpath(path, NULL); + if (real_resolved) { + resolved = flb_strdup(real_resolved); + free(real_resolved); + return resolved; + } resolved = flb_strdup(path); return resolved; } @@ -87,10 +109,22 @@ static char *get_config_path(const char *config_file) /* Try current working directory */ if (getcwd(cwd, sizeof(cwd)) != NULL) { - snprintf(path, sizeof(path), "%s/%s", cwd, config_file); - if (access(path, R_OK) == 0) { - resolved = flb_strdup(path); - return resolved; + cwd_len = strlen(cwd); + config_file_len = strlen(config_file); + if (cwd_len + 1 + config_file_len < sizeof(path)) { + ret = snprintf(path, sizeof(path), "%s/%s", cwd, config_file); + if (ret > 0 && (size_t)ret < sizeof(path)) { + if (access(path, R_OK) == 0) { + real_resolved = realpath(path, NULL); + if (real_resolved) { + resolved = flb_strdup(real_resolved); + free(real_resolved); + return resolved; + } + resolved = flb_strdup(path); + return resolved; + } + } } } @@ -158,6 +192,8 @@ static struct test_ctx *test_ctx_create(const char *config_file) char *config_path; int ret; char cwd[PATH_MAX]; + size_t cwd_len; + size_t test_dir_len; ctx = flb_calloc(1, sizeof(struct test_ctx)); if (!TEST_CHECK(ctx != NULL)) { @@ -169,7 +205,17 @@ static struct test_ctx *test_ctx_create(const char *config_file) /* Create output directory */ if (getcwd(cwd, sizeof(cwd)) != NULL) { - snprintf(ctx->output_dir, sizeof(ctx->output_dir), "%s/%s", cwd, TEST_OUTPUT_DIR); + cwd_len = strlen(cwd); + test_dir_len = strlen(TEST_OUTPUT_DIR); + if (cwd_len + 1 + test_dir_len < sizeof(ctx->output_dir)) { + ret = snprintf(ctx->output_dir, sizeof(ctx->output_dir), "%s/%s", cwd, TEST_OUTPUT_DIR); + if (ret < 0 || (size_t)ret >= sizeof(ctx->output_dir)) { + snprintf(ctx->output_dir, sizeof(ctx->output_dir), "./%s", TEST_OUTPUT_DIR); + } + } + else { + snprintf(ctx->output_dir, sizeof(ctx->output_dir), "./%s", TEST_OUTPUT_DIR); + } } else { snprintf(ctx->output_dir, sizeof(ctx->output_dir), "./%s", TEST_OUTPUT_DIR); @@ -270,10 +316,19 @@ static void cleanup_output_files(struct test_ctx *ctx, struct route_expectation { int i; char filepath[PATH_MAX]; + size_t dir_len; + size_t file_len; + int ret; for (i = 0; i < count; i++) { - snprintf(filepath, sizeof(filepath), "%s/%s", ctx->output_dir, expectations[i].output_file); - unlink(filepath); + dir_len = strlen(ctx->output_dir); + file_len = strlen(expectations[i].output_file); + if (dir_len + 1 + file_len < sizeof(filepath)) { + ret = snprintf(filepath, sizeof(filepath), "%s/%s", ctx->output_dir, expectations[i].output_file); + if (ret > 0 && (size_t)ret < sizeof(filepath)) { + unlink(filepath); + } + } } } @@ -284,16 +339,36 @@ static int verify_expectations(struct route_expectation *expectations, int count int all_passed = 1; char filepath[PATH_MAX]; int actual_count; + struct route_expectation *exp; + size_t dir_len; + size_t file_len; + int ret; for (i = 0; i < count; i++) { - struct route_expectation *exp = &expectations[i]; - - snprintf(filepath, sizeof(filepath), "%s/%s", ctx->output_dir, exp->output_file); - actual_count = count_records_in_file(filepath); + exp = &expectations[i]; + + dir_len = strlen(ctx->output_dir); + file_len = strlen(exp->output_file); + if (dir_len + 1 + file_len < sizeof(filepath)) { + ret = snprintf(filepath, sizeof(filepath), "%s/%s", ctx->output_dir, exp->output_file); + if (ret > 0 && (size_t)ret < sizeof(filepath)) { + actual_count = count_records_in_file(filepath); + } + else { + flb_error("[test] Route '%s': output file path too long", exp->route_name); + actual_count = -1; + } + } + else { + flb_error("[test] Route '%s': output file path too long", exp->route_name); + actual_count = -1; + } if (actual_count < 0) { - flb_error("[test] Route '%s': failed to read output file: %s", - exp->route_name, filepath); + if (dir_len + 1 + file_len < sizeof(filepath)) { + flb_error("[test] Route '%s': failed to read output file: %s", + exp->route_name, filepath); + } all_passed = 0; } else if (actual_count != exp->expected_count) { From b5fb24457bc24dc3f4507ca853e4b7aa9598bb40 Mon Sep 17 00:00:00 2001 From: Patrick Stephens Date: Sun, 3 May 2026 12:34:51 +0100 Subject: [PATCH 057/213] [upstream] lib: use flb_compat.h instead of unistd.h (Windows Upstream-Ref: https://github.com/fluent/fluent-bit/commit/1256bf812d69f61c7f6e9edf5e8702f893466adb Cherry-picked from Fluent Bit v4.2.4 --- source/src/flb_lib.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/src/flb_lib.c b/source/src/flb_lib.c index f679be04..29bc2006 100644 --- a/source/src/flb_lib.c +++ b/source/src/flb_lib.c @@ -20,6 +20,7 @@ #include #include +#include #include #include #include @@ -39,7 +40,6 @@ #include #include #include -#include #include #include From 6ddce12b72887c2fa9c619d94dfa9e77f5daa812 Mon Sep 17 00:00:00 2001 From: Patrick Stephens Date: Sun, 3 May 2026 12:34:51 +0100 Subject: [PATCH 058/213] [upstream] tests: internal: conditionals: NULL record should Upstream-Ref: https://github.com/fluent/fluent-bit/commit/311d93079e3e5e96f67f5d8486f8c141b5580fda Cherry-picked from Fluent Bit v4.2.4 --- source/tests/internal/conditionals.c | 31 ++++++++++++++++++++++++---- 1 file changed, 27 insertions(+), 4 deletions(-) diff --git a/source/tests/internal/conditionals.c b/source/tests/internal/conditionals.c index 81df2059..f6b78cd3 100644 --- a/source/tests/internal/conditionals.c +++ b/source/tests/internal/conditionals.c @@ -364,7 +364,7 @@ void test_condition_equals() cond = flb_condition_create(FLB_COND_OP_AND); TEST_CHECK(cond != NULL); - TEST_CHECK(flb_condition_add_rule(cond, "$level", FLB_RULE_OP_EQ, + TEST_CHECK(flb_condition_add_rule(cond, "$level", FLB_RULE_OP_EQ, "error", 0, RECORD_CONTEXT_BODY) == FLB_TRUE); result = flb_condition_evaluate(cond, &record_data->chunk); @@ -445,7 +445,7 @@ void test_condition_not_equals() cond = flb_condition_create(FLB_COND_OP_AND); TEST_CHECK(cond != NULL); - TEST_CHECK(flb_condition_add_rule(cond, "$level", FLB_RULE_OP_NEQ, + TEST_CHECK(flb_condition_add_rule(cond, "$level", FLB_RULE_OP_NEQ, "error", 0, RECORD_CONTEXT_BODY) == FLB_TRUE); result = flb_condition_evaluate(cond, &record_data->chunk); @@ -734,7 +734,7 @@ void test_condition_invalid_expressions() result = flb_condition_evaluate(NULL, &record_data->chunk); TEST_CHECK(result == FLB_TRUE); - /* Test NULL record */ + /* Test NULL record with AND condition */ cond = flb_condition_create(FLB_COND_OP_AND); TEST_CHECK(cond != NULL); @@ -742,7 +742,30 @@ void test_condition_invalid_expressions() "error", 0, RECORD_CONTEXT_BODY) == FLB_TRUE); result = flb_condition_evaluate(cond, NULL); - TEST_CHECK(result == FLB_TRUE); + TEST_CHECK(result == FLB_FALSE); /* NULL record should fail condition */ + + flb_condition_destroy(cond); + + /* Test NULL record with OR condition */ + cond = flb_condition_create(FLB_COND_OP_OR); + TEST_CHECK(cond != NULL); + + TEST_CHECK(flb_condition_add_rule(cond, "$level", FLB_RULE_OP_EQ, + "error", 0, RECORD_CONTEXT_BODY) == FLB_TRUE); + TEST_CHECK(flb_condition_add_rule(cond, "$level", FLB_RULE_OP_EQ, + "warn", 0, RECORD_CONTEXT_BODY) == FLB_TRUE); + + result = flb_condition_evaluate(cond, NULL); + TEST_CHECK(result == FLB_FALSE); /* NULL record should fail condition */ + + flb_condition_destroy(cond); + + /* Test NULL record with empty condition */ + cond = flb_condition_create(FLB_COND_OP_AND); + TEST_CHECK(cond != NULL); + + result = flb_condition_evaluate(cond, NULL); + TEST_CHECK(result == FLB_FALSE); /* NULL record should fail even with empty condition */ flb_condition_destroy(cond); From ab7098b92cefc1c6d878d1ea26bf660b25735cc0 Mon Sep 17 00:00:00 2001 From: Patrick Stephens Date: Sun, 3 May 2026 12:34:51 +0100 Subject: [PATCH 059/213] [upstream] log_event_decoder: add tests for group marker Upstream-Ref: https://github.com/fluent/fluent-bit/commit/7b87bbb8320ad6d555d3777f3c89f6bd4c7205b9 Cherry-picked from Fluent Bit v4.2.4 --- source/src/flb_log_event_decoder.c | 5 + source/tests/internal/log_event_decoder.c | 438 ++++++++++++++++++++++ 2 files changed, 443 insertions(+) diff --git a/source/src/flb_log_event_decoder.c b/source/src/flb_log_event_decoder.c index 52a39e14..977e1eb8 100644 --- a/source/src/flb_log_event_decoder.c +++ b/source/src/flb_log_event_decoder.c @@ -407,6 +407,11 @@ int flb_log_event_decoder_next(struct flb_log_event_decoder *context, } if (context->read_groups != FLB_TRUE) { + /* + * Skip group markers by recursively calling to get next record. + * msgpack_unpack_next will properly destroy and reinitialize + * unpacked_event, so no explicit cleanup needed here. + */ memset(event, 0, sizeof(struct flb_log_event)); return flb_log_event_decoder_next(context, event); } diff --git a/source/tests/internal/log_event_decoder.c b/source/tests/internal/log_event_decoder.c index 8661e4fb..94c8895d 100644 --- a/source/tests/internal/log_event_decoder.c +++ b/source/tests/internal/log_event_decoder.c @@ -22,8 +22,10 @@ #include #include #include +#include #include #include +#include #include "flb_tests_internal.h" @@ -270,6 +272,440 @@ void decoder_next() msgpack_sbuffer_destroy(&sbuf); } +static void pack_group_marker(msgpack_packer *pck, int32_t marker_type) +{ + struct flb_time tm; + + /* Set negative timestamp to indicate group marker */ + flb_time_set(&tm, marker_type, 0); + + msgpack_pack_array(pck, 2); /* Root array: [header, body] */ + msgpack_pack_array(pck, 2); /* Header array: [timestamp, metadata] */ + pack_event_time(pck, &tm); /* Group marker timestamp */ + msgpack_pack_map(pck, 1); /* Metadata: group info */ + msgpack_pack_str(pck, 5); + msgpack_pack_str_body(pck, "group", 5); + msgpack_pack_str(pck, 6); + msgpack_pack_str_body(pck, "marker", 6); + msgpack_pack_map(pck, 1); /* Body: group attributes */ + msgpack_pack_str(pck, 3); + msgpack_pack_str_body(pck, "tag", 3); + msgpack_pack_str(pck, 4); + msgpack_pack_str_body(pck, "test", 4); +} + +void decoder_skip_groups() +{ + struct flb_log_event_decoder dec; + struct flb_log_event event; + int ret; + struct flb_time tm1, tm2, tm3; + msgpack_sbuffer sbuf; + msgpack_packer pck; + char *json = NULL; + int record_count = 0; + int32_t decoded_record_type; + + /* Create timestamps for normal log records */ + flb_time_set(&tm1, 1000, 100); + flb_time_set(&tm2, 2000, 200); + flb_time_set(&tm3, 3000, 300); + + msgpack_sbuffer_init(&sbuf); + msgpack_packer_init(&pck, &sbuf, msgpack_sbuffer_write); + + /* Pack: GROUP_START, normal log1, normal log2, GROUP_END, normal log3 */ + + /* GROUP_START marker */ + pack_group_marker(&pck, FLB_LOG_EVENT_GROUP_START); + + /* Normal log 1 */ + msgpack_pack_array(&pck, 2); + msgpack_pack_array(&pck, 2); + pack_event_time(&pck, &tm1); + msgpack_pack_map(&pck, 0); + msgpack_pack_map(&pck, 1); + msgpack_pack_str(&pck, 3); + msgpack_pack_str_body(&pck, "log", 3); + msgpack_pack_str(&pck, 1); + msgpack_pack_str_body(&pck, "1", 1); + + /* Normal log 2 */ + msgpack_pack_array(&pck, 2); + msgpack_pack_array(&pck, 2); + pack_event_time(&pck, &tm2); + msgpack_pack_map(&pck, 0); + msgpack_pack_map(&pck, 1); + msgpack_pack_str(&pck, 3); + msgpack_pack_str_body(&pck, "log", 3); + msgpack_pack_str(&pck, 1); + msgpack_pack_str_body(&pck, "2", 1); + + /* GROUP_END marker */ + pack_group_marker(&pck, FLB_LOG_EVENT_GROUP_END); + + /* Normal log 3 */ + msgpack_pack_array(&pck, 2); + msgpack_pack_array(&pck, 2); + pack_event_time(&pck, &tm3); + msgpack_pack_map(&pck, 0); + msgpack_pack_map(&pck, 1); + msgpack_pack_str(&pck, 3); + msgpack_pack_str_body(&pck, "log", 3); + msgpack_pack_str(&pck, 1); + msgpack_pack_str_body(&pck, "3", 1); + + /* Initialize decoder with read_groups = false */ + ret = flb_log_event_decoder_init(&dec, (char *)sbuf.data, sbuf.size); + if (!TEST_CHECK(ret == FLB_EVENT_DECODER_SUCCESS)) { + TEST_MSG("flb_log_event_decoder_init failed. ret=%s", + flb_log_event_decoder_get_error_description(ret)); + msgpack_sbuffer_destroy(&sbuf); + return; + } + + ret = flb_log_event_decoder_read_groups(&dec, FLB_FALSE); + if (!TEST_CHECK(ret == 0)) { + TEST_MSG("flb_log_event_decoder_read_groups failed"); + flb_log_event_decoder_destroy(&dec); + msgpack_sbuffer_destroy(&sbuf); + return; + } + + /* Decode records and verify group markers are skipped */ + while ((ret = flb_log_event_decoder_next(&dec, &event)) == FLB_EVENT_DECODER_SUCCESS) { + /* Verify we never get a zeroed event (both sec and nsec should not be 0) */ + if (!TEST_CHECK(!(event.timestamp.tm.tv_sec == 0 && event.timestamp.tm.tv_nsec == 0))) { + TEST_MSG("Received zeroed event - group marker was not skipped properly"); + flb_log_event_decoder_destroy(&dec); + msgpack_sbuffer_destroy(&sbuf); + return; + } + + /* Get record type */ + ret = flb_log_event_decoder_get_record_type(&event, &decoded_record_type); + if (!TEST_CHECK(ret == 0)) { + TEST_MSG("flb_log_event_decoder_get_record_type failed"); + flb_log_event_decoder_destroy(&dec); + msgpack_sbuffer_destroy(&sbuf); + return; + } + + /* Verify we never receive group markers when read_groups is false */ + if (!TEST_CHECK(decoded_record_type == FLB_LOG_EVENT_NORMAL)) { + TEST_MSG("Received group marker (type=%d) when read_groups=false", + decoded_record_type); + flb_log_event_decoder_destroy(&dec); + msgpack_sbuffer_destroy(&sbuf); + return; + } + + record_count++; + + /* Verify expected timestamps are returned in order */ + if (record_count == 1) { + if (!TEST_CHECK(flb_time_equal(&tm1, &event.timestamp))) { + TEST_MSG("First record timestamp mismatch"); + flb_log_event_decoder_destroy(&dec); + msgpack_sbuffer_destroy(&sbuf); + return; + } + } + else if (record_count == 2) { + if (!TEST_CHECK(flb_time_equal(&tm2, &event.timestamp))) { + TEST_MSG("Second record timestamp mismatch"); + flb_log_event_decoder_destroy(&dec); + msgpack_sbuffer_destroy(&sbuf); + return; + } + } + else if (record_count == 3) { + if (!TEST_CHECK(flb_time_equal(&tm3, &event.timestamp))) { + TEST_MSG("Third record timestamp mismatch"); + flb_log_event_decoder_destroy(&dec); + msgpack_sbuffer_destroy(&sbuf); + return; + } + } + + /* Verify body is valid */ + json = flb_msgpack_to_json_str(4096, event.body, FLB_TRUE); + if (TEST_CHECK(json != NULL)) { + char expected_log[16]; + snprintf(expected_log, sizeof(expected_log), "\"log\":\"%d\"", record_count); + if (!TEST_CHECK(strstr(json, expected_log) != NULL)) { + TEST_MSG("Expected %s in body, got json=%s", expected_log, json); + } + flb_free(json); + json = NULL; + } + } + + /* Verify we got exactly 3 normal records (group markers should be skipped) */ + if (!TEST_CHECK(record_count == 3)) { + TEST_MSG("Expected 3 normal records, got %d. Group markers were not skipped properly.", + record_count); + } + + /* Verify we reached end of data, not an error */ + if (!TEST_CHECK(ret == FLB_EVENT_DECODER_ERROR_INSUFFICIENT_DATA || + ret == FLB_EVENT_DECODER_SUCCESS)) { + TEST_MSG("Unexpected decoder result: %s", + flb_log_event_decoder_get_error_description(ret)); + } + + flb_log_event_decoder_destroy(&dec); + msgpack_sbuffer_destroy(&sbuf); +} + +void decoder_skip_groups_corrupted() +{ + struct flb_log_event_decoder dec; + struct flb_log_event event; + int ret; + struct flb_time tm1, tm2; + msgpack_sbuffer sbuf; + msgpack_packer pck; + int record_count = 0; + int32_t decoded_record_type; + + flb_time_set(&tm1, 1000, 100); + flb_time_set(&tm2, 2000, 200); + + /* Test Case 1: Unmatched GROUP_START (no GROUP_END) */ + msgpack_sbuffer_init(&sbuf); + msgpack_packer_init(&pck, &sbuf, msgpack_sbuffer_write); + + pack_group_marker(&pck, FLB_LOG_EVENT_GROUP_START); + + /* Normal log */ + msgpack_pack_array(&pck, 2); + msgpack_pack_array(&pck, 2); + pack_event_time(&pck, &tm1); + msgpack_pack_map(&pck, 0); + msgpack_pack_map(&pck, 1); + msgpack_pack_str(&pck, 3); + msgpack_pack_str_body(&pck, "log", 3); + msgpack_pack_str(&pck, 1); + msgpack_pack_str_body(&pck, "1", 1); + + /* Another GROUP_START without END */ + pack_group_marker(&pck, FLB_LOG_EVENT_GROUP_START); + + /* Another normal log */ + msgpack_pack_array(&pck, 2); + msgpack_pack_array(&pck, 2); + pack_event_time(&pck, &tm2); + msgpack_pack_map(&pck, 0); + msgpack_pack_map(&pck, 1); + msgpack_pack_str(&pck, 3); + msgpack_pack_str_body(&pck, "log", 3); + msgpack_pack_str(&pck, 1); + msgpack_pack_str_body(&pck, "2", 1); + + ret = flb_log_event_decoder_init(&dec, (char *)sbuf.data, sbuf.size); + TEST_CHECK(ret == FLB_EVENT_DECODER_SUCCESS); + + ret = flb_log_event_decoder_read_groups(&dec, FLB_FALSE); + TEST_CHECK(ret == 0); + + record_count = 0; + while ((ret = flb_log_event_decoder_next(&dec, &event)) == FLB_EVENT_DECODER_SUCCESS) { + ret = flb_log_event_decoder_get_record_type(&event, &decoded_record_type); + TEST_CHECK(ret == 0); + TEST_CHECK(decoded_record_type == FLB_LOG_EVENT_NORMAL); + record_count++; + } + + /* Should get 2 normal records, skipping unmatched GROUP_START markers */ + TEST_CHECK(record_count == 2); + + flb_log_event_decoder_destroy(&dec); + msgpack_sbuffer_destroy(&sbuf); + + /* Test Case 2: Unmatched GROUP_END (no GROUP_START) */ + msgpack_sbuffer_init(&sbuf); + msgpack_packer_init(&pck, &sbuf, msgpack_sbuffer_write); + + pack_group_marker(&pck, FLB_LOG_EVENT_GROUP_END); + + /* Normal log */ + msgpack_pack_array(&pck, 2); + msgpack_pack_array(&pck, 2); + pack_event_time(&pck, &tm1); + msgpack_pack_map(&pck, 0); + msgpack_pack_map(&pck, 1); + msgpack_pack_str(&pck, 3); + msgpack_pack_str_body(&pck, "log", 3); + msgpack_pack_str(&pck, 1); + msgpack_pack_str_body(&pck, "1", 1); + + ret = flb_log_event_decoder_init(&dec, (char *)sbuf.data, sbuf.size); + TEST_CHECK(ret == FLB_EVENT_DECODER_SUCCESS); + + ret = flb_log_event_decoder_read_groups(&dec, FLB_FALSE); + TEST_CHECK(ret == 0); + + record_count = 0; + while ((ret = flb_log_event_decoder_next(&dec, &event)) == FLB_EVENT_DECODER_SUCCESS) { + ret = flb_log_event_decoder_get_record_type(&event, &decoded_record_type); + TEST_CHECK(ret == 0); + TEST_CHECK(decoded_record_type == FLB_LOG_EVENT_NORMAL); + record_count++; + } + + /* Should get 1 normal record, skipping unmatched GROUP_END */ + TEST_CHECK(record_count == 1); + + flb_log_event_decoder_destroy(&dec); + msgpack_sbuffer_destroy(&sbuf); + + /* Test Case 3: Multiple consecutive GROUP_START */ + msgpack_sbuffer_init(&sbuf); + msgpack_packer_init(&pck, &sbuf, msgpack_sbuffer_write); + + pack_group_marker(&pck, FLB_LOG_EVENT_GROUP_START); + pack_group_marker(&pck, FLB_LOG_EVENT_GROUP_START); + pack_group_marker(&pck, FLB_LOG_EVENT_GROUP_START); + + /* Normal log */ + msgpack_pack_array(&pck, 2); + msgpack_pack_array(&pck, 2); + pack_event_time(&pck, &tm1); + msgpack_pack_map(&pck, 0); + msgpack_pack_map(&pck, 1); + msgpack_pack_str(&pck, 3); + msgpack_pack_str_body(&pck, "log", 3); + msgpack_pack_str(&pck, 1); + msgpack_pack_str_body(&pck, "1", 1); + + ret = flb_log_event_decoder_init(&dec, (char *)sbuf.data, sbuf.size); + TEST_CHECK(ret == FLB_EVENT_DECODER_SUCCESS); + + ret = flb_log_event_decoder_read_groups(&dec, FLB_FALSE); + TEST_CHECK(ret == 0); + + record_count = 0; + while ((ret = flb_log_event_decoder_next(&dec, &event)) == FLB_EVENT_DECODER_SUCCESS) { + ret = flb_log_event_decoder_get_record_type(&event, &decoded_record_type); + TEST_CHECK(ret == 0); + TEST_CHECK(decoded_record_type == FLB_LOG_EVENT_NORMAL); + record_count++; + } + + /* Should get 1 normal record, skipping all GROUP_START markers */ + TEST_CHECK(record_count == 1); + + flb_log_event_decoder_destroy(&dec); + msgpack_sbuffer_destroy(&sbuf); + + /* Test Case 4: Multiple consecutive GROUP_END */ + msgpack_sbuffer_init(&sbuf); + msgpack_packer_init(&pck, &sbuf, msgpack_sbuffer_write); + + /* Normal log */ + msgpack_pack_array(&pck, 2); + msgpack_pack_array(&pck, 2); + pack_event_time(&pck, &tm1); + msgpack_pack_map(&pck, 0); + msgpack_pack_map(&pck, 1); + msgpack_pack_str(&pck, 3); + msgpack_pack_str_body(&pck, "log", 3); + msgpack_pack_str(&pck, 1); + msgpack_pack_str_body(&pck, "1", 1); + + pack_group_marker(&pck, FLB_LOG_EVENT_GROUP_END); + pack_group_marker(&pck, FLB_LOG_EVENT_GROUP_END); + pack_group_marker(&pck, FLB_LOG_EVENT_GROUP_END); + + ret = flb_log_event_decoder_init(&dec, (char *)sbuf.data, sbuf.size); + TEST_CHECK(ret == FLB_EVENT_DECODER_SUCCESS); + + ret = flb_log_event_decoder_read_groups(&dec, FLB_FALSE); + TEST_CHECK(ret == 0); + + record_count = 0; + while ((ret = flb_log_event_decoder_next(&dec, &event)) == FLB_EVENT_DECODER_SUCCESS) { + ret = flb_log_event_decoder_get_record_type(&event, &decoded_record_type); + TEST_CHECK(ret == 0); + TEST_CHECK(decoded_record_type == FLB_LOG_EVENT_NORMAL); + record_count++; + } + + /* Should get 1 normal record, skipping all GROUP_END markers */ + TEST_CHECK(record_count == 1); + + flb_log_event_decoder_destroy(&dec); + msgpack_sbuffer_destroy(&sbuf); + + /* Test Case 5: Mixed invalid states - GROUP_END, GROUP_START, GROUP_END, normal log */ + msgpack_sbuffer_init(&sbuf); + msgpack_packer_init(&pck, &sbuf, msgpack_sbuffer_write); + + pack_group_marker(&pck, FLB_LOG_EVENT_GROUP_END); + pack_group_marker(&pck, FLB_LOG_EVENT_GROUP_START); + pack_group_marker(&pck, FLB_LOG_EVENT_GROUP_END); + + /* Normal log */ + msgpack_pack_array(&pck, 2); + msgpack_pack_array(&pck, 2); + pack_event_time(&pck, &tm1); + msgpack_pack_map(&pck, 0); + msgpack_pack_map(&pck, 1); + msgpack_pack_str(&pck, 3); + msgpack_pack_str_body(&pck, "log", 3); + msgpack_pack_str(&pck, 1); + msgpack_pack_str_body(&pck, "1", 1); + + ret = flb_log_event_decoder_init(&dec, (char *)sbuf.data, sbuf.size); + TEST_CHECK(ret == FLB_EVENT_DECODER_SUCCESS); + + ret = flb_log_event_decoder_read_groups(&dec, FLB_FALSE); + TEST_CHECK(ret == 0); + + record_count = 0; + while ((ret = flb_log_event_decoder_next(&dec, &event)) == FLB_EVENT_DECODER_SUCCESS) { + /* Verify we never get a zeroed event */ + TEST_CHECK(!(event.timestamp.tm.tv_sec == 0 && event.timestamp.tm.tv_nsec == 0)); + + ret = flb_log_event_decoder_get_record_type(&event, &decoded_record_type); + TEST_CHECK(ret == 0); + TEST_CHECK(decoded_record_type == FLB_LOG_EVENT_NORMAL); + record_count++; + } + + /* Should get 1 normal record, skipping all invalid group markers */ + TEST_CHECK(record_count == 1); + + flb_log_event_decoder_destroy(&dec); + msgpack_sbuffer_destroy(&sbuf); + + /* Test Case 6: Only group markers, no normal logs */ + msgpack_sbuffer_init(&sbuf); + msgpack_packer_init(&pck, &sbuf, msgpack_sbuffer_write); + + pack_group_marker(&pck, FLB_LOG_EVENT_GROUP_START); + pack_group_marker(&pck, FLB_LOG_EVENT_GROUP_END); + pack_group_marker(&pck, FLB_LOG_EVENT_GROUP_START); + pack_group_marker(&pck, FLB_LOG_EVENT_GROUP_END); + + ret = flb_log_event_decoder_init(&dec, (char *)sbuf.data, sbuf.size); + TEST_CHECK(ret == FLB_EVENT_DECODER_SUCCESS); + + ret = flb_log_event_decoder_read_groups(&dec, FLB_FALSE); + TEST_CHECK(ret == 0); + + record_count = 0; + ret = flb_log_event_decoder_next(&dec, &event); + + /* Should get INSUFFICIENT_DATA since all records are group markers */ + TEST_CHECK(ret == FLB_EVENT_DECODER_ERROR_INSUFFICIENT_DATA); + TEST_CHECK(record_count == 0); + + flb_log_event_decoder_destroy(&dec); + msgpack_sbuffer_destroy(&sbuf); +} + TEST_LIST = { @@ -278,5 +714,7 @@ TEST_LIST = { { "decode_timestamp", decode_timestamp }, { "decode_object", decode_object }, { "decoder_next", decoder_next }, + { "decoder_skip_groups", decoder_skip_groups }, + { "decoder_skip_groups_corrupted", decoder_skip_groups_corrupted }, { 0 } }; From 0ceea91308c5fbee9f930438015d32ce7c6d1949 Mon Sep 17 00:00:00 2001 From: Patrick Stephens Date: Sun, 3 May 2026 12:34:52 +0100 Subject: [PATCH 060/213] [upstream] tests: internal: log_event_decoder: add comprehensive Upstream-Ref: https://github.com/fluent/fluent-bit/commit/0e6513f77f1176913fceb8c4b59092f7efde1dac Cherry-picked from Fluent Bit v4.2.4 --- source/tests/internal/log_event_decoder.c | 719 ++++++++++++++++++++++ 1 file changed, 719 insertions(+) diff --git a/source/tests/internal/log_event_decoder.c b/source/tests/internal/log_event_decoder.c index 94c8895d..531fce41 100644 --- a/source/tests/internal/log_event_decoder.c +++ b/source/tests/internal/log_event_decoder.c @@ -706,6 +706,720 @@ void decoder_skip_groups_corrupted() msgpack_sbuffer_destroy(&sbuf); } +void decoder_read_groups() +{ + struct flb_log_event_decoder dec; + struct flb_log_event event; + int ret; + struct flb_time tm1, tm2; + msgpack_sbuffer sbuf; + msgpack_packer pck; + int record_count = 0; + int32_t decoded_record_type; + int group_start_count = 0; + int group_end_count = 0; + int normal_count = 0; + + flb_time_set(&tm1, 1000, 100); + flb_time_set(&tm2, 2000, 200); + + msgpack_sbuffer_init(&sbuf); + msgpack_packer_init(&pck, &sbuf, msgpack_sbuffer_write); + + /* Pack: GROUP_START, normal log1, normal log2, GROUP_END, normal log3 */ + + /* GROUP_START marker */ + pack_group_marker(&pck, FLB_LOG_EVENT_GROUP_START); + + /* Normal log 1 */ + msgpack_pack_array(&pck, 2); + msgpack_pack_array(&pck, 2); + pack_event_time(&pck, &tm1); + msgpack_pack_map(&pck, 0); + msgpack_pack_map(&pck, 1); + msgpack_pack_str(&pck, 3); + msgpack_pack_str_body(&pck, "log", 3); + msgpack_pack_str(&pck, 1); + msgpack_pack_str_body(&pck, "1", 1); + + /* Normal log 2 */ + msgpack_pack_array(&pck, 2); + msgpack_pack_array(&pck, 2); + pack_event_time(&pck, &tm2); + msgpack_pack_map(&pck, 0); + msgpack_pack_map(&pck, 1); + msgpack_pack_str(&pck, 3); + msgpack_pack_str_body(&pck, "log", 3); + msgpack_pack_str(&pck, 1); + msgpack_pack_str_body(&pck, "2", 1); + + /* GROUP_END marker */ + pack_group_marker(&pck, FLB_LOG_EVENT_GROUP_END); + + /* Initialize decoder with read_groups = true */ + ret = flb_log_event_decoder_init(&dec, (char *)sbuf.data, sbuf.size); + TEST_CHECK(ret == FLB_EVENT_DECODER_SUCCESS); + + ret = flb_log_event_decoder_read_groups(&dec, FLB_TRUE); + TEST_CHECK(ret == 0); + + /* Decode records and verify group markers ARE returned */ + while ((ret = flb_log_event_decoder_next(&dec, &event)) == FLB_EVENT_DECODER_SUCCESS) { + record_count++; + + ret = flb_log_event_decoder_get_record_type(&event, &decoded_record_type); + TEST_CHECK(ret == 0); + + if (decoded_record_type == FLB_LOG_EVENT_GROUP_START) { + group_start_count++; + /* Verify GROUP_START has negative timestamp */ + TEST_CHECK(event.timestamp.tm.tv_sec == FLB_LOG_EVENT_GROUP_START); + } + else if (decoded_record_type == FLB_LOG_EVENT_GROUP_END) { + group_end_count++; + /* Verify GROUP_END has negative timestamp */ + TEST_CHECK(event.timestamp.tm.tv_sec == FLB_LOG_EVENT_GROUP_END); + } + else if (decoded_record_type == FLB_LOG_EVENT_NORMAL) { + normal_count++; + /* Normal logs should have group metadata/attributes from active group */ + if (record_count > 1 && record_count < 4) { + /* Logs 1 and 2 should have group metadata from GROUP_START */ + TEST_CHECK(event.group_metadata != NULL || event.group_attributes != NULL); + } + } + } + + /* When read_groups=true, we should get: + * 1 GROUP_START + 2 normal logs + 1 GROUP_END = 4 records total + */ + TEST_CHECK(record_count == 4); + TEST_CHECK(group_start_count == 1); + TEST_CHECK(group_end_count == 1); + TEST_CHECK(normal_count == 2); + + flb_log_event_decoder_destroy(&dec); + msgpack_sbuffer_destroy(&sbuf); +} + +void decoder_read_groups_corrupted() +{ + struct flb_log_event_decoder dec; + struct flb_log_event event; + int ret; + struct flb_time tm1; + msgpack_sbuffer sbuf; + msgpack_packer pck; + int record_count = 0; + int32_t decoded_record_type; + int group_start_count = 0; + int group_end_count = 0; + int normal_count = 0; + + flb_time_set(&tm1, 1000, 100); + + /* Test Case 1: Unmatched GROUP_START - should still return it */ + msgpack_sbuffer_init(&sbuf); + msgpack_packer_init(&pck, &sbuf, msgpack_sbuffer_write); + + pack_group_marker(&pck, FLB_LOG_EVENT_GROUP_START); + + /* Normal log */ + msgpack_pack_array(&pck, 2); + msgpack_pack_array(&pck, 2); + pack_event_time(&pck, &tm1); + msgpack_pack_map(&pck, 0); + msgpack_pack_map(&pck, 1); + msgpack_pack_str(&pck, 3); + msgpack_pack_str_body(&pck, "log", 3); + msgpack_pack_str(&pck, 1); + msgpack_pack_str_body(&pck, "1", 1); + + ret = flb_log_event_decoder_init(&dec, (char *)sbuf.data, sbuf.size); + TEST_CHECK(ret == FLB_EVENT_DECODER_SUCCESS); + + ret = flb_log_event_decoder_read_groups(&dec, FLB_TRUE); + TEST_CHECK(ret == 0); + + record_count = 0; + group_start_count = 0; + normal_count = 0; + + while ((ret = flb_log_event_decoder_next(&dec, &event)) == FLB_EVENT_DECODER_SUCCESS) { + record_count++; + ret = flb_log_event_decoder_get_record_type(&event, &decoded_record_type); + TEST_CHECK(ret == 0); + + if (decoded_record_type == FLB_LOG_EVENT_GROUP_START) { + group_start_count++; + } + else if (decoded_record_type == FLB_LOG_EVENT_NORMAL) { + normal_count++; + } + } + + /* Should get 1 GROUP_START + 1 normal log */ + TEST_CHECK(record_count == 2); + TEST_CHECK(group_start_count == 1); + TEST_CHECK(normal_count == 1); + + flb_log_event_decoder_destroy(&dec); + msgpack_sbuffer_destroy(&sbuf); + + /* Test Case 2: Unmatched GROUP_END - should still return it */ + msgpack_sbuffer_init(&sbuf); + msgpack_packer_init(&pck, &sbuf, msgpack_sbuffer_write); + + pack_group_marker(&pck, FLB_LOG_EVENT_GROUP_END); + + /* Normal log */ + msgpack_pack_array(&pck, 2); + msgpack_pack_array(&pck, 2); + pack_event_time(&pck, &tm1); + msgpack_pack_map(&pck, 0); + msgpack_pack_map(&pck, 1); + msgpack_pack_str(&pck, 3); + msgpack_pack_str_body(&pck, "log", 3); + msgpack_pack_str(&pck, 1); + msgpack_pack_str_body(&pck, "1", 1); + + ret = flb_log_event_decoder_init(&dec, (char *)sbuf.data, sbuf.size); + TEST_CHECK(ret == FLB_EVENT_DECODER_SUCCESS); + + ret = flb_log_event_decoder_read_groups(&dec, FLB_TRUE); + TEST_CHECK(ret == 0); + + record_count = 0; + group_end_count = 0; + normal_count = 0; + + while ((ret = flb_log_event_decoder_next(&dec, &event)) == FLB_EVENT_DECODER_SUCCESS) { + record_count++; + ret = flb_log_event_decoder_get_record_type(&event, &decoded_record_type); + TEST_CHECK(ret == 0); + + if (decoded_record_type == FLB_LOG_EVENT_GROUP_END) { + group_end_count++; + } + else if (decoded_record_type == FLB_LOG_EVENT_NORMAL) { + normal_count++; + } + } + + /* Should get 1 GROUP_END + 1 normal log */ + TEST_CHECK(record_count == 2); + TEST_CHECK(group_end_count == 1); + TEST_CHECK(normal_count == 1); + + flb_log_event_decoder_destroy(&dec); + msgpack_sbuffer_destroy(&sbuf); + + /* Test Case 3: Multiple consecutive GROUP_START - all should be returned */ + msgpack_sbuffer_init(&sbuf); + msgpack_packer_init(&pck, &sbuf, msgpack_sbuffer_write); + + pack_group_marker(&pck, FLB_LOG_EVENT_GROUP_START); + pack_group_marker(&pck, FLB_LOG_EVENT_GROUP_START); + pack_group_marker(&pck, FLB_LOG_EVENT_GROUP_START); + + /* Normal log */ + msgpack_pack_array(&pck, 2); + msgpack_pack_array(&pck, 2); + pack_event_time(&pck, &tm1); + msgpack_pack_map(&pck, 0); + msgpack_pack_map(&pck, 1); + msgpack_pack_str(&pck, 3); + msgpack_pack_str_body(&pck, "log", 3); + msgpack_pack_str(&pck, 1); + msgpack_pack_str_body(&pck, "1", 1); + + ret = flb_log_event_decoder_init(&dec, (char *)sbuf.data, sbuf.size); + TEST_CHECK(ret == FLB_EVENT_DECODER_SUCCESS); + + ret = flb_log_event_decoder_read_groups(&dec, FLB_TRUE); + TEST_CHECK(ret == 0); + + record_count = 0; + group_start_count = 0; + normal_count = 0; + + while ((ret = flb_log_event_decoder_next(&dec, &event)) == FLB_EVENT_DECODER_SUCCESS) { + record_count++; + ret = flb_log_event_decoder_get_record_type(&event, &decoded_record_type); + TEST_CHECK(ret == 0); + + if (decoded_record_type == FLB_LOG_EVENT_GROUP_START) { + group_start_count++; + } + else if (decoded_record_type == FLB_LOG_EVENT_NORMAL) { + normal_count++; + } + } + + /* Should get 3 GROUP_START + 1 normal log */ + TEST_CHECK(record_count == 4); + TEST_CHECK(group_start_count == 3); + TEST_CHECK(normal_count == 1); + + flb_log_event_decoder_destroy(&dec); + msgpack_sbuffer_destroy(&sbuf); + + /* Test Case 4: Mixed invalid states - all markers should be returned */ + msgpack_sbuffer_init(&sbuf); + msgpack_packer_init(&pck, &sbuf, msgpack_sbuffer_write); + + pack_group_marker(&pck, FLB_LOG_EVENT_GROUP_END); + pack_group_marker(&pck, FLB_LOG_EVENT_GROUP_START); + pack_group_marker(&pck, FLB_LOG_EVENT_GROUP_END); + + /* Normal log */ + msgpack_pack_array(&pck, 2); + msgpack_pack_array(&pck, 2); + pack_event_time(&pck, &tm1); + msgpack_pack_map(&pck, 0); + msgpack_pack_map(&pck, 1); + msgpack_pack_str(&pck, 3); + msgpack_pack_str_body(&pck, "log", 3); + msgpack_pack_str(&pck, 1); + msgpack_pack_str_body(&pck, "1", 1); + + ret = flb_log_event_decoder_init(&dec, (char *)sbuf.data, sbuf.size); + TEST_CHECK(ret == FLB_EVENT_DECODER_SUCCESS); + + ret = flb_log_event_decoder_read_groups(&dec, FLB_TRUE); + TEST_CHECK(ret == 0); + + record_count = 0; + group_start_count = 0; + group_end_count = 0; + normal_count = 0; + + while ((ret = flb_log_event_decoder_next(&dec, &event)) == FLB_EVENT_DECODER_SUCCESS) { + record_count++; + ret = flb_log_event_decoder_get_record_type(&event, &decoded_record_type); + TEST_CHECK(ret == 0); + + if (decoded_record_type == FLB_LOG_EVENT_GROUP_START) { + group_start_count++; + } + else if (decoded_record_type == FLB_LOG_EVENT_GROUP_END) { + group_end_count++; + } + else if (decoded_record_type == FLB_LOG_EVENT_NORMAL) { + normal_count++; + } + } + + /* Should get 2 GROUP_END + 1 GROUP_START + 1 normal log */ + TEST_CHECK(record_count == 4); + TEST_CHECK(group_start_count == 1); + TEST_CHECK(group_end_count == 2); + TEST_CHECK(normal_count == 1); + + flb_log_event_decoder_destroy(&dec); + msgpack_sbuffer_destroy(&sbuf); +} + +void decoder_corrupted_group_timestamps() +{ + struct flb_log_event_decoder dec; + struct flb_log_event event; + int ret; + struct flb_time tm1; + struct flb_time corrupted_tm; + msgpack_sbuffer sbuf; + msgpack_packer pck; + msgpack_sbuffer sbuf2; + msgpack_packer pck2; + int32_t decoded_record_type; + + flb_time_set(&tm1, 1000, 100); + + /* Test Case 1: Invalid negative timestamp (not -1 or -2) - should skip */ + msgpack_sbuffer_init(&sbuf); + msgpack_packer_init(&pck, &sbuf, msgpack_sbuffer_write); + + /* Create a record with corrupted group timestamp (-3) */ + flb_time_set(&corrupted_tm, -3, 0); /* Invalid group marker timestamp */ + + msgpack_pack_array(&pck, 2); /* Root array: [header, body] */ + msgpack_pack_array(&pck, 2); /* Header array: [timestamp, metadata] */ + pack_event_time(&pck, &corrupted_tm); /* Invalid group marker timestamp */ + msgpack_pack_map(&pck, 0); /* Empty metadata */ + msgpack_pack_map(&pck, 0); /* Empty body */ + + /* Normal log after corrupted marker */ + msgpack_pack_array(&pck, 2); + msgpack_pack_array(&pck, 2); + pack_event_time(&pck, &tm1); + msgpack_pack_map(&pck, 0); + msgpack_pack_map(&pck, 1); + msgpack_pack_str(&pck, 3); + msgpack_pack_str_body(&pck, "log", 3); + msgpack_pack_str(&pck, 1); + msgpack_pack_str_body(&pck, "1", 1); + + ret = flb_log_event_decoder_init(&dec, (char *)sbuf.data, sbuf.size); + TEST_CHECK(ret == FLB_EVENT_DECODER_SUCCESS); + + ret = flb_log_event_decoder_read_groups(&dec, FLB_FALSE); + TEST_CHECK(ret == 0); + + /* When read_groups=false, corrupted group marker should be skipped */ + ret = flb_log_event_decoder_next(&dec, &event); + TEST_CHECK(ret == FLB_EVENT_DECODER_SUCCESS); + ret = flb_log_event_decoder_get_record_type(&event, &decoded_record_type); + TEST_CHECK(ret == 0); + TEST_CHECK(decoded_record_type == FLB_LOG_EVENT_NORMAL); + TEST_CHECK(flb_time_equal(&tm1, &event.timestamp)); + + flb_log_event_decoder_destroy(&dec); + msgpack_sbuffer_destroy(&sbuf); + + /* Test Case 2: Invalid negative timestamp with read_groups=true - should also skip */ + msgpack_sbuffer_init(&sbuf2); + msgpack_packer_init(&pck2, &sbuf2, msgpack_sbuffer_write); + + flb_time_set(&corrupted_tm, -10, 0); /* Another invalid group marker timestamp */ + + msgpack_pack_array(&pck2, 2); + msgpack_pack_array(&pck2, 2); + pack_event_time(&pck2, &corrupted_tm); + msgpack_pack_map(&pck2, 0); + msgpack_pack_map(&pck2, 0); + + /* Normal log after corrupted marker */ + msgpack_pack_array(&pck2, 2); + msgpack_pack_array(&pck2, 2); + pack_event_time(&pck2, &tm1); + msgpack_pack_map(&pck2, 0); + msgpack_pack_map(&pck2, 1); + msgpack_pack_str(&pck2, 3); + msgpack_pack_str_body(&pck2, "log", 3); + msgpack_pack_str(&pck2, 1); + msgpack_pack_str_body(&pck2, "1", 1); + + ret = flb_log_event_decoder_init(&dec, (char *)sbuf2.data, sbuf2.size); + TEST_CHECK(ret == FLB_EVENT_DECODER_SUCCESS); + + ret = flb_log_event_decoder_read_groups(&dec, FLB_TRUE); + TEST_CHECK(ret == 0); + + /* When read_groups=true, corrupted group marker should also be skipped */ + ret = flb_log_event_decoder_next(&dec, &event); + TEST_CHECK(ret == FLB_EVENT_DECODER_SUCCESS); + ret = flb_log_event_decoder_get_record_type(&event, &decoded_record_type); + TEST_CHECK(ret == 0); + TEST_CHECK(decoded_record_type == FLB_LOG_EVENT_NORMAL); + TEST_CHECK(flb_time_equal(&tm1, &event.timestamp)); + + flb_log_event_decoder_destroy(&dec); + msgpack_sbuffer_destroy(&sbuf2); + + /* Test Case 3: Very negative timestamp - should skip */ + msgpack_sbuffer_init(&sbuf); + msgpack_packer_init(&pck, &sbuf, msgpack_sbuffer_write); + + flb_time_set(&corrupted_tm, -1000, 0); /* Very negative but invalid */ + + msgpack_pack_array(&pck, 2); + msgpack_pack_array(&pck, 2); + pack_event_time(&pck, &corrupted_tm); + msgpack_pack_map(&pck, 0); + msgpack_pack_map(&pck, 0); + + /* Normal log after corrupted marker */ + msgpack_pack_array(&pck, 2); + msgpack_pack_array(&pck, 2); + pack_event_time(&pck, &tm1); + msgpack_pack_map(&pck, 0); + msgpack_pack_map(&pck, 1); + msgpack_pack_str(&pck, 3); + msgpack_pack_str_body(&pck, "log", 3); + msgpack_pack_str(&pck, 1); + msgpack_pack_str_body(&pck, "1", 1); + + ret = flb_log_event_decoder_init(&dec, (char *)sbuf.data, sbuf.size); + TEST_CHECK(ret == FLB_EVENT_DECODER_SUCCESS); + + ret = flb_log_event_decoder_read_groups(&dec, FLB_FALSE); + TEST_CHECK(ret == 0); + + /* Corrupted marker should be skipped, normal log should be returned */ + ret = flb_log_event_decoder_next(&dec, &event); + TEST_CHECK(ret == FLB_EVENT_DECODER_SUCCESS); + ret = flb_log_event_decoder_get_record_type(&event, &decoded_record_type); + TEST_CHECK(ret == 0); + TEST_CHECK(decoded_record_type == FLB_LOG_EVENT_NORMAL); + TEST_CHECK(flb_time_equal(&tm1, &event.timestamp)); + + flb_log_event_decoder_destroy(&dec); + msgpack_sbuffer_destroy(&sbuf); +} + +void decoder_invalid_marker_preserves_group_state() +{ + struct flb_log_event_decoder dec; + struct flb_log_event event; + int ret; + struct flb_time tm1; + struct flb_time tm2; + struct flb_time corrupted_tm; + msgpack_sbuffer sbuf; + msgpack_packer pck; + int32_t decoded_record_type; + int record_count = 0; + + flb_time_set(&tm1, 1000, 100); + flb_time_set(&tm2, 2000, 200); + + /* Test: GROUP_START → normal_log1 → [corrupted -3 marker] → normal_log2 + * Expected: normal_log2 should STILL have group metadata (state preserved) */ + msgpack_sbuffer_init(&sbuf); + msgpack_packer_init(&pck, &sbuf, msgpack_sbuffer_write); + + /* GROUP_START with metadata */ + pack_group_marker(&pck, FLB_LOG_EVENT_GROUP_START); + + /* Normal log 1 - should have group metadata */ + msgpack_pack_array(&pck, 2); + msgpack_pack_array(&pck, 2); + pack_event_time(&pck, &tm1); + msgpack_pack_map(&pck, 0); + msgpack_pack_map(&pck, 1); + msgpack_pack_str(&pck, 3); + msgpack_pack_str_body(&pck, "log", 3); + msgpack_pack_str(&pck, 1); + msgpack_pack_str_body(&pck, "1", 1); + + /* Corrupted marker (-3) - should NOT clear group state */ + flb_time_set(&corrupted_tm, -3, 0); + msgpack_pack_array(&pck, 2); + msgpack_pack_array(&pck, 2); + pack_event_time(&pck, &corrupted_tm); + msgpack_pack_map(&pck, 0); + msgpack_pack_map(&pck, 0); + + /* Normal log 2 - should STILL have group metadata (state preserved) */ + msgpack_pack_array(&pck, 2); + msgpack_pack_array(&pck, 2); + pack_event_time(&pck, &tm2); + msgpack_pack_map(&pck, 0); + msgpack_pack_map(&pck, 1); + msgpack_pack_str(&pck, 3); + msgpack_pack_str_body(&pck, "log", 3); + msgpack_pack_str(&pck, 1); + msgpack_pack_str_body(&pck, "2", 1); + + ret = flb_log_event_decoder_init(&dec, (char *)sbuf.data, sbuf.size); + TEST_CHECK(ret == FLB_EVENT_DECODER_SUCCESS); + + ret = flb_log_event_decoder_read_groups(&dec, FLB_FALSE); + TEST_CHECK(ret == 0); + + /* Read normal log 1 - should have group metadata */ + ret = flb_log_event_decoder_next(&dec, &event); + TEST_CHECK(ret == FLB_EVENT_DECODER_SUCCESS); + ret = flb_log_event_decoder_get_record_type(&event, &decoded_record_type); + TEST_CHECK(ret == 0); + TEST_CHECK(decoded_record_type == FLB_LOG_EVENT_NORMAL); + TEST_CHECK(flb_time_equal(&tm1, &event.timestamp)); + TEST_CHECK(event.group_metadata != NULL || event.group_attributes != NULL); + record_count++; + + /* Read normal log 2 - should STILL have group metadata (state preserved) */ + ret = flb_log_event_decoder_next(&dec, &event); + TEST_CHECK(ret == FLB_EVENT_DECODER_SUCCESS); + ret = flb_log_event_decoder_get_record_type(&event, &decoded_record_type); + TEST_CHECK(ret == 0); + TEST_CHECK(decoded_record_type == FLB_LOG_EVENT_NORMAL); + TEST_CHECK(flb_time_equal(&tm2, &event.timestamp)); + /* CRITICAL: Group state should be preserved despite invalid marker */ + TEST_CHECK(event.group_metadata != NULL || event.group_attributes != NULL); + record_count++; + + TEST_CHECK(record_count == 2); + + flb_log_event_decoder_destroy(&dec); + msgpack_sbuffer_destroy(&sbuf); +} + +void decoder_group_end_start_sequence() +{ + struct flb_log_event_decoder dec; + struct flb_log_event event; + int ret; + struct flb_time tm1; + msgpack_sbuffer sbuf; + msgpack_packer pck; + int record_count = 0; + int32_t decoded_record_type; + int group_start_count = 0; + int group_end_count = 0; + int normal_count = 0; + + flb_time_set(&tm1, 1000, 100); + + /* Test Case: GROUP_END (unmatched) → GROUP_START → normal log */ + msgpack_sbuffer_init(&sbuf); + msgpack_packer_init(&pck, &sbuf, msgpack_sbuffer_write); + + /* GROUP_END without preceding GROUP_START */ + pack_group_marker(&pck, FLB_LOG_EVENT_GROUP_END); + + /* GROUP_START */ + pack_group_marker(&pck, FLB_LOG_EVENT_GROUP_START); + + /* Normal log */ + msgpack_pack_array(&pck, 2); + msgpack_pack_array(&pck, 2); + pack_event_time(&pck, &tm1); + msgpack_pack_map(&pck, 0); + msgpack_pack_map(&pck, 1); + msgpack_pack_str(&pck, 3); + msgpack_pack_str_body(&pck, "log", 3); + msgpack_pack_str(&pck, 1); + msgpack_pack_str_body(&pck, "1", 1); + + /* Test with read_groups = false */ + ret = flb_log_event_decoder_init(&dec, (char *)sbuf.data, sbuf.size); + TEST_CHECK(ret == FLB_EVENT_DECODER_SUCCESS); + + ret = flb_log_event_decoder_read_groups(&dec, FLB_FALSE); + TEST_CHECK(ret == 0); + + record_count = 0; + while ((ret = flb_log_event_decoder_next(&dec, &event)) == FLB_EVENT_DECODER_SUCCESS) { + record_count++; + ret = flb_log_event_decoder_get_record_type(&event, &decoded_record_type); + TEST_CHECK(ret == 0); + TEST_CHECK(decoded_record_type == FLB_LOG_EVENT_NORMAL); + + /* Verify we got the normal log */ + TEST_CHECK(flb_time_equal(&tm1, &event.timestamp)); + } + + /* Should get 1 normal log, skipping both GROUP_END and GROUP_START */ + TEST_CHECK(record_count == 1); + + flb_log_event_decoder_destroy(&dec); + msgpack_sbuffer_destroy(&sbuf); + + /* Test with read_groups = true */ + msgpack_sbuffer_init(&sbuf); + msgpack_packer_init(&pck, &sbuf, msgpack_sbuffer_write); + + pack_group_marker(&pck, FLB_LOG_EVENT_GROUP_END); + pack_group_marker(&pck, FLB_LOG_EVENT_GROUP_START); + + msgpack_pack_array(&pck, 2); + msgpack_pack_array(&pck, 2); + pack_event_time(&pck, &tm1); + msgpack_pack_map(&pck, 0); + msgpack_pack_map(&pck, 1); + msgpack_pack_str(&pck, 3); + msgpack_pack_str_body(&pck, "log", 3); + msgpack_pack_str(&pck, 1); + msgpack_pack_str_body(&pck, "1", 1); + + ret = flb_log_event_decoder_init(&dec, (char *)sbuf.data, sbuf.size); + TEST_CHECK(ret == FLB_EVENT_DECODER_SUCCESS); + + ret = flb_log_event_decoder_read_groups(&dec, FLB_TRUE); + TEST_CHECK(ret == 0); + + record_count = 0; + group_start_count = 0; + group_end_count = 0; + normal_count = 0; + + while ((ret = flb_log_event_decoder_next(&dec, &event)) == FLB_EVENT_DECODER_SUCCESS) { + record_count++; + ret = flb_log_event_decoder_get_record_type(&event, &decoded_record_type); + TEST_CHECK(ret == 0); + + if (decoded_record_type == FLB_LOG_EVENT_GROUP_START) { + group_start_count++; + } + else if (decoded_record_type == FLB_LOG_EVENT_GROUP_END) { + group_end_count++; + } + else if (decoded_record_type == FLB_LOG_EVENT_NORMAL) { + normal_count++; + /* The log should have group metadata from GROUP_START (not GROUP_END) */ + if (record_count == 3) { + /* After GROUP_END (clears state) and GROUP_START (sets state), log should have group data */ + TEST_CHECK(event.group_metadata != NULL || event.group_attributes != NULL); + } + } + } + + /* Should get: GROUP_END, GROUP_START, normal log */ + TEST_CHECK(record_count == 3); + TEST_CHECK(group_start_count == 1); + TEST_CHECK(group_end_count == 1); + TEST_CHECK(normal_count == 1); + + flb_log_event_decoder_destroy(&dec); + msgpack_sbuffer_destroy(&sbuf); + + /* Test Case 2: GROUP_START → GROUP_END → GROUP_START → log */ + msgpack_sbuffer_init(&sbuf); + msgpack_packer_init(&pck, &sbuf, msgpack_sbuffer_write); + + pack_group_marker(&pck, FLB_LOG_EVENT_GROUP_START); + pack_group_marker(&pck, FLB_LOG_EVENT_GROUP_END); + pack_group_marker(&pck, FLB_LOG_EVENT_GROUP_START); + + msgpack_pack_array(&pck, 2); + msgpack_pack_array(&pck, 2); + pack_event_time(&pck, &tm1); + msgpack_pack_map(&pck, 0); + msgpack_pack_map(&pck, 1); + msgpack_pack_str(&pck, 3); + msgpack_pack_str_body(&pck, "log", 3); + msgpack_pack_str(&pck, 1); + msgpack_pack_str_body(&pck, "1", 1); + + ret = flb_log_event_decoder_init(&dec, (char *)sbuf.data, sbuf.size); + TEST_CHECK(ret == FLB_EVENT_DECODER_SUCCESS); + + ret = flb_log_event_decoder_read_groups(&dec, FLB_TRUE); + TEST_CHECK(ret == 0); + + record_count = 0; + group_start_count = 0; + group_end_count = 0; + normal_count = 0; + + while ((ret = flb_log_event_decoder_next(&dec, &event)) == FLB_EVENT_DECODER_SUCCESS) { + record_count++; + ret = flb_log_event_decoder_get_record_type(&event, &decoded_record_type); + TEST_CHECK(ret == 0); + + if (decoded_record_type == FLB_LOG_EVENT_GROUP_START) { + group_start_count++; + } + else if (decoded_record_type == FLB_LOG_EVENT_GROUP_END) { + group_end_count++; + } + else if (decoded_record_type == FLB_LOG_EVENT_NORMAL) { + normal_count++; + /* Log should have metadata from the last GROUP_START */ + TEST_CHECK(event.group_metadata != NULL || event.group_attributes != NULL); + } + } + + /* Should get: GROUP_START, GROUP_END, GROUP_START, normal log */ + TEST_CHECK(record_count == 4); + TEST_CHECK(group_start_count == 2); + TEST_CHECK(group_end_count == 1); + TEST_CHECK(normal_count == 1); + + flb_log_event_decoder_destroy(&dec); + msgpack_sbuffer_destroy(&sbuf); +} + TEST_LIST = { @@ -716,5 +1430,10 @@ TEST_LIST = { { "decoder_next", decoder_next }, { "decoder_skip_groups", decoder_skip_groups }, { "decoder_skip_groups_corrupted", decoder_skip_groups_corrupted }, + { "decoder_read_groups", decoder_read_groups }, + { "decoder_read_groups_corrupted", decoder_read_groups_corrupted }, + { "decoder_corrupted_group_timestamps", decoder_corrupted_group_timestamps }, + { "decoder_invalid_marker_preserves_group_state", decoder_invalid_marker_preserves_group_state }, + { "decoder_group_end_start_sequence", decoder_group_end_start_sequence }, { 0 } }; From 0e7d30a476a815e766eac88fdf583ab85cd6969d Mon Sep 17 00:00:00 2001 From: Patrick Stephens Date: Sun, 3 May 2026 12:34:52 +0100 Subject: [PATCH 061/213] [upstream] log_event_decoder: improve robustness for invalid Upstream-Ref: https://github.com/fluent/fluent-bit/commit/668358c437ed206529fd5ac3351d3b1f6480ba5d Cherry-picked from Fluent Bit v4.2.4 --- source/include/fluent-bit/flb_log_event.h | 11 ++++ .../fluent-bit/flb_log_event_decoder.h | 1 + source/src/flb_log_event_decoder.c | 52 +++++++++++++++---- 3 files changed, 53 insertions(+), 11 deletions(-) diff --git a/source/include/fluent-bit/flb_log_event.h b/source/include/fluent-bit/flb_log_event.h index 2c1d99ac..89374dd9 100644 --- a/source/include/fluent-bit/flb_log_event.h +++ b/source/include/fluent-bit/flb_log_event.h @@ -33,6 +33,17 @@ #define FLB_LOG_EVENT_FORMAT_FLUENT_BIT_V1 FLB_LOG_EVENT_FORMAT_FORWARD #define FLB_LOG_EVENT_FORMAT_FLUENT_BIT_V2 4 +/* + * Log event type identification via timestamp value: + * - Non-negative timestamps (>= 0): Normal log records with actual timestamps + * - -1 (FLB_LOG_EVENT_GROUP_START): Group marker indicating start of a log group + * - -2 (FLB_LOG_EVENT_GROUP_END): Group marker indicating end of a log group + * - Other negative values: Invalid/corrupted data (will be skipped by decoder) + * + * NOTE: Negative timestamps are RESERVED for group markers. Only -1 and -2 are valid. + * Any other negative timestamp is considered invalid and will be skipped during decoding. + * Encoders must respect this contract and only use -1/-2 for group markers. + */ #define FLB_LOG_EVENT_NORMAL (int32_t) 0 #define FLB_LOG_EVENT_GROUP_START (int32_t) -1 #define FLB_LOG_EVENT_GROUP_END (int32_t) -2 diff --git a/source/include/fluent-bit/flb_log_event_decoder.h b/source/include/fluent-bit/flb_log_event_decoder.h index 7fdbaa61..6198b67b 100644 --- a/source/include/fluent-bit/flb_log_event_decoder.h +++ b/source/include/fluent-bit/flb_log_event_decoder.h @@ -62,6 +62,7 @@ struct flb_log_event_decoder { size_t length; int last_result; int read_groups; + unsigned int recursion_depth; /* Safety guard for recursion limit */ }; void flb_log_event_decoder_reset(struct flb_log_event_decoder *context, diff --git a/source/src/flb_log_event_decoder.c b/source/src/flb_log_event_decoder.c index 977e1eb8..f47111a9 100644 --- a/source/src/flb_log_event_decoder.c +++ b/source/src/flb_log_event_decoder.c @@ -20,6 +20,9 @@ #include #include #include +#include + +#define FLB_LOG_EVENT_DECODER_MAX_RECURSION_DEPTH 1000 /* Safety limit for recursion */ static int create_empty_map(struct flb_log_event_decoder *context) { msgpack_packer packer; @@ -74,6 +77,7 @@ void flb_log_event_decoder_reset(struct flb_log_event_decoder *context, context->last_result = FLB_EVENT_DECODER_ERROR_INSUFFICIENT_DATA; context->current_group_metadata = NULL; context->current_group_attributes = NULL; + context->recursion_depth = 0; /* Reset recursion counter */ msgpack_unpacked_destroy(&context->unpacked_group_record); msgpack_unpacked_init(&context->unpacked_group_record); @@ -309,6 +313,7 @@ int flb_log_event_decoder_next(struct flb_log_event_decoder *context, int result; int record_type; size_t previous_offset; + int32_t invalid_timestamp; if (context == NULL) { return FLB_EVENT_DECODER_ERROR_INVALID_CONTEXT; @@ -347,22 +352,43 @@ int flb_log_event_decoder_next(struct flb_log_event_decoder *context, &context->unpacked_event.data); if (context->last_result == FLB_EVENT_DECODER_SUCCESS) { + /* Check recursion depth limit to prevent stack overflow */ + if (context->recursion_depth >= FLB_LOG_EVENT_DECODER_MAX_RECURSION_DEPTH) { + flb_warn("[decoder] Maximum recursion depth (%d) reached, possible corruption or excessive group markers", + FLB_LOG_EVENT_DECODER_MAX_RECURSION_DEPTH); + context->last_result = FLB_EVENT_DECODER_ERROR_DESERIALIZATION_FAILURE; + return context->last_result; + } + /* get log event type */ ret = flb_log_event_decoder_get_record_type(event, &record_type); if (ret != 0) { - context->current_group_metadata = NULL; - context->current_group_attributes = NULL; - - context->last_result = FLB_EVENT_DECODER_ERROR_DESERIALIZATION_FAILURE; - return context->last_result; + /* Invalid group marker (negative timestamp but not -1 or -2). + * Log the invalid marker for debugging, but preserve group state + * to avoid losing valid group metadata if corruption occurs mid-group. + * Skip the record and continue processing. + */ + invalid_timestamp = (int32_t) event->timestamp.tm.tv_sec; + flb_debug("[decoder] Invalid group marker timestamp (%d), skipping record. " + "Group state preserved.", invalid_timestamp); + + /* Increment recursion depth before recursive call */ + context->recursion_depth++; + memset(event, 0, sizeof(struct flb_log_event)); + ret = flb_log_event_decoder_next(context, event); + context->recursion_depth--; /* Restore after return */ + return ret; } /* Meta records such as the group opener and closer are identified by negative - * timestamp values. In these cases we track the current group metadata and - * attributes in order to transparently provide them through the log_event - * structure but we also want to allow the client code raw access to such - * records which is why the read_groups decoder context property is used - * to determine the behavior. + * timestamp values (-1 for GROUP_START, -2 for GROUP_END). Only these two + * negative values are valid; any other negative timestamp is considered + * invalid and is skipped (see handling above). + * + * We track the current group metadata and attributes in order to transparently + * provide them through the log_event structure, but we also want to allow the + * client code raw access to such records, which is why the read_groups decoder + * context property is used to determine the behavior. */ if (record_type != FLB_LOG_EVENT_NORMAL) { msgpack_unpacked_destroy(&context->unpacked_group_record); @@ -411,9 +437,13 @@ int flb_log_event_decoder_next(struct flb_log_event_decoder *context, * Skip group markers by recursively calling to get next record. * msgpack_unpack_next will properly destroy and reinitialize * unpacked_event, so no explicit cleanup needed here. + * Increment recursion depth before recursive call. */ + context->recursion_depth++; memset(event, 0, sizeof(struct flb_log_event)); - return flb_log_event_decoder_next(context, event); + ret = flb_log_event_decoder_next(context, event); + context->recursion_depth--; /* Restore after return */ + return ret; } } else { From ea4a24a8fb33d5d3a6f9cedcf1f29f28921bf3ca Mon Sep 17 00:00:00 2001 From: Patrick Stephens Date: Sun, 3 May 2026 12:34:52 +0100 Subject: [PATCH 062/213] [upstream] lib: chunkio: upgrade to v1.5.4 Upstream-Ref: https://github.com/fluent/fluent-bit/commit/f98fede4006b373855c07c49c0e0dbdbe6757434 Cherry-picked from Fluent Bit v4.2.4 --- source/lib/chunkio/.github/workflows/ci.yaml | 13 +- source/lib/chunkio/CMakeLists.txt | 2 +- source/lib/chunkio/include/chunkio/cio_file.h | 3 + source/lib/chunkio/src/cio_file.c | 61 ++- source/lib/chunkio/src/cio_file_unix.c | 1 + source/lib/chunkio/src/cio_file_win32.c | 93 +++- source/lib/chunkio/tests/CMakeLists.txt | 11 +- source/lib/chunkio/tests/fs.c | 138 ++++++ source/lib/chunkio/tests/fs_windows.c | 443 ++++++++++++++++++ source/lib/chunkio/tests/metadata_update.c | 288 ++++++++++++ 10 files changed, 1026 insertions(+), 27 deletions(-) create mode 100644 source/lib/chunkio/tests/fs_windows.c create mode 100644 source/lib/chunkio/tests/metadata_update.c diff --git a/source/lib/chunkio/.github/workflows/ci.yaml b/source/lib/chunkio/.github/workflows/ci.yaml index c937a2c9..a816202e 100644 --- a/source/lib/chunkio/.github/workflows/ci.yaml +++ b/source/lib/chunkio/.github/workflows/ci.yaml @@ -15,15 +15,20 @@ jobs: max-parallel: 48 fail-fast: false matrix: - os: [windows-latest, windows-2019] + os: [windows-2025, windows-2022] steps: - uses: actions/checkout@v2 - - name: Build on ${{ matrix.os }} with vs-2019 + - name: Set up with Developer Command Prompt for Microsoft Visual C++ + uses: ilammy/msvc-dev-cmd@v1 + with: + arch: amd64 + - name: Build on ${{ matrix.os }} with MSVC run: | - .\scripts\win_build.bat + cmake -G "NMake Makefiles" -DCIO_TESTS=On . + cmake --build . - name: Run unit tests. run: | - ctest --rerun-failed --output-on-failure -C Debug --test-dir . + ctest . --rerun-failed --output-on-failure --test-dir . build-unix: name: Build sources on amd64 for ${{ matrix.os }} - ${{ matrix.compiler }} runs-on: ${{ matrix.os }} diff --git a/source/lib/chunkio/CMakeLists.txt b/source/lib/chunkio/CMakeLists.txt index 21f1c3e4..5a7f1c60 100644 --- a/source/lib/chunkio/CMakeLists.txt +++ b/source/lib/chunkio/CMakeLists.txt @@ -3,7 +3,7 @@ project(chunk-io C) set(CIO_VERSION_MAJOR 1) set(CIO_VERSION_MINOR 5) -set(CIO_VERSION_PATCH 3) +set(CIO_VERSION_PATCH 4) set(CIO_VERSION_STR "${CIO_VERSION_MAJOR}.${CIO_VERSION_MINOR}.${CIO_VERSION_PATCH}") # CFLAGS diff --git a/source/lib/chunkio/include/chunkio/cio_file.h b/source/lib/chunkio/include/chunkio/cio_file.h index 3bc65917..f05d3861 100644 --- a/source/lib/chunkio/include/chunkio/cio_file.h +++ b/source/lib/chunkio/include/chunkio/cio_file.h @@ -40,6 +40,7 @@ struct cio_file { size_t realloc_size; /* chunk size to increase alloc */ char *path; /* root path + stream */ char *map; /* map of data */ + struct cio_ctx *ctx; /* owning context */ #ifdef _WIN32 HANDLE backing_file; HANDLE backing_mapping; @@ -49,6 +50,8 @@ struct cio_file { char *st_content; crc_t crc_cur; /* crc: current value calculated */ int crc_reset; /* crc: must recalculate from the beginning ? */ + int auto_remap_warned; /* has sync auto-remap warning been emitted? */ + int map_truncated_warned; /* has RO truncation warning been emitted? */ }; size_t cio_file_real_size(struct cio_file *cf); diff --git a/source/lib/chunkio/src/cio_file.c b/source/lib/chunkio/src/cio_file.c index 56ac160a..26dc992e 100644 --- a/source/lib/chunkio/src/cio_file.c +++ b/source/lib/chunkio/src/cio_file.c @@ -324,7 +324,10 @@ static int munmap_file(struct cio_ctx *ctx, struct cio_chunk *ch) } /* Unmap file */ - cio_file_native_unmap(cf); + ret = cio_file_native_unmap(cf); + if (ret != CIO_OK) { + return -1; + } cf->data_size = 0; cf->alloc_size = 0; @@ -343,6 +346,7 @@ static int mmap_file(struct cio_ctx *ctx, struct cio_chunk *ch, size_t size) { ssize_t content_size; size_t fs_size; + size_t requested_map_size; int ret; struct cio_file *cf; @@ -413,6 +417,7 @@ static int mmap_file(struct cio_ctx *ctx, struct cio_chunk *ch, size_t size) cf->alloc_size = size; /* Map the file */ + requested_map_size = cf->alloc_size; ret = cio_file_native_map(cf, cf->alloc_size); if (ret != CIO_OK) { @@ -421,6 +426,21 @@ static int mmap_file(struct cio_ctx *ctx, struct cio_chunk *ch, size_t size) return CIO_ERROR; } + if ((cf->flags & CIO_OPEN_RD) && requested_map_size != cf->alloc_size) { + if (cf->map_truncated_warned == CIO_FALSE) { + cio_log_warn(ctx, + "[cio file] truncated read-only map from %zu to %zu bytes: %s/%s", + requested_map_size, + cf->alloc_size, + ch->st->name, + ch->name); + cf->map_truncated_warned = CIO_TRUE; + } + } + else { + cf->map_truncated_warned = CIO_FALSE; + } + /* check content data size */ if (fs_size > 0) { content_size = cio_file_st_get_content_len(cf->map, @@ -664,6 +684,9 @@ struct cio_file *cio_file_open(struct cio_ctx *ctx, cf->crc_cur = cio_crc32_init(); cf->path = path; cf->map = NULL; + cf->ctx = ctx; + cf->auto_remap_warned = CIO_FALSE; + cf->map_truncated_warned = CIO_FALSE; ch->backend = cf; #ifdef _WIN32 @@ -801,9 +824,9 @@ static int _cio_file_up(struct cio_chunk *ch, int enforced) return CIO_ERROR; } - if (cf->fd > 0) { + if (cio_file_native_is_open(cf)) { cio_log_error(ch->ctx, "[cio file] file descriptor already exists: " - "[fd=%i] %s:%s", cf->fd, ch->st->name, ch->name); + "%s:%s", ch->st->name, ch->name); return CIO_ERROR; } @@ -908,7 +931,11 @@ int cio_file_down(struct cio_chunk *ch) } /* unmap memory */ - munmap_file(ch->ctx, ch); + ret = munmap_file(ch->ctx, ch); + + if (ret != 0) { + return -1; + } /* Allocated map size is zero */ cf->alloc_size = 0; @@ -921,7 +948,12 @@ int cio_file_down(struct cio_chunk *ch) } /* Close file descriptor */ - cio_file_native_close(cf); + ret = cio_file_native_close(cf); + + if (ret != CIO_OK) { + cio_errno(); + return -1; + } return 0; } @@ -1047,7 +1079,6 @@ int cio_file_write_metadata(struct cio_chunk *ch, char *buf, size_t size) char *cur_content_data; char *new_content_data; size_t new_size; - size_t content_av; size_t meta_av; struct cio_file *cf; @@ -1082,13 +1113,11 @@ int cio_file_write_metadata(struct cio_chunk *ch, char *buf, size_t size) * where we need to increase the memory map size, move the content area * bytes to a different position and write the metadata. * - * Calculate the available space in the content area. + * Check if resize is needed before calculating content_av to avoid + * unsigned underflow. We need: header + new_metadata + content_data <= alloc_size */ - content_av = cf->alloc_size - cf->data_size; - - /* If there is no enough space, increase the file size and it memory map */ - if (content_av < size) { - new_size = (size - meta_av) + cf->data_size + CIO_FILE_HEADER_MIN; + if (cf->alloc_size < CIO_FILE_HEADER_MIN + size + cf->data_size) { + new_size = CIO_FILE_HEADER_MIN + size + cf->data_size; ret = cio_file_resize(cf, new_size); @@ -1106,7 +1135,7 @@ int cio_file_write_metadata(struct cio_chunk *ch, char *buf, size_t size) /* set new position for the content data */ cur_content_data = cio_file_st_get_content(cf->map); new_content_data = meta + size; - memmove(new_content_data, cur_content_data, size); + memmove(new_content_data, cur_content_data, cf->data_size); /* copy new metadata */ memcpy(meta, buf, size); @@ -1138,6 +1167,12 @@ int cio_file_sync(struct cio_chunk *ch) return 0; } + /* If chunk is down (unmapped), there's nothing to sync */ + /* You can only write to a chunk when it's up, so if it's down, no pending changes exist */ + if (!cio_file_native_is_mapped(cf)) { + return 0; + } + if (cf->synced == CIO_TRUE) { return 0; } diff --git a/source/lib/chunkio/src/cio_file_unix.c b/source/lib/chunkio/src/cio_file_unix.c index 72d49312..c1798a7d 100644 --- a/source/lib/chunkio/src/cio_file_unix.c +++ b/source/lib/chunkio/src/cio_file_unix.c @@ -66,6 +66,7 @@ int cio_file_native_unmap(struct cio_file *cf) cf->alloc_size = 0; cf->map = NULL; + cf->map_truncated_warned = CIO_FALSE; return CIO_OK; } diff --git a/source/lib/chunkio/src/cio_file_win32.c b/source/lib/chunkio/src/cio_file_win32.c index 18044c40..43b7a241 100644 --- a/source/lib/chunkio/src/cio_file_win32.c +++ b/source/lib/chunkio/src/cio_file_win32.c @@ -38,14 +38,14 @@ int cio_file_native_unmap(struct cio_file *cf) return CIO_ERROR; } - if (!cio_file_native_is_open(cf)) { - return CIO_OK; - } - + /* Check if already unmapped first */ if (!cio_file_native_is_mapped(cf)) { return CIO_OK; } + /* On Windows, we can unmap even if file handle is closed */ + /* The mapping handle maintains the reference */ + result = UnmapViewOfFile(cf->map); if (result == 0) { @@ -54,11 +54,18 @@ int cio_file_native_unmap(struct cio_file *cf) return CIO_ERROR; } - CloseHandle(cf->backing_mapping); + result = CloseHandle(cf->backing_mapping); + + if (result == 0) { + cio_file_native_report_os_error(); + + return CIO_ERROR; + } cf->backing_mapping = INVALID_HANDLE_VALUE; cf->alloc_size = 0; cf->map = NULL; + cf->map_truncated_warned = CIO_FALSE; return CIO_OK; } @@ -67,6 +74,9 @@ int cio_file_native_map(struct cio_file *cf, size_t map_size) { DWORD desired_protection; DWORD desired_access; + size_t file_size; + size_t actual_map_size; + int ret; if (cf == NULL) { return CIO_ERROR; @@ -92,9 +102,40 @@ int cio_file_native_map(struct cio_file *cf, size_t map_size) return CIO_ERROR; } + /* Get current file size to ensure we don't map beyond it for read-only files */ + ret = cio_file_native_get_size(cf, &file_size); + if (ret != CIO_OK) { + return CIO_ERROR; + } + + /* For read-only files, we cannot map beyond the file size */ + /* For read-write files, if map_size > file_size, we should resize first */ + if (cf->flags & CIO_OPEN_RD) { + if (map_size > file_size) { + actual_map_size = file_size; + } + else { + actual_map_size = map_size; + } + } + else { + /* For RW files, if map_size > file_size, resize the file first */ + if (map_size > file_size) { + ret = cio_file_native_resize(cf, map_size); + if (ret != CIO_OK) { + return CIO_ERROR; + } + } + actual_map_size = map_size; + } + + /* CreateFileMappingA requires size as two DWORDs (high and low) */ + /* Use actual_map_size to ensure consistency */ cf->backing_mapping = CreateFileMappingA(cf->backing_file, NULL, desired_protection, - 0, 0, NULL); + (DWORD)(actual_map_size >> 32), + (DWORD)(actual_map_size & 0xFFFFFFFFUL), + NULL); if (cf->backing_mapping == NULL) { cio_file_native_report_os_error(); @@ -102,7 +143,7 @@ int cio_file_native_map(struct cio_file *cf, size_t map_size) return CIO_ERROR; } - cf->map = MapViewOfFile(cf->backing_mapping, desired_access, 0, 0, map_size); + cf->map = MapViewOfFile(cf->backing_mapping, desired_access, 0, 0, actual_map_size); if (cf->map == NULL) { cio_file_native_report_os_error(); @@ -114,7 +155,7 @@ int cio_file_native_map(struct cio_file *cf, size_t map_size) return CIO_ERROR; } - cf->alloc_size = map_size; + cf->alloc_size = actual_map_size; return CIO_OK; } @@ -474,6 +515,38 @@ int cio_file_native_delete(struct cio_file *cf) { int result; + if (cf == NULL) { + return CIO_ERROR; + } + + if (cio_file_native_is_mapped(cf)) { + if (cf->ctx != NULL) { + cio_log_warn(cf->ctx, + "[cio file] auto-unmapping chunk prior to delete: %s", + cf->path); + } + + result = cio_file_native_unmap(cf); + + if (result != CIO_OK) { + return result; + } + } + + if (cio_file_native_is_open(cf)) { + if (cf->ctx != NULL) { + cio_log_warn(cf->ctx, + "[cio file] closing handle prior to delete: %s", + cf->path); + } + + result = cio_file_native_close(cf); + + if (result != CIO_OK) { + return result; + } + } + result = DeleteFileA(cf->path); if (result == 0) { @@ -489,6 +562,10 @@ int cio_file_native_sync(struct cio_file *cf, int sync_mode) { int result; + if (!cio_file_native_is_mapped(cf)) { + return CIO_ERROR; + } + result = FlushViewOfFile(cf->map, cf->alloc_size); if (result == 0) { diff --git a/source/lib/chunkio/tests/CMakeLists.txt b/source/lib/chunkio/tests/CMakeLists.txt index 7cee7e6c..a5cea46a 100644 --- a/source/lib/chunkio/tests/CMakeLists.txt +++ b/source/lib/chunkio/tests/CMakeLists.txt @@ -9,6 +9,15 @@ if(CIO_BACKEND_FILESYSTEM) set(UNIT_TESTS_FILES ${UNIT_TESTS_FILES} fs.c + metadata_update.c + ) +endif() + +# Windows-specific inconsistency tests +if(WIN32 AND CIO_BACKEND_FILESYSTEM) + set(UNIT_TESTS_FILES + ${UNIT_TESTS_FILES} + fs_windows.c ) endif() @@ -35,7 +44,7 @@ foreach(source_file ${UNIT_TESTS_FILES}) add_test(${source_file_we} ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/${source_file_we}) endforeach() -# Perf tests for dev purposes: note these tests are not registered, they need to +# Perf tests for dev purposes: note these tests are not registered, they need to # be executed manually set(UNIT_PERF_TESTS fs_perf.c diff --git a/source/lib/chunkio/tests/fs.c b/source/lib/chunkio/tests/fs.c index a976f46d..130921df 100644 --- a/source/lib/chunkio/tests/fs.c +++ b/source/lib/chunkio/tests/fs.c @@ -964,6 +964,143 @@ void test_legacy_failure() test_legacy_core(CIO_TRUE); } +/* + * Test case: Prevent unsigned underflow when writing large metadata to a + * chunk with small initial allocation. + * + * This test specifically validates the fix for the bug where calculating + * content_av = alloc_size - CIO_FILE_HEADER_MIN - size could underflow + * when size is large, causing the resize check to be skipped and leading + * to out-of-bounds writes. + * + * Scenario: + * - Create chunk with minimal initial size (e.g., ~100 bytes page-aligned) + * - Write small metadata (10 bytes) and small content (20 bytes) + * - Write large metadata (80 bytes) that would cause underflow: + * old code: content_av = 100 - 24 - 80 = -4 (wraps to huge unsigned) + * - Verify resize happens correctly and no buffer overrun occurs + */ +static void test_metadata_unsigned_underflow() +{ + int ret; + int err; + char *meta_buf; + int meta_len; + void *content_buf; + size_t content_size; + struct cio_ctx *ctx; + struct cio_chunk *chunk; + struct cio_stream *stream; + struct cio_options cio_opts; + + /* Test data */ + const char *small_meta = "small"; + const char *large_meta = "this-is-a-very-large-metadata-string-that-would-cause-unsigned-underflow-in-old-code"; + const char *content_data = "test-content"; + + /* Cleanup any existing test directory */ + cio_utils_recursive_delete(CIO_ENV); + + /* Initialize options */ + cio_options_init(&cio_opts); + cio_opts.root_path = CIO_ENV; + cio_opts.log_cb = log_cb; + cio_opts.log_level = CIO_LOG_INFO; + cio_opts.flags = CIO_CHECKSUM; + + /* Create context */ + ctx = cio_create(&cio_opts); + TEST_CHECK(ctx != NULL); + if (!ctx) { + printf("cannot create context\n"); + exit(1); + } + + /* Create stream */ + stream = cio_stream_create(ctx, "test_stream_underflow", CIO_STORE_FS); + TEST_CHECK(stream != NULL); + if (!stream) { + printf("cannot create stream\n"); + cio_destroy(ctx); + exit(1); + } + + /* Create chunk with minimal initial size (forces small alloc_size) */ + chunk = cio_chunk_open(ctx, stream, "test_chunk_underflow", CIO_OPEN, 100, &err); + TEST_CHECK(chunk != NULL); + if (!chunk) { + printf("cannot open chunk\n"); + cio_destroy(ctx); + exit(1); + } + + /* Step 1: Write small initial metadata */ + ret = cio_meta_write(chunk, (char *) small_meta, strlen(small_meta)); + TEST_CHECK(ret == CIO_OK); + + /* Step 2: Write some content data */ + ret = cio_chunk_write(chunk, content_data, strlen(content_data)); + TEST_CHECK(ret == CIO_OK); + + /* Verify initial state */ + ret = cio_meta_read(chunk, &meta_buf, &meta_len); + TEST_CHECK(ret == CIO_OK); + TEST_CHECK(meta_len == (int) strlen(small_meta)); + TEST_CHECK(memcmp(meta_buf, small_meta, strlen(small_meta)) == 0); + + ret = cio_chunk_get_content_copy(chunk, &content_buf, &content_size); + TEST_CHECK(ret == CIO_OK); + TEST_CHECK(content_size == strlen(content_data)); + TEST_CHECK(memcmp(content_buf, content_data, strlen(content_data)) == 0); + free(content_buf); + + /* Step 3: Write large metadata that would cause underflow in old code + * This is the critical test - the new metadata (80 bytes) is larger than + * what would fit without resize, and with a small initial alloc_size, + * the old calculation would have underflowed. + */ + ret = cio_meta_write(chunk, (char *) large_meta, strlen(large_meta)); + TEST_CHECK(ret == CIO_OK); + + /* Step 4: Verify metadata was written correctly */ + ret = cio_meta_read(chunk, &meta_buf, &meta_len); + TEST_CHECK(ret == CIO_OK); + TEST_CHECK(meta_len == (int) strlen(large_meta)); + TEST_CHECK(memcmp(meta_buf, large_meta, strlen(large_meta)) == 0); + + /* Step 5: Verify content data integrity - must be preserved */ + ret = cio_chunk_get_content_copy(chunk, &content_buf, &content_size); + TEST_CHECK(ret == CIO_OK); + TEST_CHECK(content_size == strlen(content_data)); + TEST_CHECK(memcmp(content_buf, content_data, strlen(content_data)) == 0); + + /* Step 6: Sync to disk to ensure persistence */ + ret = cio_chunk_sync(chunk); + TEST_CHECK(ret == CIO_OK); + + /* Step 7: Put chunk down and up again to test persistence */ + ret = cio_chunk_down(chunk); + TEST_CHECK(ret == CIO_OK); + + ret = cio_chunk_up(chunk); + TEST_CHECK(ret == CIO_OK); + + /* Step 8: Final validation after persistence cycle */ + ret = cio_meta_read(chunk, &meta_buf, &meta_len); + TEST_CHECK(ret == CIO_OK); + TEST_CHECK(meta_len == (int) strlen(large_meta)); + TEST_CHECK(memcmp(meta_buf, large_meta, strlen(large_meta)) == 0); + + ret = cio_chunk_get_content_copy(chunk, &content_buf, &content_size); + TEST_CHECK(ret == CIO_OK); + TEST_CHECK(content_size == strlen(content_data)); + TEST_CHECK(memcmp(content_buf, content_data, strlen(content_data)) == 0); + + /* Cleanup */ + free(content_buf); + cio_destroy(ctx); +} + TEST_LIST = { {"fs_write", test_fs_write}, {"fs_checksum", test_fs_checksum}, @@ -976,5 +1113,6 @@ TEST_LIST = { {"fs_deep_hierachy", test_deep_hierarchy}, {"legacy_success", test_legacy_success}, {"legacy_failure", test_legacy_failure}, + {"metadata_unsigned_underflow", test_metadata_unsigned_underflow}, { 0 } }; diff --git a/source/lib/chunkio/tests/fs_windows.c b/source/lib/chunkio/tests/fs_windows.c new file mode 100644 index 00000000..8f3b5a63 --- /dev/null +++ b/source/lib/chunkio/tests/fs_windows.c @@ -0,0 +1,443 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Chunk I/O + * ========= + * Copyright 2018 Eduardo Silva + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Windows File Handling Inconsistency Tests + * ========================================== + * + * This test suite highlights inconsistencies between Windows and Unix + * implementations of file handling in chunkio: + * + * 1. Delete while open/mapped: Windows allows deletion of open/mapped files + * while Unix correctly rejects it + * + * 2. Sync without mapping: Windows accesses cf->map without checking if it's + * NULL, which can cause crashes + * + * 3. File mapping size mismatch: CreateFileMapping uses current file size + * but MapViewOfFile may request a larger size, causing potential issues + * + * 4. File descriptor check: cio_file.c uses Unix-specific cf->fd check + * instead of platform-agnostic cio_file_native_is_open() + * + * These tests are designed to demonstrate the issues and verify behavior. + */ + +#ifdef _WIN32 + +#include +#include +#include +#include +#include +#include + +/* Note: We still need to include cio_file_native.h for testing native functions + * directly to verify bug fixes, but we use public APIs for state checks */ + +#include "cio_tests_internal.h" + +#define CIO_ENV "tmp" + +/* Logging callback */ +static int log_cb(struct cio_ctx *ctx, int level, const char *file, int line, + char *str) +{ + (void) ctx; + (void) level; + + printf("[cio-test-win32] %-60s => %s:%i\n", str, file, line); + return 0; +} + +/* + * ISSUE #1: Test deleting a file that is open/mapped + * + * Expected behavior: Delete should succeed after automatically releasing + * any outstanding mappings and handles. + */ +static void test_win32_delete_while_open() +{ + int ret; + int err; + struct cio_ctx *ctx; + struct cio_stream *stream; + struct cio_chunk *chunk; + struct cio_file *cf; + struct cio_options cio_opts; + + printf("\n=== Test: Delete file while open ===\n"); + + cio_utils_recursive_delete("tmp"); + + cio_options_init(&cio_opts); + cio_opts.root_path = "tmp"; + cio_opts.log_cb = log_cb; + cio_opts.log_level = CIO_LOG_DEBUG; + + ctx = cio_create(&cio_opts); + TEST_CHECK(ctx != NULL); + + stream = cio_stream_create(ctx, "test", CIO_STORE_FS); + TEST_CHECK(stream != NULL); + + /* Open and map a file */ + chunk = cio_chunk_open(ctx, stream, "test-file-open", CIO_OPEN, 1000, &err); + TEST_CHECK(chunk != NULL); + + cf = (struct cio_file *) chunk->backend; + TEST_CHECK(cf != NULL); + + /* Verify file is open (using public API) */ + TEST_CHECK(cio_chunk_is_up(chunk) == CIO_TRUE); + + /* Delete while open - should succeed and close resources automatically */ + ret = cio_file_native_delete(cf); + printf("Result of delete while open: %d (expected: CIO_OK=%d)\n", + ret, CIO_OK); + TEST_CHECK(ret == CIO_OK); + TEST_CHECK(cio_file_native_is_open(cf) == CIO_FALSE); + TEST_CHECK(cio_file_native_is_mapped(cf) == CIO_FALSE); + + cio_chunk_close(chunk, CIO_FALSE); + cio_stream_delete(stream); + cio_destroy(ctx); +} + +/* + * ISSUE #2: Test deleting a file that is mapped + * + * Expected behavior: Delete should succeed after the implementation releases + * the mapping safely. + */ +static void test_win32_delete_while_mapped() +{ + int ret; + int err; + struct cio_ctx *ctx; + struct cio_stream *stream; + struct cio_chunk *chunk; + struct cio_file *cf; + struct cio_options cio_opts; + + printf("\n=== Test: Delete file while mapped ===\n"); + + cio_utils_recursive_delete("tmp"); + + cio_options_init(&cio_opts); + cio_opts.root_path = "tmp"; + cio_opts.log_cb = log_cb; + cio_opts.log_level = CIO_LOG_DEBUG; + + ctx = cio_create(&cio_opts); + TEST_CHECK(ctx != NULL); + + stream = cio_stream_create(ctx, "test", CIO_STORE_FS); + TEST_CHECK(stream != NULL); + + /* Open and map a file */ + chunk = cio_chunk_open(ctx, stream, "test-file-mapped", CIO_OPEN, 1000, &err); + TEST_CHECK(chunk != NULL); + + cf = (struct cio_file *) chunk->backend; + TEST_CHECK(cf != NULL); + + /* Write some data to ensure mapping */ + ret = cio_chunk_write(chunk, "test data", 9); + TEST_CHECK(ret == 0); + + /* Verify file is mapped (using public API) */ + TEST_CHECK(cio_chunk_is_up(chunk) == CIO_TRUE); + + /* Delete while mapped - should succeed and release mapping */ + ret = cio_file_native_delete(cf); + printf("Result of delete while mapped: %d (expected: CIO_OK=%d)\n", + ret, CIO_OK); + TEST_CHECK(ret == CIO_OK); + TEST_CHECK(cio_file_native_is_open(cf) == CIO_FALSE); + TEST_CHECK(cio_file_native_is_mapped(cf) == CIO_FALSE); + + cio_chunk_close(chunk, CIO_FALSE); + cio_stream_delete(stream); + cio_destroy(ctx); +} + +/* + * ISSUE #3: Test syncing a file that is not mapped + * + * Expected behavior: Should check if mapped before accessing cf->map + * Current behavior: Accesses cf->map without checking, may crash + */ +static void test_win32_sync_without_map() +{ + int ret; + int err; + struct cio_ctx *ctx; + struct cio_stream *stream; + struct cio_chunk *chunk; + struct cio_file *cf; + struct cio_options cio_opts; + + printf("\n=== Test: Sync file without mapping ===\n"); + + cio_utils_recursive_delete("tmp"); + + cio_options_init(&cio_opts); + cio_opts.root_path = "tmp"; + cio_opts.log_cb = log_cb; + cio_opts.log_level = CIO_LOG_DEBUG; + + ctx = cio_create(&cio_opts); + TEST_CHECK(ctx != NULL); + + stream = cio_stream_create(ctx, "test", CIO_STORE_FS); + TEST_CHECK(stream != NULL); + + /* Open a file but don't map it */ + chunk = cio_chunk_open(ctx, stream, "test-file-sync", CIO_OPEN, 1000, &err); + TEST_CHECK(chunk != NULL); + + cf = (struct cio_file *) chunk->backend; + TEST_CHECK(cf != NULL); + + /* Manually unmap if it was auto-mapped (using public API) */ + if (cio_chunk_is_up(chunk) == CIO_TRUE) { + ret = cio_file_down(chunk); + TEST_CHECK(ret == 0); + } + + /* Verify file is not mapped (using public API) */ + TEST_CHECK(cio_chunk_is_up(chunk) == CIO_FALSE); + printf("Verified: chunk is down (not mapped)\n"); + + /* Set synced flag to FALSE to force sync path (since cio_file_down syncs before unmapping) */ + cf->synced = CIO_FALSE; + + /* Try to sync without mapping using public API */ + /* cio_file_sync should auto-remap and emit a warning */ + printf("Attempting sync on unmapped file using cio_file_sync()...\n"); + printf("cio_file_sync() should remap and warn instead of failing\n"); + + ret = cio_file_sync(chunk); + printf("Result of sync without map: %d (expected: 0 for success with warning)\n", ret); + + TEST_CHECK(ret == 0); + TEST_CHECK(cio_chunk_is_up(chunk) == CIO_FALSE); + TEST_CHECK(cio_file_native_is_open(cf) == CIO_FALSE); + + cio_chunk_close(chunk, CIO_FALSE); + cio_stream_delete(stream); + cio_destroy(ctx); +} + +/* + * ISSUE #4: Test file mapping size mismatch + * + * Expected behavior: CreateFileMapping should use map_size, not current file size + * Current behavior: Creates mapping based on file size, then tries to map larger view + */ +static void test_win32_map_size_mismatch() +{ + int ret; + int err; + size_t file_size; + size_t map_size; + struct cio_ctx *ctx; + struct cio_stream *stream; + struct cio_chunk *chunk; + struct cio_file *cf; + struct cio_options cio_opts; + + printf("\n=== Test: File mapping size mismatch ===\n"); + + cio_utils_recursive_delete("tmp"); + + cio_options_init(&cio_opts); + cio_opts.root_path = "tmp"; + cio_opts.log_cb = log_cb; + cio_opts.log_level = CIO_LOG_DEBUG; + + ctx = cio_create(&cio_opts); + TEST_CHECK(ctx != NULL); + + stream = cio_stream_create(ctx, "test", CIO_STORE_FS); + TEST_CHECK(stream != NULL); + + /* Create a small file first */ + chunk = cio_chunk_open(ctx, stream, "test-file-size", CIO_OPEN, 1024, &err); + TEST_CHECK(chunk != NULL); + + cf = (struct cio_file *) chunk->backend; + TEST_CHECK(cf != NULL); + + /* Write minimal data */ + ret = cio_chunk_write(chunk, "test", 4); + TEST_CHECK(ret == 0); + + /* Sync to ensure file is written */ + ret = cio_chunk_sync(chunk); + TEST_CHECK(ret == 0); + + /* Get actual file size */ + ret = cio_file_native_get_size(cf, &file_size); + TEST_CHECK(ret == CIO_OK); + printf("Actual file size: %zu bytes\n", file_size); + + /* Close the chunk to unmap */ + cio_chunk_close(chunk, CIO_FALSE); + + /* Reopen file */ + chunk = cio_chunk_open(ctx, stream, "test-file-size", CIO_OPEN_RD, 0, &err); + TEST_CHECK(chunk != NULL); + + cf = (struct cio_file *) chunk->backend; + TEST_CHECK(cf != NULL); + + /* Unmap if cio_chunk_open auto-mapped the file (using public API) */ + if (cio_chunk_is_up(chunk) == CIO_TRUE) { + ret = cio_file_down(chunk); + TEST_CHECK(ret == 0); + } + + /* Ensure file is still open */ + if (!cio_file_native_is_open(cf)) { + ret = cio_file_native_open(cf); + TEST_CHECK(ret == CIO_OK); + } + + /* Try to map with a size larger than the file */ + map_size = file_size + 4096; /* Request 4KB more than file size */ + printf("Attempting to map %zu bytes (file is %zu bytes)\n", map_size, file_size); + + /* This is where the issue occurs: CreateFileMapping uses current file size (0,0), + * but MapViewOfFile tries to map a larger size */ + ret = cio_file_native_map(cf, map_size); + printf("Result of mapping %zu bytes to %zu byte file: %d\n", + map_size, file_size, ret); + + /* For read-only files, mapping beyond file size is not possible on Windows. + * The mapping should be limited to file_size, and alloc_size should reflect + * the actual mapped size (file_size), not the requested size (map_size). + * This ensures consistency between CreateFileMappingA and MapViewOfFile sizes. */ + if (ret == CIO_OK) { + printf("Mapping succeeded\n"); + printf("Requested map_size: %zu, file_size: %zu\n", map_size, file_size); + + /* Verify what was actually mapped (using public API) */ + if (cio_chunk_is_up(chunk) == CIO_TRUE) { + printf("File is mapped, alloc_size: %zu\n", cf->alloc_size); + + /* For read-only files, alloc_size should match file_size (the actual mapped size), + * not the requested map_size (which exceeds file size) */ + /* This ensures consistency: CreateFileMappingA and MapViewOfFile both use actual_map_size */ + TEST_CHECK(cf->alloc_size == file_size); + + if (cf->alloc_size != file_size) { + printf("ISSUE DETECTED: alloc_size (%zu) doesn't match file_size (%zu)\n", + cf->alloc_size, file_size); + printf("For read-only files, mapping is limited to file_size\n"); + } + } + + ret = cio_file_native_unmap(cf); + TEST_CHECK(ret == CIO_OK); + } + else { + printf("Mapping failed when size mismatch occurs\n"); + /* For read-only files, this might be expected if map_size > file_size */ + } + + cio_file_native_close(cf); + cio_chunk_close(chunk, CIO_FALSE); + cio_stream_delete(stream); + cio_destroy(ctx); +} + +/* + * Test accessing file descriptor check inconsistency + * This tests the issue in cio_file.c line 804 where it checks cf->fd > 0 + * instead of using cio_file_native_is_open(cf) + */ +static void test_win32_fd_check_inconsistency() +{ + int ret; + int err; + struct cio_ctx *ctx; + struct cio_stream *stream; + struct cio_chunk *chunk; + struct cio_file *cf; + struct cio_options cio_opts; + + printf("\n=== Test: File descriptor check inconsistency ===\n"); + + cio_utils_recursive_delete("tmp"); + + cio_options_init(&cio_opts); + cio_opts.root_path = "tmp"; + cio_opts.log_cb = log_cb; + cio_opts.log_level = CIO_LOG_DEBUG; + + ctx = cio_create(&cio_opts); + TEST_CHECK(ctx != NULL); + + stream = cio_stream_create(ctx, "test", CIO_STORE_FS); + TEST_CHECK(stream != NULL); + + /* Open a file */ + chunk = cio_chunk_open(ctx, stream, "test-file-fd", CIO_OPEN, 1000, &err); + TEST_CHECK(chunk != NULL); + + cf = (struct cio_file *) chunk->backend; + TEST_CHECK(cf != NULL); + + /* Verify file is open (using public API) */ + ret = cio_chunk_is_up(chunk); + TEST_CHECK(ret == CIO_TRUE); + printf("cio_chunk_is_up(chunk): %d\n", ret); + + /* Check cf->fd value on Windows (internal check for documentation) */ + /* On Windows, cf->fd is typically -1, but the file is still open via backing_file */ + printf("cf->fd value: %d (internal, not used on Windows)\n", cf->fd); + printf("Note: cio_file.c now uses cio_file_native_is_open() instead of cf->fd > 0\n"); + + cio_chunk_close(chunk, CIO_FALSE); + cio_stream_delete(stream); + cio_destroy(ctx); +} + +TEST_LIST = { + {"win32_delete_while_open", test_win32_delete_while_open}, + {"win32_delete_while_mapped", test_win32_delete_while_mapped}, + {"win32_sync_without_map", test_win32_sync_without_map}, + {"win32_map_size_mismatch", test_win32_map_size_mismatch}, + {"win32_fd_check_inconsistency", test_win32_fd_check_inconsistency}, + {NULL, NULL} +}; + +#else /* _WIN32 */ + +#include "cio_tests_internal.h" + +/* Empty test list for non-Windows platforms */ +TEST_LIST = { + {0} +}; + +#endif /* _WIN32 */ + diff --git a/source/lib/chunkio/tests/metadata_update.c b/source/lib/chunkio/tests/metadata_update.c new file mode 100644 index 00000000..fc425f39 --- /dev/null +++ b/source/lib/chunkio/tests/metadata_update.c @@ -0,0 +1,288 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Chunk I/O + * ========= + * Copyright 2018-2019 Eduardo Silva + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "cio_tests_internal.h" + +#define CIO_ENV_META_TEST "/tmp/cio-metadata-update-test/" + +/* Logging callback */ +static int log_cb(struct cio_ctx *ctx, int level, const char *file, int line, + char *str) +{ + (void) ctx; + (void) level; + (void) file; + (void) line; + + printf("[cio-test-metadata] %s\n", str); + return 0; +} + +/* + * Test case: Validate that updating metadata after writing content data + * correctly moves the content data and preserves both metadata and content + * integrity. + * + * This test specifically validates the fix for the bug where memmove() + * was using metadata size instead of content data size when moving content + * after metadata update. + */ +static void test_metadata_update_with_content() +{ + int ret; + int err; + char *meta_buf; + int meta_len; + void *content_buf; + size_t content_size; + size_t expected_content_size; + struct cio_ctx *ctx; + struct cio_chunk *chunk; + struct cio_stream *stream; + struct cio_options cio_opts; + + /* Test data */ + const char *initial_meta = "initial-metadata"; + const char *updated_meta = "this-is-a-much-longer-metadata-string-that-will-require-content-to-be-moved"; + const char *content_data = "This is test content data that must be preserved when metadata is updated."; + const char *more_content = " Additional content appended after metadata update."; + + /* Expected final content */ + char *expected_content; + size_t expected_content_len; + + /* Cleanup any existing test directory */ + cio_utils_recursive_delete(CIO_ENV_META_TEST); + + /* Initialize options */ + cio_options_init(&cio_opts); + cio_opts.root_path = CIO_ENV_META_TEST; + cio_opts.log_cb = log_cb; + cio_opts.log_level = CIO_LOG_INFO; + cio_opts.flags = CIO_CHECKSUM; + + /* Create context */ + ctx = cio_create(&cio_opts); + TEST_CHECK(ctx != NULL); + if (!ctx) { + printf("cannot create context\n"); + exit(1); + } + + /* Create stream */ + stream = cio_stream_create(ctx, "test_stream", CIO_STORE_FS); + TEST_CHECK(stream != NULL); + if (!stream) { + printf("cannot create stream\n"); + cio_destroy(ctx); + exit(1); + } + + /* Create chunk */ + chunk = cio_chunk_open(ctx, stream, "test_chunk", CIO_OPEN, 1000, &err); + TEST_CHECK(chunk != NULL); + if (!chunk) { + printf("cannot open chunk\n"); + cio_destroy(ctx); + exit(1); + } + + /* Step 1: Write initial metadata */ + ret = cio_meta_write(chunk, (char *) initial_meta, strlen(initial_meta)); + TEST_CHECK(ret == CIO_OK); + + /* Step 2: Write some content data */ + ret = cio_chunk_write(chunk, content_data, strlen(content_data)); + TEST_CHECK(ret == CIO_OK); + + expected_content_size = strlen(content_data); + + /* Step 3: Update metadata to a larger size (this triggers content move) */ + /* This is the critical test case - when metadata grows, content data + * must be moved correctly using cf->data_size, not the metadata size */ + ret = cio_meta_write(chunk, (char *) updated_meta, strlen(updated_meta)); + TEST_CHECK(ret == CIO_OK); + + /* Step 4: Write more content after metadata update */ + ret = cio_chunk_write(chunk, more_content, strlen(more_content)); + TEST_CHECK(ret == CIO_OK); + + expected_content_size += strlen(more_content); + + /* Build expected content */ + expected_content_len = strlen(content_data) + strlen(more_content); + expected_content = malloc(expected_content_len + 1); + TEST_CHECK(expected_content != NULL); + if (!expected_content) { + cio_destroy(ctx); + exit(1); + } + memcpy(expected_content, content_data, strlen(content_data)); + memcpy(expected_content + strlen(content_data), more_content, strlen(more_content)); + expected_content[expected_content_len] = '\0'; + + /* Step 5: Sync to disk */ + ret = cio_chunk_sync(chunk); + TEST_CHECK(ret == CIO_OK); + + /* Step 6: Put chunk down */ + ret = cio_chunk_down(chunk); + TEST_CHECK(ret == CIO_OK); + + /* Verify chunk is down */ + ret = cio_chunk_is_up(chunk); + TEST_CHECK(ret == CIO_FALSE); + + /* Step 7: Put chunk up again */ + ret = cio_chunk_up(chunk); + TEST_CHECK(ret == CIO_OK); + + /* Verify chunk is up */ + ret = cio_chunk_is_up(chunk); + TEST_CHECK(ret == CIO_TRUE); + + /* Step 8: Validate metadata */ + ret = cio_meta_read(chunk, &meta_buf, &meta_len); + TEST_CHECK(ret == CIO_OK); + TEST_CHECK(meta_len == (int) strlen(updated_meta)); + TEST_CHECK(memcmp(meta_buf, updated_meta, strlen(updated_meta)) == 0); + + /* Step 9: Validate content data */ + ret = cio_chunk_get_content_copy(chunk, &content_buf, &content_size); + TEST_CHECK(ret == CIO_OK); + TEST_CHECK(content_size == expected_content_size); + TEST_CHECK(memcmp(content_buf, expected_content, expected_content_len) == 0); + + /* Cleanup */ + free(expected_content); + free(content_buf); + cio_destroy(ctx); +} + +/* + * Test case: Update metadata multiple times with varying sizes to ensure + * content data integrity is maintained throughout. + */ +static void test_metadata_multiple_updates() +{ + int ret; + int err; + char *meta_buf; + int meta_len; + void *content_buf; + size_t content_size; + struct cio_ctx *ctx; + struct cio_chunk *chunk; + struct cio_stream *stream; + struct cio_options cio_opts; + const char *test_strings[] = { + "small", + "medium-sized-metadata", + "very-long-metadata-string-that-exceeds-previous-sizes", + "tiny", + "another-medium-metadata-string" + }; + const char *content = "Test content that must remain intact"; + int i; + + /* Cleanup any existing test directory */ + cio_utils_recursive_delete(CIO_ENV_META_TEST); + + /* Initialize options */ + cio_options_init(&cio_opts); + cio_opts.root_path = CIO_ENV_META_TEST; + cio_opts.log_cb = log_cb; + cio_opts.log_level = CIO_LOG_INFO; + cio_opts.flags = CIO_CHECKSUM; + + /* Create context */ + ctx = cio_create(&cio_opts); + TEST_CHECK(ctx != NULL); + + /* Create stream */ + stream = cio_stream_create(ctx, "test_stream", CIO_STORE_FS); + TEST_CHECK(stream != NULL); + + /* Create chunk */ + chunk = cio_chunk_open(ctx, stream, "test_chunk2", CIO_OPEN, 1000, &err); + TEST_CHECK(chunk != NULL); + + /* Write initial content */ + ret = cio_chunk_write(chunk, content, strlen(content)); + TEST_CHECK(ret == CIO_OK); + + /* Update metadata multiple times with different sizes */ + for (i = 0; i < 5; i++) { + ret = cio_meta_write(chunk, (char *) test_strings[i], strlen(test_strings[i])); + TEST_CHECK(ret == CIO_OK); + + /* Verify metadata after each update */ + ret = cio_meta_read(chunk, &meta_buf, &meta_len); + TEST_CHECK(ret == CIO_OK); + TEST_CHECK(meta_len == (int) strlen(test_strings[i])); + TEST_CHECK(memcmp(meta_buf, test_strings[i], strlen(test_strings[i])) == 0); + + /* Verify content remains intact */ + ret = cio_chunk_get_content_copy(chunk, &content_buf, &content_size); + TEST_CHECK(ret == CIO_OK); + TEST_CHECK(content_size == strlen(content)); + TEST_CHECK(memcmp(content_buf, content, strlen(content)) == 0); + free(content_buf); + } + + /* Sync and test persistence */ + ret = cio_chunk_sync(chunk); + TEST_CHECK(ret == CIO_OK); + + ret = cio_chunk_down(chunk); + TEST_CHECK(ret == CIO_OK); + + ret = cio_chunk_up(chunk); + TEST_CHECK(ret == CIO_OK); + + /* Final validation after up/down cycle */ + ret = cio_meta_read(chunk, &meta_buf, &meta_len); + TEST_CHECK(ret == CIO_OK); + TEST_CHECK(meta_len == (int) strlen(test_strings[4])); + TEST_CHECK(memcmp(meta_buf, test_strings[4], strlen(test_strings[4])) == 0); + + ret = cio_chunk_get_content_copy(chunk, &content_buf, &content_size); + TEST_CHECK(ret == CIO_OK); + TEST_CHECK(content_size == strlen(content)); + TEST_CHECK(memcmp(content_buf, content, strlen(content)) == 0); + + /* Cleanup */ + free(content_buf); + cio_destroy(ctx); +} + +TEST_LIST = { + {"metadata_update_with_content", test_metadata_update_with_content}, + {"metadata_multiple_updates", test_metadata_multiple_updates}, + { 0 } +}; From bf8117baa1f322c41d3b5b44926828adbb24fed2 Mon Sep 17 00:00:00 2001 From: Patrick Stephens Date: Sun, 3 May 2026 12:34:52 +0100 Subject: [PATCH 063/213] [upstream] input_chunk: extend chunk layer to support routing Upstream-Ref: https://github.com/fluent/fluent-bit/commit/c56dc3d96827453ad9e19356a65b9e51821209c7 Cherry-picked from Fluent Bit v4.2.4 --- source/include/fluent-bit/flb_input_chunk.h | 26 + source/src/flb_input_chunk.c | 718 +++++++++++++++++++- 2 files changed, 709 insertions(+), 35 deletions(-) diff --git a/source/include/fluent-bit/flb_input_chunk.h b/source/include/fluent-bit/flb_input_chunk.h index b9b55abe..adb0bd14 100644 --- a/source/include/fluent-bit/flb_input_chunk.h +++ b/source/include/fluent-bit/flb_input_chunk.h @@ -24,6 +24,9 @@ #include #include #include +#include + +struct cio_chunk; #include #include @@ -42,6 +45,17 @@ /* Number of bytes reserved for Metadata Header on Chunks */ #define FLB_INPUT_CHUNK_META_HEADER 4 +/* Chunk metadata flags */ +#define FLB_CHUNK_FLAG_DIRECT_ROUTES (1 << 0) +#define FLB_CHUNK_FLAG_DIRECT_ROUTE_LABELS (1 << 1) +#define FLB_CHUNK_FLAG_DIRECT_ROUTE_WIDE_IDS (1 << 2) + +struct flb_chunk_direct_route { + uint32_t id; + uint16_t label_length; + const char *label; +}; + /* Chunks magic bytes (starting from Fluent Bit v1.8.10) */ #define FLB_INPUT_CHUNK_MAGIC_BYTE_0 (unsigned char) 0xF1 #define FLB_INPUT_CHUNK_MAGIC_BYTE_1 (unsigned char) 0x77 @@ -110,6 +124,18 @@ int flb_input_chunk_get_event_type(struct flb_input_chunk *ic); int flb_input_chunk_get_tag(struct flb_input_chunk *ic, const char **tag_buf, int *tag_len); +int flb_input_chunk_write_header_v2(struct cio_chunk *chunk, + int event_type, + char *tag, int tag_len, + const struct flb_chunk_direct_route *routes, + int route_count); +int flb_input_chunk_has_direct_routes(struct flb_input_chunk *ic); +int flb_input_chunk_get_direct_routes(struct flb_input_chunk *ic, + struct flb_chunk_direct_route **routes, + int *route_count); +void flb_input_chunk_destroy_direct_routes(struct flb_chunk_direct_route *routes, + int route_count); + void flb_input_chunk_ring_buffer_cleanup(struct flb_input_instance *ins); void flb_input_chunk_ring_buffer_collector(struct flb_config *ctx, void *data); ssize_t flb_input_chunk_get_size(struct flb_input_chunk *ic); diff --git a/source/src/flb_input_chunk.c b/source/src/flb_input_chunk.c index 6458cb6d..db999c5f 100644 --- a/source/src/flb_input_chunk.c +++ b/source/src/flb_input_chunk.c @@ -39,6 +39,8 @@ #include #include #include +#include +#include @@ -57,6 +59,114 @@ struct input_chunk_raw { size_t buf_size; }; +struct flb_input_chunk_meta_view { + char *buffer; + int length; + const char *tag; + int tag_length; + uint8_t flags; + uint8_t *routing_data; + uint16_t routing_data_length; +}; + +static inline int input_chunk_has_magic_bytes(char *buf, int len) +{ + unsigned char *p; + + if (len < FLB_INPUT_CHUNK_META_HEADER) { + return FLB_FALSE; + } + + p = (unsigned char *) buf; + if (p[0] == FLB_INPUT_CHUNK_MAGIC_BYTE_0 && + p[1] == FLB_INPUT_CHUNK_MAGIC_BYTE_1) { + return FLB_TRUE; + } + + return FLB_FALSE; +} + +static int input_chunk_metadata_view(struct flb_input_chunk *ic, + struct flb_input_chunk_meta_view *view) +{ + int ret; + int len; + int has_magic; + int payload_length; + int offset; + int computed_tag_len; + uint8_t flags; + uint16_t routing_length; + char *buf; + char *terminator; + const char *tag_start; + + ret = cio_meta_read(ic->chunk, &buf, &len); + if (ret == -1) { + return -1; + } + + view->buffer = buf; + view->length = len; + view->flags = 0; + view->routing_data = NULL; + view->routing_data_length = 0; + view->tag = buf; + view->tag_length = len; + + has_magic = input_chunk_has_magic_bytes(buf, len); + if (has_magic == FLB_FALSE) { + return 0; + } + + payload_length = len - FLB_INPUT_CHUNK_META_HEADER; + if (payload_length < 0) { + payload_length = 0; + } + + tag_start = buf + FLB_INPUT_CHUNK_META_HEADER; + view->tag = tag_start; + view->tag_length = payload_length; + flags = (uint8_t) buf[3]; + view->flags = flags; + + terminator = memchr(tag_start, '\0', payload_length); + if (terminator) { + view->tag_length = (int) (terminator - tag_start); + } + + if ((flags & FLB_CHUNK_FLAG_DIRECT_ROUTES) == 0) { + return 0; + } + + if (payload_length <= 0) { + return 0; + } + + if (!terminator) { + return 1; + } + + computed_tag_len = view->tag_length; + offset = computed_tag_len + 1; + + if (payload_length < offset + (int) sizeof(uint16_t)) { + return 1; + } + + routing_length = (uint16_t) (((unsigned char) tag_start[offset] << 8) | + (unsigned char) tag_start[offset + 1]); + + if (payload_length < offset + (int) sizeof(uint16_t) + routing_length) { + return 1; + } + + view->routing_data_length = routing_length; + view->routing_data = ((uint8_t *) tag_start) + offset + (int) sizeof(uint16_t); + + return 0; +} + #ifdef FLB_HAVE_IN_STORAGE_BACKLOG extern ssize_t sb_get_releasable_output_queue_space(struct flb_output_instance *output_plugin, @@ -599,6 +709,63 @@ int flb_input_chunk_place_new_chunk(struct flb_input_chunk *ic, size_t chunk_siz i_ins->config); } +static struct flb_output_instance *input_chunk_find_output_reference(struct flb_config *config, + const struct flb_chunk_direct_route *route) +{ + int alias_length; + int label_length; + int name_length; + const char *label; + uint32_t stored_id; + struct mk_list *head; + struct flb_output_instance *o_ins; + + if (!config || !route) { + return NULL; + } + + label = route->label; + label_length = 0; + stored_id = route->id; + if (label != NULL) { + label_length = route->label_length; + if (label_length == 0) { + label_length = (int) strlen(label); + } + } + + if (label != NULL && label_length > 0) { + mk_list_foreach(head, &config->outputs) { + o_ins = mk_list_entry(head, struct flb_output_instance, _head); + if (o_ins->alias != NULL) { + alias_length = (int) strlen(o_ins->alias); + if (alias_length == label_length && + strncmp(o_ins->alias, label, (size_t) label_length) == 0) { + return o_ins; + } + } + } + + mk_list_foreach(head, &config->outputs) { + o_ins = mk_list_entry(head, struct flb_output_instance, _head); + name_length = (int) strlen(o_ins->name); + if (name_length == label_length && + strncmp(o_ins->name, label, (size_t) label_length) == 0) { + return o_ins; + } + } + } + + mk_list_foreach(head, &config->outputs) { + o_ins = mk_list_entry(head, struct flb_output_instance, _head); + if ((uint32_t) o_ins->id == stored_id) { + return o_ins; + } + } + + return NULL; +} + /* Create an input chunk using a Chunk I/O */ struct flb_input_chunk *flb_input_chunk_map(struct flb_input_instance *in, int event_type, @@ -608,12 +775,22 @@ struct flb_input_chunk *flb_input_chunk_map(struct flb_input_instance *in, int tag_len; int has_routes; int ret; + int direct_status; + int direct_loaded; + int direct_index; + int direct_missing; + uint32_t missing_id; + uint16_t missing_label_length; + struct flb_chunk_direct_route *direct_routes; + const char *missing_label; + int direct_count; uint64_t ts; char *buf_data; size_t buf_size; size_t offset; ssize_t bytes; const char *tag_buf; + struct flb_output_instance *direct_output; struct flb_input_chunk *ic; /* Create context for the input instance */ @@ -775,6 +952,64 @@ struct flb_input_chunk *flb_input_chunk_map(struct flb_input_instance *in, return NULL; } + direct_routes = NULL; + direct_count = 0; + direct_loaded = FLB_FALSE; + missing_label = NULL; + missing_label_length = 0; + direct_status = flb_input_chunk_get_direct_routes(ic, &direct_routes, &direct_count); + if (direct_status == 0 && direct_count > 0) { + direct_missing = FLB_FALSE; + missing_id = 0; + for (direct_index = 0; direct_index < direct_count; direct_index++) { + direct_output = input_chunk_find_output_reference(in->config, + &direct_routes[direct_index]); + if (!direct_output) { + direct_missing = FLB_TRUE; + missing_id = direct_routes[direct_index].id; + missing_label = direct_routes[direct_index].label; + missing_label_length = direct_routes[direct_index].label_length; + break; + } + } + + if (direct_missing == FLB_FALSE) { + memset(ic->routes_mask, 0, + sizeof(flb_route_mask_element) * in->config->route_mask_size); + has_routes = 0; + for (direct_index = 0; direct_index < direct_count; direct_index++) { + direct_output = input_chunk_find_output_reference(in->config, + &direct_routes[direct_index]); + if (!direct_output) { + continue; + } + flb_routes_mask_set_bit(ic->routes_mask, + direct_output->id, + in->config); + has_routes++; + } + direct_loaded = FLB_TRUE; + } + else { + flb_plg_warn(in, + "direct route output id=%u label=%.*s not found for chunk %s, falling back to tag routing", + (unsigned int) missing_id, + (int) missing_label_length, + missing_label ? missing_label : "", + flb_input_chunk_get_name(ic)); + } + } + else if (direct_status == -2) { + flb_plg_warn(in, + "invalid direct routing metadata for chunk %s, falling back to tag routing", + flb_input_chunk_get_name(ic)); + } + + if (direct_routes) { + flb_input_chunk_destroy_direct_routes(direct_routes, direct_count); + direct_routes = NULL; + } + bytes = flb_input_chunk_get_real_size(ic); if (bytes < 0) { flb_warn("[input chunk] could not retrieve chunk real size"); @@ -783,10 +1018,22 @@ struct flb_input_chunk *flb_input_chunk_map(struct flb_input_instance *in, return NULL; } - has_routes = flb_routes_mask_set_by_tag(ic->routes_mask, tag_buf, tag_len, in); - if (has_routes == 0) { - flb_warn("[input chunk] no matching route for backoff log chunk %s", - flb_input_chunk_get_name(ic)); + if (direct_loaded == FLB_FALSE) { + has_routes = flb_routes_mask_set_by_tag(ic->routes_mask, tag_buf, tag_len, in); + if (has_routes == 0) { + flb_warn("[input chunk] no matching route for backoff log chunk %s", + flb_input_chunk_get_name(ic)); + } + } + else if (has_routes == 0) { + flb_plg_warn(in, + "direct routing metadata for chunk %s produced no routes, falling back to tag routing", + flb_input_chunk_get_name(ic)); + has_routes = flb_routes_mask_set_by_tag(ic->routes_mask, tag_buf, tag_len, in); + if (has_routes == 0) { + flb_warn("[input chunk] no matching route for backoff log chunk %s", + flb_input_chunk_get_name(ic)); + } } mk_list_add(&ic->_head, &in->chunks); @@ -868,6 +1115,433 @@ static int input_chunk_write_header(struct cio_chunk *chunk, int event_type, return 0; } +int flb_input_chunk_write_header_v2(struct cio_chunk *chunk, + int event_type, + char *tag, int tag_len, + const struct flb_chunk_direct_route *routes, + int route_count) +{ + int has_labels; + int wide_ids; + int index; + int max_tag_len; + int meta_size; + int offset; + int ret; + int id_offset; + int label_offset; + int id_bytes; + uint16_t label_length; + uint16_t routing_length; + uint16_t stored_count; + uint16_t *resolved_lengths; + size_t labels_total; + size_t routing_payload_bytes; + size_t computed_length; + uint8_t flags; + char *meta; + + has_labels = FLB_FALSE; + wide_ids = FLB_FALSE; + index = 0; + max_tag_len = 0; + meta_size = 0; + offset = 0; + ret = 0; + id_offset = 0; + label_offset = 0; + id_bytes = (int) sizeof(uint16_t); + label_length = 0; + routing_length = 0; + stored_count = 0; + resolved_lengths = NULL; + labels_total = 0; + routing_payload_bytes = 0; + computed_length = 0; + flags = 0; + meta = NULL; + + if (!chunk || !tag || !routes || route_count <= 0) { + return -1; + } + + if (route_count > UINT16_MAX) { + return -1; + } + + resolved_lengths = flb_calloc((size_t) route_count, sizeof(uint16_t)); + if (!resolved_lengths) { + flb_errno(); + return -1; + } + + for (index = 0; index < route_count; index++) { + if (routes[index].id > UINT16_MAX) { + wide_ids = FLB_TRUE; + } + label_length = routes[index].label_length; + if (routes[index].label != NULL) { + if (label_length == 0) { + computed_length = strlen(routes[index].label); + if (computed_length > UINT16_MAX) { + computed_length = UINT16_MAX; + } + label_length = (uint16_t) computed_length; + } + else if (label_length > UINT16_MAX) { + label_length = UINT16_MAX; + } + + if (label_length > 0) { + has_labels = FLB_TRUE; + } + } + else { + label_length = 0; + } + + resolved_lengths[index] = label_length; + labels_total += (size_t) label_length; + } + + if (wide_ids == FLB_TRUE) { + id_bytes = (int) sizeof(uint32_t); + } + else { + id_bytes = (int) sizeof(uint16_t); + } + + routing_payload_bytes = sizeof(uint16_t) + + ((size_t) route_count * (size_t) id_bytes); + if (has_labels == FLB_TRUE) { + routing_payload_bytes += ((size_t) route_count * sizeof(uint16_t)) + + labels_total; + } + + if (routing_payload_bytes > UINT16_MAX) { + flb_free(resolved_lengths); + return -1; + } + + routing_length = (uint16_t) routing_payload_bytes; + max_tag_len = 65535 - (int) (FLB_INPUT_CHUNK_META_HEADER + 1 + sizeof(uint16_t) + routing_length); + if (max_tag_len < 0) { + max_tag_len = 0; + } + if (tag_len > max_tag_len) { + tag_len = max_tag_len; + } + + meta_size = FLB_INPUT_CHUNK_META_HEADER + tag_len + 1 + sizeof(uint16_t) + routing_length; + meta = flb_calloc(1, meta_size); + if (!meta) { + flb_errno(); + flb_free(resolved_lengths); + return -1; + } + + meta[0] = FLB_INPUT_CHUNK_MAGIC_BYTE_0; + meta[1] = FLB_INPUT_CHUNK_MAGIC_BYTE_1; + + if (event_type == FLB_INPUT_LOGS) { + meta[2] = FLB_INPUT_CHUNK_TYPE_LOGS; + } + else if (event_type == FLB_INPUT_METRICS) { + meta[2] = FLB_INPUT_CHUNK_TYPE_METRICS; + } + else if (event_type == FLB_INPUT_TRACES) { + meta[2] = FLB_INPUT_CHUNK_TYPE_TRACES; + } + else if (event_type == FLB_INPUT_PROFILES) { + meta[2] = FLB_INPUT_CHUNK_TYPE_PROFILES; + } + + flags = FLB_CHUNK_FLAG_DIRECT_ROUTES; + if (has_labels == FLB_TRUE) { + flags |= FLB_CHUNK_FLAG_DIRECT_ROUTE_LABELS; + } + if (wide_ids == FLB_TRUE) { + flags |= FLB_CHUNK_FLAG_DIRECT_ROUTE_WIDE_IDS; + } + meta[3] = (char) flags; + + memcpy(meta + FLB_INPUT_CHUNK_META_HEADER, tag, tag_len); + meta[FLB_INPUT_CHUNK_META_HEADER + tag_len] = '\0'; + + offset = FLB_INPUT_CHUNK_META_HEADER + tag_len + 1; + meta[offset] = (uint8_t) (routing_length >> 8); + meta[offset + 1] = (uint8_t) (routing_length & 0xFF); + + stored_count = (uint16_t) route_count; + meta[offset + 2] = (uint8_t) (stored_count >> 8); + meta[offset + 3] = (uint8_t) (stored_count & 0xFF); + + id_offset = offset + 4; + for (index = 0; index < route_count; index++) { + if (wide_ids == FLB_TRUE) { + meta[id_offset] = (uint8_t) ((routes[index].id >> 24) & 0xFF); + meta[id_offset + 1] = (uint8_t) ((routes[index].id >> 16) & 0xFF); + meta[id_offset + 2] = (uint8_t) ((routes[index].id >> 8) & 0xFF); + meta[id_offset + 3] = (uint8_t) (routes[index].id & 0xFF); + } + else { + meta[id_offset] = (uint8_t) ((routes[index].id >> 8) & 0xFF); + meta[id_offset + 1] = (uint8_t) (routes[index].id & 0xFF); + } + id_offset += id_bytes; + } + + if (has_labels == FLB_TRUE) { + label_offset = offset + 4 + (route_count * id_bytes); + for (index = 0; index < route_count; index++) { + label_length = resolved_lengths[index]; + meta[label_offset] = (uint8_t) (label_length >> 8); + meta[label_offset + 1] = (uint8_t) (label_length & 0xFF); + label_offset += sizeof(uint16_t); + } + + for (index = 0; index < route_count; index++) { + label_length = resolved_lengths[index]; + if (label_length > 0 && routes[index].label != NULL) { + memcpy(meta + label_offset, routes[index].label, label_length); + } + label_offset += label_length; + } + } + + ret = cio_meta_write(chunk, (char *) meta, meta_size); + if (ret == -1) { + flb_error("[input chunk] could not write metadata"); + flb_free(resolved_lengths); + flb_free(meta); + return -1; + } + + flb_free(resolved_lengths); + flb_free(meta); + + return 0; +} + +int flb_input_chunk_has_direct_routes(struct flb_input_chunk *ic) +{ + int ret; + struct flb_input_chunk_meta_view view; + + ret = input_chunk_metadata_view(ic, &view); + if (ret == -1) { + return FLB_FALSE; + } + + if ((view.flags & FLB_CHUNK_FLAG_DIRECT_ROUTES) == 0) { + return FLB_FALSE; + } + + if (view.routing_data == NULL) { + return FLB_FALSE; + } + + if (view.routing_data_length < sizeof(uint16_t)) { + return FLB_FALSE; + } + + return FLB_TRUE; +} + +int flb_input_chunk_get_direct_routes(struct flb_input_chunk *ic, + struct flb_chunk_direct_route **routes, + int *route_count) +{ + int index; + int labels_present; + int wide_ids; + int ret; + int id_offset; + int lengths_offset; + int label_data_offset; + int id_bytes; + size_t remaining; + uint16_t routing_length; + uint16_t stored_count; + uint16_t *label_lengths; + struct flb_chunk_direct_route *result; + struct flb_input_chunk_meta_view view; + uint32_t read_id; + + index = 0; + labels_present = FLB_FALSE; + wide_ids = FLB_FALSE; + ret = 0; + id_offset = 0; + lengths_offset = 0; + label_data_offset = 0; + id_bytes = (int) sizeof(uint16_t); + remaining = 0; + routing_length = 0; + stored_count = 0; + label_lengths = NULL; + result = NULL; + read_id = 0; + + if (!routes || !route_count) { + return -1; + } + + *routes = NULL; + *route_count = 0; + + ret = input_chunk_metadata_view(ic, &view); + if (ret == -1) { + return -1; + } + + if ((view.flags & FLB_CHUNK_FLAG_DIRECT_ROUTES) == 0) { + return 0; + } + + if (view.routing_data == NULL || view.routing_data_length < sizeof(uint16_t)) { + return -2; + } + + routing_length = view.routing_data_length; + stored_count = (uint16_t) (((unsigned char) view.routing_data[0] << 8) | + (unsigned char) view.routing_data[1]); + + if (stored_count == 0) { + return 0; + } + + wide_ids = ((view.flags & FLB_CHUNK_FLAG_DIRECT_ROUTE_WIDE_IDS) != 0); + if (wide_ids == FLB_TRUE) { + id_bytes = (int) sizeof(uint32_t); + } + else { + id_bytes = (int) sizeof(uint16_t); + } + + if ((size_t) routing_length < (sizeof(uint16_t) + + ((size_t) stored_count * (size_t) id_bytes))) { + return -2; + } + + labels_present = ((view.flags & FLB_CHUNK_FLAG_DIRECT_ROUTE_LABELS) != 0); + + if (labels_present == FLB_TRUE) { + if ((size_t) routing_length < (sizeof(uint16_t) + + ((size_t) stored_count * (size_t) id_bytes) + + ((size_t) stored_count * sizeof(uint16_t)))) { + return -2; + } + } + + result = flb_calloc((size_t) stored_count, sizeof(struct flb_chunk_direct_route)); + if (!result) { + flb_errno(); + return -1; + } + + id_offset = sizeof(uint16_t); + for (index = 0; index < stored_count; index++) { + if (wide_ids == FLB_TRUE) { + read_id = ((uint32_t) ((unsigned char) view.routing_data[id_offset]) << 24) | + ((uint32_t) ((unsigned char) view.routing_data[id_offset + 1]) << 16) | + ((uint32_t) ((unsigned char) view.routing_data[id_offset + 2]) << 8) | + (uint32_t) ((unsigned char) view.routing_data[id_offset + 3]); + } + else { + read_id = ((uint32_t) ((unsigned char) view.routing_data[id_offset]) << 8) | + (uint32_t) ((unsigned char) view.routing_data[id_offset + 1]); + } + result[index].id = read_id; + result[index].label = NULL; + result[index].label_length = 0; + id_offset += id_bytes; + } + + if (labels_present == FLB_TRUE) { + label_lengths = flb_calloc((size_t) stored_count, sizeof(uint16_t)); + if (!label_lengths) { + flb_errno(); + flb_input_chunk_destroy_direct_routes(result, stored_count); + return -1; + } + + lengths_offset = sizeof(uint16_t) + (stored_count * id_bytes); + for (index = 0; index < stored_count; index++) { + label_lengths[index] = (uint16_t) (((unsigned char) view.routing_data[lengths_offset] << 8) | + (unsigned char) view.routing_data[lengths_offset + 1]); + lengths_offset += sizeof(uint16_t); + } + + label_data_offset = sizeof(uint16_t) + + (stored_count * id_bytes) + + (stored_count * (int) sizeof(uint16_t)); + if ((size_t) label_data_offset > routing_length) { + flb_free(label_lengths); + flb_input_chunk_destroy_direct_routes(result, stored_count); + return -2; + } + remaining = routing_length - (size_t) label_data_offset; + + for (index = 0; index < stored_count; index++) { + if (label_lengths[index] == 0) { + result[index].label = NULL; + result[index].label_length = 0; + continue; + } + + if (label_lengths[index] > remaining) { + flb_free(label_lengths); + flb_input_chunk_destroy_direct_routes(result, stored_count); + return -2; + } + + result[index].label = flb_malloc((size_t) label_lengths[index] + 1); + if (!result[index].label) { + flb_errno(); + flb_free(label_lengths); + flb_input_chunk_destroy_direct_routes(result, stored_count); + return -1; + } + + memcpy((char *) result[index].label, + view.routing_data + label_data_offset, + label_lengths[index]); + ((char *) result[index].label)[label_lengths[index]] = '\0'; + result[index].label_length = label_lengths[index]; + label_data_offset += label_lengths[index]; + remaining -= label_lengths[index]; + } + + flb_free(label_lengths); + } + + *routes = result; + *route_count = stored_count; + + return 0; +} + +void flb_input_chunk_destroy_direct_routes(struct flb_chunk_direct_route *routes, + int route_count) +{ + int index; + + index = 0; + + if (!routes) { + return; + } + + for (index = 0; index < route_count; index++) { + if (routes[index].label != NULL && routes[index].label_length > 0) { + flb_free((void *) routes[index].label); + } + } + + flb_free(routes); +} + struct flb_input_chunk *flb_input_chunk_create(struct flb_input_instance *in, int event_type, const char *tag, int tag_len) { @@ -2090,23 +2764,6 @@ flb_sds_t flb_input_chunk_get_name(struct flb_input_chunk *ic) return ch->name; } -static inline int input_chunk_has_magic_bytes(char *buf, int len) -{ - unsigned char *p; - - if (len < FLB_INPUT_CHUNK_META_HEADER) { - return FLB_FALSE; - } - - p = (unsigned char *) buf; - if (p[0] == FLB_INPUT_CHUNK_MAGIC_BYTE_0 && - p[1] == FLB_INPUT_CHUNK_MAGIC_BYTE_1 && p[3] == 0) { - return FLB_TRUE; - } - - return FLB_FALSE; -} - /* * Get the event type by retrieving metadata header. NOTE: this function only event type discovery by looking at the * headers bytes of a chunk that exists on disk. @@ -2152,29 +2809,20 @@ int flb_input_chunk_get_event_type(struct flb_input_chunk *ic) int flb_input_chunk_get_tag(struct flb_input_chunk *ic, const char **tag_buf, int *tag_len) { - int len; int ret; - char *buf; + struct flb_input_chunk_meta_view view; - ret = cio_meta_read(ic->chunk, &buf, &len); + ret = input_chunk_metadata_view(ic, &view); if (ret == -1) { *tag_len = -1; *tag_buf = NULL; return -1; } - /* If magic bytes exists, just set the offset */ - if (input_chunk_has_magic_bytes(buf, len)) { - *tag_len = len - FLB_INPUT_CHUNK_META_HEADER; - *tag_buf = buf + FLB_INPUT_CHUNK_META_HEADER; - } - else { - /* Old Chunk version without magic bytes */ - *tag_len = len; - *tag_buf = buf; - } + *tag_buf = view.tag; + *tag_len = view.tag_length; - return ret; + return 0; } /* From 2d446b413959a142a109dea847efaeecf28a7626 Mon Sep 17 00:00:00 2001 From: Patrick Stephens Date: Sun, 3 May 2026 12:34:53 +0100 Subject: [PATCH 064/213] [upstream] tests: internal: add new input_chunk_routes.c Upstream-Ref: https://github.com/fluent/fluent-bit/commit/139bd2bd6d761f8c25f124902df5e6041a37d01a Cherry-picked from Fluent Bit v4.2.4 --- source/tests/internal/CMakeLists.txt | 1 + source/tests/internal/input_chunk_routes.c | 194 +++++++++++++++++++++ 2 files changed, 195 insertions(+) create mode 100644 source/tests/internal/input_chunk_routes.c diff --git a/source/tests/internal/CMakeLists.txt b/source/tests/internal/CMakeLists.txt index 3c4296cf..dfe88d87 100644 --- a/source/tests/internal/CMakeLists.txt +++ b/source/tests/internal/CMakeLists.txt @@ -25,6 +25,7 @@ set(UNIT_TESTS_FILES mp.c mp_chunk_cobj.c input_chunk.c + input_chunk_routes.c flb_time.c file.c csv.c diff --git a/source/tests/internal/input_chunk_routes.c b/source/tests/internal/input_chunk_routes.c new file mode 100644 index 00000000..f141526a --- /dev/null +++ b/source/tests/internal/input_chunk_routes.c @@ -0,0 +1,194 @@ +#include +#include +#include +#include +#include +#include + +#include "flb_tests_internal.h" + +#define TEST_STREAM_PATH "/tmp/flb-chunk-direct-test" + +static int write_legacy_chunk_metadata(struct cio_chunk *chunk, + int event_type, + const char *tag, + int tag_len) +{ + int ret; + int meta_size; + char *meta; + + meta_size = FLB_INPUT_CHUNK_META_HEADER + tag_len; + meta = flb_malloc(meta_size); + if (!meta) { + flb_errno(); + return -1; + } + + meta[0] = FLB_INPUT_CHUNK_MAGIC_BYTE_0; + meta[1] = FLB_INPUT_CHUNK_MAGIC_BYTE_1; + + if (event_type == FLB_INPUT_LOGS) { + meta[2] = FLB_INPUT_CHUNK_TYPE_LOGS; + } + else if (event_type == FLB_INPUT_METRICS) { + meta[2] = FLB_INPUT_CHUNK_TYPE_METRICS; + } + else if (event_type == FLB_INPUT_TRACES) { + meta[2] = FLB_INPUT_CHUNK_TYPE_TRACES; + } + else if (event_type == FLB_INPUT_PROFILES) { + meta[2] = FLB_INPUT_CHUNK_TYPE_PROFILES; + } + else { + meta[2] = FLB_INPUT_CHUNK_TYPE_LOGS; + } + + meta[3] = 0; + + memcpy(meta + FLB_INPUT_CHUNK_META_HEADER, tag, tag_len); + + ret = cio_meta_write(chunk, meta, meta_size); + + flb_free(meta); + + return ret; +} + +static void test_chunk_metadata_direct_routes() +{ + struct cio_options opts; + struct cio_ctx *ctx; + struct cio_stream *stream; + struct cio_chunk *chunk; + struct flb_input_chunk ic; + struct flb_chunk_direct_route output_routes[2]; + struct flb_chunk_direct_route *loaded_routes; + char *content_buf; + const char *tag_buf; + const char *tag_string; + const char payload[] = "direct route payload validation string"; + int tag_len; + int route_count; + int ret; + int err; + int expected_tag_len; + size_t content_size; + size_t payload_size; + + payload_size = sizeof(payload) - 1; + tag_string = "test.tag"; + expected_tag_len = strlen(tag_string); + + cio_utils_recursive_delete(TEST_STREAM_PATH); + memset(&opts, 0, sizeof(opts)); + cio_options_init(&opts); + opts.root_path = TEST_STREAM_PATH; + opts.flags = CIO_OPEN; + ctx = cio_create(&opts); + TEST_CHECK(ctx != NULL); + if (!ctx) { + return; + } + + stream = cio_stream_create(ctx, "direct", CIO_STORE_FS); + TEST_CHECK(stream != NULL); + if (!stream) { + cio_destroy(ctx); + return; + } + + chunk = cio_chunk_open(ctx, stream, "meta", CIO_OPEN, 1024, &err); + TEST_CHECK(chunk != NULL); + if (!chunk) { + cio_destroy(ctx); + return; + } + + ret = cio_chunk_is_up(chunk); + if (ret == CIO_FALSE) { + ret = cio_chunk_up_force(chunk); + TEST_CHECK(ret == CIO_OK); + } + + tag_len = expected_tag_len; + ret = write_legacy_chunk_metadata(chunk, FLB_INPUT_LOGS, + tag_string, tag_len); + TEST_CHECK(ret == 0); + + ret = cio_chunk_write(chunk, payload, payload_size); + TEST_CHECK(ret == 0); + + ret = cio_chunk_get_content(chunk, &content_buf, &content_size); + TEST_CHECK(ret == 0); + if (ret == 0) { + TEST_CHECK(content_buf != NULL); + TEST_CHECK(content_size == payload_size); + if (content_size == payload_size) { + TEST_CHECK(memcmp(content_buf, payload, payload_size) == 0); + } + } + + output_routes[0].id = 511; + output_routes[0].label = "alpha"; + output_routes[0].label_length = 5; + output_routes[1].id = 70000; + output_routes[1].label = "beta"; + output_routes[1].label_length = 4; + ret = flb_input_chunk_write_header_v2(chunk, + FLB_INPUT_LOGS, + (char *) tag_string, + tag_len, + output_routes, + 2); + TEST_CHECK(ret == 0); + + memset(&ic, 0, sizeof(ic)); + ic.chunk = chunk; + + TEST_CHECK(flb_input_chunk_has_direct_routes(&ic) == FLB_TRUE); + + ret = cio_chunk_get_content(chunk, &content_buf, &content_size); + TEST_CHECK(ret == 0); + if (ret == 0) { + TEST_CHECK(content_buf != NULL); + TEST_CHECK(content_size == payload_size); + if (content_size == payload_size) { + TEST_CHECK(memcmp(content_buf, payload, payload_size) == 0); + } + } + + ret = flb_input_chunk_get_direct_routes(&ic, &loaded_routes, &route_count); + TEST_CHECK(ret == 0); + TEST_CHECK(route_count == 2); + if (ret == 0 && route_count == 2) { + TEST_CHECK(loaded_routes != NULL); + if (loaded_routes) { + TEST_CHECK(loaded_routes[0].id == 511); + TEST_CHECK(loaded_routes[1].id == 70000); + TEST_CHECK(loaded_routes[0].label != NULL); + TEST_CHECK(loaded_routes[1].label != NULL); + if (loaded_routes[0].label && loaded_routes[1].label) { + TEST_CHECK(strcmp(loaded_routes[0].label, "alpha") == 0); + TEST_CHECK(strcmp(loaded_routes[1].label, "beta") == 0); + } + flb_input_chunk_destroy_direct_routes(loaded_routes, route_count); + } + } + + ret = flb_input_chunk_get_tag(&ic, &tag_buf, &tag_len); + TEST_CHECK(ret == 0); + TEST_CHECK(tag_len == expected_tag_len); + if (ret == 0 && tag_len == expected_tag_len) { + TEST_CHECK(memcmp(tag_buf, tag_string, expected_tag_len) == 0); + } + + cio_chunk_close(chunk, CIO_TRUE); + cio_destroy(ctx); + cio_utils_recursive_delete(TEST_STREAM_PATH); +} + +TEST_LIST = { + { "chunk_metadata_direct_routes", test_chunk_metadata_direct_routes }, + { 0 } +}; From 0c0bff279eddd9adaf7cd06536fdaecfb9961a35 Mon Sep 17 00:00:00 2001 From: Patrick Stephens Date: Sun, 3 May 2026 12:34:53 +0100 Subject: [PATCH 065/213] [upstream] input_chunk: add direct route labels and plugin name Upstream-Ref: https://github.com/fluent/fluent-bit/commit/c701590b454d2fe8b201e90be560e910bcf576cb Cherry-picked from Fluent Bit v4.2.4 --- source/include/fluent-bit/flb_input_chunk.h | 11 + source/src/flb_input_chunk.c | 470 ++++++++++++++++++-- 2 files changed, 437 insertions(+), 44 deletions(-) diff --git a/source/include/fluent-bit/flb_input_chunk.h b/source/include/fluent-bit/flb_input_chunk.h index adb0bd14..21b6b768 100644 --- a/source/include/fluent-bit/flb_input_chunk.h +++ b/source/include/fluent-bit/flb_input_chunk.h @@ -49,11 +49,20 @@ struct cio_chunk; #define FLB_CHUNK_FLAG_DIRECT_ROUTES (1 << 0) #define FLB_CHUNK_FLAG_DIRECT_ROUTE_LABELS (1 << 1) #define FLB_CHUNK_FLAG_DIRECT_ROUTE_WIDE_IDS (1 << 2) +#define FLB_CHUNK_FLAG_DIRECT_ROUTE_PLUGIN_IDS (1 << 3) + +#define FLB_CHUNK_DIRECT_ROUTE_LABEL_ALIAS_FLAG 0x8000 +#define FLB_CHUNK_DIRECT_ROUTE_LABEL_LENGTH_MASK 0x7FFF + +struct flb_output_instance; struct flb_chunk_direct_route { uint32_t id; uint16_t label_length; const char *label; + uint8_t label_is_alias; + uint16_t plugin_name_length; + const char *plugin_name; }; /* Chunks magic bytes (starting from Fluent Bit v1.8.10) */ @@ -129,6 +138,8 @@ int flb_input_chunk_write_header_v2(struct cio_chunk *chunk, char *tag, int tag_len, const struct flb_chunk_direct_route *routes, int route_count); +int flb_chunk_route_plugin_matches(struct flb_output_instance *o_ins, + const struct flb_chunk_direct_route *route); int flb_input_chunk_has_direct_routes(struct flb_input_chunk *ic); int flb_input_chunk_get_direct_routes(struct flb_input_chunk *ic, struct flb_chunk_direct_route **routes, diff --git a/source/src/flb_input_chunk.c b/source/src/flb_input_chunk.c index db999c5f..61d6b950 100644 --- a/source/src/flb_input_chunk.c +++ b/source/src/flb_input_chunk.c @@ -167,6 +167,47 @@ static int input_chunk_metadata_view(struct flb_input_chunk *ic, return 0; } +#define ROUTE_PLUGIN_NAME_LEN(route) \ + ((route)->plugin_name_length > 0 ? (int) (route)->plugin_name_length : \ + (route)->plugin_name != NULL ? (int) strlen((route)->plugin_name) : 0) + +int flb_chunk_route_plugin_matches(struct flb_output_instance *o_ins, + const struct flb_chunk_direct_route *route) +{ + int stored_length; + int candidate_length; + + if (!route) { + return FLB_FALSE; + } + + if (!route->plugin_name || route->plugin_name[0] == '\0') { + return FLB_TRUE; + } + + if (!o_ins || !o_ins->p || !o_ins->p->name) { + return FLB_FALSE; + } + + stored_length = ROUTE_PLUGIN_NAME_LEN(route); + if (stored_length <= 0) { + return FLB_FALSE; + } + + candidate_length = (int) strlen(o_ins->p->name); + if (candidate_length != stored_length) { + return FLB_FALSE; + } + + if (strncmp(o_ins->p->name, route->plugin_name, (size_t) stored_length) != 0) { + return FLB_FALSE; + } + + return FLB_TRUE; +} + +#undef ROUTE_PLUGIN_NAME_LEN + #ifdef FLB_HAVE_IN_STORAGE_BACKLOG extern ssize_t sb_get_releasable_output_queue_space(struct flb_output_instance *output_plugin, @@ -709,9 +750,13 @@ int flb_input_chunk_place_new_chunk(struct flb_input_chunk *ic, size_t chunk_siz i_ins->config); } -static struct flb_output_instance *input_chunk_find_output_reference(struct flb_config *config, - const struct flb_chunk_direct_route *route) +static int input_chunk_collect_output_references(struct flb_config *config, + const struct flb_chunk_direct_route *route, + struct flb_output_instance ***out_matches, + size_t *out_count) { + size_t index; + size_t count; int alias_length; int label_length; int name_length; @@ -719,11 +764,15 @@ static struct flb_output_instance *input_chunk_find_output_reference(struct flb_ uint32_t stored_id; struct mk_list *head; struct flb_output_instance *o_ins; + struct flb_output_instance **matches; - if (!config || !route) { - return NULL; + if (!config || !route || !out_matches || !out_count) { + return -1; } + *out_matches = NULL; + *out_count = 0; + label = route->label; label_length = 0; stored_id = route->id; @@ -734,14 +783,16 @@ static struct flb_output_instance *input_chunk_find_output_reference(struct flb_ } } + count = 0; if (label != NULL && label_length > 0) { mk_list_foreach(head, &config->outputs) { o_ins = mk_list_entry(head, struct flb_output_instance, _head); if (o_ins->alias != NULL) { alias_length = (int) strlen(o_ins->alias); if (alias_length == label_length && - strncmp(o_ins->alias, label, (size_t) label_length) == 0) { - return o_ins; + strncmp(o_ins->alias, label, (size_t) label_length) == 0 && + flb_chunk_route_plugin_matches(o_ins, route) == FLB_TRUE) { + count++; } } } @@ -750,20 +801,88 @@ static struct flb_output_instance *input_chunk_find_output_reference(struct flb_ o_ins = mk_list_entry(head, struct flb_output_instance, _head); name_length = (int) strlen(o_ins->name); if (name_length == label_length && - strncmp(o_ins->name, label, (size_t) label_length) == 0) { - return o_ins; + strncmp(o_ins->name, label, (size_t) label_length) == 0 && + flb_chunk_route_plugin_matches(o_ins, route) == FLB_TRUE) { + if (o_ins->alias != NULL) { + alias_length = (int) strlen(o_ins->alias); + if (alias_length == label_length && + strncmp(o_ins->alias, label, (size_t) label_length) == 0) { + continue; + } + } + count++; + } + } + + if (count == 0) { + return 0; + } + } + else { + mk_list_foreach(head, &config->outputs) { + o_ins = mk_list_entry(head, struct flb_output_instance, _head); + if ((uint32_t) o_ins->id == stored_id && + flb_chunk_route_plugin_matches(o_ins, route) == FLB_TRUE) { + count++; } } + + if (count == 0) { + return 0; + } } - mk_list_foreach(head, &config->outputs) { - o_ins = mk_list_entry(head, struct flb_output_instance, _head); - if ((uint32_t) o_ins->id == stored_id) { - return o_ins; + matches = flb_calloc(count, sizeof(struct flb_output_instance *)); + if (!matches) { + flb_errno(); + return -1; + } + + index = 0; + if (label != NULL && label_length > 0) { + mk_list_foreach(head, &config->outputs) { + o_ins = mk_list_entry(head, struct flb_output_instance, _head); + if (o_ins->alias != NULL) { + alias_length = (int) strlen(o_ins->alias); + if (alias_length == label_length && + strncmp(o_ins->alias, label, (size_t) label_length) == 0 && + flb_chunk_route_plugin_matches(o_ins, route) == FLB_TRUE) { + matches[index++] = o_ins; + } + } + } + + mk_list_foreach(head, &config->outputs) { + o_ins = mk_list_entry(head, struct flb_output_instance, _head); + name_length = (int) strlen(o_ins->name); + if (name_length == label_length && + strncmp(o_ins->name, label, (size_t) label_length) == 0 && + flb_chunk_route_plugin_matches(o_ins, route) == FLB_TRUE) { + if (o_ins->alias != NULL) { + alias_length = (int) strlen(o_ins->alias); + if (alias_length == label_length && + strncmp(o_ins->alias, label, (size_t) label_length) == 0) { + continue; + } + } + matches[index++] = o_ins; + } + } + } + else { + mk_list_foreach(head, &config->outputs) { + o_ins = mk_list_entry(head, struct flb_output_instance, _head); + if ((uint32_t) o_ins->id == stored_id && + flb_chunk_route_plugin_matches(o_ins, route) == FLB_TRUE) { + matches[index++] = o_ins; + } } } - return NULL; + *out_matches = matches; + *out_count = index; + + return 0; } /* Create an input chunk using a Chunk I/O */ @@ -771,7 +890,7 @@ struct flb_input_chunk *flb_input_chunk_map(struct flb_input_instance *in, int event_type, void *chunk) { - int records = 0; + int records; int tag_len; int has_routes; int ret; @@ -790,9 +909,13 @@ struct flb_input_chunk *flb_input_chunk_map(struct flb_input_instance *in, size_t offset; ssize_t bytes; const char *tag_buf; - struct flb_output_instance *direct_output; + struct flb_output_instance **direct_matches; + size_t direct_match_count; + size_t direct_match_index; struct flb_input_chunk *ic; + records = 0; + /* Create context for the input instance */ ic = flb_calloc(1, sizeof(struct flb_input_chunk)); if (!ic) { @@ -962,15 +1085,35 @@ struct flb_input_chunk *flb_input_chunk_map(struct flb_input_instance *in, direct_missing = FLB_FALSE; missing_id = 0; for (direct_index = 0; direct_index < direct_count; direct_index++) { - direct_output = input_chunk_find_output_reference(in->config, - &direct_routes[direct_index]); - if (!direct_output) { + direct_matches = NULL; + direct_match_count = 0; + ret = input_chunk_collect_output_references(in->config, + &direct_routes[direct_index], + &direct_matches, + &direct_match_count); + if (ret == -1) { + flb_plg_error(in, + "failed collecting restored routes for chunk %s", + flb_input_chunk_get_name(ic)); + } + + if (ret != 0 || direct_match_count == 0) { direct_missing = FLB_TRUE; missing_id = direct_routes[direct_index].id; missing_label = direct_routes[direct_index].label; missing_label_length = direct_routes[direct_index].label_length; + if (missing_label_length == 0 && missing_label != NULL) { + missing_label_length = (uint16_t) strlen(missing_label); + } + if (direct_matches != NULL) { + flb_free(direct_matches); + } break; } + + if (direct_matches != NULL) { + flb_free(direct_matches); + } } if (direct_missing == FLB_FALSE) { @@ -978,15 +1121,29 @@ struct flb_input_chunk *flb_input_chunk_map(struct flb_input_instance *in, sizeof(flb_route_mask_element) * in->config->route_mask_size); has_routes = 0; for (direct_index = 0; direct_index < direct_count; direct_index++) { - direct_output = input_chunk_find_output_reference(in->config, - &direct_routes[direct_index]); - if (!direct_output) { + direct_matches = NULL; + direct_match_count = 0; + ret = input_chunk_collect_output_references(in->config, + &direct_routes[direct_index], + &direct_matches, + &direct_match_count); + if (ret != 0 || direct_match_count == 0 || direct_matches == NULL) { + if (direct_matches != NULL) { + flb_free(direct_matches); + } continue; } - flb_routes_mask_set_bit(ic->routes_mask, - direct_output->id, - in->config); - has_routes++; + + for (direct_match_index = 0; + direct_match_index < direct_match_count; + direct_match_index++) { + flb_routes_mask_set_bit(ic->routes_mask, + direct_matches[direct_match_index]->id, + in->config); + has_routes++; + } + + flb_free(direct_matches); } direct_loaded = FLB_TRUE; } @@ -1122,6 +1279,7 @@ int flb_input_chunk_write_header_v2(struct cio_chunk *chunk, int route_count) { int has_labels; + int has_plugins; int wide_ids; int index; int max_tag_len; @@ -1130,18 +1288,25 @@ int flb_input_chunk_write_header_v2(struct cio_chunk *chunk, int ret; int id_offset; int label_offset; + int plugin_lengths_offset; + int plugin_data_offset; int id_bytes; uint16_t label_length; + uint16_t plugin_length; uint16_t routing_length; uint16_t stored_count; uint16_t *resolved_lengths; + uint16_t *resolved_plugin_lengths; + uint16_t stored_label_length; size_t labels_total; + size_t plugins_total; size_t routing_payload_bytes; size_t computed_length; uint8_t flags; char *meta; has_labels = FLB_FALSE; + has_plugins = FLB_FALSE; wide_ids = FLB_FALSE; index = 0; max_tag_len = 0; @@ -1150,12 +1315,17 @@ int flb_input_chunk_write_header_v2(struct cio_chunk *chunk, ret = 0; id_offset = 0; label_offset = 0; + plugin_lengths_offset = 0; + plugin_data_offset = 0; id_bytes = (int) sizeof(uint16_t); label_length = 0; + plugin_length = 0; routing_length = 0; stored_count = 0; resolved_lengths = NULL; + resolved_plugin_lengths = NULL; labels_total = 0; + plugins_total = 0; routing_payload_bytes = 0; computed_length = 0; flags = 0; @@ -1175,11 +1345,19 @@ int flb_input_chunk_write_header_v2(struct cio_chunk *chunk, return -1; } + resolved_plugin_lengths = flb_calloc((size_t) route_count, sizeof(uint16_t)); + if (!resolved_plugin_lengths) { + flb_errno(); + flb_free(resolved_lengths); + return -1; + } + for (index = 0; index < route_count; index++) { if (routes[index].id > UINT16_MAX) { wide_ids = FLB_TRUE; } label_length = routes[index].label_length; + plugin_length = routes[index].plugin_name_length; if (routes[index].label != NULL) { if (label_length == 0) { computed_length = strlen(routes[index].label); @@ -1192,6 +1370,11 @@ int flb_input_chunk_write_header_v2(struct cio_chunk *chunk, label_length = UINT16_MAX; } + if (routes[index].label_is_alias != 0 && + label_length > FLB_CHUNK_DIRECT_ROUTE_LABEL_LENGTH_MASK) { + label_length = FLB_CHUNK_DIRECT_ROUTE_LABEL_LENGTH_MASK; + } + if (label_length > 0) { has_labels = FLB_TRUE; } @@ -1202,6 +1385,29 @@ int flb_input_chunk_write_header_v2(struct cio_chunk *chunk, resolved_lengths[index] = label_length; labels_total += (size_t) label_length; + + if (routes[index].plugin_name != NULL) { + if (plugin_length == 0) { + computed_length = strlen(routes[index].plugin_name); + if (computed_length > UINT16_MAX) { + computed_length = UINT16_MAX; + } + plugin_length = (uint16_t) computed_length; + } + else if (plugin_length > UINT16_MAX) { + plugin_length = UINT16_MAX; + } + + if (plugin_length > 0) { + has_plugins = FLB_TRUE; + } + } + else { + plugin_length = 0; + } + + resolved_plugin_lengths[index] = plugin_length; + plugins_total += (size_t) plugin_length; } if (wide_ids == FLB_TRUE) { @@ -1217,9 +1423,14 @@ int flb_input_chunk_write_header_v2(struct cio_chunk *chunk, routing_payload_bytes += ((size_t) route_count * sizeof(uint16_t)) + labels_total; } + if (has_plugins == FLB_TRUE) { + routing_payload_bytes += ((size_t) route_count * sizeof(uint16_t)) + + plugins_total; + } if (routing_payload_bytes > UINT16_MAX) { flb_free(resolved_lengths); + flb_free(resolved_plugin_lengths); return -1; } @@ -1237,6 +1448,7 @@ int flb_input_chunk_write_header_v2(struct cio_chunk *chunk, if (!meta) { flb_errno(); flb_free(resolved_lengths); + flb_free(resolved_plugin_lengths); return -1; } @@ -1263,6 +1475,9 @@ int flb_input_chunk_write_header_v2(struct cio_chunk *chunk, if (wide_ids == FLB_TRUE) { flags |= FLB_CHUNK_FLAG_DIRECT_ROUTE_WIDE_IDS; } + if (has_plugins == FLB_TRUE) { + flags |= FLB_CHUNK_FLAG_DIRECT_ROUTE_PLUGIN_IDS; + } meta[3] = (char) flags; memcpy(meta + FLB_INPUT_CHUNK_META_HEADER, tag, tag_len); @@ -1291,12 +1506,19 @@ int flb_input_chunk_write_header_v2(struct cio_chunk *chunk, id_offset += id_bytes; } + label_offset = offset + 4 + (route_count * id_bytes); + if (has_labels == FLB_TRUE) { - label_offset = offset + 4 + (route_count * id_bytes); for (index = 0; index < route_count; index++) { - label_length = resolved_lengths[index]; - meta[label_offset] = (uint8_t) (label_length >> 8); - meta[label_offset + 1] = (uint8_t) (label_length & 0xFF); + stored_label_length = resolved_lengths[index]; + if (routes[index].label_is_alias != 0 && stored_label_length > 0) { + if (stored_label_length > FLB_CHUNK_DIRECT_ROUTE_LABEL_LENGTH_MASK) { + stored_label_length = FLB_CHUNK_DIRECT_ROUTE_LABEL_LENGTH_MASK; + } + stored_label_length |= FLB_CHUNK_DIRECT_ROUTE_LABEL_ALIAS_FLAG; + } + meta[label_offset] = (uint8_t) (stored_label_length >> 8); + meta[label_offset + 1] = (uint8_t) (stored_label_length & 0xFF); label_offset += sizeof(uint16_t); } @@ -1309,15 +1531,39 @@ int flb_input_chunk_write_header_v2(struct cio_chunk *chunk, } } + plugin_lengths_offset = label_offset; + + if (has_plugins == FLB_TRUE) { + for (index = 0; index < route_count; index++) { + plugin_length = resolved_plugin_lengths[index]; + meta[plugin_lengths_offset] = (uint8_t) (plugin_length >> 8); + meta[plugin_lengths_offset + 1] = (uint8_t) (plugin_length & 0xFF); + plugin_lengths_offset += sizeof(uint16_t); + } + + plugin_data_offset = plugin_lengths_offset; + for (index = 0; index < route_count; index++) { + plugin_length = resolved_plugin_lengths[index]; + if (plugin_length > 0 && routes[index].plugin_name != NULL) { + memcpy(meta + plugin_data_offset, + routes[index].plugin_name, + plugin_length); + } + plugin_data_offset += plugin_length; + } + } + ret = cio_meta_write(chunk, (char *) meta, meta_size); if (ret == -1) { flb_error("[input chunk] could not write metadata"); flb_free(resolved_lengths); + flb_free(resolved_plugin_lengths); flb_free(meta); return -1; } flb_free(resolved_lengths); + flb_free(resolved_plugin_lengths); flb_free(meta); return 0; @@ -1354,32 +1600,44 @@ int flb_input_chunk_get_direct_routes(struct flb_input_chunk *ic, { int index; int labels_present; + int plugins_present; int wide_ids; int ret; int id_offset; int lengths_offset; int label_data_offset; + int plugin_lengths_offset; + int plugin_data_offset; int id_bytes; size_t remaining; + size_t plugin_remaining; uint16_t routing_length; uint16_t stored_count; uint16_t *label_lengths; + uint8_t *label_alias_flags; + uint16_t *plugin_lengths; struct flb_chunk_direct_route *result; struct flb_input_chunk_meta_view view; uint32_t read_id; index = 0; labels_present = FLB_FALSE; + plugins_present = FLB_FALSE; wide_ids = FLB_FALSE; ret = 0; id_offset = 0; lengths_offset = 0; label_data_offset = 0; + plugin_lengths_offset = 0; + plugin_data_offset = 0; id_bytes = (int) sizeof(uint16_t); remaining = 0; + plugin_remaining = 0; routing_length = 0; stored_count = 0; label_lengths = NULL; + label_alias_flags = NULL; + plugin_lengths = NULL; result = NULL; read_id = 0; @@ -1425,14 +1683,7 @@ int flb_input_chunk_get_direct_routes(struct flb_input_chunk *ic, } labels_present = ((view.flags & FLB_CHUNK_FLAG_DIRECT_ROUTE_LABELS) != 0); - - if (labels_present == FLB_TRUE) { - if ((size_t) routing_length < (sizeof(uint16_t) + - ((size_t) stored_count * (size_t) id_bytes) + - ((size_t) stored_count * sizeof(uint16_t)))) { - return -2; - } - } + plugins_present = ((view.flags & FLB_CHUNK_FLAG_DIRECT_ROUTE_PLUGIN_IDS) != 0); result = flb_calloc((size_t) stored_count, sizeof(struct flb_chunk_direct_route)); if (!result) { @@ -1455,9 +1706,19 @@ int flb_input_chunk_get_direct_routes(struct flb_input_chunk *ic, result[index].id = read_id; result[index].label = NULL; result[index].label_length = 0; + result[index].plugin_name = NULL; + result[index].plugin_name_length = 0; id_offset += id_bytes; } + lengths_offset = sizeof(uint16_t) + (stored_count * id_bytes); + if ((size_t) lengths_offset > routing_length) { + flb_input_chunk_destroy_direct_routes(result, stored_count); + return -2; + } + + label_data_offset = lengths_offset; + if (labels_present == FLB_TRUE) { label_lengths = flb_calloc((size_t) stored_count, sizeof(uint16_t)); if (!label_lengths) { @@ -1466,31 +1727,69 @@ int flb_input_chunk_get_direct_routes(struct flb_input_chunk *ic, return -1; } - lengths_offset = sizeof(uint16_t) + (stored_count * id_bytes); + label_alias_flags = flb_calloc((size_t) stored_count, sizeof(uint8_t)); + if (!label_alias_flags) { + flb_errno(); + flb_free(label_lengths); + flb_input_chunk_destroy_direct_routes(result, stored_count); + return -1; + } + for (index = 0; index < stored_count; index++) { - label_lengths[index] = (uint16_t) (((unsigned char) view.routing_data[lengths_offset] << 8) | - (unsigned char) view.routing_data[lengths_offset + 1]); - lengths_offset += sizeof(uint16_t); + if ((size_t) (label_data_offset + (int) sizeof(uint16_t)) > routing_length) { + flb_free(label_alias_flags); + flb_free(label_lengths); + flb_input_chunk_destroy_direct_routes(result, stored_count); + return -2; + } + label_lengths[index] = (uint16_t) (((unsigned char) view.routing_data[label_data_offset] << 8) | + (unsigned char) view.routing_data[label_data_offset + 1]); + if (plugins_present == FLB_TRUE) { + if ((label_lengths[index] & FLB_CHUNK_DIRECT_ROUTE_LABEL_ALIAS_FLAG) != 0) { + label_alias_flags[index] = FLB_TRUE; + label_lengths[index] &= FLB_CHUNK_DIRECT_ROUTE_LABEL_LENGTH_MASK; + } + else { + label_alias_flags[index] = FLB_FALSE; + } + } + else { + /* Even when plugins_present is FALSE, the alias flag may still be encoded */ + if ((label_lengths[index] & FLB_CHUNK_DIRECT_ROUTE_LABEL_ALIAS_FLAG) != 0) { + label_alias_flags[index] = FLB_TRUE; + label_lengths[index] &= FLB_CHUNK_DIRECT_ROUTE_LABEL_LENGTH_MASK; + } + else { + label_alias_flags[index] = FLB_TRUE; + } + } + label_data_offset += sizeof(uint16_t); } - label_data_offset = sizeof(uint16_t) + - (stored_count * id_bytes) + - (stored_count * (int) sizeof(uint16_t)); if ((size_t) label_data_offset > routing_length) { + flb_free(label_alias_flags); flb_free(label_lengths); flb_input_chunk_destroy_direct_routes(result, stored_count); return -2; } + remaining = routing_length - (size_t) label_data_offset; for (index = 0; index < stored_count; index++) { if (label_lengths[index] == 0) { result[index].label = NULL; result[index].label_length = 0; + if (label_alias_flags) { + result[index].label_is_alias = label_alias_flags[index]; + } + else if (plugins_present == FLB_FALSE) { + result[index].label_is_alias = FLB_TRUE; + } continue; } if (label_lengths[index] > remaining) { + flb_free(label_alias_flags); flb_free(label_lengths); flb_input_chunk_destroy_direct_routes(result, stored_count); return -2; @@ -1499,6 +1798,7 @@ int flb_input_chunk_get_direct_routes(struct flb_input_chunk *ic, result[index].label = flb_malloc((size_t) label_lengths[index] + 1); if (!result[index].label) { flb_errno(); + flb_free(label_alias_flags); flb_free(label_lengths); flb_input_chunk_destroy_direct_routes(result, stored_count); return -1; @@ -1509,12 +1809,91 @@ int flb_input_chunk_get_direct_routes(struct flb_input_chunk *ic, label_lengths[index]); ((char *) result[index].label)[label_lengths[index]] = '\0'; result[index].label_length = label_lengths[index]; + if (label_alias_flags) { + result[index].label_is_alias = label_alias_flags[index]; + } + else if (plugins_present == FLB_FALSE) { + result[index].label_is_alias = FLB_TRUE; + } label_data_offset += label_lengths[index]; remaining -= label_lengths[index]; } + flb_free(label_alias_flags); flb_free(label_lengths); } + else { + remaining = routing_length - (size_t) label_data_offset; + } + + plugin_lengths_offset = label_data_offset; + + if (plugins_present == FLB_TRUE) { + plugin_lengths = flb_calloc((size_t) stored_count, sizeof(uint16_t)); + if (!plugin_lengths) { + flb_errno(); + flb_input_chunk_destroy_direct_routes(result, stored_count); + return -1; + } + + for (index = 0; index < stored_count; index++) { + if ((size_t) (plugin_lengths_offset + (int) sizeof(uint16_t)) > routing_length) { + flb_free(plugin_lengths); + flb_input_chunk_destroy_direct_routes(result, stored_count); + return -2; + } + plugin_lengths[index] = (uint16_t) (((unsigned char) view.routing_data[plugin_lengths_offset] << 8) | + (unsigned char) view.routing_data[plugin_lengths_offset + 1]); + plugin_lengths_offset += sizeof(uint16_t); + } + + plugin_data_offset = plugin_lengths_offset; + if ((size_t) plugin_data_offset > routing_length) { + flb_free(plugin_lengths); + flb_input_chunk_destroy_direct_routes(result, stored_count); + return -2; + } + + plugin_remaining = routing_length - (size_t) plugin_data_offset; + + for (index = 0; index < stored_count; index++) { + if (plugin_lengths[index] == 0) { + result[index].plugin_name = NULL; + result[index].plugin_name_length = 0; + continue; + } + + if (plugin_lengths[index] > plugin_remaining) { + flb_free(plugin_lengths); + flb_input_chunk_destroy_direct_routes(result, stored_count); + return -2; + } + + result[index].plugin_name = flb_malloc((size_t) plugin_lengths[index] + 1); + if (!result[index].plugin_name) { + flb_errno(); + flb_free(plugin_lengths); + flb_input_chunk_destroy_direct_routes(result, stored_count); + return -1; + } + + memcpy((char *) result[index].plugin_name, + view.routing_data + plugin_data_offset, + plugin_lengths[index]); + ((char *) result[index].plugin_name)[plugin_lengths[index]] = '\0'; + result[index].plugin_name_length = plugin_lengths[index]; + plugin_data_offset += plugin_lengths[index]; + plugin_remaining -= plugin_lengths[index]; + } + + flb_free(plugin_lengths); + } + else { + for (index = 0; index < stored_count; index++) { + result[index].plugin_name = NULL; + result[index].plugin_name_length = 0; + } + } *routes = result; *route_count = stored_count; @@ -1537,6 +1916,9 @@ void flb_input_chunk_destroy_direct_routes(struct flb_chunk_direct_route *routes if (routes[index].label != NULL && routes[index].label_length > 0) { flb_free((void *) routes[index].label); } + if (routes[index].plugin_name != NULL) { + flb_free((void *) routes[index].plugin_name); + } } flb_free(routes); From 67dcd15b677c8d3421abb3d7781d4ee7ba197955 Mon Sep 17 00:00:00 2001 From: Patrick Stephens Date: Sun, 3 May 2026 12:34:53 +0100 Subject: [PATCH 066/213] [upstream] tests: internal: add direct route persistence tests Upstream-Ref: https://github.com/fluent/fluent-bit/commit/babbf92353366029121008e034e242f8e098bd8b Cherry-picked from Fluent Bit v4.2.4 --- source/tests/internal/input_chunk_routes.c | 641 +++++++++++++++++++++ 1 file changed, 641 insertions(+) diff --git a/source/tests/internal/input_chunk_routes.c b/source/tests/internal/input_chunk_routes.c index f141526a..6b32044c 100644 --- a/source/tests/internal/input_chunk_routes.c +++ b/source/tests/internal/input_chunk_routes.c @@ -1,6 +1,16 @@ +#include #include #include +#include +#include #include +#include +#include +#include +#include +#include +#include +#include #include #include #include @@ -8,6 +18,157 @@ #include "flb_tests_internal.h" #define TEST_STREAM_PATH "/tmp/flb-chunk-direct-test" +#define TEST_STREAM_PATH_MATCH "/tmp/flb-chunk-direct-test-match" +#define TEST_STREAM_PATH_NULL "/tmp/flb-chunk-direct-test-null" + +static int write_test_log_payload(struct cio_chunk *chunk) +{ + msgpack_sbuffer sbuf; + msgpack_packer pck; + int ret; + + msgpack_sbuffer_init(&sbuf); + msgpack_packer_init(&pck, &sbuf, msgpack_sbuffer_write); + + /* + * Compose a single Fluent Bit log record: [timestamp, map] + * Using a simple positive integer timestamp keeps validation minimal. + */ + msgpack_pack_array(&pck, 2); + msgpack_pack_uint64(&pck, 0); + msgpack_pack_map(&pck, 1); + msgpack_pack_str(&pck, 3); + msgpack_pack_str_body(&pck, "key", 3); + msgpack_pack_str(&pck, 5); + msgpack_pack_str_body(&pck, "value", 5); + + ret = cio_chunk_write(chunk, sbuf.data, sbuf.size); + msgpack_sbuffer_destroy(&sbuf); + + return ret; +} + +static int init_test_config(struct flb_config *config, + struct flb_input_instance *in, + struct flb_input_plugin *plugin) +{ + int ret; + + memset(config, 0, sizeof(*config)); + mk_list_init(&config->outputs); + mk_list_init(&config->inputs); + + /* Initialize environment (required by flb_input_instance_init) */ + config->env = flb_env_create(); + if (config->env == NULL) { + return -1; + } + + ret = flb_routes_mask_set_size(64, config); + if (ret != 0) { + flb_env_destroy(config->env); + config->env = NULL; + return -1; + } + + memset(in, 0, sizeof(*in)); + in->config = config; + in->p = plugin; + in->log_level = FLB_LOG_OFF; + snprintf(in->name, sizeof(in->name), "dummy.0"); + in->routable = FLB_TRUE; + mk_list_init(&in->_head); + mk_list_init(&in->chunks); + mk_list_init(&in->chunks_up); + mk_list_init(&in->chunks_down); + mk_list_init(&in->tasks); + mk_list_init(&in->collectors); + cfl_list_init(&in->routes_direct); + cfl_list_init(&in->routes); + + /* Add instance to config inputs list (required by flb_input_instance_destroy) */ + mk_list_add(&in->_head, &config->inputs); + + /* Initialize properties list (required by flb_input_instance_init) */ + mk_list_init(&in->properties); + mk_list_init(&in->net_properties); + + /* Initialize hash tables for chunks (required by flb_input_chunk_destroy) */ + in->ht_log_chunks = flb_hash_table_create(FLB_HASH_TABLE_EVICT_NONE, 512, 0); + if (!in->ht_log_chunks) { + return -1; + } + + in->ht_metric_chunks = flb_hash_table_create(FLB_HASH_TABLE_EVICT_NONE, 512, 0); + if (!in->ht_metric_chunks) { + flb_hash_table_destroy(in->ht_log_chunks); + in->ht_log_chunks = NULL; + return -1; + } + + in->ht_trace_chunks = flb_hash_table_create(FLB_HASH_TABLE_EVICT_NONE, 512, 0); + if (!in->ht_trace_chunks) { + flb_hash_table_destroy(in->ht_log_chunks); + flb_hash_table_destroy(in->ht_metric_chunks); + in->ht_log_chunks = NULL; + in->ht_metric_chunks = NULL; + return -1; + } + + in->ht_profile_chunks = flb_hash_table_create(FLB_HASH_TABLE_EVICT_NONE, 512, 0); + if (!in->ht_profile_chunks) { + flb_hash_table_destroy(in->ht_log_chunks); + flb_hash_table_destroy(in->ht_metric_chunks); + flb_hash_table_destroy(in->ht_trace_chunks); + in->ht_log_chunks = NULL; + in->ht_metric_chunks = NULL; + in->ht_trace_chunks = NULL; + return -1; + } + + return 0; +} + +static int add_test_output(struct flb_config *config, + struct flb_output_instance *out, + struct flb_output_plugin *plugin, + int id, + const char *alias) +{ + memset(out, 0, sizeof(*out)); + out->config = config; + out->p = plugin; + out->log_level = FLB_LOG_OFF; + out->id = id; + snprintf(out->name, sizeof(out->name), "%s.%d", + plugin->name ? plugin->name : "out", id); + + if (alias) { + out->alias = flb_strdup(alias); + if (!out->alias) { + return -1; + } + } + + mk_list_init(&out->_head); + mk_list_init(&out->properties); + mk_list_init(&out->net_properties); + mk_list_init(&out->upstreams); + mk_list_init(&out->flush_list); + mk_list_init(&out->flush_list_destroy); + + mk_list_add(&out->_head, &config->outputs); + + return 0; +} + +static void cleanup_test_output(struct flb_output_instance *out) +{ + if (out->alias) { + flb_free(out->alias); + out->alias = NULL; + } +} static int write_legacy_chunk_metadata(struct cio_chunk *chunk, int event_type, @@ -129,12 +290,20 @@ static void test_chunk_metadata_direct_routes() } } + memset(output_routes, 0, sizeof(output_routes)); + output_routes[0].id = 511; output_routes[0].label = "alpha"; output_routes[0].label_length = 5; + output_routes[0].label_is_alias = FLB_TRUE; + output_routes[0].plugin_name = "stdout"; + output_routes[0].plugin_name_length = 6; output_routes[1].id = 70000; output_routes[1].label = "beta"; output_routes[1].label_length = 4; + output_routes[1].label_is_alias = FLB_FALSE; + output_routes[1].plugin_name = "http"; + output_routes[1].plugin_name_length = 4; ret = flb_input_chunk_write_header_v2(chunk, FLB_INPUT_LOGS, (char *) tag_string, @@ -172,6 +341,14 @@ static void test_chunk_metadata_direct_routes() TEST_CHECK(strcmp(loaded_routes[0].label, "alpha") == 0); TEST_CHECK(strcmp(loaded_routes[1].label, "beta") == 0); } + TEST_CHECK(loaded_routes[0].label_is_alias != 0); + TEST_CHECK(loaded_routes[1].label_is_alias == 0); + TEST_CHECK(loaded_routes[0].plugin_name != NULL); + TEST_CHECK(loaded_routes[1].plugin_name != NULL); + if (loaded_routes[0].plugin_name && loaded_routes[1].plugin_name) { + TEST_CHECK(strcmp(loaded_routes[0].plugin_name, "stdout") == 0); + TEST_CHECK(strcmp(loaded_routes[1].plugin_name, "http") == 0); + } flb_input_chunk_destroy_direct_routes(loaded_routes, route_count); } } @@ -188,7 +365,471 @@ static void test_chunk_metadata_direct_routes() cio_utils_recursive_delete(TEST_STREAM_PATH); } +static void test_chunk_restore_alias_plugin_match_multiple() +{ + struct cio_options opts; + struct cio_ctx *ctx; + struct cio_stream *stream; + struct cio_chunk *chunk; + struct flb_input_chunk *ic; + struct flb_config config; + struct flb_input_instance in; + struct flb_input_plugin input_plugin; + struct flb_output_instance stdout_one; + struct flb_output_instance stdout_two; + struct flb_output_instance http_out; + struct flb_output_plugin stdout_plugin; + struct flb_output_plugin http_plugin; + struct flb_chunk_direct_route route; + const char *tag_string; + int tag_len; + int ret; + int err; + int config_ready; + + ctx = NULL; + stream = NULL; + chunk = NULL; + ic = NULL; + config_ready = FLB_FALSE; + tag_string = "test.tag"; + tag_len = (int) strlen(tag_string); + + cio_utils_recursive_delete(TEST_STREAM_PATH_MATCH); + memset(&opts, 0, sizeof(opts)); + cio_options_init(&opts); + opts.root_path = TEST_STREAM_PATH_MATCH; + opts.flags = CIO_OPEN; + + ctx = cio_create(&opts); + TEST_CHECK(ctx != NULL); + if (!ctx) { + return; + } + + stream = cio_stream_create(ctx, "direct", CIO_STORE_FS); + TEST_CHECK(stream != NULL); + if (!stream) { + goto cleanup; + } + + chunk = cio_chunk_open(ctx, stream, "meta", CIO_OPEN, 1024, &err); + TEST_CHECK(chunk != NULL); + if (!chunk) { + goto cleanup; + } + + ret = cio_chunk_is_up(chunk); + if (ret == CIO_FALSE) { + ret = cio_chunk_up_force(chunk); + TEST_CHECK(ret == CIO_OK); + if (ret != CIO_OK) { + goto cleanup; + } + } + + ret = write_legacy_chunk_metadata(chunk, FLB_INPUT_LOGS, + tag_string, tag_len); + TEST_CHECK(ret == 0); + if (ret != 0) { + goto cleanup; + } + + ret = write_test_log_payload(chunk); + TEST_CHECK(ret == 0); + if (ret != 0) { + goto cleanup; + } + + memset(&route, 0, sizeof(route)); + route.id = 25; + route.label = "shared"; + route.label_length = 6; + route.label_is_alias = FLB_TRUE; + route.plugin_name = "stdout"; + route.plugin_name_length = 6; + + ret = flb_input_chunk_write_header_v2(chunk, + FLB_INPUT_LOGS, + (char *) tag_string, + tag_len, + &route, + 1); + TEST_CHECK(ret == 0); + if (ret != 0) { + goto cleanup; + } + + memset(&input_plugin, 0, sizeof(input_plugin)); + input_plugin.name = (char *) "dummy"; + memset(&stdout_plugin, 0, sizeof(stdout_plugin)); + stdout_plugin.name = (char *) "stdout"; + memset(&http_plugin, 0, sizeof(http_plugin)); + http_plugin.name = (char *) "http"; + + ret = init_test_config(&config, &in, &input_plugin); + TEST_CHECK(ret == 0); + if (ret != 0) { + goto cleanup; + } + config_ready = FLB_TRUE; + + ret = flb_input_instance_init(&in, &config); + TEST_CHECK(ret == 0); + if (ret != 0) { + goto cleanup; + } + + ret = add_test_output(&config, &stdout_one, &stdout_plugin, 1, "shared"); + TEST_CHECK(ret == 0); + if (ret != 0) { + goto cleanup; + } + + ret = add_test_output(&config, &stdout_two, &stdout_plugin, 2, "shared"); + TEST_CHECK(ret == 0); + if (ret != 0) { + goto cleanup; + } + + ret = add_test_output(&config, &http_out, &http_plugin, 3, "shared"); + TEST_CHECK(ret == 0); + if (ret != 0) { + goto cleanup; + } + + ic = flb_input_chunk_map(&in, FLB_INPUT_LOGS, chunk); + TEST_CHECK(ic != NULL); + if (!ic) { + goto cleanup; + } + + chunk = NULL; + + TEST_CHECK(flb_routes_mask_get_bit(ic->routes_mask, + stdout_one.id, + &config) == 1); + TEST_CHECK(flb_routes_mask_get_bit(ic->routes_mask, + stdout_two.id, + &config) == 1); + TEST_CHECK(flb_routes_mask_get_bit(ic->routes_mask, + http_out.id, + &config) == 0); + +cleanup: + if (ic) { + flb_input_chunk_destroy(ic, FLB_TRUE); + ic = NULL; + } + + cleanup_test_output(&stdout_one); + cleanup_test_output(&stdout_two); + cleanup_test_output(&http_out); + + if (config_ready == FLB_TRUE) { + flb_input_instance_exit(&in, &config); + + /* Manual cleanup for stack-allocated instance */ + /* Remove from list first (before destroying hash tables) */ + mk_list_del(&in._head); + + /* Destroy hash tables */ + if (in.ht_log_chunks) { + flb_hash_table_destroy(in.ht_log_chunks); + in.ht_log_chunks = NULL; + } + if (in.ht_metric_chunks) { + flb_hash_table_destroy(in.ht_metric_chunks); + in.ht_metric_chunks = NULL; + } + if (in.ht_trace_chunks) { + flb_hash_table_destroy(in.ht_trace_chunks); + in.ht_trace_chunks = NULL; + } + if (in.ht_profile_chunks) { + flb_hash_table_destroy(in.ht_profile_chunks); + in.ht_profile_chunks = NULL; + } + + /* Release properties */ + flb_kv_release(&in.properties); + flb_kv_release(&in.net_properties); + + /* Destroy metrics (created by flb_input_instance_init) */ +#ifdef FLB_HAVE_METRICS + if (in.cmt) { + cmt_destroy(in.cmt); + in.cmt = NULL; + } + if (in.metrics) { + flb_metrics_destroy(in.metrics); + in.metrics = NULL; + } +#endif + + /* Destroy config map if created */ + if (in.tls_config_map) { + flb_config_map_destroy(in.tls_config_map); + in.tls_config_map = NULL; + } + if (in.net_config_map) { + flb_config_map_destroy(in.net_config_map); + in.net_config_map = NULL; + } + + flb_routes_empty_mask_destroy(&config); + if (config.env) { + flb_env_destroy(config.env); + config.env = NULL; + } + } + + if (chunk) { + cio_chunk_close(chunk, CIO_TRUE); + chunk = NULL; + } + + if (ctx) { + cio_destroy(ctx); + } + + cio_utils_recursive_delete(TEST_STREAM_PATH_MATCH); +} + +static void test_chunk_restore_alias_plugin_null_matches_all() +{ + struct cio_options opts; + struct cio_ctx *ctx; + struct cio_stream *stream; + struct cio_chunk *chunk; + struct flb_input_chunk *ic; + struct flb_config config; + struct flb_input_instance in; + struct flb_input_plugin input_plugin; + struct flb_output_instance stdout_one; + struct flb_output_instance stdout_two; + struct flb_output_instance http_out; + struct flb_output_plugin stdout_plugin; + struct flb_output_plugin http_plugin; + struct flb_chunk_direct_route route; + const char *tag_string; + int tag_len; + int ret; + int err; + int config_ready; + + ctx = NULL; + stream = NULL; + chunk = NULL; + ic = NULL; + config_ready = FLB_FALSE; + tag_string = "test.tag"; + tag_len = (int) strlen(tag_string); + + cio_utils_recursive_delete(TEST_STREAM_PATH_NULL); + memset(&opts, 0, sizeof(opts)); + cio_options_init(&opts); + opts.root_path = TEST_STREAM_PATH_NULL; + opts.flags = CIO_OPEN; + + ctx = cio_create(&opts); + TEST_CHECK(ctx != NULL); + if (!ctx) { + return; + } + + stream = cio_stream_create(ctx, "direct", CIO_STORE_FS); + TEST_CHECK(stream != NULL); + if (!stream) { + goto cleanup; + } + + chunk = cio_chunk_open(ctx, stream, "meta", CIO_OPEN, 1024, &err); + TEST_CHECK(chunk != NULL); + if (!chunk) { + goto cleanup; + } + + ret = cio_chunk_is_up(chunk); + if (ret == CIO_FALSE) { + ret = cio_chunk_up_force(chunk); + TEST_CHECK(ret == CIO_OK); + if (ret != CIO_OK) { + goto cleanup; + } + } + + ret = write_legacy_chunk_metadata(chunk, FLB_INPUT_LOGS, + tag_string, tag_len); + TEST_CHECK(ret == 0); + if (ret != 0) { + goto cleanup; + } + + ret = write_test_log_payload(chunk); + TEST_CHECK(ret == 0); + if (ret != 0) { + goto cleanup; + } + + memset(&route, 0, sizeof(route)); + route.id = 30; + route.label = "shared"; + route.label_length = 6; + route.label_is_alias = FLB_TRUE; + route.plugin_name = NULL; + route.plugin_name_length = 0; + + ret = flb_input_chunk_write_header_v2(chunk, + FLB_INPUT_LOGS, + (char *) tag_string, + tag_len, + &route, + 1); + TEST_CHECK(ret == 0); + if (ret != 0) { + goto cleanup; + } + + memset(&input_plugin, 0, sizeof(input_plugin)); + input_plugin.name = (char *) "dummy"; + memset(&stdout_plugin, 0, sizeof(stdout_plugin)); + stdout_plugin.name = (char *) "stdout"; + memset(&http_plugin, 0, sizeof(http_plugin)); + http_plugin.name = (char *) "http"; + + ret = init_test_config(&config, &in, &input_plugin); + TEST_CHECK(ret == 0); + if (ret != 0) { + goto cleanup; + } + config_ready = FLB_TRUE; + + ret = flb_input_instance_init(&in, &config); + TEST_CHECK(ret == 0); + if (ret != 0) { + goto cleanup; + } + + ret = add_test_output(&config, &stdout_one, &stdout_plugin, 4, "shared"); + TEST_CHECK(ret == 0); + if (ret != 0) { + goto cleanup; + } + + ret = add_test_output(&config, &stdout_two, &stdout_plugin, 5, "shared"); + TEST_CHECK(ret == 0); + if (ret != 0) { + goto cleanup; + } + + ret = add_test_output(&config, &http_out, &http_plugin, 6, "shared"); + TEST_CHECK(ret == 0); + if (ret != 0) { + goto cleanup; + } + + ic = flb_input_chunk_map(&in, FLB_INPUT_LOGS, chunk); + TEST_CHECK(ic != NULL); + if (!ic) { + goto cleanup; + } + + chunk = NULL; + + TEST_CHECK(flb_routes_mask_get_bit(ic->routes_mask, + stdout_one.id, + &config) == 1); + TEST_CHECK(flb_routes_mask_get_bit(ic->routes_mask, + stdout_two.id, + &config) == 1); + TEST_CHECK(flb_routes_mask_get_bit(ic->routes_mask, + http_out.id, + &config) == 1); + +cleanup: + if (ic) { + flb_input_chunk_destroy(ic, FLB_TRUE); + ic = NULL; + } + + cleanup_test_output(&stdout_one); + cleanup_test_output(&stdout_two); + cleanup_test_output(&http_out); + + if (config_ready == FLB_TRUE) { + flb_input_instance_exit(&in, &config); + + /* Manual cleanup for stack-allocated instance */ + /* Remove from list first (before destroying hash tables) */ + mk_list_del(&in._head); + + /* Destroy hash tables */ + if (in.ht_log_chunks) { + flb_hash_table_destroy(in.ht_log_chunks); + in.ht_log_chunks = NULL; + } + if (in.ht_metric_chunks) { + flb_hash_table_destroy(in.ht_metric_chunks); + in.ht_metric_chunks = NULL; + } + if (in.ht_trace_chunks) { + flb_hash_table_destroy(in.ht_trace_chunks); + in.ht_trace_chunks = NULL; + } + if (in.ht_profile_chunks) { + flb_hash_table_destroy(in.ht_profile_chunks); + in.ht_profile_chunks = NULL; + } + + /* Release properties */ + flb_kv_release(&in.properties); + flb_kv_release(&in.net_properties); + + /* Destroy metrics (created by flb_input_instance_init) */ +#ifdef FLB_HAVE_METRICS + if (in.cmt) { + cmt_destroy(in.cmt); + in.cmt = NULL; + } + if (in.metrics) { + flb_metrics_destroy(in.metrics); + in.metrics = NULL; + } +#endif + + /* Destroy config map if created */ + if (in.tls_config_map) { + flb_config_map_destroy(in.tls_config_map); + in.tls_config_map = NULL; + } + if (in.net_config_map) { + flb_config_map_destroy(in.net_config_map); + in.net_config_map = NULL; + } + + flb_routes_empty_mask_destroy(&config); + if (config.env) { + flb_env_destroy(config.env); + config.env = NULL; + } + } + + if (chunk) { + cio_chunk_close(chunk, CIO_TRUE); + chunk = NULL; + } + + if (ctx) { + cio_destroy(ctx); + } + + cio_utils_recursive_delete(TEST_STREAM_PATH_NULL); +} + TEST_LIST = { { "chunk_metadata_direct_routes", test_chunk_metadata_direct_routes }, + { "chunk_restore_alias_plugin_match_multiple", test_chunk_restore_alias_plugin_match_multiple }, + { "chunk_restore_alias_plugin_null_matches_all", test_chunk_restore_alias_plugin_null_matches_all }, { 0 } }; From 7b23470d9d2c27a249aaf1f0705c7894e82f7360 Mon Sep 17 00:00:00 2001 From: Patrick Stephens Date: Sun, 3 May 2026 12:34:54 +0100 Subject: [PATCH 067/213] [upstream] tests: internal: input_chunk_route: add missing cmt Upstream-Ref: https://github.com/fluent/fluent-bit/commit/5a15b125cd2067295bd12a7f483e6f1cfd50bb68 Cherry-picked from Fluent Bit v4.2.4 --- source/tests/internal/input_chunk_routes.c | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/source/tests/internal/input_chunk_routes.c b/source/tests/internal/input_chunk_routes.c index 6b32044c..ddf56327 100644 --- a/source/tests/internal/input_chunk_routes.c +++ b/source/tests/internal/input_chunk_routes.c @@ -14,6 +14,7 @@ #include #include #include +#include #include "flb_tests_internal.h" @@ -474,6 +475,10 @@ static void test_chunk_restore_alias_plugin_match_multiple() } config_ready = FLB_TRUE; +#ifdef FLB_HAVE_METRICS + cmt_initialize(); +#endif + ret = flb_input_instance_init(&in, &config); TEST_CHECK(ret == 0); if (ret != 0) { @@ -705,6 +710,10 @@ static void test_chunk_restore_alias_plugin_null_matches_all() } config_ready = FLB_TRUE; +#ifdef FLB_HAVE_METRICS + cmt_initialize(); +#endif + ret = flb_input_instance_init(&in, &config); TEST_CHECK(ret == 0); if (ret != 0) { From c565184ab0a7859c5805f1565277560766bbe519 Mon Sep 17 00:00:00 2001 From: Patrick Stephens Date: Sun, 3 May 2026 12:34:54 +0100 Subject: [PATCH 068/213] [upstream] tests: internal: input_chunk_route: simplify cleanup Upstream-Ref: https://github.com/fluent/fluent-bit/commit/fb24892909db162cd209583c91f4a3cea7c9ab1d Cherry-picked from Fluent Bit v4.2.4 --- source/tests/internal/input_chunk_routes.c | 248 ++++++++------------- 1 file changed, 94 insertions(+), 154 deletions(-) diff --git a/source/tests/internal/input_chunk_routes.c b/source/tests/internal/input_chunk_routes.c index ddf56327..49ad9517 100644 --- a/source/tests/internal/input_chunk_routes.c +++ b/source/tests/internal/input_chunk_routes.c @@ -171,6 +171,94 @@ static void cleanup_test_output(struct flb_output_instance *out) } } +static void cleanup_test_routing_scenario(struct flb_input_chunk *ic, + struct flb_output_instance *stdout_one, + struct flb_output_instance *stdout_two, + struct flb_output_instance *http_out, + struct flb_input_instance *in, + struct flb_config *config, + struct cio_chunk *chunk, + struct cio_ctx *ctx, + int config_ready, + const char *stream_path) +{ + if (ic) { + flb_input_chunk_destroy(ic, FLB_TRUE); + } + + cleanup_test_output(stdout_one); + cleanup_test_output(stdout_two); + cleanup_test_output(http_out); + + if (config_ready == FLB_TRUE) { + flb_input_instance_exit(in, config); + + /* Manual cleanup for stack-allocated instance */ + /* Remove from list first (before destroying hash tables) */ + mk_list_del(&in->_head); + + /* Destroy hash tables */ + if (in->ht_log_chunks) { + flb_hash_table_destroy(in->ht_log_chunks); + in->ht_log_chunks = NULL; + } + if (in->ht_metric_chunks) { + flb_hash_table_destroy(in->ht_metric_chunks); + in->ht_metric_chunks = NULL; + } + if (in->ht_trace_chunks) { + flb_hash_table_destroy(in->ht_trace_chunks); + in->ht_trace_chunks = NULL; + } + if (in->ht_profile_chunks) { + flb_hash_table_destroy(in->ht_profile_chunks); + in->ht_profile_chunks = NULL; + } + + /* Release properties */ + flb_kv_release(&in->properties); + flb_kv_release(&in->net_properties); + + /* Destroy metrics (created by flb_input_instance_init) */ +#ifdef FLB_HAVE_METRICS + if (in->cmt) { + cmt_destroy(in->cmt); + in->cmt = NULL; + } + if (in->metrics) { + flb_metrics_destroy(in->metrics); + in->metrics = NULL; + } +#endif + + /* Destroy config map if created */ + if (in->tls_config_map) { + flb_config_map_destroy(in->tls_config_map); + in->tls_config_map = NULL; + } + if (in->net_config_map) { + flb_config_map_destroy(in->net_config_map); + in->net_config_map = NULL; + } + + flb_routes_empty_mask_destroy(config); + if (config->env) { + flb_env_destroy(config->env); + config->env = NULL; + } + } + + if (chunk) { + cio_chunk_close(chunk, CIO_TRUE); + } + + if (ctx) { + cio_destroy(ctx); + } + + cio_utils_recursive_delete(stream_path); +} + static int write_legacy_chunk_metadata(struct cio_chunk *chunk, int event_type, const char *tag, @@ -522,83 +610,9 @@ static void test_chunk_restore_alias_plugin_match_multiple() &config) == 0); cleanup: - if (ic) { - flb_input_chunk_destroy(ic, FLB_TRUE); - ic = NULL; - } - - cleanup_test_output(&stdout_one); - cleanup_test_output(&stdout_two); - cleanup_test_output(&http_out); - - if (config_ready == FLB_TRUE) { - flb_input_instance_exit(&in, &config); - - /* Manual cleanup for stack-allocated instance */ - /* Remove from list first (before destroying hash tables) */ - mk_list_del(&in._head); - - /* Destroy hash tables */ - if (in.ht_log_chunks) { - flb_hash_table_destroy(in.ht_log_chunks); - in.ht_log_chunks = NULL; - } - if (in.ht_metric_chunks) { - flb_hash_table_destroy(in.ht_metric_chunks); - in.ht_metric_chunks = NULL; - } - if (in.ht_trace_chunks) { - flb_hash_table_destroy(in.ht_trace_chunks); - in.ht_trace_chunks = NULL; - } - if (in.ht_profile_chunks) { - flb_hash_table_destroy(in.ht_profile_chunks); - in.ht_profile_chunks = NULL; - } - - /* Release properties */ - flb_kv_release(&in.properties); - flb_kv_release(&in.net_properties); - - /* Destroy metrics (created by flb_input_instance_init) */ -#ifdef FLB_HAVE_METRICS - if (in.cmt) { - cmt_destroy(in.cmt); - in.cmt = NULL; - } - if (in.metrics) { - flb_metrics_destroy(in.metrics); - in.metrics = NULL; - } -#endif - - /* Destroy config map if created */ - if (in.tls_config_map) { - flb_config_map_destroy(in.tls_config_map); - in.tls_config_map = NULL; - } - if (in.net_config_map) { - flb_config_map_destroy(in.net_config_map); - in.net_config_map = NULL; - } - - flb_routes_empty_mask_destroy(&config); - if (config.env) { - flb_env_destroy(config.env); - config.env = NULL; - } - } - - if (chunk) { - cio_chunk_close(chunk, CIO_TRUE); - chunk = NULL; - } - - if (ctx) { - cio_destroy(ctx); - } - - cio_utils_recursive_delete(TEST_STREAM_PATH_MATCH); + cleanup_test_routing_scenario(ic, &stdout_one, &stdout_two, &http_out, + &in, &config, chunk, ctx, config_ready, + TEST_STREAM_PATH_MATCH); } static void test_chunk_restore_alias_plugin_null_matches_all() @@ -757,83 +771,9 @@ static void test_chunk_restore_alias_plugin_null_matches_all() &config) == 1); cleanup: - if (ic) { - flb_input_chunk_destroy(ic, FLB_TRUE); - ic = NULL; - } - - cleanup_test_output(&stdout_one); - cleanup_test_output(&stdout_two); - cleanup_test_output(&http_out); - - if (config_ready == FLB_TRUE) { - flb_input_instance_exit(&in, &config); - - /* Manual cleanup for stack-allocated instance */ - /* Remove from list first (before destroying hash tables) */ - mk_list_del(&in._head); - - /* Destroy hash tables */ - if (in.ht_log_chunks) { - flb_hash_table_destroy(in.ht_log_chunks); - in.ht_log_chunks = NULL; - } - if (in.ht_metric_chunks) { - flb_hash_table_destroy(in.ht_metric_chunks); - in.ht_metric_chunks = NULL; - } - if (in.ht_trace_chunks) { - flb_hash_table_destroy(in.ht_trace_chunks); - in.ht_trace_chunks = NULL; - } - if (in.ht_profile_chunks) { - flb_hash_table_destroy(in.ht_profile_chunks); - in.ht_profile_chunks = NULL; - } - - /* Release properties */ - flb_kv_release(&in.properties); - flb_kv_release(&in.net_properties); - - /* Destroy metrics (created by flb_input_instance_init) */ -#ifdef FLB_HAVE_METRICS - if (in.cmt) { - cmt_destroy(in.cmt); - in.cmt = NULL; - } - if (in.metrics) { - flb_metrics_destroy(in.metrics); - in.metrics = NULL; - } -#endif - - /* Destroy config map if created */ - if (in.tls_config_map) { - flb_config_map_destroy(in.tls_config_map); - in.tls_config_map = NULL; - } - if (in.net_config_map) { - flb_config_map_destroy(in.net_config_map); - in.net_config_map = NULL; - } - - flb_routes_empty_mask_destroy(&config); - if (config.env) { - flb_env_destroy(config.env); - config.env = NULL; - } - } - - if (chunk) { - cio_chunk_close(chunk, CIO_TRUE); - chunk = NULL; - } - - if (ctx) { - cio_destroy(ctx); - } - - cio_utils_recursive_delete(TEST_STREAM_PATH_NULL); + cleanup_test_routing_scenario(ic, &stdout_one, &stdout_two, &http_out, + &in, &config, chunk, ctx, config_ready, + TEST_STREAM_PATH_NULL); } TEST_LIST = { From 015f363a142d6b4a3ff6849025ff9ee2c2975293 Mon Sep 17 00:00:00 2001 From: Patrick Stephens Date: Sun, 3 May 2026 12:34:54 +0100 Subject: [PATCH 069/213] [upstream] plugins: kafka: fix cmake cross compile error Upstream-Ref: https://github.com/fluent/fluent-bit/commit/7eab11a3b290187483b92cfb72a0e324b61aa790 Cherry-picked from Fluent Bit v4.2.4 --- source/plugins/in_kafka/CMakeLists.txt | 4 +++- source/plugins/out_kafka/CMakeLists.txt | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/source/plugins/in_kafka/CMakeLists.txt b/source/plugins/in_kafka/CMakeLists.txt index 928266b5..dc251276 100644 --- a/source/plugins/in_kafka/CMakeLists.txt +++ b/source/plugins/in_kafka/CMakeLists.txt @@ -3,5 +3,7 @@ set(src ) FLB_PLUGIN(in_kafka "${src}" ${KAFKA_LIBRARIES} flb-aws) -target_include_directories(flb-plugin-in_kafka PUBLIC ${KAFKA_INCLUDEDIR}/librdkafka) +if(DEFINED KAFKA_INCLUDEDIR) + target_include_directories(flb-plugin-in_kafka PUBLIC ${KAFKA_INCLUDEDIR}/librdkafka) +endif() target_link_libraries(flb-plugin-in_kafka -lpthread) diff --git a/source/plugins/out_kafka/CMakeLists.txt b/source/plugins/out_kafka/CMakeLists.txt index 9d8b3464..1c967329 100644 --- a/source/plugins/out_kafka/CMakeLists.txt +++ b/source/plugins/out_kafka/CMakeLists.txt @@ -5,5 +5,7 @@ set(src kafka.c) FLB_PLUGIN(out_kafka "${src}" ${KAFKA_LIBRARIES} flb-aws) -target_include_directories(flb-plugin-out_kafka PUBLIC ${KAFKA_INCLUDEDIR}/librdkafka) +if(DEFINED KAFKA_INCLUDEDIR) + target_include_directories(flb-plugin-out_kafka PUBLIC ${KAFKA_INCLUDEDIR}/librdkafka) +endif() target_link_libraries(flb-plugin-out_kafka -lpthread) From e21dcbd1cc189e0076e946311ebd00c339baadae Mon Sep 17 00:00:00 2001 From: Patrick Stephens Date: Sun, 3 May 2026 12:34:55 +0100 Subject: [PATCH 070/213] [upstream] build: Implement strict pointer types option Upstream-Ref: https://github.com/fluent/fluent-bit/commit/cc15e1bc8a3cfd22031962babcb5392863f68014 Cherry-picked from Fluent Bit v4.2.4 --- source/CMakeLists.txt | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/source/CMakeLists.txt b/source/CMakeLists.txt index fbd4aaf0..af51a0cc 100644 --- a/source/CMakeLists.txt +++ b/source/CMakeLists.txt @@ -140,6 +140,7 @@ option(FLB_SMALL "Optimise for small size" No) set(FLB_SECURITY "ReleaseOnly" CACHE STRING "Build with security optimizations") set_property(CACHE FLB_SECURITY PROPERTY STRINGS "On;Off;ReleaseOnly") option(FLB_COVERAGE "Build with code-coverage" No) +option(FLB_COMPILER_STRICT_POINTER_TYPES "Build with -Werror=incompatible-pointer-types" No) option(FLB_JEMALLOC "Build with Jemalloc support" No) option(FLB_REGEX "Build with Regex support" Yes) option(FLB_UTF8_ENCODER "Build with UTF8 encoding support" Yes) @@ -445,6 +446,17 @@ if(FLB_COVERAGE) set(CMAKE_BUILD_TYPE "Debug") endif() +if(FLB_COMPILER_STRICT_POINTER_TYPES) + if(CMAKE_C_COMPILER_ID MATCHES "GNU|Clang|AppleClang") + add_compile_options(-Werror=incompatible-pointer-types) + endif() + # Currently, AppleClang has more struct rules when using -Werror=incompatible-pointer-types. + # We still permit to discarding const qualifiers warnings. + if(CMAKE_C_COMPILER_ID MATCHES "AppleClang") + add_compile_options(-Wno-incompatible-pointer-types-discards-qualifiers) + endif() +endif() + # Enable Debug symbols if specified if(MSVC) set(CMAKE_BUILD_TYPE None) # Avoid flag conflicts (See CMakeList.txt:L18) From ba7adc8a62f37ce4c0632dbfc3deafb11f2d9668 Mon Sep 17 00:00:00 2001 From: Patrick Stephens Date: Sun, 3 May 2026 12:34:55 +0100 Subject: [PATCH 071/213] [upstream] tls: openssl: Implement certificates" thumbprint Upstream-Ref: https://github.com/fluent/fluent-bit/commit/baa01530f6fc58749f9b594f05df2d0cbb23f1f6 Cherry-picked from Fluent Bit v4.2.4 --- source/include/fluent-bit/tls/flb_tls.h | 2 + source/src/tls/flb_tls.c | 7 + source/src/tls/openssl.c | 227 ++++++++++++++++++++++++ 3 files changed, 236 insertions(+) diff --git a/source/include/fluent-bit/tls/flb_tls.h b/source/include/fluent-bit/tls/flb_tls.h index 7b4822a5..47c261eb 100644 --- a/source/include/fluent-bit/tls/flb_tls.h +++ b/source/include/fluent-bit/tls/flb_tls.h @@ -97,6 +97,7 @@ struct flb_tls_backend { #if defined(FLB_SYSTEM_WINDOWS) int (*set_certstore_name)(struct flb_tls *tls, const char *certstore_name); int (*set_use_enterprise_store)(struct flb_tls *tls, int use_enterprise); + int (*set_client_thumbprints)(struct flb_tls *tls, const char *thumbprints); #endif }; @@ -135,6 +136,7 @@ int flb_tls_set_verify_hostname(struct flb_tls *tls, int verify_hostname); #if defined(FLB_SYSTEM_WINDOWS) int flb_tls_set_certstore_name(struct flb_tls *tls, const char *certstore_name); int flb_tls_set_use_enterprise_store(struct flb_tls *tls, int use_enterprise); +int flb_tls_set_client_thumbprints(struct flb_tls *tls, const char *thumbprints); #endif int flb_tls_load_system_certificates(struct flb_tls *tls); diff --git a/source/src/tls/flb_tls.c b/source/src/tls/flb_tls.c index bba94650..53d6cc53 100644 --- a/source/src/tls/flb_tls.c +++ b/source/src/tls/flb_tls.c @@ -314,6 +314,13 @@ int flb_tls_set_use_enterprise_store(struct flb_tls *tls, int use_enterprise) return 0; } + +int flb_tls_set_client_thumbprints(struct flb_tls *tls, const char *thumbprints) { + if (tls && tls->api->set_client_thumbprints) { + return tls->api->set_client_thumbprints(tls, thumbprints); + } + return -1; +} #endif int flb_tls_net_read(struct flb_tls_session *session, void *buf, size_t len) diff --git a/source/src/tls/openssl.c b/source/src/tls/openssl.c index fbd213fa..16241b7e 100644 --- a/source/src/tls/openssl.c +++ b/source/src/tls/openssl.c @@ -40,6 +40,11 @@ #ifdef FLB_SYSTEM_WINDOWS #define strtok_r(str, delimiter, context) \ strtok_s(str, delimiter, context) + #include + #ifndef CERT_FIND_SHA256_HASH + /* Older SDKs may not define this */ + #define CERT_FIND_SHA256_HASH 0x0001000d + #endif #endif /* @@ -59,6 +64,8 @@ struct tls_context { #if defined(FLB_SYSTEM_WINDOWS) char *certstore_name; int use_enterprise_store; + CRYPT_HASH_BLOB *allowed_thumbprints; + size_t allowed_thumbprints_count; #endif pthread_mutex_t mutex; }; @@ -158,6 +165,17 @@ static void tls_context_destroy(void *ctx_backend) ctx->certstore_name = NULL; } + if (ctx->allowed_thumbprints) { + /* We allocated each blob->pbData; free them too */ + for (size_t i = 0; i < ctx->allowed_thumbprints_count; i++) { + if (ctx->allowed_thumbprints[i].pbData) { + flb_free(ctx->allowed_thumbprints[i].pbData); + } + } + flb_free(ctx->allowed_thumbprints); + ctx->allowed_thumbprints = NULL; + ctx->allowed_thumbprints_count = 0; + } #endif pthread_mutex_unlock(&ctx->mutex); @@ -323,6 +341,62 @@ static int windows_load_system_certificates(struct tls_context *ctx) return -1; } + if (ctx->allowed_thumbprints_count > 0) { + size_t loaded = 0; + DWORD find_type = 0; + size_t i; + + for (i = 0; i < ctx->allowed_thumbprints_count; i++) { + find_type = (ctx->allowed_thumbprints[i].cbData == 20) + ? CERT_FIND_SHA1_HASH + : CERT_FIND_SHA256_HASH; + + win_cert = NULL; + while ((win_cert = CertFindCertificateInStore(win_store, + X509_ASN_ENCODING, + 0, + find_type, + &ctx->allowed_thumbprints[i], + win_cert)) != NULL) { + + win_cert_data = win_cert->pbCertEncoded; + ossl_cert = d2i_X509(NULL, &win_cert_data, win_cert->cbCertEncoded); + if (!ossl_cert) { + flb_debug("[tls] parse failed for matched certificate (thumbprint idx %zu)", i); + continue; + } + + ret = X509_STORE_add_cert(ossl_store, ossl_cert); + if (ret != 1) { + unsigned long err = ERR_get_error(); + if (ERR_GET_REASON(err) == X509_R_CERT_ALREADY_IN_HASH_TABLE) { + flb_debug("[tls] certificate already present (thumbprint idx %zu).", i); + } + else { + flb_warn("[tls] add_cert failed: %s", ERR_error_string(err, NULL)); + } + } + else { + loaded++; + } + X509_free(ossl_cert); + } + } + + if (!CertCloseStore(win_store, 0)) { + flb_error("[tls] cannot close windows certificate store: %lu", GetLastError()); + return -1; + } + + if (loaded == 0) { + flb_warn("[tls] no certificates loaded by thumbprint from '%s'.", certstore_name); + } + else { + flb_debug("[tls] loaded %zu certificate(s) by thumbprint from '%s'.", loaded, certstore_name); + } + return 0; + } + /* Iterate over certificates in the store */ while ((win_cert = CertEnumCertificatesInStore(win_store, win_cert)) != NULL) { /* Check if the certificate is encoded in ASN.1 DER format */ @@ -597,6 +671,12 @@ static void *tls_context_create(int verify, ctx->mode = mode; ctx->alpn = NULL; ctx->debug_level = debug; +#if defined(FLB_SYSTEM_WINDOWS) + ctx->certstore_name = NULL; + ctx->use_enterprise_store = 0; + ctx->allowed_thumbprints = NULL; + ctx->allowed_thumbprints_count = 0; +#endif pthread_mutex_init(&ctx->mutex, NULL); /* Verify peer: by default OpenSSL always verify peer */ @@ -812,6 +892,152 @@ static int tls_set_use_enterprise_store(struct flb_tls *tls, int use_enterprise) return 0; } + +static int hex_nibble(int c) { + if (c >= '0' && c <= '9') { + return c - '0'; + } + else if (c >= 'a' && c <= 'f') { + return c - 'a' + 10; + } + else if (c >= 'A' && c <= 'F') { + return c - 'A' + 10; + } + return -1; +} + +static char *compact_hex(const char *s) { + size_t n = 0; + size_t i; + char *out = flb_calloc(1, strlen(s) + 1); + + if (!out) { + return NULL; + } + + for (i = 0; s[i]; i++) { + int c = s[i]; + if ((c >= '0' && c <= '9') || + (c >= 'a' && c <= 'f') || + (c >= 'A' && c <= 'F')) { + out[n++] = (char)c; + } + } + out[n] = '\0'; + return out; +} + +static unsigned char *hex_to_bytes(const char *hex, size_t *out_len) { + unsigned char *buf = NULL; + size_t i; + size_t len = strlen(hex); + if (len % 2 != 0) { + return NULL; + } + + buf = flb_calloc(1, len / 2); + if (!buf) { + return NULL; + } + + for (i = 0; i < len; i += 2) { + int hi = hex_nibble(hex[i]); + int lo = hex_nibble(hex[i+1]); + if (hi < 0 || lo < 0) { + flb_free(buf); + return NULL; + } + buf[i/2] = (unsigned char)((hi << 4) | lo); + } + *out_len = len / 2; + return buf; +} + +static int windows_set_allowed_thumbprints(struct tls_context *ctx, const char *thumbprints) +{ + char *token_ctx = NULL, *tok = NULL; + size_t cap = 4, count = 0; + char *hex = NULL; + struct cfl_list *kvs; + struct cfl_list *head; + struct cfl_split_entry *cur; + CRYPT_HASH_BLOB *arr; + size_t bytes_len = 0; + unsigned char *bytes = NULL; + + if (!thumbprints || !*thumbprints) { + return 0; + } + + arr = flb_calloc(cap, sizeof(*arr)); + if (!arr) { + return -1; + } + + kvs = cfl_utils_split(thumbprints, ',', -1); + cfl_list_foreach(head, kvs) { + cur = cfl_list_entry(head, struct cfl_split_entry, _head); + tok = cur->value; + hex = compact_hex(tok); + if (hex && *hex) { + bytes = hex_to_bytes(hex, &bytes_len); + if (bytes && (bytes_len == 20 || bytes_len == 32)) { + if (count == cap) { + cap *= 2; + CRYPT_HASH_BLOB *tmp = flb_realloc(arr, cap * sizeof(*arr)); + if (!tmp) { + flb_free(bytes); + break; + } + arr = tmp; + } + arr[count].cbData = (DWORD)bytes_len; + arr[count].pbData = bytes; + count++; + } + else { + flb_warn("[tls] ignoring thumbprint '%s' (length must be 40 or 64 hex chars after stripping).", tok); + if (bytes) { + flb_free(bytes); + } + } + } + if (hex) { + flb_free(hex); + } + } + cfl_utils_split_free(kvs); + + if (count == 0) { + if (arr) { + flb_free(arr); + } + flb_warn("[tls] no valid thumbprints parsed."); + return -1; + } + + ctx->allowed_thumbprints = arr; + ctx->allowed_thumbprints_count = count; + flb_debug("[tls] parsed %zu allowed thumbprint(s).", count); + + return 0; +} + +static int tls_set_client_thumbprints(struct flb_tls *tls, const char *thumbprints) { + struct tls_context *ctx = tls->ctx; + int rc = 0; + + pthread_mutex_lock(&ctx->mutex); + + if (ctx->allowed_thumbprints || ctx->allowed_thumbprints_count) { + pthread_mutex_unlock(&ctx->mutex); + return -1; + } + rc = windows_set_allowed_thumbprints(ctx, thumbprints); + pthread_mutex_unlock(&ctx->mutex); + return rc; +} + #endif static void *tls_session_create(struct flb_tls *tls, @@ -1243,5 +1469,6 @@ static struct flb_tls_backend tls_openssl = { #if defined(FLB_SYSTEM_WINDOWS) .set_certstore_name = tls_set_certstore_name, .set_use_enterprise_store = tls_set_use_enterprise_store, + .set_client_thumbprints = tls_set_client_thumbprints, #endif }; From 97c66bdb699cf045f1b8b809bc0b2896f8b00abc Mon Sep 17 00:00:00 2001 From: Patrick Stephens Date: Sun, 3 May 2026 12:34:55 +0100 Subject: [PATCH 072/213] [upstream] output: Handle to load certificates with their Upstream-Ref: https://github.com/fluent/fluent-bit/commit/3d3db60befe07da8cdbc9a82f229db2897e10e78 Cherry-picked from Fluent Bit v4.2.4 --- source/include/fluent-bit/flb_output.h | 1 + source/src/flb_output.c | 22 ++++++++++++++++++++++ 2 files changed, 23 insertions(+) diff --git a/source/include/fluent-bit/flb_output.h b/source/include/fluent-bit/flb_output.h index d9675e48..c54a7a32 100644 --- a/source/include/fluent-bit/flb_output.h +++ b/source/include/fluent-bit/flb_output.h @@ -374,6 +374,7 @@ struct flb_output_instance { # if defined(FLB_SYSTEM_WINDOWS) char *tls_win_certstore_name; /* CertStore Name (Windows) */ int tls_win_use_enterprise_certstore; /* Use enterprise CertStore */ + char *tls_win_thumbprints; /* CertStore Thumbprints (Windows) */ # endif #endif diff --git a/source/src/flb_output.c b/source/src/flb_output.c index 1d7f1d80..307905b3 100644 --- a/source/src/flb_output.c +++ b/source/src/flb_output.c @@ -98,6 +98,11 @@ struct flb_config_map output_global_properties[] = { 0, FLB_FALSE, 0, "Sets whether using enterprise certstore or not on an output (Windows)" }, + { + FLB_CONFIG_MAP_STR, "tls.windows.client_thumbprints", NULL, + 0, FLB_FALSE, 0, + "Comma-separated list of certificate thumbprints (SHA1/SHA256) to trust from the Windows store (Windows)" + }, {0} }; @@ -193,6 +198,9 @@ static void flb_output_free_properties(struct flb_output_instance *ins) if (ins->tls_win_certstore_name) { flb_sds_destroy(ins->tls_win_certstore_name); } + if (ins->tls_win_thumbprints) { + flb_sds_destroy(ins->tls_win_thumbprints); + } # endif #endif } @@ -774,6 +782,7 @@ struct flb_output_instance *flb_output_new(struct flb_config *config, # if defined(FLB_SYSTEM_WINDOWS) instance->tls_win_certstore_name = NULL; instance->tls_win_use_enterprise_certstore = FLB_FALSE; + instance->tls_win_thumbprints = NULL; # endif #endif @@ -1007,6 +1016,9 @@ int flb_output_set_property(struct flb_output_instance *ins, ins->tls_win_use_enterprise_certstore = flb_utils_bool(tmp); flb_sds_destroy(tmp); } + else if (prop_key_check("tls.windows.client_thumbprints", k, len) == 0 && tmp) { + flb_utils_set_plugin_string_property("tls.windows.client_thumbprints", &ins->tls_win_thumbprints, tmp); + } # endif #endif else if (prop_key_check("storage.total_limit_size", k, len) == 0 && tmp) { @@ -1421,6 +1433,16 @@ int flb_output_init_all(struct flb_config *config) } } + if (ins->tls_win_thumbprints) { + ret = flb_tls_set_client_thumbprints(ins->tls, ins->tls_win_thumbprints); + if (ret == -1) { + flb_error("[input %s] error set up to use thumbprints of certificates in TLS context", + ins->name); + + return -1; + } + } + if (ins->tls_win_certstore_name) { flb_debug("[output %s] starting to load %s certstore in TLS context", ins->name, ins->tls_win_certstore_name); From ad17974317b3fd32abe1f4ed4f85822643284655 Mon Sep 17 00:00:00 2001 From: Patrick Stephens Date: Sun, 3 May 2026 12:34:55 +0100 Subject: [PATCH 073/213] [upstream] in_syslog: Handle octent_counting Upstream-Ref: https://github.com/fluent/fluent-bit/commit/a2917b899575f33eb01e0b356a4b812df9a9d857 Cherry-picked from Fluent Bit v4.2.4 --- source/plugins/in_syslog/syslog.c | 6 ++- source/plugins/in_syslog/syslog.h | 8 +++ source/plugins/in_syslog/syslog_conf.c | 13 +++++ source/plugins/in_syslog/syslog_conn.c | 2 + source/plugins/in_syslog/syslog_conn.h | 3 ++ source/plugins/in_syslog/syslog_prot.c | 67 +++++++++++++++++++++----- 6 files changed, 87 insertions(+), 12 deletions(-) diff --git a/source/plugins/in_syslog/syslog.c b/source/plugins/in_syslog/syslog.c index 793ea0a0..6e76ac08 100644 --- a/source/plugins/in_syslog/syslog.c +++ b/source/plugins/in_syslog/syslog.c @@ -244,7 +244,11 @@ static struct flb_config_map config_map[] = { 0, FLB_TRUE, offsetof(struct flb_syslog, source_address_key), "Key where the source address will be injected" }, - + { + FLB_CONFIG_MAP_STR, "frame", (char *) NULL, + 0, FLB_TRUE, offsetof(struct flb_syslog, frame_str), + "TCP framing: newline (default) or octet_counting (RFC 6587)" + }, /* EOF */ {0} diff --git a/source/plugins/in_syslog/syslog.h b/source/plugins/in_syslog/syslog.h index 7e42daa2..ba3ee937 100644 --- a/source/plugins/in_syslog/syslog.h +++ b/source/plugins/in_syslog/syslog.h @@ -33,6 +33,10 @@ /* 32KB chunk size */ #define FLB_SYSLOG_CHUNK "32768" +/* TCP framing */ +#define FLB_SYSLOG_FRAME_NEWLINE 0 +#define FLB_SYSLOG_FRAME_OCTET_COUNTING 1 + struct syslog_conn; /* Context / Config*/ @@ -67,6 +71,10 @@ struct flb_syslog { flb_sds_t raw_message_key; flb_sds_t source_address_key; + /* TCP framing */ + flb_sds_t frame_str; + int frame_type; + int dgram_mode_flag; int collector_id; struct mk_event *collector_event; diff --git a/source/plugins/in_syslog/syslog_conf.c b/source/plugins/in_syslog/syslog_conf.c index a20d78c6..60db3ad9 100644 --- a/source/plugins/in_syslog/syslog_conf.c +++ b/source/plugins/in_syslog/syslog_conf.c @@ -107,6 +107,19 @@ struct flb_syslog *syslog_conf_create(struct flb_input_instance *ins, ctx->mode = FLB_SYSLOG_UNIX_UDP; } + /* TCP Frame type (only applies to stream modes; default newline) */ + ctx->frame_type = FLB_SYSLOG_FRAME_NEWLINE; + if (ctx->frame_str != NULL) { + if (strcasecmp(ctx->frame_str, "octet_counting") == 0 || + strcasecmp(ctx->frame_str, "octet") == 0) { + ctx->frame_type = FLB_SYSLOG_FRAME_OCTET_COUNTING; + } + else if (strcasecmp(ctx->frame_str, "newline") != 0) { + flb_plg_warn(ins, "[in_syslog] unknown frame '%s', using 'newline'", + ctx->frame_str); + } + } + /* Check if TCP mode was requested */ if (ctx->mode == FLB_SYSLOG_TCP || ctx->mode == FLB_SYSLOG_UDP) { /* Listen interface (if not set, defaults to 0.0.0.0:5140) */ diff --git a/source/plugins/in_syslog/syslog_conn.c b/source/plugins/in_syslog/syslog_conn.c index eec1ab4d..1768dba9 100644 --- a/source/plugins/in_syslog/syslog_conn.c +++ b/source/plugins/in_syslog/syslog_conn.c @@ -178,6 +178,8 @@ struct syslog_conn *syslog_conn_add(struct flb_connection *connection, conn->ins = ctx->ins; conn->buf_len = 0; conn->buf_parsed = 0; + conn->frame_expected_len = 0; + conn->frame_have_len = 0; /* Allocate read buffer */ conn->buf_data = flb_malloc(ctx->buffer_chunk_size); diff --git a/source/plugins/in_syslog/syslog_conn.h b/source/plugins/in_syslog/syslog_conn.h index 5604da68..9e20b37c 100644 --- a/source/plugins/in_syslog/syslog_conn.h +++ b/source/plugins/in_syslog/syslog_conn.h @@ -35,6 +35,9 @@ struct syslog_conn { size_t buf_size; /* Buffer size */ size_t buf_len; /* Buffer length */ size_t buf_parsed; /* Parsed buffer (offset) */ + /* Octet-counting framing state */ + size_t frame_expected_len; /* remaining message bytes needed */ + int frame_have_len; /* 0 = need length, 1 = have length */ struct flb_input_instance *ins; /* Parent plugin instance */ struct flb_syslog *ctx; /* Plugin configuration context */ struct flb_connection *connection; diff --git a/source/plugins/in_syslog/syslog_prot.c b/source/plugins/in_syslog/syslog_prot.c index 37a00534..512ae251 100644 --- a/source/plugins/in_syslog/syslog_prot.c +++ b/source/plugins/in_syslog/syslog_prot.c @@ -218,21 +218,55 @@ int syslog_prot_process(struct syslog_conn *conn) flb_log_event_encoder_reset(ctx->log_encoder); - /* Always parse while some remaining bytes exists */ + /* Always parse while some remaining bytes exist */ while (eof < end) { - /* Lookup the ending byte */ - eof = p = conn->buf_data + conn->buf_parsed; - while (*eof != '\n' && *eof != '\0' && eof < end) { - eof++; + if (ctx->frame_type == FLB_SYSLOG_FRAME_NEWLINE) { + /* newline framing (current behavior) */ + eof = p = conn->buf_data + conn->buf_parsed; + while (*eof != '\n' && *eof != '\0' && eof < end) { + eof++; + } + /* Incomplete message */ + if (eof == end || (*eof != '\n' && *eof != '\0')) { + break; + } + len = (eof - p); } - - /* Incomplete message */ - if (eof == end || (*eof != '\n' && *eof != '\0')) { - break; + else { + /* RFC 6587 octet-counting framing: SP */ + p = conn->buf_data + conn->buf_parsed; + + if (!conn->frame_have_len) { + char *sp = p; + size_t n = 0; + while (sp < end && *sp >= '0' && *sp <= '9') { + if (n > SIZE_MAX / 10) { + n = SIZE_MAX; + break; + } + n = n * 10 + (size_t)(*sp - '0'); + sp++; + } + if (sp == end) { + break; + } + if (*sp != ' ') { + flb_plg_warn(ctx->ins, "invalid octet-counting length"); + return -1; + } + conn->buf_parsed += (sp - p) + 1; + conn->frame_expected_len = n; + conn->frame_have_len = 1; + p = conn->buf_data + conn->buf_parsed; + end = conn->buf_data + conn->buf_len; + } + if ((size_t)(end - p) < conn->frame_expected_len) { + break; + } + len = (int)conn->frame_expected_len; } /* No data ? */ - len = (eof - p); if (len == 0) { consume_bytes(conn->buf_data, 1, conn->buf_len); conn->buf_len--; @@ -266,7 +300,18 @@ int syslog_prot_process(struct syslog_conn *conn) flb_plg_debug(ctx->ins, "unparsed log message: %.*s", len, p); } - conn->buf_parsed += len + 1; + if (ctx->frame_type == FLB_SYSLOG_FRAME_NEWLINE) { + conn->buf_parsed += len + 1; + } + else { + conn->buf_parsed += len; + conn->frame_expected_len = 0; + conn->frame_have_len = 0; + if (conn->buf_parsed < conn->buf_len && + conn->buf_data[conn->buf_parsed] == '\n') { + conn->buf_parsed += 1; + } + } end = conn->buf_data + conn->buf_len; eof = conn->buf_data + conn->buf_parsed; } From 6843dbc6d9da96c3f0d3c5bb9e83e175d0a4fe2d Mon Sep 17 00:00:00 2001 From: Patrick Stephens Date: Sun, 3 May 2026 12:34:55 +0100 Subject: [PATCH 074/213] [upstream] in_syslog: Add test cases for octet counting Upstream-Ref: https://github.com/fluent/fluent-bit/commit/5bf9544d23ad4e89b1b18b6d4f8253602dc25caa Cherry-picked from Fluent Bit v4.2.4 --- source/tests/runtime/in_syslog.c | 323 ++++++++++++++++++++++++++++++- 1 file changed, 322 insertions(+), 1 deletion(-) diff --git a/source/tests/runtime/in_syslog.c b/source/tests/runtime/in_syslog.c index 532bf069..e5d6be06 100644 --- a/source/tests/runtime/in_syslog.c +++ b/source/tests/runtime/in_syslog.c @@ -305,6 +305,61 @@ static int init_udp(char *in_host, int in_port, struct sockaddr_in *addr) return fd; } +/* Copy src into dst, stripping a single trailing '\n' if present; return len */ +static size_t rstrip_nl_copy(char *dst, size_t dstsz, const char *src) +{ + size_t n = strlen(src); + if (n > 0 && src[n - 1] == '\n') { + n -= 1; + } + if (n + 1 > dstsz) { + n = dstsz - 1; + } + memcpy(dst, src, n); + dst[n] = '\0'; + return n; +} + +/* Build one octet-counted frame into 'out' as: " " + msg [+ '\n' if add_lf] */ +/* Returns total bytes written (excluding terminal '\0' in 'out') */ +static size_t build_octet_frame(char *out, size_t outsz, + const char *msg, int add_lf) +{ + char tmp[2048]; + size_t mlen = rstrip_nl_copy(tmp, sizeof(tmp), msg); + char hdr[64]; + int hlen = snprintf(hdr, sizeof(hdr), "%zu ", mlen); + size_t need = (size_t)hlen + mlen + (add_lf ? 1 : 0); + + if (need + 1 > outsz) { + /* truncate conservatively if buffer too small (shouldn't happen in tests) */ + need = outsz - 1; + add_lf = 0; + if ((size_t)hlen > need) { + hlen = (int)need; + } + } + + memcpy(out, hdr, (size_t)hlen); + memcpy(out + hlen, tmp, mlen); + if (add_lf) { + out[hlen + mlen] = '\n'; + } + out[need] = '\0'; + return need; +} + +/* Build two consecutive octet-counted frames into one buffer */ +static size_t build_two_frames(char *out, size_t outsz, + const char *msg1, const char *msg2, + int add_lf_for_each) +{ + size_t off = 0; + off += build_octet_frame(out + off, outsz - off, msg1, add_lf_for_each); + off += build_octet_frame(out + off, outsz - off, msg2, add_lf_for_each); + return off; +} + void flb_test_syslog_tcp() { struct flb_lib_out_cb cb_data; @@ -1002,6 +1057,269 @@ void flb_test_syslog_rfc3164() test_ctx_destroy(ctx); } +void flb_test_syslog_tcp_octet_counting() +{ + struct flb_lib_out_cb cb_data; + struct test_ctx *ctx; + flb_sockfd_t fd; + int ret; + int num; + ssize_t w_size; + + struct str_list expected = { + .size = sizeof(RFC5424_EXPECTED_STRS_1)/sizeof(char*), + .lists = &RFC5424_EXPECTED_STRS_1[0], + }; + + char frame[4096]; + size_t fsize = build_octet_frame(frame, sizeof(frame), RFC5424_EXAMPLE_1, /*add_lf=*/0); + + clear_output_num(); + cb_data.cb = cb_check_json_str_list; + cb_data.data = &expected; + + ctx = test_ctx_create(&cb_data); + if (!TEST_CHECK(ctx != NULL)) { + TEST_MSG("test_ctx_create failed"); + exit(EXIT_FAILURE); + } + + ret = flb_input_set(ctx->flb, ctx->i_ffd, + "mode", "tcp", + "frame", "octet_counting", + "parser", PARSER_NAME_RFC5424, + NULL); + TEST_CHECK(ret == 0); + + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + fd = connect_tcp(NULL, -1); + if (!TEST_CHECK(fd >= 0)) { + test_ctx_destroy(ctx); + exit(EXIT_FAILURE); + } + + w_size = send(fd, frame, fsize, 0); + if (!TEST_CHECK(w_size == (ssize_t)fsize)) { + TEST_MSG("failed to send, errno=%d", errno); + flb_socket_close(fd); + test_ctx_destroy(ctx); + exit(EXIT_FAILURE); + } + + flb_time_msleep(500); + num = get_output_num(); + if (!TEST_CHECK(num > 0)) { + TEST_MSG("no outputs (octet_counting single)"); + } + + flb_socket_close(fd); + test_ctx_destroy(ctx); +} + +/* -------- TCP + RFC6587 octet-counting: frame with trailing LF -------- */ +void flb_test_syslog_tcp_octet_counting_lf() +{ + struct flb_lib_out_cb cb_data; + struct test_ctx *ctx; + flb_sockfd_t fd; + int ret; + int num; + ssize_t w_size; + + struct str_list expected = { + .size = sizeof(RFC5424_EXPECTED_STRS_1)/sizeof(char*), + .lists = &RFC5424_EXPECTED_STRS_1[0], + }; + + char frame[4096]; + size_t fsize = build_octet_frame(frame, sizeof(frame), RFC5424_EXAMPLE_1, /*add_lf=*/1); + + clear_output_num(); + cb_data.cb = cb_check_json_str_list; + cb_data.data = &expected; + + ctx = test_ctx_create(&cb_data); + if (!TEST_CHECK(ctx != NULL)) { + TEST_MSG("test_ctx_create failed"); + exit(EXIT_FAILURE); + } + + ret = flb_input_set(ctx->flb, ctx->i_ffd, + "mode", "tcp", + "frame", "octet_counting", + "parser", PARSER_NAME_RFC5424, + NULL); + TEST_CHECK(ret == 0); + + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + fd = connect_tcp(NULL, -1); + if (!TEST_CHECK(fd >= 0)) { + test_ctx_destroy(ctx); + exit(EXIT_FAILURE); + } + + w_size = send(fd, frame, fsize, 0); + if (!TEST_CHECK(w_size == (ssize_t)fsize)) { + TEST_MSG("failed to send, errno=%d", errno); + flb_socket_close(fd); + test_ctx_destroy(ctx); + exit(EXIT_FAILURE); + } + + flb_time_msleep(500); + num = get_output_num(); + if (!TEST_CHECK(num > 0)) { + TEST_MSG("no outputs (octet_counting + LF)"); + } + + flb_socket_close(fd); + test_ctx_destroy(ctx); +} + +/* -------- TCP + RFC6587 octet-counting: fragmented send (header then body) -------- */ +void flb_test_syslog_tcp_octet_counting_fragmented() +{ + struct flb_lib_out_cb cb_data; + struct test_ctx *ctx; + flb_sockfd_t fd; + int ret; + int num; + ssize_t w_size; + + struct str_list expected = { + .size = sizeof(RFC5424_EXPECTED_STRS_1)/sizeof(char*), + .lists = &RFC5424_EXPECTED_STRS_1[0], + }; + + char msg[2048]; + size_t mlen = rstrip_nl_copy(msg, sizeof(msg), RFC5424_EXAMPLE_1); + char hdr[64]; + int hlen = snprintf(hdr, sizeof(hdr), "%zu ", mlen); + + clear_output_num(); + cb_data.cb = cb_check_json_str_list; + cb_data.data = &expected; + + ctx = test_ctx_create(&cb_data); + if (!TEST_CHECK(ctx != NULL)) { + TEST_MSG("test_ctx_create failed"); + exit(EXIT_FAILURE); + } + + ret = flb_input_set(ctx->flb, ctx->i_ffd, + "mode", "tcp", + "frame", "octet_counting", + "parser", PARSER_NAME_RFC5424, + NULL); + TEST_CHECK(ret == 0); + + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + fd = connect_tcp(NULL, -1); + if (!TEST_CHECK(fd >= 0)) { + test_ctx_destroy(ctx); + exit(EXIT_FAILURE); + } + + /* Send header only first */ + w_size = send(fd, hdr, (size_t)hlen, 0); + if (!TEST_CHECK(w_size == hlen)) { + TEST_MSG("failed to send header, errno=%d", errno); + flb_socket_close(fd); + test_ctx_destroy(ctx); + exit(EXIT_FAILURE); + } + /* Give the input a moment to hit 'need more bytes' path */ + flb_time_msleep(50); + + /* Now send body */ + w_size = send(fd, msg, mlen, 0); + if (!TEST_CHECK(w_size == (ssize_t)mlen)) { + TEST_MSG("failed to send body, errno=%d", errno); + flb_socket_close(fd); + test_ctx_destroy(ctx); + exit(EXIT_FAILURE); + } + + flb_time_msleep(500); + num = get_output_num(); + if (!TEST_CHECK(num > 0)) { + TEST_MSG("no outputs (octet_counting fragmented)"); + } + + flb_socket_close(fd); + test_ctx_destroy(ctx); +} + +/* -------- TCP + RFC6587 octet-counting: two frames back-to-back -------- */ +void flb_test_syslog_tcp_octet_counting_multi() +{ + struct flb_lib_out_cb cb_data; + struct test_ctx *ctx; + flb_sockfd_t fd; + int ret; + int num; + ssize_t w_size; + + struct str_list expected = { + .size = sizeof(RFC5424_EXPECTED_STRS_1)/sizeof(char*), + .lists = &RFC5424_EXPECTED_STRS_1[0], + }; + + char frames[8192]; + size_t fsize = build_two_frames(frames, sizeof(frames), + RFC5424_EXAMPLE_1, RFC5424_EXAMPLE_1, + /*add_lf_for_each=*/0); + + clear_output_num(); + cb_data.cb = cb_check_json_str_list; + cb_data.data = &expected; + + ctx = test_ctx_create(&cb_data); + if (!TEST_CHECK(ctx != NULL)) { + TEST_MSG("test_ctx_create failed"); + exit(EXIT_FAILURE); + } + + ret = flb_input_set(ctx->flb, ctx->i_ffd, + "mode", "tcp", + "frame", "octet_counting", + "parser", PARSER_NAME_RFC5424, + NULL); + TEST_CHECK(ret == 0); + + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + fd = connect_tcp(NULL, -1); + if (!TEST_CHECK(fd >= 0)) { + test_ctx_destroy(ctx); + exit(EXIT_FAILURE); + } + + w_size = send(fd, frames, fsize, 0); + if (!TEST_CHECK(w_size == (ssize_t)fsize)) { + TEST_MSG("failed to send frames, errno=%d", errno); + flb_socket_close(fd); + test_ctx_destroy(ctx); + exit(EXIT_FAILURE); + } + + flb_time_msleep(500); + num = get_output_num(); + if (!TEST_CHECK(num >= 2)) { + TEST_MSG("expected at least 2 outputs (octet_counting multi), got %d", num); + } + + flb_socket_close(fd); + test_ctx_destroy(ctx); +} + TEST_LIST = { {"syslog_tcp", flb_test_syslog_tcp}, {"syslog_udp", flb_test_syslog_udp}, @@ -1020,6 +1338,9 @@ TEST_LIST = { {"syslog_udp_unix", flb_test_syslog_udp_unix}, #endif #endif + {"syslog_tcp_octet_counting", flb_test_syslog_tcp_octet_counting}, + {"syslog_tcp_octet_counting_lf", flb_test_syslog_tcp_octet_counting_lf}, + {"syslog_tcp_octet_counting_fragmented", flb_test_syslog_tcp_octet_counting_fragmented}, + {"syslog_tcp_octet_counting_multi", flb_test_syslog_tcp_octet_counting_multi}, {NULL, NULL} }; - From 086aafd206b8c8991a6f81e286adf5c71c0b6d27 Mon Sep 17 00:00:00 2001 From: Patrick Stephens Date: Sun, 3 May 2026 12:34:55 +0100 Subject: [PATCH 075/213] [upstream] in_syslog: Include a header explicitly Upstream-Ref: https://github.com/fluent/fluent-bit/commit/478943719727213ac7969152674ccd404b22b366 Cherry-picked from Fluent Bit v4.2.4 --- source/plugins/in_syslog/syslog_prot.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/source/plugins/in_syslog/syslog_prot.h b/source/plugins/in_syslog/syslog_prot.h index 9b28c474..5d47e3cc 100644 --- a/source/plugins/in_syslog/syslog_prot.h +++ b/source/plugins/in_syslog/syslog_prot.h @@ -22,6 +22,8 @@ #include +#include + #include "syslog.h" #define FLB_MAP_EXPAND_SUCCESS 0 From 4e13eee95e238c634817b00cae3a00dfbe5c18a1 Mon Sep 17 00:00:00 2001 From: Patrick Stephens Date: Sun, 3 May 2026 12:34:55 +0100 Subject: [PATCH 076/213] [upstream] in_syslog: tests: Split declaration and applying Upstream-Ref: https://github.com/fluent/fluent-bit/commit/16e2dd83b8da51e7a73253bc8f458fdc36ac7045 Cherry-picked from Fluent Bit v4.2.4 --- source/tests/runtime/in_syslog.c | 33 +++++++++++++++++++++----------- 1 file changed, 22 insertions(+), 11 deletions(-) diff --git a/source/tests/runtime/in_syslog.c b/source/tests/runtime/in_syslog.c index e5d6be06..f77277c8 100644 --- a/source/tests/runtime/in_syslog.c +++ b/source/tests/runtime/in_syslog.c @@ -326,10 +326,14 @@ static size_t build_octet_frame(char *out, size_t outsz, const char *msg, int add_lf) { char tmp[2048]; - size_t mlen = rstrip_nl_copy(tmp, sizeof(tmp), msg); + size_t mlen = 0; char hdr[64]; - int hlen = snprintf(hdr, sizeof(hdr), "%zu ", mlen); - size_t need = (size_t)hlen + mlen + (add_lf ? 1 : 0); + int hlen = 0; + size_t need = 0; + + mlen = rstrip_nl_copy(tmp, sizeof(tmp), msg); + hlen = snprintf(hdr, sizeof(hdr), "%zu ", mlen); + need = (size_t)hlen + mlen + (add_lf ? 1 : 0); if (need + 1 > outsz) { /* truncate conservatively if buffer too small (shouldn't happen in tests) */ @@ -340,7 +344,7 @@ static size_t build_octet_frame(char *out, size_t outsz, } } - memcpy(out, hdr, (size_t)hlen); + memcpy(out, hdr, hlen); memcpy(out + hlen, tmp, mlen); if (add_lf) { out[hlen + mlen] = '\n'; @@ -1072,8 +1076,9 @@ void flb_test_syslog_tcp_octet_counting() }; char frame[4096]; - size_t fsize = build_octet_frame(frame, sizeof(frame), RFC5424_EXAMPLE_1, /*add_lf=*/0); + size_t fsize = 0; + fsize = build_octet_frame(frame, sizeof(frame), RFC5424_EXAMPLE_1, /*add_lf=*/0); clear_output_num(); cb_data.cb = cb_check_json_str_list; cb_data.data = &expected; @@ -1134,8 +1139,9 @@ void flb_test_syslog_tcp_octet_counting_lf() }; char frame[4096]; - size_t fsize = build_octet_frame(frame, sizeof(frame), RFC5424_EXAMPLE_1, /*add_lf=*/1); + size_t fsize = 0; + fsize = build_octet_frame(frame, sizeof(frame), RFC5424_EXAMPLE_1, /*add_lf=*/1); clear_output_num(); cb_data.cb = cb_check_json_str_list; cb_data.data = &expected; @@ -1196,9 +1202,12 @@ void flb_test_syslog_tcp_octet_counting_fragmented() }; char msg[2048]; - size_t mlen = rstrip_nl_copy(msg, sizeof(msg), RFC5424_EXAMPLE_1); + size_t mlen = 0; char hdr[64]; - int hlen = snprintf(hdr, sizeof(hdr), "%zu ", mlen); + int hlen = 0; + + mlen = rstrip_nl_copy(msg, sizeof(msg), RFC5424_EXAMPLE_1); + hlen = snprintf(hdr, sizeof(hdr), "%zu ", mlen); clear_output_num(); cb_data.cb = cb_check_json_str_list; @@ -1272,9 +1281,11 @@ void flb_test_syslog_tcp_octet_counting_multi() }; char frames[8192]; - size_t fsize = build_two_frames(frames, sizeof(frames), - RFC5424_EXAMPLE_1, RFC5424_EXAMPLE_1, - /*add_lf_for_each=*/0); + size_t fsize = 0; + + fsize = build_two_frames(frames, sizeof(frames), + RFC5424_EXAMPLE_1, RFC5424_EXAMPLE_1, + /*add_lf_for_each=*/0); clear_output_num(); cb_data.cb = cb_check_json_str_list; From 16a7d4cf579b5ffe7204cb941d95d5a1ea7f5d37 Mon Sep 17 00:00:00 2001 From: Patrick Stephens Date: Sun, 3 May 2026 12:34:56 +0100 Subject: [PATCH 077/213] [upstream] in_syslog: tests: Use format instead of frame Upstream-Ref: https://github.com/fluent/fluent-bit/commit/9f8c816007af14a2c62789fbd409c78eeec4440c Cherry-picked from Fluent Bit v4.2.4 --- source/plugins/in_syslog/syslog.c | 6 +++--- source/plugins/in_syslog/syslog.h | 2 +- source/plugins/in_syslog/syslog_conf.c | 10 +++++----- source/tests/runtime/in_syslog.c | 8 ++++---- 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/source/plugins/in_syslog/syslog.c b/source/plugins/in_syslog/syslog.c index 6e76ac08..45d0fa2b 100644 --- a/source/plugins/in_syslog/syslog.c +++ b/source/plugins/in_syslog/syslog.c @@ -245,9 +245,9 @@ static struct flb_config_map config_map[] = { "Key where the source address will be injected" }, { - FLB_CONFIG_MAP_STR, "frame", (char *) NULL, - 0, FLB_TRUE, offsetof(struct flb_syslog, frame_str), - "TCP framing: newline (default) or octet_counting (RFC 6587)" + FLB_CONFIG_MAP_STR, "format", (char *) NULL, + 0, FLB_TRUE, offsetof(struct flb_syslog, format_str), + "Format of TCP framing: newline (default) or octet_counting (RFC 6587)" }, /* EOF */ diff --git a/source/plugins/in_syslog/syslog.h b/source/plugins/in_syslog/syslog.h index ba3ee937..aa6900ec 100644 --- a/source/plugins/in_syslog/syslog.h +++ b/source/plugins/in_syslog/syslog.h @@ -72,7 +72,7 @@ struct flb_syslog { flb_sds_t source_address_key; /* TCP framing */ - flb_sds_t frame_str; + flb_sds_t format_str; int frame_type; int dgram_mode_flag; diff --git a/source/plugins/in_syslog/syslog_conf.c b/source/plugins/in_syslog/syslog_conf.c index 60db3ad9..a2ab053f 100644 --- a/source/plugins/in_syslog/syslog_conf.c +++ b/source/plugins/in_syslog/syslog_conf.c @@ -109,14 +109,14 @@ struct flb_syslog *syslog_conf_create(struct flb_input_instance *ins, /* TCP Frame type (only applies to stream modes; default newline) */ ctx->frame_type = FLB_SYSLOG_FRAME_NEWLINE; - if (ctx->frame_str != NULL) { - if (strcasecmp(ctx->frame_str, "octet_counting") == 0 || - strcasecmp(ctx->frame_str, "octet") == 0) { + if (ctx->format_str != NULL) { + if (strcasecmp(ctx->format_str, "octet_counting") == 0 || + strcasecmp(ctx->format_str, "octet") == 0) { ctx->frame_type = FLB_SYSLOG_FRAME_OCTET_COUNTING; } - else if (strcasecmp(ctx->frame_str, "newline") != 0) { + else if (strcasecmp(ctx->format_str, "newline") != 0) { flb_plg_warn(ins, "[in_syslog] unknown frame '%s', using 'newline'", - ctx->frame_str); + ctx->format_str); } } diff --git a/source/tests/runtime/in_syslog.c b/source/tests/runtime/in_syslog.c index f77277c8..26911924 100644 --- a/source/tests/runtime/in_syslog.c +++ b/source/tests/runtime/in_syslog.c @@ -1091,7 +1091,7 @@ void flb_test_syslog_tcp_octet_counting() ret = flb_input_set(ctx->flb, ctx->i_ffd, "mode", "tcp", - "frame", "octet_counting", + "format", "octet_counting", "parser", PARSER_NAME_RFC5424, NULL); TEST_CHECK(ret == 0); @@ -1154,7 +1154,7 @@ void flb_test_syslog_tcp_octet_counting_lf() ret = flb_input_set(ctx->flb, ctx->i_ffd, "mode", "tcp", - "frame", "octet_counting", + "format", "octet_counting", "parser", PARSER_NAME_RFC5424, NULL); TEST_CHECK(ret == 0); @@ -1221,7 +1221,7 @@ void flb_test_syslog_tcp_octet_counting_fragmented() ret = flb_input_set(ctx->flb, ctx->i_ffd, "mode", "tcp", - "frame", "octet_counting", + "format", "octet_counting", "parser", PARSER_NAME_RFC5424, NULL); TEST_CHECK(ret == 0); @@ -1299,7 +1299,7 @@ void flb_test_syslog_tcp_octet_counting_multi() ret = flb_input_set(ctx->flb, ctx->i_ffd, "mode", "tcp", - "frame", "octet_counting", + "format", "octet_counting", "parser", PARSER_NAME_RFC5424, NULL); TEST_CHECK(ret == 0); From 37b4e4661bf46d38c514ee3b9893629489cfdfcd Mon Sep 17 00:00:00 2001 From: Patrick Stephens Date: Sun, 3 May 2026 12:34:56 +0100 Subject: [PATCH 078/213] [upstream] scheduler: engine: Plug macOS abort on dry-run Upstream-Ref: https://github.com/fluent/fluent-bit/commit/b67db0285846907520d6c7bc2a295b126b2151dd Cherry-picked from Fluent Bit v4.2.4 --- source/include/fluent-bit/flb_scheduler.h | 4 ++++ source/src/flb_engine.c | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/source/include/fluent-bit/flb_scheduler.h b/source/include/fluent-bit/flb_scheduler.h index 11b526d3..ab1adf4e 100644 --- a/source/include/fluent-bit/flb_scheduler.h +++ b/source/include/fluent-bit/flb_scheduler.h @@ -192,11 +192,14 @@ struct flb_sched_timer_coro { struct cfl_list _head; }; +#define FLB_SCHED_TLS_MAGIC 0x53544350u + /* parameter for timer callback running under a co-routine */ struct flb_sched_timer_coro_cb_params { struct flb_sched_timer_coro *stc; struct flb_config *config; void *data; + int magic; struct flb_coro *coro; }; @@ -259,6 +262,7 @@ static FLB_INLINE void sched_timer_cb_params_set(struct flb_sched_timer_coro *st params->config = config; params->data = data; params->coro = coro; + params->magic = FLB_SCHED_TLS_MAGIC; FLB_TLS_SET(sched_timer_coro_cb_params, params); co_switch(coro->callee); diff --git a/source/src/flb_engine.c b/source/src/flb_engine.c index 40a06b19..cd927057 100644 --- a/source/src/flb_engine.c +++ b/source/src/flb_engine.c @@ -1229,7 +1229,7 @@ int flb_engine_shutdown(struct flb_config *config) /* scheduler */ sched_params = (struct flb_sched_timer_coro_cb_params *) FLB_TLS_GET(sched_timer_coro_cb_params); - if (sched_params != NULL) { + if (sched_params && sched_params->magic == FLB_SCHED_TLS_MAGIC) { flb_free(sched_params); FLB_TLS_SET(sched_timer_coro_cb_params, NULL); } From 8499aa2d94684b6dc6c67f67fe12e0f310094ac9 Mon Sep 17 00:00:00 2001 From: Patrick Stephens Date: Sun, 3 May 2026 12:34:56 +0100 Subject: [PATCH 079/213] [upstream] output_thread: Validate a magic for cleaning up Upstream-Ref: https://github.com/fluent/fluent-bit/commit/ef58c5382aca3d126b36c065a031d146de6b2b39 Cherry-picked from Fluent Bit v4.2.4 --- source/src/flb_output_thread.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/src/flb_output_thread.c b/source/src/flb_output_thread.c index bb146665..56b38967 100644 --- a/source/src/flb_output_thread.c +++ b/source/src/flb_output_thread.c @@ -402,7 +402,7 @@ static void output_thread(void *data) } sched_params = (struct flb_sched_timer_coro_cb_params *) FLB_TLS_GET(sched_timer_coro_cb_params); - if (sched_params != NULL) { + if (sched_params != NULL && sched_params->magic == FLB_SCHED_TLS_MAGIC) { flb_free(sched_params); FLB_TLS_SET(sched_timer_coro_cb_params, NULL); } From 0b9d00489541a014f482805652e144f03fa2377f Mon Sep 17 00:00:00 2001 From: Patrick Stephens Date: Sun, 3 May 2026 12:34:56 +0100 Subject: [PATCH 080/213] [upstream] scheduler: Use a correct type on a member Upstream-Ref: https://github.com/fluent/fluent-bit/commit/be2f2be2c91c1e9eb9075577315223c486935de9 Cherry-picked from Fluent Bit v4.2.4 --- source/include/fluent-bit/flb_scheduler.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/include/fluent-bit/flb_scheduler.h b/source/include/fluent-bit/flb_scheduler.h index ab1adf4e..227d8a5a 100644 --- a/source/include/fluent-bit/flb_scheduler.h +++ b/source/include/fluent-bit/flb_scheduler.h @@ -199,7 +199,7 @@ struct flb_sched_timer_coro_cb_params { struct flb_sched_timer_coro *stc; struct flb_config *config; void *data; - int magic; + uint32_t magic; struct flb_coro *coro; }; From 4f16be8ea67ea117e07dc4e944fc89d26952ec39 Mon Sep 17 00:00:00 2001 From: Patrick Stephens Date: Sun, 3 May 2026 12:34:56 +0100 Subject: [PATCH 081/213] [upstream] router: add router context structure and move routing Upstream-Ref: https://github.com/fluent/fluent-bit/commit/7ce0d1606b4f361c6428c8be868af2cf0067da41 Cherry-picked from Fluent Bit v4.2.4 --- source/include/fluent-bit/flb_router.h | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/source/include/fluent-bit/flb_router.h b/source/include/fluent-bit/flb_router.h index 78824b8d..7a66d7af 100644 --- a/source/include/fluent-bit/flb_router.h +++ b/source/include/fluent-bit/flb_router.h @@ -28,6 +28,8 @@ #include #include #include +#include +#include #include struct flb_router_path { @@ -36,6 +38,22 @@ struct flb_router_path { struct cfl_list _head; }; +struct flb_router { + /* Routing masks */ + size_t route_mask_size; + size_t route_mask_slots; + uint64_t *route_empty_mask; + + /* metrics context */ + struct cmt *cmt; + + /* logs routing metrics */ + struct cmt_counter *logs_records_total; + struct cmt_counter *logs_bytes_total; + struct cmt_counter *logs_drop_records_total; + struct cmt_counter *logs_drop_bytes_total; +}; + static inline int flb_router_match_type(int in_event_type, struct flb_output_instance *o_ins) { @@ -154,5 +172,9 @@ int flb_router_config_parse(struct flb_cf *cf, void flb_router_routes_destroy(struct cfl_list *input_routes); int flb_router_apply_config(struct flb_config *config); +int flb_router_metrics_create(struct flb_config *config, struct flb_router *router); +struct flb_router *flb_router_create(struct flb_config *config); +void flb_router_destroy(struct flb_router *router); + #endif From bc1b687d5f4b20da0bacad55b8e343769ca33ad4 Mon Sep 17 00:00:00 2001 From: Patrick Stephens Date: Sun, 3 May 2026 12:34:57 +0100 Subject: [PATCH 082/213] [upstream] router: implement router context creation and Upstream-Ref: https://github.com/fluent/fluent-bit/commit/0fc19d17fe8af3bd6d209c05d61aea472db681f4 Cherry-picked from Fluent Bit v4.2.4 --- source/src/flb_router.c | 102 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 102 insertions(+) diff --git a/source/src/flb_router.c b/source/src/flb_router.c index 47251d52..a4ce99c1 100644 --- a/source/src/flb_router.c +++ b/source/src/flb_router.c @@ -25,6 +25,7 @@ #include #include #include +#include #ifdef FLB_HAVE_REGEX #include @@ -298,3 +299,104 @@ void flb_router_exit(struct flb_config *config) } } } + +static int router_metrics_create(struct flb_router *router) +{ + if (!router || !router->cmt) { + return -1; + } + + router->logs_records_total = cmt_counter_create(router->cmt, + "fluentbit", + "routing_logs", + "records_total", + "Total log records routed from input to output", + 2, + (char *[]) {"input", "output"}); + if (!router->logs_records_total) { + return -1; + } + + router->logs_bytes_total = cmt_counter_create(router->cmt, + "fluentbit", + "routing_logs", + "bytes_total", + "Total bytes routed from input to output (logs)", + 2, + (char *[]) {"input", "output"}); + if (!router->logs_bytes_total) { + return -1; + } + + router->logs_drop_records_total = cmt_counter_create(router->cmt, + "fluentbit", + "routing_logs", + "drop_records_total", + "Total log records dropped during routing", + 2, + (char *[]) {"input", "output"}); + if (!router->logs_drop_records_total) { + return -1; + } + + router->logs_drop_bytes_total = cmt_counter_create(router->cmt, + "fluentbit", + "routing_logs", + "drop_bytes_total", + "Total bytes dropped during routing (logs)", + 2, + (char *[]) {"input", "output"}); + if (!router->logs_drop_bytes_total) { + return -1; + } + + return 0; +} + +int flb_router_metrics_create(struct flb_config *config, struct flb_router *router) +{ + (void) config; + + return router_metrics_create(router); +} + +struct flb_router *flb_router_create(struct flb_config *config) +{ + struct flb_router *router; + + (void) config; + + router = flb_calloc(1, sizeof(struct flb_router)); + if (!router) { + flb_errno(); + return NULL; + } + + router->cmt = cmt_create(); + if (!router->cmt) { + flb_free(router); + return NULL; + } + + if (router_metrics_create(router) != 0) { + flb_router_destroy(router); + return NULL; + } + + return router; +} + +void flb_router_destroy(struct flb_router *router) +{ + if (!router) { + return; + } + + flb_routes_empty_mask_destroy(router); + + if (router->cmt) { + cmt_destroy(router->cmt); + } + + flb_free(router); +} From cac95915fda647de9881481a4ee24e55c3faf1c3 Mon Sep 17 00:00:00 2001 From: Patrick Stephens Date: Sun, 3 May 2026 12:34:57 +0100 Subject: [PATCH 083/213] [upstream] routes_mask: migrate API to use router context Upstream-Ref: https://github.com/fluent/fluent-bit/commit/b361654b5378652c7d4ad4cda141f03ee7d3def3 Cherry-picked from Fluent Bit v4.2.4 --- source/include/fluent-bit/flb_routes_mask.h | 25 +++-- source/src/flb_routes_mask.c | 107 ++++++++++++++------ 2 files changed, 92 insertions(+), 40 deletions(-) diff --git a/source/include/fluent-bit/flb_routes_mask.h b/source/include/fluent-bit/flb_routes_mask.h index 3f47f549..5be19140 100644 --- a/source/include/fluent-bit/flb_routes_mask.h +++ b/source/include/fluent-bit/flb_routes_mask.h @@ -46,23 +46,26 @@ typedef uint64_t flb_route_mask_element; /* forward declaration */ struct flb_input_instance; struct flb_config; +struct flb_router; int flb_routes_mask_set_by_tag(flb_route_mask_element *routes_mask, const char *tag, int tag_len, struct flb_input_instance *in); -int flb_routes_mask_get_bit(flb_route_mask_element *routes_mask, int value, - struct flb_config *config); -void flb_routes_mask_set_bit(flb_route_mask_element *routes_mask, int value, - struct flb_config *config); -void flb_routes_mask_clear_bit(flb_route_mask_element *routes_mask, int value, - struct flb_config *config); -int flb_routes_mask_is_empty(flb_route_mask_element *routes_mask, - struct flb_config *config); +int flb_routes_mask_get_bit(flb_route_mask_element *routes_mask, int value, + struct flb_router *router); +void flb_routes_mask_set_bit(flb_route_mask_element *routes_mask, int value, + struct flb_router *router); +void flb_routes_mask_clear_bit(flb_route_mask_element *routes_mask, int value, + struct flb_router *router); +int flb_routes_mask_is_empty(flb_route_mask_element *routes_mask, + struct flb_router *router); -int flb_routes_empty_mask_create(struct flb_config *config); -void flb_routes_empty_mask_destroy(struct flb_config *config); +int flb_routes_empty_mask_create(struct flb_router *router); +void flb_routes_empty_mask_destroy(struct flb_router *router); -int flb_routes_mask_set_size(size_t mask_size, struct flb_config *config); +int flb_routes_mask_set_size(size_t mask_size, struct flb_router *router); +size_t flb_routes_mask_get_size(struct flb_router *router); +size_t flb_routes_mask_get_slots(struct flb_router *router); #endif diff --git a/source/src/flb_routes_mask.c b/source/src/flb_routes_mask.c index 4ff29587..42db8999 100644 --- a/source/src/flb_routes_mask.c +++ b/source/src/flb_routes_mask.c @@ -24,6 +24,24 @@ #include +size_t flb_routes_mask_get_size(struct flb_router *router) +{ + if (router == NULL) { + return 0; + } + + return router->route_mask_size; +} + +size_t flb_routes_mask_get_slots(struct flb_router *router) +{ + if (router == NULL) { + return 0; + } + + return router->route_mask_slots; +} + /* * Set the routes_mask for input chunk with a router_match on tag, return a * non-zero value if any routes matched @@ -34,17 +52,20 @@ int flb_routes_mask_set_by_tag(flb_route_mask_element *routes_mask, struct flb_input_instance *in) { int has_routes = 0; + size_t size; struct mk_list *o_head; struct flb_output_instance *o_ins; if (!in) { return 0; } + if (in->config == NULL || in->config->router == NULL) { + return 0; + } + /* Clear the bit field */ - memset(routes_mask, - 0, - sizeof(flb_route_mask_element) * - in->config->route_mask_size); + size = flb_routes_mask_get_size(in->config->router); + memset(routes_mask, 0, sizeof(flb_route_mask_element) * size); /* Find all matching routes for the given tag */ mk_list_foreach(o_head, &in->config->outputs) { @@ -58,7 +79,7 @@ int flb_routes_mask_set_by_tag(flb_route_mask_element *routes_mask, , NULL #endif )) { - flb_routes_mask_set_bit(routes_mask, o_ins->id, o_ins->config); + flb_routes_mask_set_bit(routes_mask, o_ins->id, o_ins->config->router); has_routes = 1; } } @@ -74,13 +95,17 @@ int flb_routes_mask_set_by_tag(flb_route_mask_element *routes_mask, * */ void flb_routes_mask_set_bit(flb_route_mask_element *routes_mask, int value, - struct flb_config *config) + struct flb_router *router) { int index; uint64_t bit; - if (value < 0 || value >= config->route_mask_slots) { - flb_warn("[routes_mask] Can't set bit (%d) past limits of bitfield", + if (router == NULL) { + return; + } + + if (value < 0 || value >= router->route_mask_slots) { + flb_warn("[routes_mask] Can't clear bit (%d) past limits of bitfield", value); return; } @@ -98,12 +123,16 @@ void flb_routes_mask_set_bit(flb_route_mask_element *routes_mask, int value, * */ void flb_routes_mask_clear_bit(flb_route_mask_element *routes_mask, int value, - struct flb_config *config) + struct flb_router *router) { int index; uint64_t bit; - if (value < 0 || value >= config->route_mask_slots) { + if (router == NULL) { + return; + } + + if (value < 0 || value >= router->route_mask_slots) { flb_warn("[routes_mask] Can't set bit (%d) past limits of bitfield", value); return; @@ -123,12 +152,16 @@ void flb_routes_mask_clear_bit(flb_route_mask_element *routes_mask, int value, * */ int flb_routes_mask_get_bit(flb_route_mask_element *routes_mask, int value, - struct flb_config *config) + struct flb_router *router) { int index; uint64_t bit; - if (value < 0 || value >= config->route_mask_slots) { + if (router == NULL) { + return 0; + } + + if (value < 0 || value >= router->route_mask_slots) { flb_warn("[routes_mask] Can't get bit (%d) past limits of bitfield", value); return 0; @@ -140,47 +173,63 @@ int flb_routes_mask_get_bit(flb_route_mask_element *routes_mask, int value, } int flb_routes_mask_is_empty(flb_route_mask_element *routes_mask, - struct flb_config *config) + struct flb_router *router) { + if (router == NULL || router->route_empty_mask == NULL) { + return 0; + } + return memcmp(routes_mask, - config->route_empty_mask, - config->route_mask_size * sizeof(flb_route_mask_element)) == 0; + router->route_empty_mask, + router->route_mask_size * sizeof(flb_route_mask_element)) == 0; } -int flb_routes_empty_mask_create(struct flb_config *config) +int flb_routes_empty_mask_create(struct flb_router *router) { - flb_routes_empty_mask_destroy(config); + if (router == NULL) { + return -1; + } + + flb_routes_empty_mask_destroy(router); - config->route_empty_mask = flb_calloc(config->route_mask_size, + router->route_empty_mask = flb_calloc(router->route_mask_size, sizeof(flb_route_mask_element)); - if (config->route_empty_mask == NULL) { + if (router->route_empty_mask == NULL) { return -1; } return 0; } -void flb_routes_empty_mask_destroy(struct flb_config *config) +void flb_routes_empty_mask_destroy(struct flb_router *router) { - if (config->route_empty_mask != NULL) { - flb_free(config->route_empty_mask); + if (router == NULL) { + return; + } - config->route_empty_mask = NULL; + if (router->route_empty_mask != NULL) { + flb_free(router->route_empty_mask); + + router->route_empty_mask = NULL; } } -int flb_routes_mask_set_size(size_t mask_size, struct flb_config *config) +int flb_routes_mask_set_size(size_t mask_size, struct flb_router *router) { + if (router == NULL) { + return -1; + } + if (mask_size < 1) { mask_size = 1; } - mask_size = (mask_size / FLB_ROUTES_MASK_ELEMENT_BITS) + - (mask_size % FLB_ROUTES_MASK_ELEMENT_BITS); + mask_size = (mask_size + FLB_ROUTES_MASK_ELEMENT_BITS - 1) / + FLB_ROUTES_MASK_ELEMENT_BITS; - config->route_mask_size = mask_size; - config->route_mask_slots = mask_size * FLB_ROUTES_MASK_ELEMENT_BITS; + router->route_mask_size = mask_size; + router->route_mask_slots = mask_size * FLB_ROUTES_MASK_ELEMENT_BITS; - return flb_routes_empty_mask_create(config); + return flb_routes_empty_mask_create(router); } From 9deb338edb43a965f579e20f08174b8630227efb Mon Sep 17 00:00:00 2001 From: Patrick Stephens Date: Sun, 3 May 2026 12:34:57 +0100 Subject: [PATCH 084/213] [upstream] config: integrate router context creation and Upstream-Ref: https://github.com/fluent/fluent-bit/commit/0adf3e48a668bb6e1025132e72efb6e0fe4e0a63 Cherry-picked from Fluent Bit v4.2.4 --- source/include/fluent-bit/flb_config.h | 6 +++--- source/src/flb_config.c | 12 ++++++++++-- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/source/include/fluent-bit/flb_config.h b/source/include/fluent-bit/flb_config.h index 10fe72a4..4ee6ec1a 100644 --- a/source/include/fluent-bit/flb_config.h +++ b/source/include/fluent-bit/flb_config.h @@ -31,6 +31,8 @@ #include +struct flb_router; + #define FLB_CONFIG_FLUSH_SECS 1 #define FLB_CONFIG_HTTP_LISTEN "0.0.0.0" #define FLB_CONFIG_HTTP_PORT "2020" @@ -293,9 +295,7 @@ struct flb_config { int hot_reload_watchdog_timeout_seconds; /* Routing */ - size_t route_mask_size; - size_t route_mask_slots; - uint64_t *route_empty_mask; + struct flb_router *router; #ifdef FLB_SYSTEM_WINDOWS /* maxstdio (Windows) */ int win_maxstdio; diff --git a/source/src/flb_config.c b/source/src/flb_config.c index aefc04f9..250c950a 100644 --- a/source/src/flb_config.c +++ b/source/src/flb_config.c @@ -299,7 +299,14 @@ struct flb_config *flb_config_init() } /* Routing */ - flb_routes_mask_set_size(1, config); + config->router = flb_router_create(config); + if (!config->router) { + flb_error("[config] could not create router"); + flb_cf_destroy(cf); + flb_free(config); + return NULL; + } + flb_routes_mask_set_size(1, config->router); config->cio = NULL; config->storage_path = NULL; @@ -608,7 +615,8 @@ void flb_config_exit(struct flb_config *config) /* release task map */ flb_config_task_map_resize(config, 0); - flb_routes_empty_mask_destroy(config); + + flb_router_destroy(config->router); /* Clean up router input routes */ flb_router_routes_destroy(&config->input_routes); From 4a18bea58f560d32874cf03747303af8f87c6b7a Mon Sep 17 00:00:00 2001 From: Patrick Stephens Date: Sun, 3 May 2026 12:34:57 +0100 Subject: [PATCH 085/213] [upstream] engine: publish routing metrics on log batch flush Upstream-Ref: https://github.com/fluent/fluent-bit/commit/350a944510c2a62ab1d1314adc7db2ee6bebf12d Cherry-picked from Fluent Bit v4.2.4 --- source/src/flb_engine.c | 89 +++++++++++++++++++++++++++++------------ 1 file changed, 64 insertions(+), 25 deletions(-) diff --git a/source/src/flb_engine.c b/source/src/flb_engine.c index cd927057..17ca4b37 100644 --- a/source/src/flb_engine.c +++ b/source/src/flb_engine.c @@ -38,6 +38,7 @@ #include #include #include +#include #include #include #include @@ -243,7 +244,8 @@ static inline int handle_output_event(uint64_t ts, uint32_t type; uint32_t key; double latency_seconds; - char *name; + char *in_name; + char *out_name; struct flb_task *task; struct flb_task_retry *retry; struct flb_output_instance *ins; @@ -289,7 +291,8 @@ static inline int handle_output_event(uint64_t ts, if (flb_output_is_threaded(ins) == FLB_FALSE) { flb_output_flush_finished(config, out_id); } - name = (char *) flb_output_name(ins); + in_name = (char *) flb_input_name(task->i_ins); + out_name = (char *) flb_output_name(ins); /* If we are in synchronous mode, flush the next waiting task */ if (ins->flags & FLB_OUTPUT_SYNCHRONOUS) { @@ -302,16 +305,26 @@ static inline int handle_output_event(uint64_t ts, if (ret == FLB_OK) { /* cmetrics */ cmt_counter_add(ins->cmt_proc_records, ts, task->event_chunk->total_events, - 1, (char *[]) {name}); + 1, (char *[]) {out_name}); cmt_counter_add(ins->cmt_proc_bytes, ts, task->event_chunk->size, - 1, (char *[]) {name}); + 1, (char *[]) {out_name}); + + if (config->router && task->event_chunk->type == FLB_EVENT_TYPE_LOGS) { + cmt_counter_add(config->router->logs_records_total, ts, + task->event_chunk->total_events, + 2, (char *[]) {in_name, out_name}); + + cmt_counter_add(config->router->logs_bytes_total, ts, + task->event_chunk->size, + 2, (char *[]) {in_name, out_name}); + } /* latency histogram */ if (ins->cmt_latency) { latency_seconds = flb_time_now() - ((struct flb_input_chunk *) task->ic)->create_time; cmt_histogram_observe(ins->cmt_latency, ts, latency_seconds, 2, - (char *[]) {(char *) flb_input_name(task->i_ins), name}); + (char *[]) {in_name, out_name}); } /* [OLD API] Update metrics */ @@ -346,7 +359,7 @@ static inline int handle_output_event(uint64_t ts, cmt_gauge_set(ins->cmt_chunk_available_capacity_percent, ts, calculate_chunk_capacity_percent(ins), - 1, (char *[]) {name}); + 1, (char *[]) {out_name}); flb_task_retry_clean(task, ins); flb_task_users_dec(task, FLB_TRUE); @@ -355,11 +368,22 @@ static inline int handle_output_event(uint64_t ts, if (ins->retry_limit == FLB_OUT_RETRY_NONE) { /* cmetrics: output_dropped_records_total */ cmt_counter_add(ins->cmt_dropped_records, ts, task->records, - 1, (char *[]) {name}); + 1, (char *[]) {out_name}); + + if (config->router && task->event_chunk && + task->event_chunk->type == FLB_EVENT_TYPE_LOGS) { + cmt_counter_add(config->router->logs_drop_records_total, ts, + task->records, + 2, (char *[]) {in_name, out_name}); + + cmt_counter_add(config->router->logs_drop_bytes_total, ts, + task->event_chunk->size, + 2, (char *[]) {in_name, out_name}); + } cmt_gauge_set(ins->cmt_chunk_available_capacity_percent, ts, calculate_chunk_capacity_percent(ins), - 1, (char *[]) {name}); + 1, (char *[]) {out_name}); /* OLD metrics API */ #ifdef FLB_HAVE_METRICS @@ -389,13 +413,24 @@ static inline int handle_output_event(uint64_t ts, */ /* cmetrics */ - cmt_counter_inc(ins->cmt_retries_failed, ts, 1, (char *[]) {name}); + cmt_counter_inc(ins->cmt_retries_failed, ts, 1, (char *[]) {out_name}); cmt_counter_add(ins->cmt_dropped_records, ts, task->records, - 1, (char *[]) {name}); + 1, (char *[]) {out_name}); + + if (config->router && task->event_chunk && + task->event_chunk->type == FLB_EVENT_TYPE_LOGS) { + cmt_counter_add(config->router->logs_drop_records_total, ts, + task->records, + 2, (char *[]) {in_name, out_name}); + + cmt_counter_add(config->router->logs_drop_bytes_total, ts, + task->event_chunk->size, + 2, (char *[]) {in_name, out_name}); + } cmt_gauge_set(ins->cmt_chunk_available_capacity_percent, ts, calculate_chunk_capacity_percent(ins), - 1, (char *[]) {name}); + 1, (char *[]) {out_name}); /* OLD metrics API */ #ifdef FLB_HAVE_METRICS @@ -449,13 +484,13 @@ static inline int handle_output_event(uint64_t ts, flb_output_name(ins), out_id); /* cmetrics */ - cmt_counter_inc(ins->cmt_retries, ts, 1, (char *[]) {name}); + cmt_counter_inc(ins->cmt_retries, ts, 1, (char *[]) {out_name}); cmt_counter_add(ins->cmt_retried_records, ts, task->records, - 1, (char *[]) {name}); + 1, (char *[]) {out_name}); cmt_gauge_set(ins->cmt_chunk_available_capacity_percent, ts, calculate_chunk_capacity_percent(ins), - 1, (char *[]) {name}); + 1, (char *[]) {out_name}); /* OLD metrics API: update the metrics since a new retry is coming */ #ifdef FLB_HAVE_METRICS @@ -466,13 +501,24 @@ static inline int handle_output_event(uint64_t ts, } else if (ret == FLB_ERROR) { /* cmetrics */ - cmt_counter_inc(ins->cmt_errors, ts, 1, (char *[]) {name}); + cmt_counter_inc(ins->cmt_errors, ts, 1, (char *[]) {out_name}); cmt_counter_add(ins->cmt_dropped_records, ts, task->records, - 1, (char *[]) {name}); + 1, (char *[]) {out_name}); + + if (config->router && task->event_chunk && + task->event_chunk->type == FLB_EVENT_TYPE_LOGS) { + cmt_counter_add(config->router->logs_drop_records_total, ts, + task->records, + 2, (char *[]) {in_name, out_name}); + + cmt_counter_add(config->router->logs_drop_bytes_total, ts, + task->event_chunk->size, + 2, (char *[]) {in_name, out_name}); + } cmt_gauge_set(ins->cmt_chunk_available_capacity_percent, ts, calculate_chunk_capacity_percent(ins), - 1, (char *[]) {name}); + 1, (char *[]) {out_name}); /* OLD API */ #ifdef FLB_HAVE_METRICS @@ -811,14 +857,7 @@ int flb_engine_start(struct flb_config *config) config->notification_channels_initialized = FLB_TRUE; config->notification_event.type = FLB_ENGINE_EV_NOTIFICATION; - ret = flb_routes_mask_set_size(mk_list_size(&config->outputs), config); - - if (ret != 0) { - flb_error("[engine] routing mask dimensioning failed"); - return -1; - } - - ret = flb_routes_mask_set_size(mk_list_size(&config->outputs), config); + ret = flb_routes_mask_set_size(mk_list_size(&config->outputs), config->router); if (ret != 0) { flb_error("[engine] routing mask dimensioning failed"); From 7339f8fec83aded861f34a79412bda0a5c532f8a Mon Sep 17 00:00:00 2001 From: Patrick Stephens Date: Sun, 3 May 2026 12:34:57 +0100 Subject: [PATCH 086/213] [upstream] input_chunk: update routing mask operations to use Upstream-Ref: https://github.com/fluent/fluent-bit/commit/b89124854afc86fc6126a203e4053f374b4655da Cherry-picked from Fluent Bit v4.2.4 --- source/src/flb_input_chunk.c | 56 +++++++++++++++++++++++++----------- 1 file changed, 40 insertions(+), 16 deletions(-) diff --git a/source/src/flb_input_chunk.c b/source/src/flb_input_chunk.c index 61d6b950..d33b9ff6 100644 --- a/source/src/flb_input_chunk.c +++ b/source/src/flb_input_chunk.c @@ -309,7 +309,7 @@ static int flb_input_chunk_release_space( if (!flb_routes_mask_get_bit(old_input_chunk->routes_mask, output_plugin->id, - input_plugin->config)) { + input_plugin->config->router)) { continue; } @@ -332,14 +332,14 @@ static int flb_input_chunk_release_space( if (release_scope == FLB_INPUT_CHUNK_RELEASE_SCOPE_LOCAL) { flb_routes_mask_clear_bit(old_input_chunk->routes_mask, output_plugin->id, - input_plugin->config); + input_plugin->config->router); FS_CHUNK_SIZE_DEBUG_MOD(output_plugin, old_input_chunk, chunk_size); output_plugin->fs_chunks_size -= chunk_size; chunk_destroy_flag = flb_routes_mask_is_empty( old_input_chunk->routes_mask, - input_plugin->config); + input_plugin->config->router); chunk_released = FLB_TRUE; } @@ -361,6 +361,25 @@ static int flb_input_chunk_release_space( dropped_record_count, 1, (char *[]) {(char *) flb_output_name(output_plugin)}); + if (input_plugin->config && input_plugin->config->router && + old_input_chunk->event_type == FLB_INPUT_LOGS) { + struct flb_router *router = input_plugin->config->router; + + cmt_counter_add(router->logs_drop_records_total, + cfl_time_now(), + (double) dropped_record_count, + 2, + (char *[]){(char *) flb_input_name(old_input_chunk->in), + (char *) flb_output_name(output_plugin)}); + + cmt_counter_add(router->logs_drop_bytes_total, + cfl_time_now(), + (double) chunk_size, + 2, + (char *[]){(char *) flb_input_name(old_input_chunk->in), + (char *) flb_output_name(output_plugin)}); + } + flb_metrics_sum(FLB_METRIC_OUT_DROPPED_RECORDS, dropped_record_count, output_plugin->metrics); @@ -464,6 +483,7 @@ int flb_input_chunk_write(void *data, const char *buf, size_t len) { int ret; struct flb_input_chunk *ic; + size_t mask_size; ic = (struct flb_input_chunk *) data; @@ -558,7 +578,7 @@ static int flb_input_chunk_safe_delete(struct flb_input_chunk *ic, */ if (flb_routes_mask_get_bit(old_ic->routes_mask, o_id, - ic->in->config) == 0) { + ic->in->config->router) == 0) { return FLB_FALSE; } @@ -663,7 +683,7 @@ int flb_input_chunk_find_space_new_data(struct flb_input_chunk *ic, if ((o_ins->total_limit_size == -1) || ((1 << o_ins->id) & overlimit) == 0 || (flb_routes_mask_get_bit(ic->routes_mask, o_ins->id, - o_ins->config) == 0)) { + o_ins->config->router) == 0)) { continue; } @@ -705,7 +725,7 @@ int flb_input_chunk_has_overlimit_routes(struct flb_input_chunk *ic, if ((o_ins->total_limit_size == -1) || (flb_routes_mask_get_bit(ic->routes_mask, o_ins->id, - o_ins->config) == 0)) { + o_ins->config->router) == 0)) { continue; } @@ -747,7 +767,7 @@ int flb_input_chunk_place_new_chunk(struct flb_input_chunk *ic, size_t chunk_siz } } return !flb_routes_mask_is_empty(ic->routes_mask, - i_ins->config); + i_ins->config->router); } static int input_chunk_collect_output_references(struct flb_config *config, @@ -913,6 +933,7 @@ struct flb_input_chunk *flb_input_chunk_map(struct flb_input_instance *in, size_t direct_match_count; size_t direct_match_index; struct flb_input_chunk *ic; + size_t mask_size; records = 0; @@ -937,8 +958,9 @@ struct flb_input_chunk *flb_input_chunk_map(struct flb_input_instance *in, return NULL; } + mask_size = flb_routes_mask_get_size(in->config->router); ic->routes_mask = (flb_route_mask_element *) - flb_calloc(in->config->route_mask_size, + flb_calloc(mask_size, sizeof(flb_route_mask_element)); if (ic->routes_mask == NULL) { @@ -1118,7 +1140,7 @@ struct flb_input_chunk *flb_input_chunk_map(struct flb_input_instance *in, if (direct_missing == FLB_FALSE) { memset(ic->routes_mask, 0, - sizeof(flb_route_mask_element) * in->config->route_mask_size); + sizeof(flb_route_mask_element) * mask_size); has_routes = 0; for (direct_index = 0; direct_index < direct_count; direct_index++) { direct_matches = NULL; @@ -1139,7 +1161,7 @@ struct flb_input_chunk *flb_input_chunk_map(struct flb_input_instance *in, direct_match_index++) { flb_routes_mask_set_bit(ic->routes_mask, direct_matches[direct_match_index]->id, - in->config); + in->config->router); has_routes++; } @@ -1931,6 +1953,7 @@ struct flb_input_chunk *flb_input_chunk_create(struct flb_input_instance *in, in int err; int set_down = FLB_FALSE; int has_routes; + size_t mask_size; char name[64]; struct cio_chunk *chunk; struct flb_storage_input *storage; @@ -1995,8 +2018,9 @@ struct flb_input_chunk *flb_input_chunk_create(struct flb_input_instance *in, in #ifdef FLB_HAVE_METRICS ic->total_records = 0; #endif + mask_size = flb_routes_mask_get_size(in->config->router); ic->routes_mask = (flb_route_mask_element *) - flb_calloc(in->config->route_mask_size, + flb_calloc(mask_size, sizeof(flb_route_mask_element)); if (ic->routes_mask == NULL) { @@ -2060,7 +2084,7 @@ int flb_input_chunk_destroy_corrupted(struct flb_input_chunk *ic, if (flb_routes_mask_get_bit(ic->routes_mask, o_ins->id, - o_ins->config) != 0) { + o_ins->config->router) != 0) { if (ic->fs_counted == FLB_TRUE) { FS_CHUNK_SIZE_DEBUG_MOD(o_ins, ic, -bytes); o_ins->fs_chunks_size -= bytes; @@ -2139,7 +2163,7 @@ int flb_input_chunk_destroy(struct flb_input_chunk *ic, int del) if (flb_routes_mask_get_bit(ic->routes_mask, o_ins->id, - o_ins->config) != 0) { + o_ins->config->router) != 0) { if (ic->fs_counted == FLB_TRUE) { FS_CHUNK_SIZE_DEBUG_MOD(o_ins, ic, -bytes); o_ins->fs_chunks_size -= bytes; @@ -2292,7 +2316,7 @@ static struct flb_input_chunk *input_chunk_get(struct flb_input_instance *in, * that the chunk will flush to, we need to modify the routes_mask of the oldest chunks * (based in creation time) to get enough space for the incoming chunk. */ - if (!flb_routes_mask_is_empty(ic->routes_mask, ic->in->config) + if (!flb_routes_mask_is_empty(ic->routes_mask, ic->in->config->router) && flb_input_chunk_place_new_chunk(ic, chunk_size) == 0) { /* * If the chunk is not newly created, the chunk might already have logs inside. @@ -2301,7 +2325,7 @@ static struct flb_input_chunk *input_chunk_get(struct flb_input_instance *in, * the chunk. */ if (new_chunk || - flb_routes_mask_is_empty(ic->routes_mask, ic->in->config) == FLB_TRUE) { + flb_routes_mask_is_empty(ic->routes_mask, ic->in->config->router) == FLB_TRUE) { flb_input_chunk_destroy(ic, FLB_TRUE); } return NULL; @@ -3226,7 +3250,7 @@ void flb_input_chunk_update_output_instances(struct flb_input_chunk *ic, if (flb_routes_mask_get_bit(ic->routes_mask, o_ins->id, - o_ins->config) != 0) { + o_ins->config->router) != 0) { /* * if there is match on any index of 1's in the binary, it indicates * that the input chunk will flush to this output instance From ea2101bc42aab70e7910680ca9b64b2563b7f1ba Mon Sep 17 00:00:00 2001 From: Patrick Stephens Date: Sun, 3 May 2026 12:34:57 +0100 Subject: [PATCH 087/213] [upstream] in_storage_backlog: update routing mask operations to Upstream-Ref: https://github.com/fluent/fluent-bit/commit/d05826fa2bba0408b0c645c174ecc89c943bb233 Cherry-picked from Fluent Bit v4.2.4 --- source/plugins/in_storage_backlog/sb.c | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/source/plugins/in_storage_backlog/sb.c b/source/plugins/in_storage_backlog/sb.c index de3f0b46..5e0ecba8 100644 --- a/source/plugins/in_storage_backlog/sb.c +++ b/source/plugins/in_storage_backlog/sb.c @@ -281,12 +281,14 @@ static int sb_append_chunk_to_segregated_backlogs(struct cio_chunk *target_chun int tag_len; const char * tag_buf; int result; + size_t slots; memset(&dummy_input_chunk, 0, sizeof(struct flb_input_chunk)); + slots = flb_routes_mask_get_slots(context->ins->config->router); memset(context->dummy_routes_mask, 0, - context->ins->config->route_mask_slots * sizeof(flb_route_mask_element)); + slots * sizeof(flb_route_mask_element)); dummy_input_chunk.in = context->ins; dummy_input_chunk.chunk = target_chunk; @@ -317,7 +319,7 @@ static int sb_append_chunk_to_segregated_backlogs(struct cio_chunk *target_chun backlog = mk_list_entry(head, struct sb_out_queue, _head); if (flb_routes_mask_get_bit(dummy_input_chunk.routes_mask, backlog->ins->id, - backlog->ins->config)) { + backlog->ins->config->router)) { result = sb_append_chunk_to_segregated_backlog(target_chunk, stream, chunk_size, backlog); if (result) { @@ -656,6 +658,7 @@ static int cb_sb_init(struct flb_input_instance *in, int ret; char mem[32]; struct flb_sb *ctx; + size_t slots; ctx = flb_calloc(1, sizeof(struct flb_sb)); @@ -664,7 +667,8 @@ static int cb_sb_init(struct flb_input_instance *in, return -1; } - ctx->dummy_routes_mask = flb_calloc(in->config->route_mask_slots, + slots = flb_routes_mask_get_slots(config->router); + ctx->dummy_routes_mask = flb_calloc(slots, sizeof(flb_route_mask_element)); if (ctx->dummy_routes_mask == NULL) { From a2c3461b3769fa98298dd25b84bf24c99ff5c050 Mon Sep 17 00:00:00 2001 From: Patrick Stephens Date: Sun, 3 May 2026 12:34:58 +0100 Subject: [PATCH 088/213] [upstream] task: update routing mask checks to use router Upstream-Ref: https://github.com/fluent/fluent-bit/commit/2d3ce50977b13c2269990f3c20bb74a440066928 Cherry-picked from Fluent Bit v4.2.4 --- source/src/flb_task.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/src/flb_task.c b/source/src/flb_task.c index 74b9223d..b796595a 100644 --- a/source/src/flb_task.c +++ b/source/src/flb_task.c @@ -425,7 +425,7 @@ struct flb_task *flb_task_create(uint64_t ref_id, if (task_ic->routes_mask) { if (flb_routes_mask_get_bit(task_ic->routes_mask, o_ins->id, - o_ins->config) == 0) { + o_ins->config->router) == 0) { continue; } } @@ -469,7 +469,7 @@ struct flb_task *flb_task_create(uint64_t ref_id, if (flb_routes_mask_get_bit(task_ic->routes_mask, o_ins->id, - o_ins->config) != 0) { + o_ins->config->router) != 0) { route = flb_calloc(1, sizeof(struct flb_task_route)); if (!route) { flb_errno(); From 3c0edb7974446216ff1500073d5188d1a2d228f7 Mon Sep 17 00:00:00 2001 From: Patrick Stephens Date: Sun, 3 May 2026 12:34:58 +0100 Subject: [PATCH 089/213] [upstream] tests: internal: routing: update routing tests to use Upstream-Ref: https://github.com/fluent/fluent-bit/commit/b0cdb170efbd4e127feba5fa1f02e152f5b39aff Cherry-picked from Fluent Bit v4.2.4 --- source/tests/internal/conditional_routing.c | 32 ++++++++++++++------- source/tests/internal/input_chunk_routes.c | 30 +++++++++++++------ 2 files changed, 44 insertions(+), 18 deletions(-) diff --git a/source/tests/internal/conditional_routing.c b/source/tests/internal/conditional_routing.c index 430580e6..38c4c19c 100644 --- a/source/tests/internal/conditional_routing.c +++ b/source/tests/internal/conditional_routing.c @@ -315,7 +315,7 @@ void test_conditional_routing_per_record() /* Cleanup */ flb_router_exit(&config); - cleanup_conditional_routing_instances(&input, &output1, &output2, &output3, + cleanup_conditional_routing_instances(&config, &input, &output1, &output2, &output3, &input_routes, &route1, &route2, &route3, &route_output1, &route_output2, &route_output3); } @@ -374,7 +374,7 @@ void test_conditional_routing_default_route() /* Cleanup */ flb_router_exit(&config); - cleanup_conditional_routing_instances(&input, &output1, &output2, &output3, + cleanup_conditional_routing_instances(&config, &input, &output1, &output2, &output3, &input_routes, &route1, &route2, &route3, &route_output1, &route_output2, &route_output3); } @@ -414,17 +414,17 @@ void test_conditional_routing_route_mask() if (strcmp(record->level, "info") == 0) { /* Create a test chunk */ - chunk = flb_input_chunk_create(&input, "test_tag", 8, NULL, 0); + chunk = flb_input_chunk_create(&input, FLB_INPUT_LOGS, "test_tag", 8); TEST_CHECK(chunk != NULL); if (chunk) { /* Set route mask for info output only */ routes_mask = chunk->routes_mask; - flb_routes_mask_set_bit(routes_mask, output1.id, &config); + flb_routes_mask_set_bit(routes_mask, output1.id, config.router); /* Verify route mask is set correctly */ - TEST_CHECK(flb_routes_mask_get_bit(routes_mask, output1.id, &config) == 1); - TEST_CHECK(flb_routes_mask_get_bit(routes_mask, output2.id, &config) == 0); - TEST_CHECK(flb_routes_mask_get_bit(routes_mask, output3.id, &config) == 0); + TEST_CHECK(flb_routes_mask_get_bit(routes_mask, output1.id, config.router) == 1); + TEST_CHECK(flb_routes_mask_get_bit(routes_mask, output2.id, config.router) == 0); + TEST_CHECK(flb_routes_mask_get_bit(routes_mask, output3.id, config.router) == 0); flb_input_chunk_destroy(chunk); } @@ -433,7 +433,7 @@ void test_conditional_routing_route_mask() /* Cleanup */ flb_router_exit(&config); - cleanup_conditional_routing_instances(&input, &output1, &output2, &output3, + cleanup_conditional_routing_instances(&config, &input, &output1, &output2, &output3, &input_routes, &route1, &route2, &route3, &route_output1, &route_output2, &route_output3); } @@ -503,7 +503,7 @@ void test_conditional_routing_no_duplicates() /* Cleanup */ flb_router_exit(&config); - cleanup_conditional_routing_instances(&input, &output1, &output2, &output3, + cleanup_conditional_routing_instances(&config, &input, &output1, &output2, &output3, &input_routes, &route1, &route2, &route3, &route_output1, &route_output2, &route_output3); } @@ -814,6 +814,12 @@ static void setup_conditional_routing_instances(struct flb_config *config, mk_list_init(&config->outputs); cfl_list_init(&config->input_routes); + config->router = flb_router_create(config); + TEST_CHECK(config->router != NULL); + if (config->router) { + flb_routes_mask_set_size(1, config->router); + } + memset(input, 0, sizeof(struct flb_input_instance)); mk_list_init(&input->_head); cfl_list_init(&input->routes_direct); @@ -947,7 +953,8 @@ static int test_record_routing(struct flb_input_instance *input, return found; } -static void cleanup_conditional_routing_instances(struct flb_input_instance *input, +static void cleanup_conditional_routing_instances(struct flb_config *config, + struct flb_input_instance *input, struct flb_output_instance *output1, struct flb_output_instance *output2, struct flb_output_instance *output3, @@ -970,6 +977,11 @@ static void cleanup_conditional_routing_instances(struct flb_input_instance *inp flb_sds_destroy(route_output1->name); flb_sds_destroy(route_output2->name); flb_sds_destroy(route_output3->name); + + if (config && config->router) { + flb_router_destroy(config->router); + config->router = NULL; + } } TEST_LIST = { diff --git a/source/tests/internal/input_chunk_routes.c b/source/tests/internal/input_chunk_routes.c index 49ad9517..27285a7e 100644 --- a/source/tests/internal/input_chunk_routes.c +++ b/source/tests/internal/input_chunk_routes.c @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -65,8 +66,18 @@ static int init_test_config(struct flb_config *config, return -1; } - ret = flb_routes_mask_set_size(64, config); + /* Create router context */ + config->router = flb_router_create(config); + if (config->router == NULL) { + flb_env_destroy(config->env); + config->env = NULL; + return -1; + } + + ret = flb_routes_mask_set_size(64, config->router); if (ret != 0) { + flb_router_destroy(config->router); + config->router = NULL; flb_env_destroy(config->env); config->env = NULL; return -1; @@ -241,7 +252,10 @@ static void cleanup_test_routing_scenario(struct flb_input_chunk *ic, in->net_config_map = NULL; } - flb_routes_empty_mask_destroy(config); + if (config->router) { + flb_router_destroy(config->router); + config->router = NULL; + } if (config->env) { flb_env_destroy(config->env); config->env = NULL; @@ -601,13 +615,13 @@ static void test_chunk_restore_alias_plugin_match_multiple() TEST_CHECK(flb_routes_mask_get_bit(ic->routes_mask, stdout_one.id, - &config) == 1); + config.router) == 1); TEST_CHECK(flb_routes_mask_get_bit(ic->routes_mask, stdout_two.id, - &config) == 1); + config.router) == 1); TEST_CHECK(flb_routes_mask_get_bit(ic->routes_mask, http_out.id, - &config) == 0); + config.router) == 0); cleanup: cleanup_test_routing_scenario(ic, &stdout_one, &stdout_two, &http_out, @@ -762,13 +776,13 @@ static void test_chunk_restore_alias_plugin_null_matches_all() TEST_CHECK(flb_routes_mask_get_bit(ic->routes_mask, stdout_one.id, - &config) == 1); + config.router) == 1); TEST_CHECK(flb_routes_mask_get_bit(ic->routes_mask, stdout_two.id, - &config) == 1); + config.router) == 1); TEST_CHECK(flb_routes_mask_get_bit(ic->routes_mask, http_out.id, - &config) == 1); + config.router) == 1); cleanup: cleanup_test_routing_scenario(ic, &stdout_one, &stdout_two, &http_out, From 8ba1d22c93ab365994554094146e8d743e60786b Mon Sep 17 00:00:00 2001 From: Patrick Stephens Date: Sun, 3 May 2026 12:34:58 +0100 Subject: [PATCH 090/213] [upstream] input_chunk: small cleanup Upstream-Ref: https://github.com/fluent/fluent-bit/commit/f73850f31de7dba3e9936d5c520bde58f26c5a05 Cherry-picked from Fluent Bit v4.2.4 --- source/src/flb_input_chunk.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/src/flb_input_chunk.c b/source/src/flb_input_chunk.c index d33b9ff6..2e813e8d 100644 --- a/source/src/flb_input_chunk.c +++ b/source/src/flb_input_chunk.c @@ -293,6 +293,7 @@ static int flb_input_chunk_release_space( { struct mk_list *input_chunk_iterator_tmp; struct mk_list *input_chunk_iterator; + struct flb_router *router; ssize_t dropped_record_count; int chunk_destroy_flag; struct flb_input_chunk *old_input_chunk; @@ -363,7 +364,7 @@ static int flb_input_chunk_release_space( if (input_plugin->config && input_plugin->config->router && old_input_chunk->event_type == FLB_INPUT_LOGS) { - struct flb_router *router = input_plugin->config->router; + router = input_plugin->config->router; cmt_counter_add(router->logs_drop_records_total, cfl_time_now(), @@ -483,7 +484,6 @@ int flb_input_chunk_write(void *data, const char *buf, size_t len) { int ret; struct flb_input_chunk *ic; - size_t mask_size; ic = (struct flb_input_chunk *) data; From 4c97567dc9867f71e5c7297934d1d88d0b1a8acd Mon Sep 17 00:00:00 2001 From: Patrick Stephens Date: Sun, 3 May 2026 12:34:58 +0100 Subject: [PATCH 091/213] [upstream] router: remove unnecessary function Upstream-Ref: https://github.com/fluent/fluent-bit/commit/6c487f098b9f92e14ce12ef7a22af443bf1c71f9 Cherry-picked from Fluent Bit v4.2.4 --- source/src/flb_router.c | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/source/src/flb_router.c b/source/src/flb_router.c index a4ce99c1..5b9c0813 100644 --- a/source/src/flb_router.c +++ b/source/src/flb_router.c @@ -353,17 +353,10 @@ static int router_metrics_create(struct flb_router *router) return 0; } -int flb_router_metrics_create(struct flb_config *config, struct flb_router *router) -{ - (void) config; - - return router_metrics_create(router); -} - struct flb_router *flb_router_create(struct flb_config *config) { + int ret; struct flb_router *router; - (void) config; router = flb_calloc(1, sizeof(struct flb_router)); @@ -378,7 +371,9 @@ struct flb_router *flb_router_create(struct flb_config *config) return NULL; } - if (router_metrics_create(router) != 0) { + ret = router_metrics_create(router); + if (ret != 0) { + flb_error("[router] failed to create metrics"); flb_router_destroy(router); return NULL; } From afe0e53d4bf20d605ac75de10b355829d8c15862 Mon Sep 17 00:00:00 2001 From: Patrick Stephens Date: Sun, 3 May 2026 12:34:58 +0100 Subject: [PATCH 092/213] [upstream] tests: internal: input_chunk_routes: fix compiler Upstream-Ref: https://github.com/fluent/fluent-bit/commit/a62268e3296d5b1ced980d06946b652ffbcdc140 Cherry-picked from Fluent Bit v4.2.4 --- source/tests/internal/input_chunk_routes.c | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/source/tests/internal/input_chunk_routes.c b/source/tests/internal/input_chunk_routes.c index 27285a7e..489e04a0 100644 --- a/source/tests/internal/input_chunk_routes.c +++ b/source/tests/internal/input_chunk_routes.c @@ -1,3 +1,7 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +#include "flb_tests_internal.h" + #include #include #include @@ -5,6 +9,14 @@ #include #include #include + +/* Dummy workaround: undefine input plugin macros to avoid redefinition warnings */ +#undef flb_plg_error +#undef flb_plg_warn +#undef flb_plg_info +#undef flb_plg_debug +#undef flb_plg_trace + #include #include #include @@ -17,7 +29,6 @@ #include #include -#include "flb_tests_internal.h" #define TEST_STREAM_PATH "/tmp/flb-chunk-direct-test" #define TEST_STREAM_PATH_MATCH "/tmp/flb-chunk-direct-test-match" From 7dcea6634567bf89b8fc4b337ad46c2016e36d41 Mon Sep 17 00:00:00 2001 From: Patrick Stephens Date: Sun, 3 May 2026 12:34:58 +0100 Subject: [PATCH 093/213] [upstream] tests: internal: fuzzers: aws_credentials_fuzzer: fix Upstream-Ref: https://github.com/fluent/fluent-bit/commit/3eb6c5b54466b0a7842a5b304b3605dcd3bb5804 Cherry-picked from Fluent Bit v4.2.4 --- .../internal/fuzzers/aws_credentials_fuzzer.c | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/source/tests/internal/fuzzers/aws_credentials_fuzzer.c b/source/tests/internal/fuzzers/aws_credentials_fuzzer.c index d6ccb22e..51df0a3e 100644 --- a/source/tests/internal/fuzzers/aws_credentials_fuzzer.c +++ b/source/tests/internal/fuzzers/aws_credentials_fuzzer.c @@ -18,6 +18,7 @@ */ #include +#include #include #include #include @@ -57,7 +58,7 @@ void fuzz_sts(const uint8_t *data, size_t size) { flb_sds_t s1 = flb_sts_uri(action, role_arn, session_name, external_id, identity_token); if (s1 != NULL) { - flb_sds_destroy(s1); + flb_sds_destroy(s1); } flb_free(action); @@ -76,13 +77,17 @@ void fuzz_sts(const uint8_t *data, size_t size) { void fuzz_http(const uint8_t *data, size_t size) { time_t expiration; struct flb_aws_credentials *creds = NULL; - - char *response = get_null_terminated(250, &data, &size); - creds = flb_parse_http_credentials(response, 250, &expiration); - if (creds != NULL) { - flb_aws_credentials_destroy(creds); + size_t response_len; + + response_len = (size > 250) ? 250 : size; + char *response = get_null_terminated(response_len, &data, &size); + if (response != NULL) { + creds = flb_parse_http_credentials(response, strlen(response), &expiration); + if (creds != NULL) { + flb_aws_credentials_destroy(creds); + } + flb_free(response); } - flb_free(response); } From 5180ce7e8217497bb2261257e918a32d4cffbabe Mon Sep 17 00:00:00 2001 From: Patrick Stephens Date: Sun, 3 May 2026 12:34:58 +0100 Subject: [PATCH 094/213] [upstream] config: release kernel context on router creation Upstream-Ref: https://github.com/fluent/fluent-bit/commit/de813524de70d0aea512776ad1d0e8614f9dad13 Cherry-picked from Fluent Bit v4.2.4 --- source/src/flb_config.c | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/source/src/flb_config.c b/source/src/flb_config.c index 250c950a..e54b839e 100644 --- a/source/src/flb_config.c +++ b/source/src/flb_config.c @@ -302,11 +302,33 @@ struct flb_config *flb_config_init() config->router = flb_router_create(config); if (!config->router) { flb_error("[config] could not create router"); + if (config->kernel) { + flb_kernel_destroy(config->kernel); + } +#ifdef FLB_HAVE_HTTP_SERVER + if (config->http_listen) { + flb_free(config->http_listen); + } + + if (config->http_port) { + flb_free(config->http_port); + } +#endif + flb_cf_destroy(cf); + flb_free(config); + return NULL; + } + ret = flb_routes_mask_set_size(1, config->router); + if (ret != 0) { + flb_error("[config] routing mask dimensioning failed"); + flb_router_destroy(config->router); + if (config->kernel) { + flb_kernel_destroy(config->kernel); + } flb_cf_destroy(cf); flb_free(config); return NULL; } - flb_routes_mask_set_size(1, config->router); config->cio = NULL; config->storage_path = NULL; From 6f1107bad9baaea63d4b99555be11e96ff494b0c Mon Sep 17 00:00:00 2001 From: Patrick Stephens Date: Sun, 3 May 2026 12:34:59 +0100 Subject: [PATCH 095/213] [upstream] bin: destroy cf opts on failure Upstream-Ref: https://github.com/fluent/fluent-bit/commit/9fbf2e146f258dfaee2a7103ff33708df099d92e Cherry-picked from Fluent Bit v4.2.4 --- source/src/fluent-bit.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/source/src/fluent-bit.c b/source/src/fluent-bit.c index 58c92619..c97f83c1 100644 --- a/source/src/fluent-bit.c +++ b/source/src/fluent-bit.c @@ -880,6 +880,12 @@ int flb_main(int argc, char **argv) /* Create Fluent Bit context */ ctx = flb_create(); if (!ctx) { + flb_cf_destroy(cf_opts); +#ifdef FLB_HAVE_CHUNK_TRACE + if (trace_output) { + flb_free(trace_output); + } +#endif exit(EXIT_FAILURE); } config = ctx->config; From 91b074bd0c6fb2f9aed2bd7753ae1bca1694d488 Mon Sep 17 00:00:00 2001 From: Patrick Stephens Date: Sun, 3 May 2026 12:34:59 +0100 Subject: [PATCH 096/213] [upstream] config: release http resources on exit Upstream-Ref: https://github.com/fluent/fluent-bit/commit/02c9e9f478247e92ade88fe71806757202c67d22 Cherry-picked from Fluent Bit v4.2.4 --- source/src/flb_config.c | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/source/src/flb_config.c b/source/src/flb_config.c index e54b839e..8637009a 100644 --- a/source/src/flb_config.c +++ b/source/src/flb_config.c @@ -325,6 +325,15 @@ struct flb_config *flb_config_init() if (config->kernel) { flb_kernel_destroy(config->kernel); } +#ifdef FLB_HAVE_HTTP_SERVER + if (config->http_listen) { + flb_free(config->http_listen); + } + + if (config->http_port) { + flb_free(config->http_port); + } +#endif flb_cf_destroy(cf); flb_free(config); return NULL; From 216534c666727918597205be3a165995134173f5 Mon Sep 17 00:00:00 2001 From: Patrick Stephens Date: Sun, 3 May 2026 12:34:59 +0100 Subject: [PATCH 097/213] [upstream] out_s3: Add a NULL check for plugging SEGV on dry_run Upstream-Ref: https://github.com/fluent/fluent-bit/commit/05c8c3aba77d9fb50c2cdacdd57abbfe206f1e56 Cherry-picked from Fluent Bit v4.2.4 --- source/plugins/out_s3/s3.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/source/plugins/out_s3/s3.c b/source/plugins/out_s3/s3.c index f9705b59..7dc07170 100644 --- a/source/plugins/out_s3/s3.c +++ b/source/plugins/out_s3/s3.c @@ -1112,7 +1112,11 @@ static int cb_s3_worker_exit(void *data, struct flb_config *config) struct worker_info *info; struct flb_s3 *ctx = data; - flb_plg_info(ctx->ins, "initializing worker"); + if (!ctx) { + return 0; + } + + flb_plg_info(ctx->ins, "terminating worker"); info = FLB_TLS_GET(s3_worker_info); if (info != NULL) { From 6ac091e1d114d0e5e04214b6d42f9815480d0dd4 Mon Sep 17 00:00:00 2001 From: Patrick Stephens Date: Sun, 3 May 2026 12:34:59 +0100 Subject: [PATCH 098/213] [upstream] build: Add workdir option for the default Upstream-Ref: https://github.com/fluent/fluent-bit/commit/7970be427e832a5ab919bc4b648e18bfc135b59c Cherry-picked from Fluent Bit v4.2.4 --- source/cpack/wix/WIX.template.in.cmakein | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/cpack/wix/WIX.template.in.cmakein b/source/cpack/wix/WIX.template.in.cmakein index 2f839408..752546b0 100644 --- a/source/cpack/wix/WIX.template.in.cmakein +++ b/source/cpack/wix/WIX.template.in.cmakein @@ -83,7 +83,7 @@ - + From 94c4dd74764eb2594c29909ae9da086df933dbbb Mon Sep 17 00:00:00 2001 From: Patrick Stephens Date: Sun, 3 May 2026 12:34:59 +0100 Subject: [PATCH 099/213] [upstream] thread_storage: Use pthread_once to ensure Upstream-Ref: https://github.com/fluent/fluent-bit/commit/a4c158dd018a2afaba3971caaf828b59f346f096 Cherry-picked from Fluent Bit v4.2.4 --- source/include/fluent-bit/flb_log.h | 6 +++- source/include/fluent-bit/flb_output.h | 4 +++ source/include/fluent-bit/flb_scheduler.h | 4 +++ .../include/fluent-bit/flb_thread_storage.h | 34 +++++++++++++++++-- 4 files changed, 44 insertions(+), 4 deletions(-) diff --git a/source/include/fluent-bit/flb_log.h b/source/include/fluent-bit/flb_log.h index 75de5060..90f145ef 100644 --- a/source/include/fluent-bit/flb_log.h +++ b/source/include/fluent-bit/flb_log.h @@ -35,7 +35,11 @@ #include /* FIXME: this extern should be auto-populated from flb_thread_storage.h */ -extern FLB_TLS_DEFINE(struct flb_log, flb_log_ctx) +#ifndef FLB_HAVE_C_TLS +FLB_TLS_DECLARE(struct flb_log, flb_log_ctx); +#else +extern FLB_TLS_DEFINE(struct flb_log, flb_log_ctx); +#endif /* Message types */ #define FLB_LOG_OFF 0 diff --git a/source/include/fluent-bit/flb_output.h b/source/include/fluent-bit/flb_output.h index c54a7a32..6ad593a3 100644 --- a/source/include/fluent-bit/flb_output.h +++ b/source/include/fluent-bit/flb_output.h @@ -593,7 +593,11 @@ struct flb_out_flush_params { struct flb_coro *coro; /* coroutine context */ }; +#ifndef FLB_HAVE_C_TLS +FLB_TLS_DECLARE(struct flb_out_flush_params, out_flush_params); +#else extern FLB_TLS_DEFINE(struct flb_out_flush_params, out_flush_params); +#endif #define FLB_OUTPUT_RETURN(x) \ flb_output_return_do(x); \ diff --git a/source/include/fluent-bit/flb_scheduler.h b/source/include/fluent-bit/flb_scheduler.h index 227d8a5a..588dbeeb 100644 --- a/source/include/fluent-bit/flb_scheduler.h +++ b/source/include/fluent-bit/flb_scheduler.h @@ -203,7 +203,11 @@ struct flb_sched_timer_coro_cb_params { struct flb_coro *coro; }; +#ifndef FLB_HAVE_C_TLS +FLB_TLS_DECLARE(struct flb_sched_timer_coro_cb_params, sched_timer_coro_cb_params); +#else extern FLB_TLS_DEFINE(struct flb_sched_timer_coro_cb_params, sched_timer_coro_cb_params); +#endif struct flb_timer_cb_coro_params { diff --git a/source/include/fluent-bit/flb_thread_storage.h b/source/include/fluent-bit/flb_thread_storage.h index 7ec9df94..a442929f 100644 --- a/source/include/fluent-bit/flb_thread_storage.h +++ b/source/include/fluent-bit/flb_thread_storage.h @@ -40,13 +40,41 @@ /* Fallback mode using pthread_*() for Thread-Local-Storage usage */ #define FLB_TLS_SET(key, val) pthread_setspecific(key, (void *) val) #define FLB_TLS_GET(key) pthread_getspecific(key) -#define FLB_TLS_INIT(key) pthread_key_create(&key, NULL) -#define FLB_TLS_DEFINE(type, name) pthread_key_t name; + +/* + * Thread-safe idempotent initialization using pthread_once. + * This ensures pthread_key_create is only called once even if FLB_TLS_INIT + * is called from multiple locations (different compilation units, hot reload, etc). + */ +#define FLB_TLS_INIT(key) \ + do { \ + extern pthread_once_t key##_once; \ + void key##_init_func(void); \ + pthread_once(&key##_once, key##_init_func); \ + } while(0) + +/* Define a TLS key with its pthread_once control and init function */ +#define FLB_TLS_DEFINE(type, name) \ + pthread_key_t name; \ + pthread_once_t name##_once = PTHREAD_ONCE_INIT; \ + void name##_init_func(void) { \ + pthread_key_create(&name, NULL); \ + } + +/* Declare a TLS key that's defined elsewhere */ +#define FLB_TLS_DECLARE(type, name) \ + extern pthread_key_t name; \ + extern pthread_once_t name##_once; \ + void name##_init_func(void); #endif /* FIXME: this extern should be auto-populated from flb_thread_storage.h */ -extern FLB_TLS_DEFINE(struct flb_worker, flb_worker_ctx) +#ifndef FLB_HAVE_C_TLS +FLB_TLS_DECLARE(struct flb_worker, flb_worker_ctx); +#else +extern FLB_TLS_DEFINE(struct flb_worker, flb_worker_ctx); +#endif #endif /* !FLB_THREAD_STORAGE_H */ From f0d5c120860c345efbe8d311feddce1880c1d6a3 Mon Sep 17 00:00:00 2001 From: Patrick Stephens Date: Sun, 3 May 2026 12:34:59 +0100 Subject: [PATCH 100/213] [upstream] out_opensearch: release aws signature a retry or Upstream-Ref: https://github.com/fluent/fluent-bit/commit/c26a2d1c4381f8c1cf1e7819fd8ba6957d919536 Cherry-picked from Fluent Bit v4.2.4 --- source/plugins/out_opensearch/opensearch.c | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/source/plugins/out_opensearch/opensearch.c b/source/plugins/out_opensearch/opensearch.c index 3f52a7ab..1037987b 100644 --- a/source/plugins/out_opensearch/opensearch.c +++ b/source/plugins/out_opensearch/opensearch.c @@ -974,6 +974,10 @@ static void cb_opensearch_flush(struct flb_event_chunk *event_chunk, ret = flb_http_do(c, &b_sent); if (ret != 0) { flb_plg_warn(ctx->ins, "http_do=%i URI=%s", ret, ctx->uri); + if (signature) { + flb_sds_destroy(signature); + signature = NULL; + } goto retry; } else { @@ -988,6 +992,10 @@ static void cb_opensearch_flush(struct flb_event_chunk *event_chunk, flb_plg_error(ctx->ins, "HTTP status=%i URI=%s", c->resp.status, ctx->uri); } + if (signature) { + flb_sds_destroy(signature); + signature = NULL; + } goto retry; } @@ -1021,6 +1029,10 @@ static void cb_opensearch_flush(struct flb_event_chunk *event_chunk, fflush(stderr); } } + if (signature) { + flb_sds_destroy(signature); + signature = NULL; + } goto retry; } else { @@ -1029,6 +1041,10 @@ static void cb_opensearch_flush(struct flb_event_chunk *event_chunk, } } else { + if (signature) { + flb_sds_destroy(signature); + signature = NULL; + } goto retry; } } From b5f6a639198e23a7b4b44be41f6fe9dddd54de88 Mon Sep 17 00:00:00 2001 From: Patrick Stephens Date: Sun, 3 May 2026 12:35:00 +0100 Subject: [PATCH 101/213] [upstream] router: for direct routes improve the handling of Upstream-Ref: https://github.com/fluent/fluent-bit/commit/00baa18badb71fab495fe48b1db298fe18398bc8 Cherry-picked from Fluent Bit v4.2.4 --- source/include/fluent-bit/flb_router.h | 3 + source/src/flb_router_config.c | 132 +++++++++++++++++++++---- 2 files changed, 118 insertions(+), 17 deletions(-) diff --git a/source/include/fluent-bit/flb_router.h b/source/include/fluent-bit/flb_router.h index 7a66d7af..5071513d 100644 --- a/source/include/fluent-bit/flb_router.h +++ b/source/include/fluent-bit/flb_router.h @@ -136,6 +136,9 @@ struct flb_route { struct flb_input_routes { flb_sds_t input_name; + flb_sds_t plugin_name; + int has_alias; + struct flb_input_instance *instance; struct cfl_list processors; struct cfl_list routes; struct cfl_list _head; diff --git a/source/src/flb_router_config.c b/source/src/flb_router_config.c index d5a6687e..9da87ef7 100644 --- a/source/src/flb_router_config.c +++ b/source/src/flb_router_config.c @@ -388,6 +388,10 @@ static void input_routes_destroy(struct flb_input_routes *input) flb_sds_destroy(input->input_name); } + if (input->plugin_name) { + flb_sds_destroy(input->plugin_name); + } + flb_free(input); } @@ -984,6 +988,8 @@ static int parse_input_section(struct flb_cf_section *section, struct cfl_list *input_routes, struct flb_config *config) { + uint32_t mask; + size_t before_count; struct flb_input_routes *input; struct cfl_kvlist *kvlist; struct cfl_variant *name_var; @@ -992,8 +998,7 @@ static int parse_input_section(struct flb_cf_section *section, struct cfl_kvlist *routes_kvlist; struct cfl_list *head; struct cfl_kvpair *pair; - uint32_t mask; - size_t before_count; + struct cfl_variant *alias_var; if (!section || !input_routes) { return -1; @@ -1034,13 +1039,29 @@ static int parse_input_section(struct flb_cf_section *section, cfl_list_init(&input->_head); cfl_list_init(&input->processors); cfl_list_init(&input->routes); + input->has_alias = FLB_FALSE; + input->instance = NULL; - input->input_name = copy_from_cfl_sds(name_var->data.as_string); - if (!input->input_name) { + input->plugin_name = copy_from_cfl_sds(name_var->data.as_string); + if (!input->plugin_name) { flb_free(input); return -1; } + alias_var = cfl_kvlist_fetch(kvlist, "alias"); + if (alias_var && alias_var->type == CFL_VARIANT_STRING && + cfl_sds_len(alias_var->data.as_string) > 0) { + input->input_name = copy_from_cfl_sds(alias_var->data.as_string); + input->has_alias = FLB_TRUE; + } + else { + input->input_name = copy_from_cfl_sds(name_var->data.as_string); + } + if (!input->input_name) { + input_routes_destroy(input); + return -1; + } + processors_var = cfl_kvlist_fetch(kvlist, "processors"); if (processors_var) { if (parse_processors(processors_var, &input->processors, config) != 0) { @@ -1127,33 +1148,110 @@ int flb_router_config_parse(struct flb_cf *cf, } /* Apply parsed router configuration to actual input/output instances */ +static int input_instance_already_selected(struct flb_config *config, + struct flb_input_routes *current, + struct flb_input_instance *candidate) +{ + struct cfl_list *head; + struct flb_input_routes *routes; + + if (!config || !candidate) { + return FLB_FALSE; + } + + cfl_list_foreach(head, &config->input_routes) { + routes = cfl_list_entry(head, struct flb_input_routes, _head); + + if (routes == current) { + continue; + } + + if (routes->instance == candidate) { + return FLB_TRUE; + } + } + + return FLB_FALSE; +} + static struct flb_input_instance *find_input_instance(struct flb_config *config, - flb_sds_t name) + struct flb_input_routes *routes) { struct mk_list *head; struct flb_input_instance *ins; + size_t key_len; - if (!config || !name) { + if (!config || !routes) { return NULL; } - mk_list_foreach(head, &config->inputs) { - ins = mk_list_entry(head, struct flb_input_instance, _head); + if (routes->instance) { + return routes->instance; + } - if (!ins->p) { - continue; + if (routes->has_alias && routes->input_name) { + mk_list_foreach(head, &config->inputs) { + ins = mk_list_entry(head, struct flb_input_instance, _head); + + if (!ins->p || !ins->alias) { + continue; + } + + if (strcmp(ins->alias, routes->input_name) == 0 && + input_instance_already_selected(config, routes, ins) == FLB_FALSE) { + routes->instance = ins; + return ins; + } } + } - if (ins->alias && strcmp(ins->alias, name) == 0) { - return ins; + if (routes->input_name) { + mk_list_foreach(head, &config->inputs) { + ins = mk_list_entry(head, struct flb_input_instance, _head); + + if (!ins->p) { + continue; + } + + if (strcmp(ins->name, routes->input_name) == 0 && + input_instance_already_selected(config, routes, ins) == FLB_FALSE) { + routes->instance = ins; + return ins; + } } + } - if (strcmp(ins->name, name) == 0) { - return ins; + if (routes->plugin_name) { + mk_list_foreach(head, &config->inputs) { + ins = mk_list_entry(head, struct flb_input_instance, _head); + + if (!ins->p || !ins->p->name) { + continue; + } + + if (strcmp(ins->p->name, routes->plugin_name) == 0 && + input_instance_already_selected(config, routes, ins) == FLB_FALSE) { + routes->instance = ins; + return ins; + } } + } - if (ins->p->name && strcmp(ins->p->name, name) == 0) { - return ins; + if (routes->input_name) { + key_len = flb_sds_len(routes->input_name); + + mk_list_foreach(head, &config->inputs) { + ins = mk_list_entry(head, struct flb_input_instance, _head); + + if (!ins->p || key_len == 0) { + continue; + } + + if (strncmp(ins->name, routes->input_name, key_len) == 0 && + input_instance_already_selected(config, routes, ins) == FLB_FALSE) { + routes->instance = ins; + return ins; + } } } @@ -1259,7 +1357,7 @@ int flb_router_apply_config(struct flb_config *config) cfl_list_foreach(input_head, &config->input_routes) { input_routes = cfl_list_entry(input_head, struct flb_input_routes, _head); - input_ins = find_input_instance(config, input_routes->input_name); + input_ins = find_input_instance(config, input_routes); if (!input_ins) { flb_warn("[router] could not find input instance '%s' for routes", input_routes->input_name ? input_routes->input_name : "(null)"); From 3fb65b2650e9d97d96e4f1e8a1d2b97158889dc1 Mon Sep 17 00:00:00 2001 From: Patrick Stephens Date: Sun, 3 May 2026 12:35:00 +0100 Subject: [PATCH 102/213] [upstream] motd: add 4.2 banner Upstream-Ref: https://github.com/fluent/fluent-bit/commit/b63bcf40cc46c6b7a98df5eddc942d033bda5e0d Cherry-picked from Fluent Bit v4.2.4 --- source/include/fluent-bit/flb_motd.h | 72 +++++++++++++++++----------- 1 file changed, 44 insertions(+), 28 deletions(-) diff --git a/source/include/fluent-bit/flb_motd.h b/source/include/fluent-bit/flb_motd.h index ea2a5734..1ff2fdec 100644 --- a/source/include/fluent-bit/flb_motd.h +++ b/source/include/fluent-bit/flb_motd.h @@ -26,34 +26,50 @@ static unsigned char flb_motd_text[] = { 0x20, 0x20, 0x5f, 0x20, 0x20, 0x20, 0x20, 0x5f, 0x5f, 0x5f, 0x5f, 0x5f, 0x5f, 0x20, 0x5f, 0x20, 0x5f, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x5f, 0x5f, 0x5f, 0x20, 0x20, 0x20, - 0x5f, 0x5f, 0x20, 0x20, 0x0a, 0x7c, 0x20, 0x20, 0x5f, 0x5f, 0x5f, 0x7c, - 0x20, 0x7c, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7c, 0x20, 0x7c, 0x20, 0x20, 0x20, - 0x7c, 0x20, 0x5f, 0x5f, 0x5f, 0x20, 0x28, 0x5f, 0x29, 0x20, 0x7c, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x2f, 0x20, - 0x20, 0x20, 0x7c, 0x20, 0x2f, 0x20, 0x20, 0x7c, 0x20, 0x0a, 0x7c, 0x20, - 0x7c, 0x5f, 0x20, 0x20, 0x7c, 0x20, 0x7c, 0x5f, 0x20, 0x20, 0x20, 0x5f, - 0x20, 0x20, 0x5f, 0x5f, 0x5f, 0x20, 0x5f, 0x20, 0x5f, 0x5f, 0x20, 0x7c, - 0x20, 0x7c, 0x5f, 0x20, 0x20, 0x7c, 0x20, 0x7c, 0x5f, 0x2f, 0x20, 0x2f, - 0x5f, 0x7c, 0x20, 0x7c, 0x5f, 0x20, 0x20, 0x5f, 0x5f, 0x20, 0x20, 0x20, - 0x5f, 0x5f, 0x2f, 0x20, 0x2f, 0x7c, 0x20, 0x7c, 0x20, 0x60, 0x7c, 0x20, - 0x7c, 0x20, 0x0a, 0x7c, 0x20, 0x20, 0x5f, 0x7c, 0x20, 0x7c, 0x20, 0x7c, - 0x20, 0x7c, 0x20, 0x7c, 0x20, 0x7c, 0x2f, 0x20, 0x5f, 0x20, 0x5c, 0x20, - 0x27, 0x5f, 0x20, 0x5c, 0x7c, 0x20, 0x5f, 0x5f, 0x7c, 0x20, 0x7c, 0x20, - 0x5f, 0x5f, 0x5f, 0x20, 0x5c, 0x20, 0x7c, 0x20, 0x5f, 0x5f, 0x7c, 0x20, - 0x5c, 0x20, 0x5c, 0x20, 0x2f, 0x20, 0x2f, 0x20, 0x2f, 0x5f, 0x7c, 0x20, - 0x7c, 0x20, 0x20, 0x7c, 0x20, 0x7c, 0x20, 0x0a, 0x7c, 0x20, 0x7c, 0x20, - 0x20, 0x20, 0x7c, 0x20, 0x7c, 0x20, 0x7c, 0x5f, 0x7c, 0x20, 0x7c, 0x20, - 0x20, 0x5f, 0x5f, 0x2f, 0x20, 0x7c, 0x20, 0x7c, 0x20, 0x7c, 0x20, 0x7c, - 0x5f, 0x20, 0x20, 0x7c, 0x20, 0x7c, 0x5f, 0x2f, 0x20, 0x2f, 0x20, 0x7c, - 0x20, 0x7c, 0x5f, 0x20, 0x20, 0x20, 0x5c, 0x20, 0x56, 0x20, 0x2f, 0x5c, - 0x5f, 0x5f, 0x5f, 0x20, 0x20, 0x7c, 0x5f, 0x5f, 0x7c, 0x20, 0x7c, 0x5f, - 0x0a, 0x5c, 0x5f, 0x7c, 0x20, 0x20, 0x20, 0x7c, 0x5f, 0x7c, 0x5c, 0x5f, - 0x5f, 0x2c, 0x5f, 0x7c, 0x5c, 0x5f, 0x5f, 0x5f, 0x7c, 0x5f, 0x7c, 0x20, - 0x7c, 0x5f, 0x7c, 0x5c, 0x5f, 0x5f, 0x7c, 0x20, 0x5c, 0x5f, 0x5f, 0x5f, - 0x5f, 0x2f, 0x7c, 0x5f, 0x7c, 0x5c, 0x5f, 0x5f, 0x7c, 0x20, 0x20, 0x20, - 0x5c, 0x5f, 0x2f, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7c, 0x5f, 0x28, 0x5f, - 0x29, 0x5f, 0x5f, 0x5f, 0x2f, 0x0a, 0x0a, 0x0a, 0x00 + 0x5f, 0x5f, 0x5f, 0x5f, 0x5f, 0x20, 0x0a, 0x7c, 0x20, 0x20, 0x5f, 0x5f, + 0x5f, 0x7c, 0x20, 0x7c, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7c, 0x20, 0x7c, 0x20, + 0x20, 0x20, 0x7c, 0x20, 0x5f, 0x5f, 0x5f, 0x20, 0x28, 0x5f, 0x29, 0x20, + 0x7c, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x2f, 0x20, 0x20, 0x20, 0x7c, 0x20, 0x2f, 0x20, 0x5f, 0x5f, 0x20, 0x20, + 0x5c, 0x0a, 0x7c, 0x20, 0x7c, 0x5f, 0x20, 0x20, 0x7c, 0x20, 0x7c, 0x5f, + 0x20, 0x20, 0x20, 0x5f, 0x20, 0x20, 0x5f, 0x5f, 0x5f, 0x20, 0x5f, 0x20, + 0x5f, 0x5f, 0x20, 0x7c, 0x20, 0x7c, 0x5f, 0x20, 0x20, 0x7c, 0x20, 0x7c, + 0x5f, 0x2f, 0x20, 0x2f, 0x5f, 0x7c, 0x20, 0x7c, 0x5f, 0x20, 0x20, 0x5f, + 0x5f, 0x20, 0x20, 0x20, 0x5f, 0x5f, 0x2f, 0x20, 0x2f, 0x7c, 0x20, 0x7c, + 0x20, 0x60, 0x27, 0x20, 0x2f, 0x20, 0x2f, 0x27, 0x0a, 0x7c, 0x20, 0x20, + 0x5f, 0x7c, 0x20, 0x7c, 0x20, 0x7c, 0x20, 0x7c, 0x20, 0x7c, 0x20, 0x7c, + 0x2f, 0x20, 0x5f, 0x20, 0x5c, 0x20, 0x27, 0x5f, 0x20, 0x5c, 0x7c, 0x20, + 0x5f, 0x5f, 0x7c, 0x20, 0x7c, 0x20, 0x5f, 0x5f, 0x5f, 0x20, 0x5c, 0x20, + 0x7c, 0x20, 0x5f, 0x5f, 0x7c, 0x20, 0x5c, 0x20, 0x5c, 0x20, 0x2f, 0x20, + 0x2f, 0x20, 0x2f, 0x5f, 0x7c, 0x20, 0x7c, 0x20, 0x20, 0x20, 0x2f, 0x20, + 0x2f, 0x20, 0x20, 0x0a, 0x7c, 0x20, 0x7c, 0x20, 0x20, 0x20, 0x7c, 0x20, + 0x7c, 0x20, 0x7c, 0x5f, 0x7c, 0x20, 0x7c, 0x20, 0x20, 0x5f, 0x5f, 0x2f, + 0x20, 0x7c, 0x20, 0x7c, 0x20, 0x7c, 0x20, 0x7c, 0x5f, 0x20, 0x20, 0x7c, + 0x20, 0x7c, 0x5f, 0x2f, 0x20, 0x2f, 0x20, 0x7c, 0x20, 0x7c, 0x5f, 0x20, + 0x20, 0x20, 0x5c, 0x20, 0x56, 0x20, 0x2f, 0x5c, 0x5f, 0x5f, 0x5f, 0x20, + 0x20, 0x7c, 0x5f, 0x2e, 0x2f, 0x20, 0x2f, 0x5f, 0x5f, 0x5f, 0x0a, 0x5c, + 0x5f, 0x7c, 0x20, 0x20, 0x20, 0x7c, 0x5f, 0x7c, 0x5c, 0x5f, 0x5f, 0x2c, + 0x5f, 0x7c, 0x5c, 0x5f, 0x5f, 0x5f, 0x7c, 0x5f, 0x7c, 0x20, 0x7c, 0x5f, + 0x7c, 0x5c, 0x5f, 0x5f, 0x7c, 0x20, 0x5c, 0x5f, 0x5f, 0x5f, 0x5f, 0x2f, + 0x7c, 0x5f, 0x7c, 0x5c, 0x5f, 0x5f, 0x7c, 0x20, 0x20, 0x20, 0x5c, 0x5f, + 0x2f, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7c, 0x5f, 0x28, 0x5f, 0x29, 0x5f, + 0x5f, 0x5f, 0x5f, 0x5f, 0x2f, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x46, 0x6c, 0x75, 0x65, 0x6e, 0x74, 0x20, 0x42, 0x69, 0x74, + 0x20, 0x76, 0x34, 0x2e, 0x32, 0x20, 0xe2, 0x80, 0x93, 0x20, 0x44, 0x69, + 0x72, 0x65, 0x63, 0x74, 0x20, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x73, 0x20, + 0x41, 0x68, 0x65, 0x61, 0x64, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x43, 0x65, 0x6c, 0x65, 0x62, 0x72, 0x61, 0x74, 0x69, + 0x6e, 0x67, 0x20, 0x31, 0x30, 0x20, 0x59, 0x65, 0x61, 0x72, 0x73, 0x20, + 0x6f, 0x66, 0x20, 0x4f, 0x70, 0x65, 0x6e, 0x2c, 0x20, 0x46, 0x6c, 0x75, + 0x65, 0x6e, 0x74, 0x20, 0x49, 0x6e, 0x6e, 0x6f, 0x76, 0x61, 0x74, 0x69, + 0x6f, 0x6e, 0x21, 0x0a, 0x0a, 0x00 }; #endif From 2190ba60874d613ded0a9751a74e8ddd9672d6d6 Mon Sep 17 00:00:00 2001 From: Patrick Stephens Date: Sun, 3 May 2026 12:35:00 +0100 Subject: [PATCH 103/213] [upstream] out_http: Plug a NULL dereference Upstream-Ref: https://github.com/fluent/fluent-bit/commit/a8d39bb931f99c99630887d8dc11871794a796da Cherry-picked from Fluent Bit v4.2.4 --- source/plugins/out_http/http.c | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/source/plugins/out_http/http.c b/source/plugins/out_http/http.c index 88b62673..1c912989 100644 --- a/source/plugins/out_http/http.c +++ b/source/plugins/out_http/http.c @@ -178,6 +178,18 @@ static int http_post(struct flb_out_http *ctx, ctx->host, ctx->port, ctx->proxy, 0); + if (c == NULL) { + flb_plg_error(ctx->ins, "[http_client] failed to create HTTP client"); + if (payload_buf != body) { + flb_free(payload_buf); + } + + if (u_conn) { + flb_upstream_conn_release(u_conn); + } + + return FLB_RETRY; + } if (c->proxy.host) { flb_plg_debug(ctx->ins, "[http_client] proxy host: %s port: %i", From 9dcd2b0e69242446ee6a7560e592bf2ea158d8a9 Mon Sep 17 00:00:00 2001 From: Patrick Stephens Date: Sun, 3 May 2026 12:35:00 +0100 Subject: [PATCH 104/213] [upstream] out_http: added PUT support (#10882) Upstream-Ref: https://github.com/fluent/fluent-bit/commit/b6fcaabc138595923abe44d949f51a9363cb7f31 Cherry-picked from Fluent Bit v4.2.4 --- source/plugins/out_http/http.c | 37 +++++++++++++++++------------ source/plugins/out_http/http.h | 3 +++ source/plugins/out_http/http_conf.c | 18 ++++++++++++++ 3 files changed, 43 insertions(+), 15 deletions(-) diff --git a/source/plugins/out_http/http.c b/source/plugins/out_http/http.c index 1c912989..bfaa9d74 100644 --- a/source/plugins/out_http/http.c +++ b/source/plugins/out_http/http.c @@ -109,10 +109,10 @@ static void append_headers(struct flb_http_client *c, } } -static int http_post(struct flb_out_http *ctx, - const void *body, size_t body_len, - const char *tag, int tag_len, - char **headers) +static int http_request(struct flb_out_http *ctx, + const void *body, size_t body_len, + const char *tag, int tag_len, + char **headers) { int ret = 0; int out_ret = FLB_OK; @@ -173,7 +173,7 @@ static int http_post(struct flb_out_http *ctx, /* Create HTTP client context */ - c = flb_http_client(u_conn, FLB_HTTP_POST, ctx->uri, + c = flb_http_client(u_conn, ctx->http_method, ctx->uri, payload_buf, payload_size, ctx->host, ctx->port, ctx->proxy, 0); @@ -541,7 +541,7 @@ static char **extract_headers(msgpack_object *obj) { return NULL; } -static int post_all_requests(struct flb_out_http *ctx, +static int send_all_requests(struct flb_out_http *ctx, const char *data, size_t size, flb_sds_t body_key, flb_sds_t headers_key, @@ -610,8 +610,10 @@ static int post_all_requests(struct flb_out_http *ctx, } if (body_found && headers_found) { - flb_plg_trace(ctx->ins, "posting record %zu", record_count++); - ret = http_post(ctx, body, body_size, event_chunk->tag, + flb_plg_trace(ctx->ins, "sending record %zu via %s", + record_count++, + ctx->http_method == FLB_HTTP_POST ? "POST" : "PUT"); + ret = http_request(ctx, body, body_size, event_chunk->tag, flb_sds_len(event_chunk->tag), headers); } else { @@ -643,11 +645,11 @@ static void cb_http_flush(struct flb_event_chunk *event_chunk, (void) i_ins; if (ctx->body_key) { - ret = post_all_requests(ctx, event_chunk->data, event_chunk->size, + ret = send_all_requests(ctx, event_chunk->data, event_chunk->size, ctx->body_key, ctx->headers_key, event_chunk); if (ret < 0) { flb_plg_error(ctx->ins, - "failed to post requests body key \"%s\"", ctx->body_key); + "failed to send requests using body key \"%s\"", ctx->body_key); } } else { @@ -661,15 +663,15 @@ static void cb_http_flush(struct flb_event_chunk *event_chunk, (ctx->out_format == FLB_PACK_JSON_FORMAT_STREAM) || (ctx->out_format == FLB_PACK_JSON_FORMAT_LINES) || (ctx->out_format == FLB_HTTP_OUT_GELF)) { - ret = http_post(ctx, out_body, out_size, - event_chunk->tag, flb_sds_len(event_chunk->tag), NULL); + ret = http_request(ctx, out_body, out_size, + event_chunk->tag, flb_sds_len(event_chunk->tag), NULL); flb_sds_destroy(out_body); } else { /* msgpack */ - ret = http_post(ctx, - event_chunk->data, event_chunk->size, - event_chunk->tag, flb_sds_len(event_chunk->tag), NULL); + ret = http_request(ctx, + event_chunk->data, event_chunk->size, + event_chunk->tag, flb_sds_len(event_chunk->tag), NULL); } } @@ -771,6 +773,11 @@ static struct flb_config_map config_map[] = { 0, FLB_TRUE, offsetof(struct flb_out_http, uri), "Specify an optional HTTP URI for the target web server, e.g: /something" }, + { + FLB_CONFIG_MAP_STR, "http_method", "POST", + 0, FLB_FALSE, 0, + "Specify the HTTP method to use. Supported methods are POST and PUT" + }, /* Gelf Properties */ { diff --git a/source/plugins/out_http/http.h b/source/plugins/out_http/http.h index 4373f2c2..2a0b3ca5 100644 --- a/source/plugins/out_http/http.h +++ b/source/plugins/out_http/http.h @@ -67,6 +67,9 @@ struct flb_out_http { char *host; int port; + /* HTTP method */ + int http_method; + /* GELF fields */ struct flb_gelf_fields gelf_fields; diff --git a/source/plugins/out_http/http_conf.c b/source/plugins/out_http/http_conf.c index 24f93ca9..a9aa2596 100644 --- a/source/plugins/out_http/http_conf.c +++ b/source/plugins/out_http/http_conf.c @@ -22,6 +22,7 @@ #include #include #include +#include #include #ifdef FLB_HAVE_SIGNV4 @@ -272,6 +273,23 @@ struct flb_out_http *flb_http_conf_create(struct flb_output_instance *ins, } } + /* HTTP method */ + ctx->http_method = FLB_HTTP_POST; + tmp = flb_output_get_property("http_method", ins); + if (tmp) { + if (strcasecmp(tmp, "POST") == 0) { + ctx->http_method = FLB_HTTP_POST; + } + else if (strcasecmp(tmp, "PUT") == 0) { + ctx->http_method = FLB_HTTP_PUT; + } + else { + flb_plg_error(ctx->ins, "invalid http_method option '%s'. Supported methods are POST and PUT", tmp); + flb_free(ctx); + return NULL; + } + } + ctx->u = upstream; ctx->uri = uri; ctx->host = ins->host.name; From 8dd7b2b64329517326ac99e2450c4a32fb5212cf Mon Sep 17 00:00:00 2001 From: Patrick Stephens Date: Sun, 3 May 2026 12:35:01 +0100 Subject: [PATCH 105/213] [upstream] in_tail: Implement long ling truncation Upstream-Ref: https://github.com/fluent/fluent-bit/commit/7612a09acc1110f0f26964f57ed6ab51f87d502a Cherry-picked from Fluent Bit v4.2.4 --- source/plugins/in_tail/tail.c | 6 ++ source/plugins/in_tail/tail_config.c | 28 +++++-- source/plugins/in_tail/tail_config.h | 3 + source/plugins/in_tail/tail_file.c | 115 ++++++++++++++++++++++++++- 4 files changed, 143 insertions(+), 9 deletions(-) diff --git a/source/plugins/in_tail/tail.c b/source/plugins/in_tail/tail.c index 41ac94a9..82bf1dc6 100644 --- a/source/plugins/in_tail/tail.c +++ b/source/plugins/in_tail/tail.c @@ -719,6 +719,12 @@ static struct flb_config_map config_map[] = { 0, FLB_TRUE, offsetof(struct flb_tail_config, skip_empty_lines), "Allows to skip empty lines." }, + + { + FLB_CONFIG_MAP_BOOL, "truncate_long_lines", "false", + 0, FLB_TRUE, offsetof(struct flb_tail_config, truncate_long_lines), + "Truncate overlong lines after input encoding to UTF-8" + }, #ifdef __linux__ { FLB_CONFIG_MAP_BOOL, "file_cache_advise", "true", diff --git a/source/plugins/in_tail/tail_config.c b/source/plugins/in_tail/tail_config.c index 929834b7..2778b7af 100644 --- a/source/plugins/in_tail/tail_config.c +++ b/source/plugins/in_tail/tail_config.c @@ -170,7 +170,7 @@ struct flb_tail_config *flb_tail_config_create(struct flb_input_instance *ins, if (sec == 0 && nsec == 0) { flb_plg_error(ctx->ins, "invalid 'refresh_interval' config " "value (%s)", tmp); - flb_free(ctx); + flb_tail_config_destroy(ctx); return NULL; } @@ -192,7 +192,7 @@ struct flb_tail_config *flb_tail_config_create(struct flb_input_instance *ins, /* Config: seconds interval to monitor file after rotation */ if (ctx->rotate_wait <= 0) { flb_plg_error(ctx->ins, "invalid 'rotate_wait' config value"); - flb_free(ctx); + flb_tail_config_destroy(ctx); return NULL; } @@ -215,7 +215,7 @@ struct flb_tail_config *flb_tail_config_create(struct flb_input_instance *ins, } else { flb_plg_error(ctx->ins, "invalid encoding 'unicode.encoding' value"); - flb_free(ctx); + flb_tail_config_destroy(ctx); return NULL; } } @@ -230,11 +230,20 @@ struct flb_tail_config *flb_tail_config_create(struct flb_input_instance *ins, } else { flb_plg_error(ctx->ins, "invalid encoding 'generic.encoding' value %s", tmp); - flb_free(ctx); + flb_tail_config_destroy(ctx); return NULL; } } +#ifdef FLB_HAVE_UNICODE_ENCODER + if (ctx->preferred_input_encoding != FLB_UNICODE_ENCODING_UNSPECIFIED && + ctx->generic_input_encoding_type != FLB_GENERIC_UNSPECIFIED) { + flb_plg_error(ctx->ins, + "'unicode.encoding' and 'generic.encoding' cannot be specified at the same time"); + flb_tail_config_destroy(ctx); + return NULL; + } +#endif #ifdef FLB_HAVE_PARSER /* Config: multi-line support */ if (ctx->multiline == FLB_TRUE) { @@ -258,7 +267,7 @@ struct flb_tail_config *flb_tail_config_create(struct flb_input_instance *ins, /* Validate buffer limit */ if (ctx->buf_chunk_size > ctx->buf_max_size) { flb_plg_error(ctx->ins, "buffer_max_size must be >= buffer_chunk"); - flb_free(ctx); + flb_tail_config_destroy(ctx); return NULL; } @@ -485,6 +494,13 @@ struct flb_tail_config *flb_tail_config_create(struct flb_input_instance *ins, "multiline_truncated_total", "Total number of truncated occurences for multilines", 1, (char *[]) {"name"}); + ctx->cmt_long_line_truncated = \ + cmt_counter_create(ins->cmt, + "fluentbit", "input", + "long_line_truncated_total", + "Total number of truncated occurences for long lines", + 1, (char *[]) {"name"}); + /* OLD metrics */ flb_metrics_add(FLB_TAIL_METRIC_F_OPENED, "files_opened", ctx->ins->metrics); @@ -494,6 +510,8 @@ struct flb_tail_config *flb_tail_config_create(struct flb_input_instance *ins, "files_rotated", ctx->ins->metrics); flb_metrics_add(FLB_TAIL_METRIC_M_TRUNCATED, "multiline_truncated", ctx->ins->metrics); + flb_metrics_add(FLB_TAIL_METRIC_L_TRUNCATED, + "long_line_truncated", ctx->ins->metrics); #endif return ctx; diff --git a/source/plugins/in_tail/tail_config.h b/source/plugins/in_tail/tail_config.h index dfd5f919..00615448 100644 --- a/source/plugins/in_tail/tail_config.h +++ b/source/plugins/in_tail/tail_config.h @@ -42,6 +42,7 @@ #define FLB_TAIL_METRIC_F_CLOSED 101 /* number of closed files */ #define FLB_TAIL_METRIC_F_ROTATED 102 /* number of rotated files */ #define FLB_TAIL_METRIC_M_TRUNCATED 103 /* number of truncated occurrences of multiline */ +#define FLB_TAIL_METRIC_L_TRUNCATED 104 /* number of truncated occurrences of long lines */ #endif struct flb_tail_config { @@ -54,6 +55,7 @@ struct flb_tail_config { /* Buffer Config */ size_t buf_chunk_size; /* allocation chunks */ size_t buf_max_size; /* max size of a buffer */ + int truncate_long_lines; /* truncate long lines after re-encode */ /* Static files processor */ size_t static_batch_size; @@ -169,6 +171,7 @@ struct flb_tail_config { struct cmt_counter *cmt_files_closed; struct cmt_counter *cmt_files_rotated; struct cmt_counter *cmt_multiline_truncated; + struct cmt_counter *cmt_long_line_truncated; /* Hash: hash tables for quick acess to registered files */ struct flb_hash_table *static_hash; diff --git a/source/plugins/in_tail/tail_file.c b/source/plugins/in_tail/tail_file.c index ae2c4204..1624bd94 100644 --- a/source/plugins/in_tail/tail_file.c +++ b/source/plugins/in_tail/tail_file.c @@ -457,6 +457,24 @@ static FLB_INLINE const char *flb_skip_leading_zeros_simd(const char *data, cons return data; } +/* Return a UTF-8 safe cut position <= max */ +static size_t utf8_safe_truncate_pos(const char *s, size_t len, size_t max) +{ + size_t cut = 0; + + cut = (len <= max) ? len : max; + if (cut == len) { + return cut; + } + + /* backtrack over continuation bytes 10xxxxxx */ + while (cut > 0 && ((unsigned char)s[cut] & 0xC0) == 0x80) { + cut--; + } + + return cut; +} + static int process_content(struct flb_tail_file *file, size_t *bytes) { size_t len; @@ -481,6 +499,13 @@ static int process_content(struct flb_tail_file *file, size_t *bytes) #ifdef FLB_HAVE_UNICODE_ENCODER size_t decoded_len; #endif + size_t cut = 0; + size_t eff_max = 0; + size_t dec_len = 0; + size_t window = 0; + int truncation_happened = FLB_FALSE; + size_t bytes_override = 0; + void *nl = NULL; #ifdef FLB_HAVE_METRICS uint64_t ts; char *name; @@ -530,7 +555,8 @@ static int process_content(struct flb_tail_file *file, size_t *bytes) end - data); if (ret > 0) { data = decoded; - end = data + strlen(decoded); + /* Generic encoding conversion returns decoded length precisely with ret. */ + end = data + (size_t) ret; } else { flb_plg_error(ctx->ins, "encoding failed '%.*s' with status %d", end - data, data, ret); @@ -542,6 +568,58 @@ static int process_content(struct flb_tail_file *file, size_t *bytes) data = (char *)flb_skip_leading_zeros_simd(data, end, &processed_bytes); } + if (ctx->truncate_long_lines == FLB_TRUE && + file->buf_size >= ctx->buf_max_size) { + /* Use buf_max_size as the truncation threshold */ + if (ctx->buf_max_size > 0) { + eff_max = ctx->buf_max_size - 1; + } + else { + eff_max = 0; + } + dec_len = (size_t)(end - data); + window = ctx->buf_max_size + 1; + if (window > dec_len) { + window = dec_len; + } + + nl = memchr(data, '\n', window); + if (nl == NULL && eff_max > 0 && dec_len >= eff_max) { + if (file->skip_next == FLB_TRUE) { + bytes_override = (original_len > 0) ? original_len : file->buf_len; + goto truncation_end; + } + cut = utf8_safe_truncate_pos(data, dec_len, eff_max); + + if (cut > 0) { + if (ctx->multiline == FLB_TRUE) { + flb_tail_mult_flush(file, ctx); + } + + flb_tail_file_pack_line(NULL, data, cut, file, processed_bytes); + file->skip_next = FLB_TRUE; + + #ifdef FLB_HAVE_METRICS + cmt_counter_inc(ctx->cmt_long_line_truncated, + cfl_time_now(), 1, + (char*[]){ (char*) flb_input_name(ctx->ins) }); + /* Old api */ + flb_metrics_sum(FLB_TAIL_METRIC_L_TRUNCATED, 1, ctx->ins->metrics); + #endif + if (original_len > 0) { + bytes_override = original_len; + } + else { + bytes_override = file->buf_len; + } + truncation_happened = FLB_TRUE; + + lines++; + goto truncation_end; + } + } + } + while (data < end && (p = memchr(data, '\n', end - data))) { len = (p - data); crlf = 0; @@ -700,6 +778,7 @@ static int process_content(struct flb_tail_file *file, size_t *bytes) file->last_processed_bytes = processed_bytes; } +truncation_end: if (decoded) { flb_free(decoded); decoded = NULL; @@ -709,9 +788,13 @@ static int process_content(struct flb_tail_file *file, size_t *bytes) if (lines > 0) { /* Append buffer content to a chunk */ - if (original_len > 0) { + if (truncation_happened) { + *bytes = bytes_override; + } + else if (original_len > 0) { *bytes = original_len; - } else { + } + else { *bytes = processed_bytes; } @@ -1506,12 +1589,13 @@ int flb_tail_file_chunk(struct flb_tail_file *file) size_t file_buffer_capacity; size_t stream_data_length; ssize_t raw_data_length; - size_t processed_bytes; + size_t processed_bytes = 0; uint8_t *read_buffer; size_t read_size; size_t size; char *tmp; int ret; + int lines; struct flb_tail_config *ctx; /* Check if we the engine issued a pause */ @@ -1529,6 +1613,29 @@ int flb_tail_file_chunk(struct flb_tail_file *file) * If there is no more room for more data, try to increase the * buffer under the limit of buffer_max_size. */ + if (ctx->truncate_long_lines == FLB_TRUE) { + lines = process_content(file, &processed_bytes); + if (lines < 0) { + flb_plg_debug(ctx->ins, "inode=%"PRIu64" file=%s process content ERROR", + file->inode, file->name); + return FLB_TAIL_ERROR; + } + + if (lines > 0) { + file->stream_offset += processed_bytes; + file->last_processed_bytes = 0; + consume_bytes(file->buf_data, processed_bytes, file->buf_len); + file->buf_len -= processed_bytes; + file->buf_data[file->buf_len] = '\0'; + +#ifdef FLB_HAVE_SQLDB + if (file->config->db) { + flb_tail_db_file_offset(file, file->config); + } +#endif + return adjust_counters(ctx, file); + } + } if (file->buf_size >= ctx->buf_max_size) { if (ctx->skip_long_lines == FLB_FALSE) { flb_plg_error(ctx->ins, "file=%s requires a larger buffer size, " From 6c8b0a5b40fc40813f1bfe8025bc4ce959c297c9 Mon Sep 17 00:00:00 2001 From: Patrick Stephens Date: Sun, 3 May 2026 12:35:01 +0100 Subject: [PATCH 106/213] [upstream] tests: runtime: in_tail: Add test cases for long line Upstream-Ref: https://github.com/fluent/fluent-bit/commit/e9f63e602acec61a23b5f1b135830b671244ef5b Cherry-picked from Fluent Bit v4.2.4 --- source/tests/runtime/in_tail.c | 232 +++++++++++++++++++++++++++++++++ 1 file changed, 232 insertions(+) diff --git a/source/tests/runtime/in_tail.c b/source/tests/runtime/in_tail.c index c83e2b01..2b1ebb52 100644 --- a/source/tests/runtime/in_tail.c +++ b/source/tests/runtime/in_tail.c @@ -991,6 +991,236 @@ void flb_test_in_tail_skip_long_lines() unlink(path); } +static int write_long_ascii_line(int fd, size_t total_bytes) +{ + const char *chunk = "0123456789abcdef0123456789abcdef"; /* 32 bytes */ + size_t chunk_len = strlen(chunk); + size_t written = 0; + ssize_t ret; + size_t rest = 0; + + while (written + chunk_len <= total_bytes) { + ret = write(fd, chunk, chunk_len); + if (ret < 0) { + flb_errno(); + return -1; + } + written += (size_t) ret; + } + if (written < total_bytes) { + rest = total_bytes - written; + ret = write(fd, chunk, rest); + if (ret < 0) { + flb_errno(); + return -1; + } + written += (size_t) ret; + } + if (write(fd, "\n", 1) != 1) { + flb_errno(); + return -1; + } + return 0; +} + +static int write_long_utf8_line(int fd, size_t total_bytes) +{ + const char *u8_aa = "あ"; + size_t u8_len = strlen(u8_aa); /* 3 */ + size_t written = 0; + ssize_t ret; + const char *ascii = "XYZ"; + size_t rest = 0; + + while (written + u8_len <= total_bytes) { + ret = write(fd, u8_aa, u8_len); + if (ret < 0) { + flb_errno(); + return -1; + } + written += (size_t) ret; + } + + if (written < total_bytes) { + rest = total_bytes - written; + if (rest > strlen(ascii)) { + rest = strlen(ascii); + } + ret = write(fd, ascii, rest); + if (ret < 0) { + flb_errno(); + return -1; + } + written += (size_t) ret; + } + if (write(fd, "\n", 1) != 1) { + flb_errno(); + return -1; + } + return 0; +} + +void flb_test_in_tail_truncate_long_lines() +{ + int64_t ret; + flb_ctx_t *ctx = NULL; + int in_ffd, out_ffd; + char path[PATH_MAX]; + struct tail_test_result result = {0}; + int fd; + + const char *target = "truncate_long_lines_basic"; + int nExpected = 3; /* before + truncated long line + after */ + int nExpectedNotMatched = 0; /* unused */ + int nExpectedLines = 0; /* unused */ + + struct flb_lib_out_cb cb; + int unused = 0; + int num = 0; + + cb.cb = cb_count_msgpack; + cb.data = &unused; + + clear_output_num(); + + ctx = flb_create(); + TEST_CHECK_(ctx != NULL, "flb_create failed"); + + TEST_CHECK_(flb_service_set(ctx, "Log_Level", "error", NULL) == 0, + "setting service options"); + + in_ffd = flb_input(ctx, "tail", NULL); + TEST_CHECK(in_ffd >= 0); + TEST_CHECK(flb_input_set(ctx, in_ffd, "tag", "test", NULL) == 0); + + snprintf(path, sizeof(path) - 1, DPATH "/log/%s.log", target); + fd = creat(path, S_IRWXU | S_IRGRP); + TEST_CHECK(fd >= 0); + + write(fd, "before_long_line\n", strlen("before_long_line\n")); + + TEST_CHECK(write_long_ascii_line(fd, 10 * 1024) == 0); + + write(fd, "after_long_line\n", strlen("after_long_line\n")); + close(fd); + + TEST_CHECK_(access(path, R_OK) == 0, "accessing log file: %s", path); + + TEST_CHECK(flb_input_set(ctx, in_ffd, + "path", path, + "read_from_head", "true", + "truncate_long_lines", "on", + "skip_long_lines", "off", + "Buffer_Chunk_Size", "1k", + "Buffer_Max_Size", "4k", + NULL) == 0); + + out_ffd = flb_output(ctx, "lib", &cb); + TEST_CHECK(out_ffd >= 0); + TEST_CHECK(flb_output_set(ctx, out_ffd, + "match", "test", + NULL) == 0); + + TEST_CHECK(flb_service_set(ctx, "Flush", "0.5", + "Grace", "1", + NULL) == 0); + + ret = flb_start(ctx); + TEST_CHECK_(ret == 0, "starting engine"); + + wait_num_with_timeout(5000, &num); + + num = get_output_num(); + TEST_CHECK(num == nExpected); + TEST_MSG("output count (truncate basic): got=%d expected=%d", num, nExpected); + + ret = flb_stop(ctx); + TEST_CHECK_(ret == 0, "stopping engine"); + + if (ctx) { + flb_destroy(ctx); + } + + unlink(path); +} + +void flb_test_in_tail_truncate_long_lines_utf8() +{ + int64_t ret; + flb_ctx_t *ctx = NULL; + int in_ffd, out_ffd; + char path[PATH_MAX]; + int fd; + + const char *target = "truncate_long_lines_utf8"; + int nExpected = 1; + + struct flb_lib_out_cb cb; + int unused = 0; + int num = 0; + + cb.cb = cb_count_msgpack; + cb.data = &unused; + + clear_output_num(); + + ctx = flb_create(); + TEST_CHECK_(ctx != NULL, "flb_create failed"); + + TEST_CHECK_(flb_service_set(ctx, "Log_Level", "error", NULL) == 0, + "setting service options"); + + in_ffd = flb_input(ctx, "tail", NULL); + TEST_CHECK(in_ffd >= 0); + TEST_CHECK(flb_input_set(ctx, in_ffd, "tag", "test", NULL) == 0); + + snprintf(path, sizeof(path) - 1, DPATH "/log/%s.log", target); + fd = creat(path, S_IRWXU | S_IRGRP); + TEST_CHECK(fd >= 0); + + TEST_CHECK(write_long_utf8_line(fd, 10 * 1024) == 0); + close(fd); + + TEST_CHECK_(access(path, R_OK) == 0, "accessing log file: %s", path); + + TEST_CHECK(flb_input_set(ctx, in_ffd, + "path", path, + "read_from_head", "true", + "truncate_long_lines", "on", + "skip_long_lines", "off", + "Buffer_Chunk_Size", "1k", + "Buffer_Max_Size", "4k", + NULL) == 0); + + out_ffd = flb_output(ctx, "lib", &cb); + TEST_CHECK(out_ffd >= 0); + TEST_CHECK(flb_output_set(ctx, out_ffd, + "match", "test", + NULL) == 0); + + TEST_CHECK(flb_service_set(ctx, "Flush", "0.5", + "Grace", "1", + NULL) == 0); + + ret = flb_start(ctx); + TEST_CHECK_(ret == 0, "starting engine"); + + wait_num_with_timeout(5000, &num); + + num = get_output_num(); + TEST_CHECK(num == nExpected); + TEST_MSG("output count (truncate utf8): got=%d expected=%d", num, nExpected); + + ret = flb_stop(ctx); + TEST_CHECK_(ret == 0, "stopping engine"); + + if (ctx) { + flb_destroy(ctx); + } + + unlink(path); +} + /* * test case for https://github.com/fluent/fluent-bit/issues/3943 * @@ -2426,6 +2656,8 @@ TEST_LIST = { {"issue_3943", flb_test_in_tail_issue_3943}, /* Properties */ {"skip_long_lines", flb_test_in_tail_skip_long_lines}, + {"truncate_long_lines", flb_test_in_tail_truncate_long_lines}, + {"truncate_long_lines_utf8", flb_test_in_tail_truncate_long_lines_utf8}, {"path_comma", flb_test_path_comma}, {"path_key", flb_test_path_key}, {"exclude_path", flb_test_exclude_path}, From 118a08188397b5ca5782b204c92d76158755910b Mon Sep 17 00:00:00 2001 From: Patrick Stephens Date: Sun, 3 May 2026 12:35:01 +0100 Subject: [PATCH 107/213] [upstream] in_systemd: add explicit restarts for the journal Upstream-Ref: https://github.com/fluent/fluent-bit/commit/9ed44f0219f762c0fa0a91f20ecfe03e5ad3548f Cherry-picked from Fluent Bit v4.2.4 --- source/plugins/in_systemd/systemd.c | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/source/plugins/in_systemd/systemd.c b/source/plugins/in_systemd/systemd.c index 58ce4e4c..e1063542 100644 --- a/source/plugins/in_systemd/systemd.c +++ b/source/plugins/in_systemd/systemd.c @@ -308,6 +308,15 @@ static int in_systemd_collect(struct flb_input_instance *ins, } while ((ret_j = sd_journal_next(ctx->j)) > 0) { + /* + * Reset the journal data cursor as soon as we advance to the next + * entry. Newer libsystemd releases keep Zstandard decompression + * state across data lookups, so carrying over the state from a + * previous entry can trigger use-after-free bugs while we fetch the + * first fields (for example when retrieving _SYSTEMD_UNIT for + * dynamic tags). + */ + sd_journal_restart_data(ctx->j); /* If the tag is composed dynamically, gather the Systemd Unit name */ if (ctx->dynamic_tag) { ret = sd_journal_get_data(ctx->j, "_SYSTEMD_UNIT", &data, &length); @@ -384,6 +393,15 @@ static int in_systemd_collect(struct flb_input_instance *ins, /* Pack every field in the entry */ entries = 0; skip_entries = 0; + + /* + * Restart the journal data cursor before enumerating the fields for + * this entry. sd_journal_get_data() above may advance the cursor, so + * reset it again to ensure enumeration starts from the first field and + * that libsystemd does not reuse a stale decompression context. + */ + sd_journal_restart_data(ctx->j); + while (sd_journal_enumerate_data(ctx->j, &data, &length) > 0 && entries < ctx->max_fields) { key = (const char *) data; From 02ec4aa15a0055f8534282db5bd201b7fbb36099 Mon Sep 17 00:00:00 2001 From: Patrick Stephens Date: Sun, 3 May 2026 12:35:01 +0100 Subject: [PATCH 108/213] [upstream] config: Add dead letter queue related config Upstream-Ref: https://github.com/fluent/fluent-bit/commit/3d6f0bc099ea43f3f416e9feacef696cf515fe51 Cherry-picked from Fluent Bit v4.2.4 --- source/include/fluent-bit/flb_config.h | 7 +++++++ source/src/flb_config.c | 11 +++++++++++ 2 files changed, 18 insertions(+) diff --git a/source/include/fluent-bit/flb_config.h b/source/include/fluent-bit/flb_config.h index 4ee6ec1a..ff9a89ef 100644 --- a/source/include/fluent-bit/flb_config.h +++ b/source/include/fluent-bit/flb_config.h @@ -254,6 +254,10 @@ struct flb_config { char *storage_type; /* global storage type */ int storage_inherit; /* apply storage type to inputs */ + /* DLQ for non-retriable output failures */ + int storage_keep_rejected; /* 0/1 */ + char *storage_rejected_path; /* relative to storage_path, default "rejected" */ + /* Embedded SQL Database support (SQLite3) */ #ifdef FLB_HAVE_SQLDB struct mk_list sqldb_list; @@ -415,6 +419,9 @@ enum conf_type { #define FLB_CONF_STORAGE_TRIM_FILES "storage.trim_files" #define FLB_CONF_STORAGE_TYPE "storage.type" #define FLB_CONF_STORAGE_INHERIT "storage.inherit" +/* Storage DLQ */ +#define FLB_CONF_STORAGE_KEEP_REJECTED "storage.keep.rejected" +#define FLB_CONF_STORAGE_REJECTED_PATH "storage.rejected.path" /* Coroutines */ #define FLB_CONF_STR_CORO_STACK_SIZE "Coro_Stack_Size" diff --git a/source/src/flb_config.c b/source/src/flb_config.c index 8637009a..c71e6939 100644 --- a/source/src/flb_config.c +++ b/source/src/flb_config.c @@ -164,6 +164,13 @@ struct flb_service_config service_configs[] = { {FLB_CONF_STORAGE_INHERIT, FLB_CONF_TYPE_BOOL, offsetof(struct flb_config, storage_inherit)}, + /* Storage / DLQ */ + {FLB_CONF_STORAGE_KEEP_REJECTED, + FLB_CONF_TYPE_BOOL, + offsetof(struct flb_config, storage_keep_rejected)}, + {FLB_CONF_STORAGE_REJECTED_PATH, + FLB_CONF_TYPE_STR, + offsetof(struct flb_config, storage_rejected_path)}, /* Coroutines */ {FLB_CONF_STR_CORO_STACK_SIZE, @@ -346,6 +353,7 @@ struct flb_config *flb_config_init() config->storage_type = NULL; config->storage_inherit = FLB_FALSE; config->storage_bl_flush_on_shutdown = FLB_FALSE; + config->storage_rejected_path = NULL; config->sched_cap = FLB_SCHED_CAP; config->sched_base = FLB_SCHED_BASE; config->json_escape_unicode = FLB_TRUE; @@ -608,6 +616,9 @@ void flb_config_exit(struct flb_config *config) if (config->storage_bl_mem_limit) { flb_free(config->storage_bl_mem_limit); } + if (config->storage_rejected_path) { + flb_free(config->storage_rejected_path); + } #ifdef FLB_HAVE_STREAM_PROCESSOR if (config->stream_processor_file) { From c0f7405d9793c139e52bb8fd2ff49a09d0fbc82c Mon Sep 17 00:00:00 2001 From: Patrick Stephens Date: Sun, 3 May 2026 12:35:01 +0100 Subject: [PATCH 109/213] [upstream] config: storage: Implement dlq for filesystem chunks Upstream-Ref: https://github.com/fluent/fluent-bit/commit/6b74d5841592eed3d067d7eaa79f8b2d6290cf15 Cherry-picked from Fluent Bit v4.2.4 --- source/include/fluent-bit/flb_config.h | 1 + source/include/fluent-bit/flb_storage.h | 7 ++ source/src/flb_storage.c | 111 ++++++++++++++++++++++++ 3 files changed, 119 insertions(+) diff --git a/source/include/fluent-bit/flb_config.h b/source/include/fluent-bit/flb_config.h index ff9a89ef..a27ec0c9 100644 --- a/source/include/fluent-bit/flb_config.h +++ b/source/include/fluent-bit/flb_config.h @@ -257,6 +257,7 @@ struct flb_config { /* DLQ for non-retriable output failures */ int storage_keep_rejected; /* 0/1 */ char *storage_rejected_path; /* relative to storage_path, default "rejected" */ + void *storage_rejected_stream; /* NULL until first use */ /* Embedded SQL Database support (SQLite3) */ #ifdef FLB_HAVE_SQLDB diff --git a/source/include/fluent-bit/flb_storage.h b/source/include/fluent-bit/flb_storage.h index 8fedd9d9..a01e3024 100644 --- a/source/include/fluent-bit/flb_storage.h +++ b/source/include/fluent-bit/flb_storage.h @@ -88,4 +88,11 @@ int flb_storage_metrics_update(struct flb_config *config, struct flb_storage_met void flb_storage_chunk_count(struct flb_config *ctx, int *mem_chunks, int *fs_chunks); +/* DLQ */ +int flb_storage_quarantine_chunk(struct flb_config *ctx, + struct cio_chunk *ch, + const char *tag, + int status_code, + const char *out_name); + #endif diff --git a/source/src/flb_storage.c b/source/src/flb_storage.c index 323a9c53..f4a3af9d 100644 --- a/source/src/flb_storage.c +++ b/source/src/flb_storage.c @@ -781,6 +781,117 @@ void flb_storage_chunk_count(struct flb_config *ctx, int *mem_chunks, int *fs_ch *fs_chunks = storage_st.chunks_fs; } + +static struct cio_stream *get_or_create_rejected_stream(struct flb_config *ctx) +{ +#ifdef CIO_HAVE_BACKEND_FILESYSTEM + struct cio_stream *st; + const char *name; + + if (!ctx || !ctx->cio) { + return NULL; + } + if (!ctx->storage_keep_rejected || !ctx->storage_path) { + return NULL; + } + + if (ctx->storage_rejected_stream) { + return ctx->storage_rejected_stream; + } + + name = ctx->storage_rejected_path ? ctx->storage_rejected_path : "rejected"; + + st = cio_stream_get(ctx->cio, name); + if (!st) { + st = cio_stream_create(ctx->cio, name, FLB_STORAGE_FS); + } + if (!st) { + flb_warn("[storage] failed to create rejected stream '%s'", name); + return NULL; + } + + ctx->storage_rejected_stream = st; + return st; +#else + FLB_UNUSED(ctx); + return NULL; +#endif +} + +int flb_storage_quarantine_chunk(struct flb_config *ctx, + struct cio_chunk *src, + const char *tag, + int status_code, + const char *out_name) +{ +#ifdef CIO_HAVE_BACKEND_FILESYSTEM + struct cio_stream *dlq; + void *buf = NULL; + size_t size = 0; + int err = 0; + char name[256]; + struct cio_chunk *dst; + + if (!ctx || !src) { + return -1; + } + dlq = get_or_create_rejected_stream(ctx); + if (!dlq) { + return -1; + } + + if (cio_chunk_is_up(src) != CIO_TRUE) { + if (cio_chunk_up_force(src) != CIO_OK) { + flb_warn("[storage] cannot bring chunk up to copy into DLQ"); + return -1; + } + } + + if (cio_chunk_get_content_copy(src, &buf, &size) != CIO_OK || size == 0) { + flb_warn("[storage] cannot read content for DLQ copy (size=%zu)", size); + return -1; + } + + /* Compose a simple, unique-ish file name */ + snprintf(name, sizeof(name), + "%s_%d_%s_%p.flb", + tag ? tag : "no-tag", + status_code, + out_name ? out_name : "out", + (void *) src); + + /* Create + write the DLQ copy */ + dst = cio_chunk_open(ctx->cio, dlq, name, CIO_OPEN, size, &err); + if (!dst) { + flb_warn("[storage] DLQ open failed (err=%d)", err); + flb_free(buf); + return -1; + } + if (cio_chunk_write(dst, buf, size) != CIO_OK || + cio_chunk_sync(dst) != CIO_OK) { + flb_warn("[storage] DLQ write/sync failed"); + cio_chunk_close(dst, CIO_TRUE); + flb_free(buf); + return -1; + } + + cio_chunk_close(dst, CIO_FALSE); + flb_free(buf); + + flb_info("[storage] quarantined rejected chunk into DLQ stream (bytes=%zu)", size); + + return 0; +#else + FLB_UNUSED(ctx); + FLB_UNUSED(src); + FLB_UNUSED(tag); + FLB_UNUSED(status_code); + FLB_UNUSED(out_name); + + return -1; +#endif +} + void flb_storage_destroy(struct flb_config *ctx) { struct cio_ctx *cio; From 14e4a3cb098a0da7fd21862f8739bb97d89647b1 Mon Sep 17 00:00:00 2001 From: Patrick Stephens Date: Sun, 3 May 2026 12:35:02 +0100 Subject: [PATCH 110/213] [upstream] engine: Add a capability to handle dead letter queue Upstream-Ref: https://github.com/fluent/fluent-bit/commit/81ee67c12c1617d4174802cc7800b6683177e222 Cherry-picked from Fluent Bit v4.2.4 --- source/src/flb_engine.c | 51 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/source/src/flb_engine.c b/source/src/flb_engine.c index 17ca4b37..8ac6a856 100644 --- a/source/src/flb_engine.c +++ b/source/src/flb_engine.c @@ -232,6 +232,50 @@ static inline double calculate_chunk_capacity_percent(struct flb_output_instance ((double)ins->total_limit_size)); } +static void handle_dlq_if_available(struct flb_config *config, + struct flb_task *task, + struct flb_output_instance *ins, + int status_code /* pass 0 if unknown */) +{ + const char *tag_buf = NULL; + int tag_len = 0; + flb_sds_t tag_sds = NULL; + const char *tag = NULL; + const char *out = NULL; + struct flb_input_chunk *ic; + struct cio_chunk *cio_ch; + + if (!config || !config->storage_keep_rejected || !task || !task->ic || !ins) { + return; + } + + ic = (struct flb_input_chunk *) task->ic; + + if (!ic || !ic->chunk) { + return; + } + + /* Obtain tag from the input chunk API (no direct field available) */ + if (flb_input_chunk_get_tag(ic, &tag_buf, &tag_len) == 0 && tag_buf && tag_len > 0) { + tag_sds = flb_sds_create_len(tag_buf, tag_len); /* make it NUL-terminated */ + tag = tag_sds; + } + else { + /* Fallback: use input instance name */ + tag = flb_input_name(task->i_ins); + } + + out = flb_output_name(ins); + cio_ch = (struct cio_chunk *) ic->chunk; /* ic->chunk is a cio_chunk* under the hood */ + + /* Copy bytes into DLQ stream (filesystem) */ + (void) flb_storage_quarantine_chunk(config, cio_ch, tag, status_code, out); + + if (tag_sds) { + flb_sds_destroy(tag_sds); + } +} + static inline int handle_output_event(uint64_t ts, struct flb_config *config, uint64_t val) @@ -366,6 +410,8 @@ static inline int handle_output_event(uint64_t ts, } else if (ret == FLB_RETRY) { if (ins->retry_limit == FLB_OUT_RETRY_NONE) { + handle_dlq_if_available(config, task, ins, 0); + /* cmetrics: output_dropped_records_total */ cmt_counter_add(ins->cmt_dropped_records, ts, task->records, 1, (char *[]) {out_name}); @@ -412,6 +458,8 @@ static inline int handle_output_event(uint64_t ts, * - It reached the maximum number of re-tries */ + handle_dlq_if_available(config, task, ins, 0); + /* cmetrics */ cmt_counter_inc(ins->cmt_retries_failed, ts, 1, (char *[]) {out_name}); cmt_counter_add(ins->cmt_dropped_records, ts, task->records, @@ -464,6 +512,8 @@ static inline int handle_output_event(uint64_t ts, * memory available or we ran out of file descriptors. */ if (retry_seconds == -1) { + handle_dlq_if_available(config, task, ins, 0); + flb_warn("[engine] retry for chunk '%s' could not be scheduled: " "input=%s > output=%s", flb_input_chunk_get_name(task->ic), @@ -500,6 +550,7 @@ static inline int handle_output_event(uint64_t ts, } } else if (ret == FLB_ERROR) { + handle_dlq_if_available(config, task, ins, 0); /* cmetrics */ cmt_counter_inc(ins->cmt_errors, ts, 1, (char *[]) {out_name}); cmt_counter_add(ins->cmt_dropped_records, ts, task->records, From b629a4d291d0c234488e4c4e6b32345e4555cb42 Mon Sep 17 00:00:00 2001 From: Patrick Stephens Date: Sun, 3 May 2026 12:35:02 +0100 Subject: [PATCH 111/213] [upstream] storage: Use correct visibility of struct Upstream-Ref: https://github.com/fluent/fluent-bit/commit/5ac5b9eee55f67383659a46a1a78285deb1e3b53 Cherry-picked from Fluent Bit v4.2.4 --- source/include/fluent-bit/flb_storage.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/source/include/fluent-bit/flb_storage.h b/source/include/fluent-bit/flb_storage.h index a01e3024..38d9f2c2 100644 --- a/source/include/fluent-bit/flb_storage.h +++ b/source/include/fluent-bit/flb_storage.h @@ -74,6 +74,8 @@ static inline char *flb_storage_get_type(int type) return NULL; } +struct flb_input_instance; + int flb_storage_create(struct flb_config *ctx); int flb_storage_input_create(struct cio_ctx *cio, struct flb_input_instance *in); From 78348f5e167715e9a4ad2d792c4821ee6f0727d9 Mon Sep 17 00:00:00 2001 From: Patrick Stephens Date: Sun, 3 May 2026 12:35:02 +0100 Subject: [PATCH 112/213] [upstream] storage: tests: internal: Add DLQ internal tests Upstream-Ref: https://github.com/fluent/fluent-bit/commit/90d5ad0e12938d97e48ab4a4276e540845dd279c Cherry-picked from Fluent Bit v4.2.4 --- source/tests/internal/CMakeLists.txt | 1 + source/tests/internal/storage_dlq.c | 393 +++++++++++++++++++++++++++ 2 files changed, 394 insertions(+) create mode 100644 source/tests/internal/storage_dlq.c diff --git a/source/tests/internal/CMakeLists.txt b/source/tests/internal/CMakeLists.txt index dfe88d87..fe6c1dcf 100644 --- a/source/tests/internal/CMakeLists.txt +++ b/source/tests/internal/CMakeLists.txt @@ -55,6 +55,7 @@ set(UNIT_TESTS_FILES storage_inherit.c unicode.c opentelemetry.c + storage_dlq.c ) # TLS helpers diff --git a/source/tests/internal/storage_dlq.c b/source/tests/internal/storage_dlq.c new file mode 100644 index 00000000..7bd6851e --- /dev/null +++ b/source/tests/internal/storage_dlq.c @@ -0,0 +1,393 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "flb_tests_internal.h" + +static int mkpath(const char *p) { + if (mkdir(p, 0777) == 0) { + return 0; + } + if (errno == EEXIST) { + return 0; + } + return -1; +} + +static void tmpdir_for(char *out, size_t n, const char *name) { + snprintf(out, n, "/tmp/flb-dlq-%s-%ld", name, (long) getpid()); + mkpath(out); +} + +/* helper: open a DLQ chunk by basename and return its content copy */ +static int read_dlq_chunk_content(struct flb_config *ctx, + const char *rejected_stream_name, + const char *chunk_basename, + void **out_buf, size_t *out_size) +{ + int err = 0; + struct cio_stream *st; + struct cio_chunk *ch; + + *out_buf = NULL; + *out_size = 0; + + st = cio_stream_get(ctx->cio, rejected_stream_name); + if (!st) { + st = cio_stream_create(ctx->cio, rejected_stream_name, FLB_STORAGE_FS); + if (!st) { return -1; } + } + + /* Open existing DLQ file by name */ + ch = cio_chunk_open(ctx->cio, st, chunk_basename, CIO_OPEN, 0, &err); + if (!ch) { + return -1; + } + + /* ensure it's readable */ + if (cio_chunk_is_up(ch) != CIO_TRUE) { + if (cio_chunk_up_force(ch) != CIO_OK) { + cio_chunk_close(ch, CIO_FALSE); + return -1; + } + } + + if (cio_chunk_get_content_copy(ch, out_buf, out_size) != CIO_OK) { + cio_chunk_close(ch, CIO_FALSE); + return -1; + } + + cio_chunk_close(ch, CIO_FALSE); + return 0; +} + +/* tiny binary “contains” (since memmem is non-portable) */ +static int buf_contains(const void *hay, size_t hlen, + const void *needle, size_t nlen) +{ + if (nlen == 0 || hlen < nlen) return 0; + const unsigned char *h = (const unsigned char *) hay; + const unsigned char *n = (const unsigned char *) needle; + + for (size_t i = 0; i + nlen <= hlen; i++) { + if (h[i] == n[0] && memcmp(h + i, n, nlen) == 0) { + return 1; + } + } + return 0; +} + +/* find the most recent *.flb file in dir; write full path into out */ +static int find_latest_flb(const char *dir, char *out, size_t out_sz) +{ + struct dirent *e; + time_t best_t = 0; + char best_path[1024] = {0}; + struct stat st; + char full[1024]; + size_t len = 0; + DIR *d = opendir(dir); + + if (!d) { + return -1; + } + + while ((e = readdir(d)) != NULL) { + len = strlen(e->d_name); + if (len < 5) { + continue; + } + if (strcmp(e->d_name + (len - 4), ".flb") != 0) { + continue; + } + + snprintf(full, sizeof(full), "%s/%s", dir, e->d_name); + if (stat(full, &st) == 0) { + if (st.st_mtime >= best_t) { + best_t = st.st_mtime; + strncpy(best_path, full, sizeof(best_path) - 1); + } + } + } + closedir(d); + + if (best_path[0] == '\0') { + return -1; + } + strncpy(out, best_path, out_sz - 1); + out[out_sz-1] = '\0'; + return 0; +} + +static void free_ctx(struct flb_config *ctx) +{ + if (!ctx) { + return; + } + + if (ctx->cio) { + cio_destroy(ctx->cio); + ctx->cio = NULL; + } + + flb_config_exit(ctx); +} + +static const char *get_dlq_stream_name(struct flb_config *ctx) +{ + if (ctx->storage_rejected_stream) { + return ((struct cio_stream *)ctx->storage_rejected_stream)->name; + } + return ctx->storage_rejected_path ? ctx->storage_rejected_path : "rejected"; +} + +static void delete_all_chunks_in_stream(struct cio_ctx *cio, struct cio_stream *st) +{ + struct mk_list *head; + struct mk_list *tmp; + struct cio_chunk *ch; + + if (!cio || !st) { + return; + } + + mk_list_foreach_safe(head, tmp, &st->chunks) { + ch = mk_list_entry(head, struct cio_chunk, _head); + + char *name_copy = flb_strdup(ch->name); + if (!name_copy) { + continue; + } + + cio_chunk_close(ch, CIO_FALSE); + + (void) cio_chunk_delete(cio, st, name_copy); + + flb_free(name_copy); + } +} + +static void rmdir_stream_dir(const char *root, const char *stream_name) +{ + if (!root || !stream_name) { + return; + } + + char path[1024]; + snprintf(path, sizeof(path), "%s/%s", root, stream_name); + path[sizeof(path)-1] = '\0'; + + /* Best-effort: ignore errors */ + (void) rmdir(path); +} + +/* Minimal POSIX rm -rf for the whole temp tree after CIO is gone */ +static void rm_rf_best_effort(const char *root) +{ + DIR *d; + struct dirent *e; + char p[1024]; + struct stat st; + + if (!root) { return; } + + d = opendir(root); + if (!d) { + (void) rmdir(root); + return; + } + while ((e = readdir(d)) != NULL) { + if (!strcmp(e->d_name, ".") || !strcmp(e->d_name, "..")) { + continue; + } + snprintf(p, sizeof(p), "%s/%s", root, e->d_name); + if (lstat(p, &st) != 0) { + continue; + } + if (S_ISDIR(st.st_mode)) { + rm_rf_best_effort(p); + } + else { + (void) unlink(p); + } + } + closedir(d); + (void) rmdir(root); +} + +static void test_cleanup_with_cio(struct flb_config *ctx, const char *root) +{ + if (ctx && ctx->cio) { + struct cio_stream *st_in = cio_stream_get(ctx->cio, "in_tail"); + struct cio_stream *st_dlq = cio_stream_get(ctx->cio, get_dlq_stream_name(ctx)); + + delete_all_chunks_in_stream(ctx->cio, st_in); + delete_all_chunks_in_stream(ctx->cio, st_dlq); + + rmdir_stream_dir(root, "in_tail"); + rmdir_stream_dir(root, get_dlq_stream_name(ctx)); + } + + free_ctx(ctx); + + rm_rf_best_effort(root); +} + +static struct flb_config *make_ctx_fs(const char *root, const char *rejected) +{ + struct cio_options opts; + struct flb_config *ctx = flb_config_init(); + TEST_CHECK(ctx != NULL); + + ctx->storage_path = flb_strdup(root); + ctx->storage_keep_rejected = FLB_TRUE; + ctx->storage_rejected_path = flb_strdup(rejected); + + cio_options_init(&opts); + opts.root_path = ctx->storage_path; + opts.flags = CIO_OPEN | CIO_CHECKSUM; + opts.log_cb = NULL; + + ctx->cio = cio_create(&opts); + TEST_CHECK(ctx->cio != NULL); + + /* mimic engine behavior: load + qsort */ + TEST_CHECK(cio_load(ctx->cio, NULL) == 0); + cio_qsort(ctx->cio, NULL); + + return ctx; +} + +static struct cio_chunk *make_src_chunk(struct flb_config *ctx, + int storage_type, /* FLB_STORAGE_FS */ + const char *stream_name, + const char *file_name, + const char *payload) +{ + int err = 0; + int cio_type = storage_type; + struct cio_stream *st = NULL; + struct cio_chunk *ch = NULL; + + st = cio_stream_get(ctx->cio, stream_name); + if (!st) { + st = cio_stream_create(ctx->cio, stream_name, cio_type); + } + TEST_CHECK(st != NULL); + + ch = cio_chunk_open(ctx->cio, st, file_name, CIO_OPEN, 0, &err); + TEST_CHECK(ch != NULL); + + TEST_CHECK(cio_chunk_write(ch, payload, strlen(payload)) == CIO_OK); + TEST_CHECK(cio_chunk_sync(ch) == CIO_OK); + + return ch; +} + +static void test_dlq_copy_from_fs_chunk(void) +{ + char root[256], rejdir[256], latest[1024]; + struct cio_chunk *src = NULL; + struct flb_config *ctx = NULL; + int rc; + const char *payload = + "{\"time\":\"2024-09-03 14:51:05.064735+00:00\",\"msg\":\"oops FS\"}\n"; + char latest_copy[1024]; + void *content = NULL; + size_t content_size = 0; + char *base = NULL; + + tmpdir_for(root, sizeof(root), "fs"); + snprintf(rejdir, sizeof(rejdir), "%s/%s", root, "rejected"); + mkpath(rejdir); + + ctx = make_ctx_fs(root, "rejected"); + + src = make_src_chunk(ctx, FLB_STORAGE_FS, + "in_tail", + "t0-0-0000000000.000000000.flb", + payload); + + rc = flb_storage_quarantine_chunk(ctx, src, + "kube.var.log.containers.test", + 400, "http"); + TEST_CHECK(rc == 0); + + TEST_CHECK(find_latest_flb(rejdir, latest, sizeof(latest)) == 0); + + /* get just the filename (basename) */ + strncpy(latest_copy, latest, sizeof(latest_copy)-1); + latest_copy[sizeof(latest_copy)-1] = '\0'; + base = basename(latest_copy); + + TEST_CHECK(read_dlq_chunk_content(ctx, "rejected", base, &content, &content_size) == 0); + TEST_CHECK(content != NULL); + TEST_CHECK(content_size > 0); + TEST_CHECK(buf_contains(content, content_size, payload, strlen(payload)) == 1); + + flb_free(content); + cio_chunk_close(src, CIO_FALSE); + test_cleanup_with_cio(ctx, root); +} + +static void test_dlq_disabled_no_copy(void) +{ + char root[256], rejdir[256], latest[1024]; + struct cio_chunk *src = NULL; + struct flb_config *ctx = NULL; + struct cio_options opts; + int rc; + const char *payload = "{\"msg\":\"should not be copied\"}\n"; + + tmpdir_for(root, sizeof(root), "disabled"); + snprintf(rejdir, sizeof(rejdir), "%s/%s", root, "rejected"); + mkpath(rejdir); + + /* DLQ disabled */ + ctx = flb_config_init(); + TEST_CHECK(ctx != NULL); + + ctx->storage_path = flb_strdup(root); + ctx->storage_keep_rejected = FLB_FALSE; + ctx->storage_rejected_path = flb_strdup("rejected"); + + cio_options_init(&opts); + opts.root_path = ctx->storage_path; + opts.flags = CIO_OPEN; + ctx->cio = cio_create(&opts); + TEST_CHECK(ctx->cio != NULL); + + src = make_src_chunk(ctx, FLB_STORAGE_FS, + "in_tail", + "t1-0.flb", + payload); + + /* Attempt to copy: should fail because DLQ is disabled */ + rc = flb_storage_quarantine_chunk(ctx, src, + "tag", 400, "out"); + TEST_CHECK(rc != 0); + + TEST_CHECK(find_latest_flb(rejdir, latest, sizeof(latest)) != 0); + + cio_chunk_close(src, CIO_FALSE); + test_cleanup_with_cio(ctx, root); +} + +TEST_LIST = { + { "dlq_copy_from_fs_chunk", test_dlq_copy_from_fs_chunk }, + { "dlq_disabled_no_copy", test_dlq_disabled_no_copy }, + { NULL, NULL } +}; From 07119ea5020869535649610617231cd6a3be6532 Mon Sep 17 00:00:00 2001 From: Patrick Stephens Date: Sun, 3 May 2026 12:35:02 +0100 Subject: [PATCH 113/213] [upstream] storage: tests: internal: Make to be able to compile Upstream-Ref: https://github.com/fluent/fluent-bit/commit/d8878bee675379d22f4f3671d59bace09e20ea3d Cherry-picked from Fluent Bit v4.2.4 --- source/tests/internal/storage_dlq.c | 170 ++++++++++++++++++++++++---- 1 file changed, 148 insertions(+), 22 deletions(-) diff --git a/source/tests/internal/storage_dlq.c b/source/tests/internal/storage_dlq.c index 7bd6851e..8cfe536a 100644 --- a/source/tests/internal/storage_dlq.c +++ b/source/tests/internal/storage_dlq.c @@ -17,18 +17,49 @@ #include "flb_tests_internal.h" +#ifdef _WIN32 +# define FLB_UNLINK _unlink +# define FLB_RMDIR _rmdir +#else +# define FLB_UNLINK unlink +# define FLB_RMDIR rmdir +#endif + static int mkpath(const char *p) { +#if FLB_SYSTEM_WINDOWS + if (_mkdir(p) == 0) { + return 0; + } +#else if (mkdir(p, 0777) == 0) { return 0; } +#endif if (errno == EEXIST) { return 0; } return -1; } -static void tmpdir_for(char *out, size_t n, const char *name) { +static void join_path(char *out, size_t cap, const char *a, const char *b) +{ +#ifdef _WIN32 + _snprintf(out, cap, "%s\\%s", a, b); +#else + snprintf(out, cap, "%s/%s", a, b); +#endif + out[cap - 1] = '\0'; +} + +static void tmpdir_for(char *out, size_t n, const char *name) +{ +#ifdef _WIN32 + DWORD pid = GetCurrentProcessId(); + _snprintf(out, n, "C:\\Windows\\Temp\\flb-dlq-%s-%lu", name, (unsigned long) pid); +#else snprintf(out, n, "/tmp/flb-dlq-%s-%ld", name, (long) getpid()); +#endif + out[n-1] = '\0'; mkpath(out); } @@ -90,23 +121,59 @@ static int buf_contains(const void *hay, size_t hlen, return 0; } -/* find the most recent *.flb file in dir; write full path into out */ -static int find_latest_flb(const char *dir, char *out, size_t out_sz) +#if FLB_SYSTEM_WINDOWS +static int find_latest_flb_win32(const char *dir, char *out, size_t out_sz) { + WIN32_FIND_DATAA ffd; + HANDLE h = INVALID_HANDLE_VALUE; + char pattern[1024]; + ULONGLONG best_ts = 0ULL; + char best_name[MAX_PATH] = {0}; + + _snprintf(pattern, sizeof(pattern), "%s\\*.flb", dir); + pattern[sizeof(pattern)-1] = '\0'; + + h = FindFirstFileA(pattern, &ffd); + if (h == INVALID_HANDLE_VALUE) { + return -1; + } + + do { + if (ffd.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) { + continue; + } + ULONGLONG ts = (((ULONGLONG)ffd.ftLastWriteTime.dwHighDateTime) << 32) | + (ULONGLONG)ffd.ftLastWriteTime.dwLowDateTime; + if (ts >= best_ts) { + best_ts = ts; + strncpy(best_name, ffd.cFileName, sizeof(best_name)-1); + best_name[sizeof(best_name)-1] = '\0'; + } + } while (FindNextFileA(h, &ffd)); + + FindClose(h); + + if (best_name[0] == '\0') { + return -1; + } + + join_path(out, out_sz, dir, best_name); + return 0; +} +#else +static int find_latest_flb_unix(const char *dir, char *out, size_t out_sz) +{ + DIR *d = opendir(dir); struct dirent *e; time_t best_t = 0; char best_path[1024] = {0}; struct stat st; char full[1024]; - size_t len = 0; - DIR *d = opendir(dir); - if (!d) { - return -1; - } + if (!d) return -1; while ((e = readdir(d)) != NULL) { - len = strlen(e->d_name); + size_t len = strlen(e->d_name); if (len < 5) { continue; } @@ -114,11 +181,11 @@ static int find_latest_flb(const char *dir, char *out, size_t out_sz) continue; } - snprintf(full, sizeof(full), "%s/%s", dir, e->d_name); + join_path(full, sizeof(full), dir, e->d_name); if (stat(full, &st) == 0) { if (st.st_mtime >= best_t) { best_t = st.st_mtime; - strncpy(best_path, full, sizeof(best_path) - 1); + strncpy(best_path, full, sizeof(best_path)-1); } } } @@ -131,6 +198,17 @@ static int find_latest_flb(const char *dir, char *out, size_t out_sz) out[out_sz-1] = '\0'; return 0; } +#endif + +/* find the most recent *.flb file in dir; write full path into out */ +static int find_latest_flb(const char *dir, char *out, size_t out_sz) +{ +#if FLB_SYSTEM_WINDOWS + return find_latest_flb_win32(dir, out, out_sz); +#else + return find_latest_flb_unix(dir, out, out_sz); +#endif +} static void free_ctx(struct flb_config *ctx) { @@ -195,37 +273,85 @@ static void rmdir_stream_dir(const char *root, const char *stream_name) } /* Minimal POSIX rm -rf for the whole temp tree after CIO is gone */ -static void rm_rf_best_effort(const char *root) +#if FLB_SYSTEM_WINDOWS +static void rm_rf_best_effort_win32(const char *root) +{ + WIN32_FIND_DATAA ffd; + HANDLE h = INVALID_HANDLE_VALUE; + char pattern[1024], p[1024]; + + _snprintf(pattern, sizeof(pattern), "%s\\*", + root ? root : ""); + pattern[sizeof(pattern)-1] = '\0'; + + h = FindFirstFileA(pattern, &ffd); + if (h == INVALID_HANDLE_VALUE) { + /* try removing root itself */ + (void) FLB_RMDIR(root); + return; + } + + do { + const char *name = ffd.cFileName; + if (!strcmp(name, ".") || !strcmp(name, "..")) continue; + + join_path(p, sizeof(p), root, name); + + if (ffd.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) { + rm_rf_best_effort_win32(p); + } + else { + /* clear read-only if needed */ + if (ffd.dwFileAttributes & FILE_ATTRIBUTE_READONLY) { + SetFileAttributesA(p, + ffd.dwFileAttributes & ~FILE_ATTRIBUTE_READONLY); + } + (void) DeleteFileA(p); + } + } while (FindNextFileA(h, &ffd)); + + FindClose(h); + (void) FLB_RMDIR(root); +} +#else +static void rm_rf_best_effort_unix(const char *root) { - DIR *d; + DIR *d = opendir(root); struct dirent *e; char p[1024]; struct stat st; - if (!root) { return; } - - d = opendir(root); if (!d) { - (void) rmdir(root); + (void) FLB_RMDIR(root); return; } while ((e = readdir(d)) != NULL) { - if (!strcmp(e->d_name, ".") || !strcmp(e->d_name, "..")) { + if (!strcmp(e->d_name, ".") || !strcmp(e->d_name, "..")) { continue; } - snprintf(p, sizeof(p), "%s/%s", root, e->d_name); + join_path(p, sizeof(p), root, e->d_name); if (lstat(p, &st) != 0) { continue; } if (S_ISDIR(st.st_mode)) { - rm_rf_best_effort(p); + rm_rf_best_effort_unix(p); } else { - (void) unlink(p); + (void) FLB_UNLINK(p); } } closedir(d); - (void) rmdir(root); + (void) FLB_RMDIR(root); +} +#endif + +static void rm_rf_best_effort(const char *root) +{ +#if FLB_SYSTEM_WINDOWS + rm_rf_best_effort_win32(root); +#else + rm_rf_best_effort_unix(root); +#endif } static void test_cleanup_with_cio(struct flb_config *ctx, const char *root) From 48cf2cf918b4f0d9ec3ee705775a8116796f67d4 Mon Sep 17 00:00:00 2001 From: Patrick Stephens Date: Sun, 3 May 2026 12:35:02 +0100 Subject: [PATCH 114/213] [upstream] storage: tests: internal: Fix a compilation error on Upstream-Ref: https://github.com/fluent/fluent-bit/commit/e183f8108760f287bc8b442f5caf844c9e4ea7c7 Cherry-picked from Fluent Bit v4.2.4 --- source/tests/internal/storage_dlq.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/source/tests/internal/storage_dlq.c b/source/tests/internal/storage_dlq.c index 8cfe536a..b20358e8 100644 --- a/source/tests/internal/storage_dlq.c +++ b/source/tests/internal/storage_dlq.c @@ -109,11 +109,12 @@ static int read_dlq_chunk_content(struct flb_config *ctx, static int buf_contains(const void *hay, size_t hlen, const void *needle, size_t nlen) { + size_t i; if (nlen == 0 || hlen < nlen) return 0; const unsigned char *h = (const unsigned char *) hay; const unsigned char *n = (const unsigned char *) needle; - for (size_t i = 0; i + nlen <= hlen; i++) { + for (i = 0; i + nlen <= hlen; i++) { if (h[i] == n[0] && memcmp(h + i, n, nlen) == 0) { return 1; } From 587a22f14c29114b7df3c32c037f0ab38d5cb4ca Mon Sep 17 00:00:00 2001 From: Patrick Stephens Date: Sun, 3 May 2026 12:35:02 +0100 Subject: [PATCH 115/213] [upstream] storage: Restore the original state of chunks Upstream-Ref: https://github.com/fluent/fluent-bit/commit/15194723da143804f1cb9cd3e43b4a9b1594ec9e Cherry-picked from Fluent Bit v4.2.4 --- source/src/flb_storage.c | 48 ++++++++++++++++++++++++++++++++-------- 1 file changed, 39 insertions(+), 9 deletions(-) diff --git a/source/src/flb_storage.c b/source/src/flb_storage.c index f4a3af9d..47154e0f 100644 --- a/source/src/flb_storage.c +++ b/source/src/flb_storage.c @@ -781,6 +781,24 @@ void flb_storage_chunk_count(struct flb_config *ctx, int *mem_chunks, int *fs_ch *fs_chunks = storage_st.chunks_fs; } +/* Replace '/', '\\' and ':' with '_' to make filename components safe */ +static inline void sanitize_name_component(const char *in, char *out, size_t out_sz) +{ + size_t i; + + if (out_sz == 0) { + return; + } + + if (!in) { + in = "no-tag"; + } + + for (i = 0; i < out_sz - 1 && in[i] != '\0'; i++) { + out[i] = (in[i] == '/' || in[i] == '\\' || in[i] == ':') ? '_' : in[i]; + } + out[i] = '\0'; +} static struct cio_stream *get_or_create_rejected_stream(struct flb_config *ctx) { @@ -827,10 +845,13 @@ int flb_storage_quarantine_chunk(struct flb_config *ctx, #ifdef CIO_HAVE_BACKEND_FILESYSTEM struct cio_stream *dlq; void *buf = NULL; + int was_up = 0; size_t size = 0; int err = 0; char name[256]; struct cio_chunk *dst; + char safe_tag[128]; + char safe_out[64]; if (!ctx || !src) { return -1; @@ -840,26 +861,28 @@ int flb_storage_quarantine_chunk(struct flb_config *ctx, return -1; } - if (cio_chunk_is_up(src) != CIO_TRUE) { + /* Remember original state and bring the chunk up if needed */ + was_up = (cio_chunk_is_up(src) == CIO_TRUE); + if (!was_up) { if (cio_chunk_up_force(src) != CIO_OK) { flb_warn("[storage] cannot bring chunk up to copy into DLQ"); return -1; } } + sanitize_name_component(tag, safe_tag, sizeof(safe_tag)); + sanitize_name_component(out_name ? out_name : "out", safe_out, sizeof(safe_out)); + + /* Compose a simple, unique-ish file name with sanitized pieces */ + snprintf(name, sizeof(name), + "%s_%d_%s_%p.flb", + safe_tag, status_code, safe_out, (void *) src); + if (cio_chunk_get_content_copy(src, &buf, &size) != CIO_OK || size == 0) { flb_warn("[storage] cannot read content for DLQ copy (size=%zu)", size); return -1; } - /* Compose a simple, unique-ish file name */ - snprintf(name, sizeof(name), - "%s_%d_%s_%p.flb", - tag ? tag : "no-tag", - status_code, - out_name ? out_name : "out", - (void *) src); - /* Create + write the DLQ copy */ dst = cio_chunk_open(ctx->cio, dlq, name, CIO_OPEN, size, &err); if (!dst) { @@ -880,6 +903,13 @@ int flb_storage_quarantine_chunk(struct flb_config *ctx, flb_info("[storage] quarantined rejected chunk into DLQ stream (bytes=%zu)", size); + /* Restore original state if we brought the chunk up */ + if (!was_up) { + if (cio_chunk_down(src) != CIO_OK) { + flb_debug("[storage] failed to bring chunk back down after DLQ copy"); + } + } + return 0; #else FLB_UNUSED(ctx); From c8833afbb324fe7e8a5bd8c7f39fa61a5173d7bd Mon Sep 17 00:00:00 2001 From: Patrick Stephens Date: Sun, 3 May 2026 12:35:02 +0100 Subject: [PATCH 116/213] [upstream] storage: Restore status of chunks for error paths Upstream-Ref: https://github.com/fluent/fluent-bit/commit/eb72734f091ccea490b55807633dc7e2f009396c Cherry-picked from Fluent Bit v4.2.4 --- source/src/flb_storage.c | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/source/src/flb_storage.c b/source/src/flb_storage.c index 47154e0f..0a3c174c 100644 --- a/source/src/flb_storage.c +++ b/source/src/flb_storage.c @@ -836,6 +836,17 @@ static struct cio_stream *get_or_create_rejected_stream(struct flb_config *ctx) #endif } +static inline int flb_storage_chunk_restore_state(struct cio_chunk *src, int was_up, int ret_val) +{ + if (!was_up) { + if (cio_chunk_down(src) != CIO_OK) { + flb_debug("[storage] failed to bring chunk back down"); + } + } + + return ret_val; +} + int flb_storage_quarantine_chunk(struct flb_config *ctx, struct cio_chunk *src, const char *tag, @@ -880,7 +891,7 @@ int flb_storage_quarantine_chunk(struct flb_config *ctx, if (cio_chunk_get_content_copy(src, &buf, &size) != CIO_OK || size == 0) { flb_warn("[storage] cannot read content for DLQ copy (size=%zu)", size); - return -1; + return flb_storage_chunk_restore_state(src, was_up, -1); } /* Create + write the DLQ copy */ @@ -888,14 +899,14 @@ int flb_storage_quarantine_chunk(struct flb_config *ctx, if (!dst) { flb_warn("[storage] DLQ open failed (err=%d)", err); flb_free(buf); - return -1; + return flb_storage_chunk_restore_state(src, was_up, -1); } if (cio_chunk_write(dst, buf, size) != CIO_OK || cio_chunk_sync(dst) != CIO_OK) { flb_warn("[storage] DLQ write/sync failed"); cio_chunk_close(dst, CIO_TRUE); flb_free(buf); - return -1; + return flb_storage_chunk_restore_state(src, was_up, -1); } cio_chunk_close(dst, CIO_FALSE); @@ -903,14 +914,7 @@ int flb_storage_quarantine_chunk(struct flb_config *ctx, flb_info("[storage] quarantined rejected chunk into DLQ stream (bytes=%zu)", size); - /* Restore original state if we brought the chunk up */ - if (!was_up) { - if (cio_chunk_down(src) != CIO_OK) { - flb_debug("[storage] failed to bring chunk back down after DLQ copy"); - } - } - - return 0; + return flb_storage_chunk_restore_state(src, was_up, 0); #else FLB_UNUSED(ctx); FLB_UNUSED(src); From 53113311391e7bcd67fb2df120ebed1eeded80a7 Mon Sep 17 00:00:00 2001 From: Patrick Stephens Date: Sun, 3 May 2026 12:35:03 +0100 Subject: [PATCH 117/213] [upstream] storage: internal: tests: Confirm the initial state Upstream-Ref: https://github.com/fluent/fluent-bit/commit/615ef595c86b7831217ecb10070227a63a242b2e Cherry-picked from Fluent Bit v4.2.4 --- source/tests/internal/storage_dlq.c | 73 +++++++++++++++++++++++++++++ 1 file changed, 73 insertions(+) diff --git a/source/tests/internal/storage_dlq.c b/source/tests/internal/storage_dlq.c index b20358e8..bca4d269 100644 --- a/source/tests/internal/storage_dlq.c +++ b/source/tests/internal/storage_dlq.c @@ -513,8 +513,81 @@ static void test_dlq_disabled_no_copy(void) test_cleanup_with_cio(ctx, root); } +static void test_dlq_restores_chunk_state_when_initially_down(void) +{ + char root[256], rejdir[256]; + struct flb_config *ctx = NULL; + struct cio_chunk *src = NULL; + int rc; + const char *payload = "{\"msg\":\"state-restore-down\"}\n"; + + tmpdir_for(root, sizeof(root), "restore-down"); + snprintf(rejdir, sizeof(rejdir), "%s/%s", root, "rejected"); + mkpath(rejdir); + + ctx = make_ctx_fs(root, "rejected"); + + /* Create the chunk */ + src = make_src_chunk(ctx, FLB_STORAGE_FS, + "in_tail", + "restore-down-0-0000000000.000000000.flb", + payload); + TEST_CHECK(src != NULL); + + if (cio_chunk_is_up(src) == CIO_TRUE) { + TEST_CHECK(cio_chunk_down(src) == CIO_OK); + } + TEST_CHECK(cio_chunk_is_up(src) != CIO_TRUE); + + rc = flb_storage_quarantine_chunk(ctx, src, + "tag.down", 500, "out_http"); + TEST_CHECK(rc == 0); + + TEST_CHECK(cio_chunk_is_up(src) != CIO_TRUE); + + cio_chunk_close(src, CIO_FALSE); + test_cleanup_with_cio(ctx, root); +} + +static void test_dlq_preserves_chunk_state_when_initially_up(void) +{ + char root[256], rejdir[256]; + struct flb_config *ctx = NULL; + struct cio_chunk *src = NULL; + int rc; + const char *payload = "{\"msg\":\"state-preserve-up\"}\n"; + + tmpdir_for(root, sizeof(root), "preserve-up"); + snprintf(rejdir, sizeof(rejdir), "%s/%s", root, "rejected"); + mkpath(rejdir); + + ctx = make_ctx_fs(root, "rejected"); + + src = make_src_chunk(ctx, FLB_STORAGE_FS, + "preserve_in", + "preserve-up-0-0000000000.000000000.flb", + payload); + TEST_CHECK(src != NULL); + + if (cio_chunk_is_up(src) != CIO_TRUE) { + TEST_CHECK(cio_chunk_up_force(src) == CIO_OK); + } + TEST_CHECK(cio_chunk_is_up(src) == CIO_TRUE); + + rc = flb_storage_quarantine_chunk(ctx, src, + "tag.up", 502, "out_es"); + TEST_CHECK(rc == 0); + + TEST_CHECK(cio_chunk_is_up(src) == CIO_TRUE); + + cio_chunk_close(src, CIO_FALSE); + test_cleanup_with_cio(ctx, root); +} + TEST_LIST = { { "dlq_copy_from_fs_chunk", test_dlq_copy_from_fs_chunk }, { "dlq_disabled_no_copy", test_dlq_disabled_no_copy }, + { "dlq_restores_chunk_state_when_initially_down", test_dlq_restores_chunk_state_when_initially_down }, + { "dlq_preserves_chunk_state_when_initially_up", test_dlq_preserves_chunk_state_when_initially_up }, { NULL, NULL } }; From 4c935c98c109452e5c0edd7253e81f31e20087b9 Mon Sep 17 00:00:00 2001 From: Patrick Stephens Date: Sun, 3 May 2026 12:35:03 +0100 Subject: [PATCH 118/213] [upstream] engine: Tweak conditions for enabling DLQ Upstream-Ref: https://github.com/fluent/fluent-bit/commit/29f41f65f3b7c41d617cecbb6f1407f41e594acd Cherry-picked from Fluent Bit v4.2.4 --- source/src/flb_engine.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/source/src/flb_engine.c b/source/src/flb_engine.c index 8ac6a856..8ff891ea 100644 --- a/source/src/flb_engine.c +++ b/source/src/flb_engine.c @@ -245,7 +245,10 @@ static void handle_dlq_if_available(struct flb_config *config, struct flb_input_chunk *ic; struct cio_chunk *cio_ch; - if (!config || !config->storage_keep_rejected || !task || !task->ic || !ins) { + if (!config || + !config->storage_keep_rejected || + !config->storage_path || + !task || !task->ic || !ins) { return; } From ec901d0e55be5b92e94d0676d6814041bb03294f Mon Sep 17 00:00:00 2001 From: Patrick Stephens Date: Sun, 3 May 2026 12:35:03 +0100 Subject: [PATCH 119/213] [upstream] filter_kubernetes: use service account issuer to Upstream-Ref: https://github.com/fluent/fluent-bit/commit/f831f7ff994f009ba61de9a00a6f2ac12c5e5fdb Cherry-picked from Fluent Bit v4.2.4 --- source/plugins/filter_kubernetes/kube_conf.h | 5 +- .../filter_kubernetes/kubernetes_aws.c | 143 +++++++++++++++++- 2 files changed, 137 insertions(+), 11 deletions(-) diff --git a/source/plugins/filter_kubernetes/kube_conf.h b/source/plugins/filter_kubernetes/kube_conf.h index 1c8cad03..cb36f2d7 100644 --- a/source/plugins/filter_kubernetes/kube_conf.h +++ b/source/plugins/filter_kubernetes/kube_conf.h @@ -74,11 +74,10 @@ #define SERVICE_NAME_SOURCE_MAX_LEN 64 /* - * Configmap used for verifying whether if FluentBit is - * on EKS or native Kubernetes + * Namespace and token path used for verifying whether FluentBit is + * on EKS or native Kubernetes by inspecting serviceaccount token issuer */ #define KUBE_SYSTEM_NAMESPACE "kube-system" -#define AWS_AUTH_CONFIG_MAP "aws-auth" /* * Possible platform values for Kubernetes plugin diff --git a/source/plugins/filter_kubernetes/kubernetes_aws.c b/source/plugins/filter_kubernetes/kubernetes_aws.c index 340c1913..642f907d 100644 --- a/source/plugins/filter_kubernetes/kubernetes_aws.c +++ b/source/plugins/filter_kubernetes/kubernetes_aws.c @@ -23,6 +23,7 @@ #include #include #include +#include #include "kube_conf.h" #include "kube_meta.h" @@ -282,19 +283,145 @@ int fetch_pod_service_map(struct flb_kube *ctx, char *api_server_url, return 0; } -/* Determine platform by checking aws-auth configmap */ +/* Determine platform by checking serviceaccount token issuer */ int determine_platform(struct flb_kube *ctx) { int ret; - char *config_buf; - size_t config_size; + char *token_buf = NULL; + size_t token_size; + char *payload = NULL; + size_t payload_len; + char *issuer_start; + char *issuer_end; + char *first_dot; + char *second_dot; + size_t payload_b64_len; + size_t padded_len; + char *payload_b64; + size_t issuer_len; + char *issuer_value; + int is_eks; + + /* Read serviceaccount token */ + ret = flb_utils_read_file(FLB_KUBE_TOKEN, &token_buf, &token_size); + if (ret != 0 || !token_buf) { + return -1; + } + + /* JWT tokens have 3 parts separated by dots: header.payload.signature */ + first_dot = strchr(token_buf, '.'); + if (!first_dot) { + flb_free(token_buf); + return -1; + } + + second_dot = strchr(first_dot + 1, '.'); + if (!second_dot) { + flb_free(token_buf); + return -1; + } + + /* Extract and decode the payload (middle part) */ + payload_b64_len = second_dot - (first_dot + 1); + + /* Calculate padded length */ + padded_len = payload_b64_len; + while (padded_len % 4 != 0) padded_len++; + + payload_b64 = flb_malloc(padded_len + 1); + if (!payload_b64) { + flb_errno(); + flb_free(token_buf); + return -1; + } + + memcpy(payload_b64, first_dot + 1, payload_b64_len); + + /* Convert base64url to base64 and add padding */ + for (size_t i = 0; i < payload_b64_len; i++) { + if (payload_b64[i] == '-') { + payload_b64[i] = '+'; + } + else if (payload_b64[i] == '_') { + payload_b64[i] = '/'; + } + } + while (payload_b64_len < padded_len) { + payload_b64[payload_b64_len++] = '='; + } + payload_b64[padded_len] = '\0'; + + /* Base64 decode the payload */ + payload = flb_malloc(payload_b64_len * 3 / 4 + 4); /* Conservative size estimate */ + if (!payload) { + flb_errno(); + flb_free(token_buf); + flb_free(payload_b64); + return -1; + } + + ret = flb_base64_decode((unsigned char *)payload, padded_len * 3 / 4 + 4, + &payload_len, (unsigned char *)payload_b64, padded_len); + + flb_free(token_buf); + flb_free(payload_b64); + + if (ret != 0) { + flb_free(payload); + return -1; + } + + payload[payload_len] = '\0'; + + /* Look for "iss" field in the JSON payload */ + issuer_start = strstr(payload, "\"iss\":"); + if (!issuer_start) { + flb_free(payload); + return -1; + } + + /* Skip to the value part */ + issuer_start = strchr(issuer_start, ':'); + if (!issuer_start) { + flb_free(payload); + return -1; + } + issuer_start++; + + /* Skip whitespace and opening quote */ + while (*issuer_start == ' ' || *issuer_start == '\t') issuer_start++; + if (*issuer_start != '"') { + flb_free(payload); + return -1; + } + issuer_start++; + + /* Find closing quote */ + issuer_end = strchr(issuer_start, '"'); + if (!issuer_end) { + flb_free(payload); + return -1; + } + + /* Check if issuer contains EKS OIDC URL pattern */ + /* EKS OIDC URLs follow pattern: https://oidc.eks.{region}.amazonaws.com/id/{cluster-id} */ + issuer_len = issuer_end - issuer_start; + issuer_value = flb_strndup(issuer_start, issuer_len); + if (!issuer_value) { + flb_free(payload); + return -1; + } - ret = get_api_server_configmap(ctx, KUBE_SYSTEM_NAMESPACE, AWS_AUTH_CONFIG_MAP, &config_buf, &config_size); - if (ret != -1) { - flb_free(config_buf); - return 1; + is_eks = strstr(issuer_value, "oidc.eks.") != NULL; + flb_free(issuer_value); + + if (is_eks) { + flb_free(payload); + return 1; /* EKS detected */ } - return -1; + + flb_free(payload); + return -1; /* Not EKS */ } /* Gather pods list information from Kubelet */ From 6f791cd4a08f95df78e3bc5c3af247b34c580e90 Mon Sep 17 00:00:00 2001 From: Patrick Stephens Date: Sun, 3 May 2026 12:35:03 +0100 Subject: [PATCH 120/213] [upstream] filter_kubernetes: aws: declare variable on top of Upstream-Ref: https://github.com/fluent/fluent-bit/commit/d814153416f57d83d5924f080cb59a34976e9cf6 Cherry-picked from Fluent Bit v4.2.4 --- .../filter_kubernetes/kubernetes_aws.c | 59 ++++++++++--------- 1 file changed, 30 insertions(+), 29 deletions(-) diff --git a/source/plugins/filter_kubernetes/kubernetes_aws.c b/source/plugins/filter_kubernetes/kubernetes_aws.c index 642f907d..eb5bf583 100644 --- a/source/plugins/filter_kubernetes/kubernetes_aws.c +++ b/source/plugins/filter_kubernetes/kubernetes_aws.c @@ -81,19 +81,19 @@ static int get_pod_service_file_info(struct flb_kube *ctx, char **buffer) return packed; } -static void extract_service_attribute(msgpack_object *attr_map, const char *key_name, +static void extract_service_attribute(msgpack_object *attr_map, const char *key_name, char *dest, int max_len, int *dest_len, int *fields) { struct flb_record_accessor *ra; struct flb_ra_value *val; const char *str_val; size_t str_len; - + ra = flb_ra_create((char *)key_name, FLB_FALSE); if (!ra) { return; } - + val = flb_ra_get_value_object(ra, *attr_map); if (val && val->type == FLB_RA_STRING) { str_val = flb_ra_value_buffer(val, &str_len); @@ -149,7 +149,7 @@ static void parse_pod_service_map(struct flb_kube *ctx, char *api_buf, for (i = 0; i < api_map.via.map.size; i++) { k = api_map.via.map.ptr[i].key; v = api_map.via.map.ptr[i].val; - + if (k.type != MSGPACK_OBJECT_STR || v.type != MSGPACK_OBJECT_MAP) { flb_plg_error(ctx->ins, "key and values are not string and map"); continue; @@ -169,7 +169,7 @@ static void parse_pod_service_map(struct flb_kube *ctx, char *api_buf, KEY_ATTRIBUTES_MAX_LEN, &attrs->name_len, &attrs->fields); extract_service_attribute(&v, "$Environment", attrs->environment, KEY_ATTRIBUTES_MAX_LEN, &attrs->environment_len, &attrs->fields); - extract_service_attribute(&v, "$ServiceNameSource", attrs->name_source, + extract_service_attribute(&v, "$ServiceNameSource", attrs->name_source, SERVICE_NAME_SOURCE_MAX_LEN, &attrs->name_source_len, &attrs->fields); if (attrs->name[0] != '\0' || attrs->environment[0] != '\0') { pthread_mutex_lock(mutex); @@ -177,7 +177,7 @@ static void parse_pod_service_map(struct flb_kube *ctx, char *api_buf, attrs, sizeof(struct service_attributes)); pthread_mutex_unlock(mutex); } - + flb_free(attrs); flb_free(pod_name); } @@ -287,13 +287,14 @@ int fetch_pod_service_map(struct flb_kube *ctx, char *api_server_url, int determine_platform(struct flb_kube *ctx) { int ret; + size_t i; char *token_buf = NULL; size_t token_size; char *payload = NULL; size_t payload_len; - char *issuer_start; + char *issuer_start; char *issuer_end; - char *first_dot; + char *first_dot; char *second_dot; size_t payload_b64_len; size_t padded_len; @@ -301,44 +302,44 @@ int determine_platform(struct flb_kube *ctx) size_t issuer_len; char *issuer_value; int is_eks; - + /* Read serviceaccount token */ ret = flb_utils_read_file(FLB_KUBE_TOKEN, &token_buf, &token_size); if (ret != 0 || !token_buf) { return -1; } - + /* JWT tokens have 3 parts separated by dots: header.payload.signature */ first_dot = strchr(token_buf, '.'); if (!first_dot) { flb_free(token_buf); return -1; } - + second_dot = strchr(first_dot + 1, '.'); if (!second_dot) { flb_free(token_buf); return -1; } - + /* Extract and decode the payload (middle part) */ payload_b64_len = second_dot - (first_dot + 1); - + /* Calculate padded length */ padded_len = payload_b64_len; while (padded_len % 4 != 0) padded_len++; - + payload_b64 = flb_malloc(padded_len + 1); if (!payload_b64) { flb_errno(); flb_free(token_buf); return -1; } - + memcpy(payload_b64, first_dot + 1, payload_b64_len); - + /* Convert base64url to base64 and add padding */ - for (size_t i = 0; i < payload_b64_len; i++) { + for (i = 0; i < payload_b64_len; i++) { if (payload_b64[i] == '-') { payload_b64[i] = '+'; } @@ -350,7 +351,7 @@ int determine_platform(struct flb_kube *ctx) payload_b64[payload_b64_len++] = '='; } payload_b64[padded_len] = '\0'; - + /* Base64 decode the payload */ payload = flb_malloc(payload_b64_len * 3 / 4 + 4); /* Conservative size estimate */ if (!payload) { @@ -359,27 +360,27 @@ int determine_platform(struct flb_kube *ctx) flb_free(payload_b64); return -1; } - - ret = flb_base64_decode((unsigned char *)payload, padded_len * 3 / 4 + 4, + + ret = flb_base64_decode((unsigned char *)payload, padded_len * 3 / 4 + 4, &payload_len, (unsigned char *)payload_b64, padded_len); - + flb_free(token_buf); flb_free(payload_b64); - + if (ret != 0) { flb_free(payload); return -1; } - + payload[payload_len] = '\0'; - + /* Look for "iss" field in the JSON payload */ issuer_start = strstr(payload, "\"iss\":"); if (!issuer_start) { flb_free(payload); return -1; } - + /* Skip to the value part */ issuer_start = strchr(issuer_start, ':'); if (!issuer_start) { @@ -387,7 +388,7 @@ int determine_platform(struct flb_kube *ctx) return -1; } issuer_start++; - + /* Skip whitespace and opening quote */ while (*issuer_start == ' ' || *issuer_start == '\t') issuer_start++; if (*issuer_start != '"') { @@ -395,14 +396,14 @@ int determine_platform(struct flb_kube *ctx) return -1; } issuer_start++; - + /* Find closing quote */ issuer_end = strchr(issuer_start, '"'); if (!issuer_end) { flb_free(payload); return -1; } - + /* Check if issuer contains EKS OIDC URL pattern */ /* EKS OIDC URLs follow pattern: https://oidc.eks.{region}.amazonaws.com/id/{cluster-id} */ issuer_len = issuer_end - issuer_start; @@ -419,7 +420,7 @@ int determine_platform(struct flb_kube *ctx) flb_free(payload); return 1; /* EKS detected */ } - + flb_free(payload); return -1; /* Not EKS */ } From 985628eeffd3b8b76b97b2b106115adb42ebf355 Mon Sep 17 00:00:00 2001 From: Patrick Stephens Date: Sun, 3 May 2026 12:35:03 +0100 Subject: [PATCH 121/213] [upstream] in_forward: fix connection release on pause memory Upstream-Ref: https://github.com/fluent/fluent-bit/commit/0717103342c3bd2995da08155a22e7e433953b6a Cherry-picked from Fluent Bit v4.2.4 --- source/plugins/in_forward/fw.c | 27 ++++++++++++++++++++- source/plugins/in_forward/fw.h | 8 +++++++ source/plugins/in_forward/fw_conn.c | 37 +++++++++++++++++++++++++---- 3 files changed, 66 insertions(+), 6 deletions(-) diff --git a/source/plugins/in_forward/fw.c b/source/plugins/in_forward/fw.c index 77da340d..d33b36fb 100644 --- a/source/plugins/in_forward/fw.c +++ b/source/plugins/in_forward/fw.c @@ -124,28 +124,37 @@ static int fw_unix_create(struct flb_in_fw_config *ctx) static int in_fw_collect(struct flb_input_instance *ins, struct flb_config *config, void *in_context) { + int state_backup; struct flb_connection *connection; struct fw_conn *conn; struct flb_in_fw_config *ctx; ctx = in_context; + state_backup = ctx->state; + ctx->state = FW_INSTANCE_STATE_ACCEPTING_CLIENT; + connection = flb_downstream_conn_get(ctx->downstream); if (connection == NULL) { flb_plg_error(ctx->ins, "could not accept new connection"); + ctx->state = state_backup; return -1; } if (!config->is_ingestion_active) { flb_downstream_conn_release(connection); + ctx->state = state_backup; + return -1; } if(ctx->is_paused) { flb_downstream_conn_release(connection); flb_plg_trace(ins, "TCP connection will be closed FD=%i", connection->fd); + ctx->state = state_backup; + return -1; } @@ -154,9 +163,17 @@ static int in_fw_collect(struct flb_input_instance *ins, conn = fw_conn_add(connection, ctx); if (!conn) { flb_downstream_conn_release(connection); + ctx->state = state_backup; + return -1; } + ctx->state = state_backup; + + if (ctx->state == FW_INSTANCE_STATE_PAUSED) { + fw_conn_del_all(ctx); + } + return 0; } @@ -263,6 +280,7 @@ static int in_fw_init(struct flb_input_instance *ins, return -1; } + ctx->state = FW_INSTANCE_STATE_RUNNING; ctx->coll_fd = -1; ctx->ins = ins; mk_list_init(&ctx->connections); @@ -386,7 +404,10 @@ static void in_fw_pause(void *data, struct flb_config *config) return; } - fw_conn_del_all(ctx); + if (ctx->state == FW_INSTANCE_STATE_RUNNING) { + fw_conn_del_all(ctx); + } + ctx->is_paused = FLB_TRUE; ret = pthread_mutex_unlock(&ctx->conn_mutex); if (ret != 0) { @@ -406,6 +427,8 @@ static void in_fw_pause(void *data, struct flb_config *config) if (config->is_ingestion_active == FLB_FALSE) { fw_conn_del_all(ctx); } + + ctx->state = FW_INSTANCE_STATE_PAUSED; } static void in_fw_resume(void *data, struct flb_config *config) { @@ -427,6 +450,8 @@ static void in_fw_resume(void *data, struct flb_config *config) { flb_plg_error(ctx->ins, "cannot unlock collector mutex"); return; } + + ctx->state = FW_INSTANCE_STATE_RUNNING; } } diff --git a/source/plugins/in_forward/fw.h b/source/plugins/in_forward/fw.h index 527b4859..42557ef4 100644 --- a/source/plugins/in_forward/fw.h +++ b/source/plugins/in_forward/fw.h @@ -25,6 +25,12 @@ #include #include +#define FW_INSTANCE_STATE_RUNNING 0 +#define FW_INSTANCE_STATE_ACCEPTING_CLIENT 1 +#define FW_INSTANCE_STATE_PROCESSING_PACKET 2 +#define FW_INSTANCE_STATE_PAUSED 3 + + enum { FW_HANDSHAKE_HELO = 1, FW_HANDSHAKE_PINGPONG = 2, @@ -76,6 +82,8 @@ struct flb_in_fw_config { pthread_mutex_t conn_mutex; + int state; + /* Plugin is paused */ int is_paused; }; diff --git a/source/plugins/in_forward/fw_conn.c b/source/plugins/in_forward/fw_conn.c index 116c56fb..af503426 100644 --- a/source/plugins/in_forward/fw_conn.c +++ b/source/plugins/in_forward/fw_conn.c @@ -28,8 +28,7 @@ #include "fw_prot.h" #include "fw_conn.h" -/* Callback invoked every time an event is triggered for a connection */ -int fw_conn_event(void *data) +static int fw_conn_event_internal(struct flb_connection *connection) { int ret; int bytes; @@ -39,9 +38,6 @@ int fw_conn_event(void *data) struct fw_conn *conn; struct mk_event *event; struct flb_in_fw_config *ctx; - struct flb_connection *connection; - - connection = (struct flb_connection *) data; conn = connection->user_data; @@ -127,6 +123,37 @@ int fw_conn_event(void *data) return 0; } +/* Callback invoked every time an event is triggered for a connection */ +int fw_conn_event(void *data) +{ + struct flb_in_fw_config *ctx; + struct fw_conn *conn; + int result; + struct flb_connection *connection; + int state_backup; + + connection = (struct flb_connection *) data; + + conn = connection->user_data; + + ctx = conn->ctx; + + state_backup = ctx->state; + + ctx->state = FW_INSTANCE_STATE_PROCESSING_PACKET; + + result = fw_conn_event_internal(connection); + + if (ctx->state == FW_INSTANCE_STATE_PROCESSING_PACKET) { + ctx->state = state_backup; + } + else if (ctx->state == FW_INSTANCE_STATE_PAUSED) { + fw_conn_del_all(ctx); + } + + return result; +} + /* Create a new Forward request instance */ struct fw_conn *fw_conn_add(struct flb_connection *connection, struct flb_in_fw_config *ctx) { From 933865e3929533070207b2111d0196935e76f01a Mon Sep 17 00:00:00 2001 From: Patrick Stephens Date: Sun, 3 May 2026 12:35:03 +0100 Subject: [PATCH 122/213] [upstream] in_opentelemetry: added five missing configuration Upstream-Ref: https://github.com/fluent/fluent-bit/commit/0e56bc2b470c195e1d0dd90024e7b38493185e59 Cherry-picked from Fluent Bit v4.2.4 --- source/plugins/in_opentelemetry/opentelemetry.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/source/plugins/in_opentelemetry/opentelemetry.c b/source/plugins/in_opentelemetry/opentelemetry.c index 3b02f210..f649ada7 100644 --- a/source/plugins/in_opentelemetry/opentelemetry.c +++ b/source/plugins/in_opentelemetry/opentelemetry.c @@ -200,7 +200,7 @@ static struct flb_config_map config_map[] = { { FLB_CONFIG_MAP_BOOL, "http2", "true", 0, FLB_TRUE, offsetof(struct flb_opentelemetry, enable_http2), - NULL + "Enable HTTP/2 protocol support for the OpenTelemetry receiver" }, { @@ -219,19 +219,19 @@ static struct flb_config_map config_map[] = { { FLB_CONFIG_MAP_SIZE, "buffer_max_size", HTTP_BUFFER_MAX_SIZE, 0, FLB_TRUE, offsetof(struct flb_opentelemetry, buffer_max_size), - "" + "Maximum size of the HTTP request buffer" }, { FLB_CONFIG_MAP_SIZE, "buffer_chunk_size", HTTP_BUFFER_CHUNK_SIZE, 0, FLB_TRUE, offsetof(struct flb_opentelemetry, buffer_chunk_size), - "" + "Size of each buffer chunk allocated for HTTP requests" }, { FLB_CONFIG_MAP_STR, "tag_key", NULL, 0, FLB_TRUE, offsetof(struct flb_opentelemetry, tag_key), - "" + "Record accessor key to use for generating tags from incoming records" }, { FLB_CONFIG_MAP_BOOL, "tag_from_uri", "true", @@ -252,6 +252,7 @@ static struct flb_config_map config_map[] = { { FLB_CONFIG_MAP_STR, "logs_metadata_key", "otlp", 0, FLB_TRUE, offsetof(struct flb_opentelemetry, logs_metadata_key), + "Key name to store OpenTelemetry logs metadata in the record" }, { FLB_CONFIG_MAP_STR, "logs_body_key", NULL, From b73119c75429cb553757433167345e8a9f368ee2 Mon Sep 17 00:00:00 2001 From: Patrick Stephens Date: Sun, 3 May 2026 12:35:04 +0100 Subject: [PATCH 123/213] [upstream] out_s3: implement retry_limit parameter Upstream-Ref: https://github.com/fluent/fluent-bit/commit/f4108db65e876fadbabac459aa11990c5440550a Cherry-picked from Fluent Bit v4.2.4 --- source/plugins/out_s3/s3.c | 32 +++++++++++++++++++++----------- source/plugins/out_s3/s3.h | 7 +++++-- 2 files changed, 26 insertions(+), 13 deletions(-) diff --git a/source/plugins/out_s3/s3.c b/source/plugins/out_s3/s3.c index 7dc07170..3016b28d 100644 --- a/source/plugins/out_s3/s3.c +++ b/source/plugins/out_s3/s3.c @@ -617,6 +617,10 @@ static int cb_s3_init(struct flb_output_instance *ins, ctx->retry_time = 0; ctx->upload_queue_success = FLB_FALSE; + if(ctx->ins->retry_limit < 0) { + ctx->ins->retry_limit = MAX_UPLOAD_ERRORS; + } + /* Export context */ flb_output_set_context(ins, ctx); @@ -1367,11 +1371,10 @@ static int put_all_chunks(struct flb_s3 *ctx) continue; } - if (chunk->failures >= MAX_UPLOAD_ERRORS) { + if (chunk->failures >= ctx->ins->retry_limit) { flb_plg_warn(ctx->ins, - "Chunk for tag %s failed to send %i times, " - "will not retry", - (char *) fsf->meta_buf, MAX_UPLOAD_ERRORS); + "Chunk for tag %s failed to send %d/%d times, will not retry", + (char *) fsf->meta_buf, chunk->failures, ctx->ins->retry_limit); flb_fstore_file_inactive(ctx->fs, fsf); continue; } @@ -1656,7 +1659,7 @@ static struct multipart_upload *get_upload(struct flb_s3 *ctx, if (tmp_upload->upload_state == MULTIPART_UPLOAD_STATE_COMPLETE_IN_PROGRESS) { continue; } - if (tmp_upload->upload_errors >= MAX_UPLOAD_ERRORS) { + if (tmp_upload->upload_errors >= ctx->ins->retry_limit) { tmp_upload->upload_state = MULTIPART_UPLOAD_STATE_COMPLETE_IN_PROGRESS; flb_plg_error(ctx->ins, "Upload for %s has reached max upload errors", tmp_upload->s3_key); @@ -1902,7 +1905,7 @@ static void s3_upload_queue(struct flb_config *config, void *out_context) /* If retry limit was reached, discard file and remove file from queue */ upload_contents->retry_counter++; - if (upload_contents->retry_counter >= MAX_UPLOAD_ERRORS) { + if (upload_contents->retry_counter >= ctx->ins->retry_limit) { flb_plg_warn(ctx->ins, "Chunk file failed to send %d times, will not " "retry", upload_contents->retry_counter); s3_store_file_inactive(ctx, upload_contents->upload_file); @@ -3303,6 +3306,13 @@ static void cb_s3_upload(struct flb_config *config, void *data) if (ret != FLB_OK) { flb_plg_error(ctx->ins, "Could not send chunk with tag %s", (char *) fsf->meta_buf); + if(chunk->failures >= ctx->ins->retry_limit){ + flb_plg_warn(ctx->ins, + "Chunk for tag %s failed to send %d/%d times, will not retry", + (char *) fsf->meta_buf, chunk->failures, ctx->ins->retry_limit); + flb_fstore_file_inactive(ctx->fs, fsf); + continue; + } } } @@ -3311,7 +3321,7 @@ static void cb_s3_upload(struct flb_config *config, void *data) m_upload = mk_list_entry(head, struct multipart_upload, _head); complete = FLB_FALSE; - if (m_upload->complete_errors >= MAX_UPLOAD_ERRORS) { + if (m_upload->complete_errors >= ctx->ins->retry_limit) { flb_plg_error(ctx->ins, "Upload for %s has reached max completion errors, " "plugin will give up", m_upload->s3_key); @@ -3823,10 +3833,10 @@ static void cb_s3_flush(struct flb_event_chunk *event_chunk, m_upload_file, file_first_log_time); } - /* Discard upload_file if it has failed to upload MAX_UPLOAD_ERRORS times */ - if (upload_file != NULL && upload_file->failures >= MAX_UPLOAD_ERRORS) { - flb_plg_warn(ctx->ins, "File with tag %s failed to send %d times, will not " - "retry", event_chunk->tag, MAX_UPLOAD_ERRORS); + /* Discard upload_file if it has failed to upload retry_limit times */ + if (upload_file != NULL && upload_file->failures >= ctx->ins->retry_limit) { + flb_plg_warn(ctx->ins, "File with tag %s failed to send %d/%d times, will not retry", + event_chunk->tag, upload_file->failures, ctx->ins->retry_limit); s3_store_file_inactive(ctx, upload_file); upload_file = NULL; } diff --git a/source/plugins/out_s3/s3.h b/source/plugins/out_s3/s3.h index d1004fcf..fc30ff81 100644 --- a/source/plugins/out_s3/s3.h +++ b/source/plugins/out_s3/s3.h @@ -48,6 +48,8 @@ #define DEFAULT_UPLOAD_TIMEOUT 3600 +#define MAX_UPLOAD_ERRORS 5 + /* * If we see repeated errors on an upload/chunk, we will discard it * This saves us from scenarios where something goes wrong and an upload can @@ -56,8 +58,9 @@ * * The same is done for chunks, just to be safe, even though realistically * I can't think of a reason why a chunk could become unsendable. + * + * The retry limit is now configurable via the retry_limit parameter. */ -#define MAX_UPLOAD_ERRORS 5 struct upload_queue { struct s3_file *upload_file; @@ -96,7 +99,7 @@ struct multipart_upload { struct mk_list _head; - /* see note for MAX_UPLOAD_ERRORS */ + /* see note for retry_limit configuration */ int upload_errors; int complete_errors; }; From 5fbe6c70e8a65eac7408742b27070127dde79a8e Mon Sep 17 00:00:00 2001 From: Patrick Stephens Date: Sun, 3 May 2026 12:35:04 +0100 Subject: [PATCH 124/213] [upstream] in_blob: added missing configuration descriptions. Upstream-Ref: https://github.com/fluent/fluent-bit/commit/ae1445463ca03bac9b46147166c0ee00993363e8 Cherry-picked from Fluent Bit v4.2.4 --- source/plugins/in_blob/blob.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/source/plugins/in_blob/blob.c b/source/plugins/in_blob/blob.c index c3fb8918..c819acde 100644 --- a/source/plugins/in_blob/blob.c +++ b/source/plugins/in_blob/blob.c @@ -991,12 +991,14 @@ static struct flb_config_map config_map[] = { { FLB_CONFIG_MAP_STR, "exclude_pattern", NULL, 0, FLB_TRUE, offsetof(struct blob_ctx, exclude_pattern), + "Glob pattern to exclude files from being processed" }, #ifdef FLB_HAVE_SQLDB { FLB_CONFIG_MAP_STR, "database_file", NULL, 0, FLB_TRUE, offsetof(struct blob_ctx, database_file), + "SQLite database file path to track processed files" }, #endif @@ -1009,31 +1011,37 @@ static struct flb_config_map config_map[] = { { FLB_CONFIG_MAP_STR, "upload_success_action", NULL, 0, FLB_TRUE, offsetof(struct blob_ctx, upload_success_action_str), + "Action to perform after successful upload: \"delete\", \"emit_log\", or \"add_suffix\"" }, { FLB_CONFIG_MAP_STR, "upload_success_suffix", NULL, 0, FLB_TRUE, offsetof(struct blob_ctx, upload_success_suffix), + "Suffix to append to filename when upload_success_action is \"add_suffix\"" }, { FLB_CONFIG_MAP_STR, "upload_success_message", NULL, 0, FLB_TRUE, offsetof(struct blob_ctx, upload_success_message), + "Message to emit as log when upload_success_action is \"emit_log\"" }, { FLB_CONFIG_MAP_STR, "upload_failure_action", NULL, 0, FLB_TRUE, offsetof(struct blob_ctx, upload_failure_action_str), + "Action to perform after failed upload: \"delete\", \"emit_log\", or \"add_suffix\"" }, { FLB_CONFIG_MAP_STR, "upload_failure_suffix", NULL, 0, FLB_TRUE, offsetof(struct blob_ctx, upload_failure_suffix), + "Suffix to append to filename when upload_failure_action is \"add_suffix\"" }, { FLB_CONFIG_MAP_STR, "upload_failure_message", NULL, 0, FLB_TRUE, offsetof(struct blob_ctx, upload_failure_message), + "Message to emit as log when upload_failure_action is \"emit_log\"" }, /* EOF */ From 52ab7b9e46d12b6f1a155a9fba22beae443fe012 Mon Sep 17 00:00:00 2001 From: Patrick Stephens Date: Sun, 3 May 2026 12:35:04 +0100 Subject: [PATCH 125/213] [upstream] tls: openssl: Implement flexible certstore loading on Upstream-Ref: https://github.com/fluent/fluent-bit/commit/ef3f0750c741ae4586a093adf747bb77c932bec7 Cherry-picked from Fluent Bit v4.2.4 --- source/src/tls/openssl.c | 138 +++++++++++++++++++++++++++++++++++---- 1 file changed, 125 insertions(+), 13 deletions(-) diff --git a/source/src/tls/openssl.c b/source/src/tls/openssl.c index 16241b7e..9fafcee0 100644 --- a/source/src/tls/openssl.c +++ b/source/src/tls/openssl.c @@ -302,6 +302,100 @@ int tls_context_alpn_set(void *ctx_backend, const char *alpn) } #ifdef _MSC_VER +/* Parse certstore_name prefix like + * + * "My" -> no prefix, leave location untouched + * "CurrentUser\\My" -> CERT_SYSTEM_STORE_CURRENT_USER, "My" + * "HKCU\\My" -> CERT_SYSTEM_STORE_CURRENT_USER, "My" + * "LocalMachine\\My" -> CERT_SYSTEM_STORE_LOCAL_MACHINE, "My" + * "HKLM\\My" -> CERT_SYSTEM_STORE_LOCAL_MACHINE, "My" + * "LocalMachineEnterprise\\My"-> CERT_SYSTEM_STORE_LOCAL_MACHINE_ENTERPRISE, "My" + * "HKLME\\My" -> CERT_SYSTEM_STORE_LOCAL_MACHINE_ENTERPRISE, "My" + * + * Also accepts '/' as separator. + * + * If no known prefix is found, *store_name_out is left as-is and *location_flags + * is not modified (so legacy behavior is preserved). + */ +static void windows_resolve_certstore_location(const char *configured_name, + DWORD *location_flags, + const char **store_name_out) +{ + const char *name; + const char *sep; + size_t prefix_len; + char prefix_buf[32]; + size_t i; + size_t len = 0; + + if (!configured_name || !*configured_name) { + return; + } + + name = configured_name; + len = strlen(name); + + /* Optional "Cert:\" prefix (PowerShell style) */ + if (len >= 6 && + strncasecmp(name, "cert:", 5) == 0 && + (name[5] == '\\' || name[5] == '/')) { + name += 6; + } + + /* Find first '\' or '/' separator */ + sep = name; + while (*sep != '\0' && *sep != '\\' && *sep != '/') { + sep++; + } + + if (*sep == '\0') { + /* No prefix, only store name (e.g. "My" or "Root") + * -> keep legacy behavior (location_flags unchanged). + */ + *store_name_out = name; + return; + } + + /* Copy and lowercase prefix into buffer */ + prefix_len = (size_t)(sep - name); + if (prefix_len >= sizeof(prefix_buf)) { + prefix_len = sizeof(prefix_buf) - 1; + } + + for (i = 0; i < prefix_len; i++) { + char c = (char) name[i]; + + if (c >= 'A' && c <= 'Z') { + c = (char) (c - 'A' + 'a'); + } + prefix_buf[i] = c; + } + prefix_buf[prefix_len] = '\0'; + + /* Default: keep *location_flags as-is */ + if (strncasecmp(prefix_buf, "currentuser", 11) == 0 || + strncasecmp(prefix_buf, "hkcu", 4) == 0) { + *location_flags = CERT_SYSTEM_STORE_CURRENT_USER; + } + else if (strncasecmp(prefix_buf, "localmachine", 12) == 0 || + strncasecmp(prefix_buf, "hklm", 4) == 0) { + *location_flags = CERT_SYSTEM_STORE_LOCAL_MACHINE; + } + else if (strncasecmp(prefix_buf, "localmachineenterprise", 22) == 0 || + strncasecmp(prefix_buf, "hklme", 5) == 0) { + *location_flags = CERT_SYSTEM_STORE_LOCAL_MACHINE_ENTERPRISE; + } + else { + /* Unknown prefix -> treat entire string as store name */ + *store_name_out = configured_name; + + return; + } + + /* Store name part after the separator "\" or "/" */ + *store_name_out = sep + 1; +} + static int windows_load_system_certificates(struct tls_context *ctx) { int ret; @@ -311,7 +405,9 @@ static int windows_load_system_certificates(struct tls_context *ctx) const unsigned char *win_cert_data; X509_STORE *ossl_store = SSL_CTX_get_cert_store(ctx->ctx); X509 *ossl_cert; - char *certstore_name = "Root"; + char *configured_name = "Root"; + const char *store_name = "Root"; + DWORD store_location = CERT_SYSTEM_STORE_CURRENT_USER; /* Check if OpenSSL certificate store is available */ if (!ossl_store) { @@ -320,20 +416,36 @@ static int windows_load_system_certificates(struct tls_context *ctx) } if (ctx->certstore_name) { - certstore_name = ctx->certstore_name; + configured_name = ctx->certstore_name; + store_name = ctx->certstore_name; + } + + /* First, resolve explicit prefix if present */ + windows_resolve_certstore_location(configured_name, + &store_location, + &store_name); + + /* Backward compatibility: + * If no prefix was given (store_name == configured_name) and + * use_enterprise_store is set, override location accordingly. + */ + if (store_name == configured_name && ctx->use_enterprise_store) { + store_location = CERT_SYSTEM_STORE_LOCAL_MACHINE_ENTERPRISE; } - if (ctx->use_enterprise_store) { - /* Open the Windows system enterprise certificate store */ + /* Open the Windows certificate store for the resolved location */ + if (store_location == CERT_SYSTEM_STORE_CURRENT_USER) { + /* Keep using CertOpenSystemStoreA for current user to avoid + * changing existing behavior. + */ + win_store = CertOpenSystemStoreA(0, store_name); + } + else { win_store = CertOpenStore(CERT_STORE_PROV_SYSTEM, 0, 0, - CERT_SYSTEM_STORE_LOCAL_MACHINE_ENTERPRISE, - certstore_name); - } - else { - /* Open the Windows system certificate store */ - win_store = CertOpenSystemStoreA(0, certstore_name); + store_location, + store_name); } if (win_store == NULL) { @@ -389,10 +501,10 @@ static int windows_load_system_certificates(struct tls_context *ctx) } if (loaded == 0) { - flb_warn("[tls] no certificates loaded by thumbprint from '%s'.", certstore_name); + flb_warn("[tls] no certificates loaded by thumbprint from '%s'.", configured_name); } else { - flb_debug("[tls] loaded %zu certificate(s) by thumbprint from '%s'.", loaded, certstore_name); + flb_debug("[tls] loaded %zu certificate(s) by thumbprint from '%s'.", loaded, configured_name); } return 0; } @@ -445,7 +557,7 @@ static int windows_load_system_certificates(struct tls_context *ctx) } flb_debug("[tls] successfully loaded certificates from windows system %s store.", - certstore_name); + configured_name); return 0; } #endif From cfe4f2212568b46a9857dd3895ac8fbce8c74362 Mon Sep 17 00:00:00 2001 From: Patrick Stephens Date: Sun, 3 May 2026 12:35:04 +0100 Subject: [PATCH 126/213] [upstream] tls: openssl: Handle location prefix more strictly Upstream-Ref: https://github.com/fluent/fluent-bit/commit/923dec2f2300af8cc99216fa0c0214671710e4f9 Cherry-picked from Fluent Bit v4.2.4 --- source/src/tls/openssl.c | 39 ++++++++++++++++++++++----------------- 1 file changed, 22 insertions(+), 17 deletions(-) diff --git a/source/src/tls/openssl.c b/source/src/tls/openssl.c index 9fafcee0..84216cdc 100644 --- a/source/src/tls/openssl.c +++ b/source/src/tls/openssl.c @@ -317,9 +317,9 @@ int tls_context_alpn_set(void *ctx_backend, const char *alpn) * If no known prefix is found, *store_name_out is left as-is and *location_flags * is not modified (so legacy behavior is preserved). */ -static void windows_resolve_certstore_location(const char *configured_name, - DWORD *location_flags, - const char **store_name_out) +static int windows_resolve_certstore_location(const char *configured_name, + DWORD *location_flags, + const char **store_name_out) { const char *name; const char *sep; @@ -327,9 +327,10 @@ static void windows_resolve_certstore_location(const char *configured_name, char prefix_buf[32]; size_t i; size_t len = 0; + char c; if (!configured_name || !*configured_name) { - return; + return FLB_FALSE; } name = configured_name; @@ -353,7 +354,8 @@ static void windows_resolve_certstore_location(const char *configured_name, * -> keep legacy behavior (location_flags unchanged). */ *store_name_out = name; - return; + + return FLB_FALSE; } /* Copy and lowercase prefix into buffer */ @@ -363,7 +365,7 @@ static void windows_resolve_certstore_location(const char *configured_name, } for (i = 0; i < prefix_len; i++) { - char c = (char) name[i]; + c = (char) name[i]; if (c >= 'A' && c <= 'Z') { c = (char) (c - 'A' + 'a'); @@ -373,27 +375,29 @@ static void windows_resolve_certstore_location(const char *configured_name, prefix_buf[prefix_len] = '\0'; /* Default: keep *location_flags as-is */ - if (strncasecmp(prefix_buf, "currentuser", 11) == 0 || - strncasecmp(prefix_buf, "hkcu", 4) == 0) { + if (strcmp(prefix_buf, "currentuser") == 0 || + strcmp(prefix_buf, "hkcu") == 0) { *location_flags = CERT_SYSTEM_STORE_CURRENT_USER; } - else if (strncasecmp(prefix_buf, "localmachine", 12) == 0 || - strncasecmp(prefix_buf, "hklm", 4) == 0) { + else if (strcmp(prefix_buf, "localmachine") == 0 || + strcmp(prefix_buf, "hklm") == 0) { *location_flags = CERT_SYSTEM_STORE_LOCAL_MACHINE; } - else if (strncasecmp(prefix_buf, "localmachineenterprise", 22) == 0 || - strncasecmp(prefix_buf, "hklme", 5) == 0) { + else if (strcmp(prefix_buf, "localmachineenterprise") == 0 || + strcmp(prefix_buf, "hklme") == 0) { *location_flags = CERT_SYSTEM_STORE_LOCAL_MACHINE_ENTERPRISE; } else { /* Unknown prefix -> treat entire string as store name */ *store_name_out = configured_name; - return; + return FLB_FALSE; } /* Store name part after the separator "\" or "/" */ *store_name_out = sep + 1; + + return FLB_TRUE; } static int windows_load_system_certificates(struct tls_context *ctx) @@ -408,6 +412,7 @@ static int windows_load_system_certificates(struct tls_context *ctx) char *configured_name = "Root"; const char *store_name = "Root"; DWORD store_location = CERT_SYSTEM_STORE_CURRENT_USER; + int has_location_prefix = FLB_FALSE; /* Check if OpenSSL certificate store is available */ if (!ossl_store) { @@ -421,15 +426,15 @@ static int windows_load_system_certificates(struct tls_context *ctx) } /* First, resolve explicit prefix if present */ - windows_resolve_certstore_location(configured_name, - &store_location, - &store_name); + has_location_prefix = windows_resolve_certstore_location(configured_name, + &store_location, + &store_name); /* Backward compatibility: * If no prefix was given (store_name == configured_name) and * use_enterprise_store is set, override location accordingly. */ - if (store_name == configured_name && ctx->use_enterprise_store) { + if (has_location_prefix == FLB_FALSE && ctx->use_enterprise_store) { store_location = CERT_SYSTEM_STORE_LOCAL_MACHINE_ENTERPRISE; } From 915921c8dc994e3f5ad31be52dec9197ece8c1c7 Mon Sep 17 00:00:00 2001 From: Patrick Stephens Date: Sun, 3 May 2026 12:35:04 +0100 Subject: [PATCH 127/213] [upstream] in_kubernetes_events: add support for configuring Upstream-Ref: https://github.com/fluent/fluent-bit/commit/10ebd3a354f2a052ac865960145fda844b3b120e Cherry-picked from Fluent Bit v4.2.4 --- .../plugins/in_kubernetes_events/kubernetes_events_conf.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/source/plugins/in_kubernetes_events/kubernetes_events_conf.c b/source/plugins/in_kubernetes_events/kubernetes_events_conf.c index e40d67b4..e84872f7 100644 --- a/source/plugins/in_kubernetes_events/kubernetes_events_conf.c +++ b/source/plugins/in_kubernetes_events/kubernetes_events_conf.c @@ -128,6 +128,13 @@ static int network_init(struct k8s_events *ctx, struct flb_config *config) return -1; } + if (flb_input_upstream_set(ctx->upstream, ctx->ins) != 0) { + flb_plg_error(ctx->ins, "network upstream setup failed"); + flb_upstream_destroy(ctx->upstream); + ctx->upstream = NULL; + return -1; + } + return 0; } From ab9d623cd9b51f5e974e49b02d584dcefde1f82f Mon Sep 17 00:00:00 2001 From: Patrick Stephens Date: Sun, 3 May 2026 12:35:05 +0100 Subject: [PATCH 128/213] [upstream] dockerfile: Docker image to support large page sizes Upstream-Ref: https://github.com/fluent/fluent-bit/commit/137771bcfe6e690eb18be909278b5212580de28c Cherry-picked from Fluent Bit v4.2.4 --- source/dockerfiles/Dockerfile.largepage | 277 ++++++++++++++++++++++++ 1 file changed, 277 insertions(+) create mode 100644 source/dockerfiles/Dockerfile.largepage diff --git a/source/dockerfiles/Dockerfile.largepage b/source/dockerfiles/Dockerfile.largepage new file mode 100644 index 00000000..daef9ba1 --- /dev/null +++ b/source/dockerfiles/Dockerfile.largepage @@ -0,0 +1,277 @@ +# syntax=docker/dockerfile:1 +# check=skip=InvalidBaseImagePlatform + +# To use this container you may need to do the following: +# https://askubuntu.com/a/1369504 +# sudo add-apt-repository ppa:jacob/virtualisation #(for Ubuntu 20.04) +# sudo apt-get update && sudo apt-get install qemu qemu-user qemu-user-static +# https://stackoverflow.com/a/60667468 +# docker run --rm --privileged multiarch/qemu-user-static --reset -p yes +# docker buildx rm builder +# docker buildx create --name builder --use +# docker buildx inspect --bootstrap +# docker buildx build --platform "linux/amd64,linux/arm64,linux/arm/v7,linux/s390x" -f ./dockerfiles/Dockerfile.multiarch --build-arg FLB_TARBALL=https://github.com/fluent/fluent-bit/archive/v1.8.11.tar.gz ./dockerfiles/ + +# Set this to the current release version: it gets done so as part of the release. +ARG RELEASE_VERSION=4.2.1 + +# For multi-arch builds - assumption is running on an AMD64 host +FROM multiarch/qemu-user-static:x86_64-arm AS qemu-arm32 +FROM multiarch/qemu-user-static:x86_64-aarch64 AS qemu-arm64 + +FROM debian:bookworm-slim AS builder-base + +COPY --from=qemu-arm32 /usr/bin/qemu-arm-static /usr/bin/ +COPY --from=qemu-arm64 /usr/bin/qemu-aarch64-static /usr/bin/ + +ARG FLB_NIGHTLY_BUILD +ENV FLB_NIGHTLY_BUILD=$FLB_NIGHTLY_BUILD + +ARG FLB_CHUNK_TRACE=On +ENV FLB_CHUNK_TRACE=${FLB_CHUNK_TRACE} + +RUN mkdir -p /fluent-bit/bin /fluent-bit/etc /fluent-bit/log + +ENV DEBIAN_FRONTEND=noninteractive + +# hadolint ignore=DL3008 +RUN echo "deb http://deb.debian.org/debian bookworm-backports main" >> /etc/apt/sources.list && \ + apt-get update && \ + apt-get install -y --no-install-recommends \ + build-essential \ + curl \ + ca-certificates \ + git \ + make \ + tar \ + libssl-dev \ + libcurl4-openssl-dev \ + libsasl2-dev \ + pkg-config \ + libsystemd-dev/bookworm-backports \ + zlib1g-dev \ + libpq-dev \ + postgresql-server-dev-all \ + flex \ + bison \ + libyaml-dev \ + && apt-get satisfy -y cmake "cmake (<< 4.0)" \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + +# Must be run from root of repo +WORKDIR /src/fluent-bit/ +COPY . ./ + +# We split the builder setup out so people can target it or use as a base image without doing a full build. +FROM builder-base AS builder +WORKDIR /src/fluent-bit/build/ + +# Required to be set to ARMV7 for that target +ARG WAMR_BUILD_TARGET +ARG EXTRA_CMAKE_FLAGS +ENV EXTRA_CMAKE_FLAGS=${EXTRA_CMAKE_FLAGS} + +# Enable jemalloc large page support via CMake option introduced in 5ca1c93 +ARG FLB_JEMALLOC_OPTIONS="--with-lg-page=16 --with-lg-quantum=3" +ENV FLB_JEMALLOC_OPTIONS=${FLB_JEMALLOC_OPTIONS} + +# We do not want word splitting for EXTRA_CMAKE_FLAGS in case multiple are defined +# hadolint ignore=SC2086 +RUN [ -n "${WAMR_BUILD_TARGET:-}" ] && EXTRA_CMAKE_FLAGS="$EXTRA_CMAKE_FLAGS -DWAMR_BUILD_TARGET=$WAMR_BUILD_TARGET"; \ + cmake -DFLB_SIMD=On \ + -DFLB_RELEASE=On \ + -DFLB_JEMALLOC=On \ + -DFLB_TLS=On \ + -DFLB_SHARED_LIB=Off \ + -DFLB_EXAMPLES=Off \ + -DFLB_HTTP_SERVER=On \ + -DFLB_IN_EXEC=Off \ + -DFLB_IN_SYSTEMD=On \ + -DFLB_OUT_KAFKA=On \ + -DFLB_OUT_PGSQL=On \ + -DFLB_NIGHTLY_BUILD="$FLB_NIGHTLY_BUILD" \ + -DFLB_LOG_NO_CONTROL_CHARS=On \ + -DFLB_CHUNK_TRACE="$FLB_CHUNK_TRACE" \ + -DFLB_JEMALLOC_OPTIONS="$FLB_JEMALLOC_OPTIONS" \ + $EXTRA_CMAKE_FLAGS \ + .. + +ARG CFLAGS="-v" +ENV CFLAGS=${CFLAGS} + +RUN make -j "$(getconf _NPROCESSORS_ONLN)" +RUN install bin/fluent-bit /fluent-bit/bin/ + +# Configuration files +COPY conf/fluent-bit.conf \ + conf/parsers.conf \ + conf/parsers_ambassador.conf \ + conf/parsers_java.conf \ + conf/parsers_extra.conf \ + conf/parsers_openstack.conf \ + conf/parsers_cinder.conf \ + conf/plugins.conf \ + /fluent-bit/etc/ + +# Generate schema and include as part of the container image +RUN /fluent-bit/bin/fluent-bit -J > /fluent-bit/etc/schema.json + +# Simple example of how to properly extract packages for reuse in distroless +# Taken from: https://github.com/GoogleContainerTools/distroless/issues/863 +FROM debian:bookworm-slim AS deb-extractor +COPY --from=qemu-arm32 /usr/bin/qemu-arm-static /usr/bin/ +COPY --from=qemu-arm64 /usr/bin/qemu-aarch64-static /usr/bin/ + +# We download all debs locally then extract them into a directory we can use as the root for distroless. +# We also include some extra handling for the status files that some tooling uses for scanning, etc. +WORKDIR /tmp +SHELL ["/bin/bash", "-o", "pipefail", "-c"] +RUN echo "deb http://deb.debian.org/debian bookworm-backports main" >> /etc/apt/sources.list && \ + apt-get update && \ + apt-get download \ + libssl3 \ + libcurl4 \ + libnghttp2-14 \ + librtmp1 \ + libssh2-1 \ + libpsl5 \ + libbrotli1 \ + libsasl2-2 \ + pkg-config \ + libpq5 \ + libsystemd0/bookworm-backports \ + zlib1g \ + ca-certificates \ + libatomic1 \ + libgcrypt20 \ + libzstd1 \ + liblz4-1 \ + libgssapi-krb5-2 \ + libldap-2.5 \ + libgpg-error0 \ + libkrb5-3 \ + libk5crypto3 \ + libcom-err2 \ + libkrb5support0 \ + libgnutls30 \ + libkeyutils1 \ + libp11-kit0 \ + libidn2-0 \ + libunistring2 \ + libtasn1-6 \ + libnettle8 \ + libhogweed6 \ + libgmp10 \ + libffi8 \ + liblzma5 \ + libyaml-0-2 \ + libcap2 \ + && \ + mkdir -p /dpkg/var/lib/dpkg/status.d/ && \ + for deb in *.deb; do \ + package_name=$(dpkg-deb -I "${deb}" | awk '/^ Package: .*$/ {print $2}'); \ + echo "Processing: ${package_name}"; \ + dpkg --ctrl-tarfile "$deb" | tar -Oxf - ./control > "/dpkg/var/lib/dpkg/status.d/${package_name}"; \ + dpkg --extract "$deb" /dpkg || exit 10; \ + done + +# Remove unnecessary files extracted from deb packages like man pages and docs etc. +RUN find /dpkg/ -type d -empty -delete && \ + rm -r /dpkg/usr/share/doc/ + +# We want latest at time of build +# hadolint ignore=DL3006 +FROM gcr.io/distroless/cc-debian12 AS production +ARG RELEASE_VERSION +ENV FLUENT_BIT_VERSION=${RELEASE_VERSION} +LABEL description="Fluent Bit multi-architecture container image" \ + vendor="Fluent Organization" \ + version="${RELEASE_VERSION}" \ + author="Eduardo Silva " \ + org.opencontainers.image.description="Fluent Bit container image" \ + org.opencontainers.image.title="Fluent Bit" \ + org.opencontainers.image.licenses="Apache-2.0" \ + org.opencontainers.image.vendor="Fluent Organization" \ + org.opencontainers.image.version="${RELEASE_VERSION}" \ + org.opencontainers.image.source="https://github.com/fluent/fluent-bit" \ + org.opencontainers.image.documentation="https://docs.fluentbit.io/" \ + org.opencontainers.image.authors="Eduardo Silva " + +# Copy the libraries from the extractor stage into root +COPY --from=deb-extractor /dpkg / + +# Copy certificates +COPY --from=builder /etc/ssl/certs /etc/ssl/certs + +# Finally the binaries as most likely to change +COPY --from=builder /fluent-bit /fluent-bit + +EXPOSE 2020 + +# Entry point +ENTRYPOINT [ "/fluent-bit/bin/fluent-bit" ] +CMD ["/fluent-bit/bin/fluent-bit", "-c", "/fluent-bit/etc/fluent-bit.conf"] + +FROM debian:bookworm-slim AS debug +ARG RELEASE_VERSION +ENV FLUENT_BIT_VERSION=${RELEASE_VERSION} +LABEL description="Fluent Bit multi-architecture debug container image" \ + vendor="Fluent Organization" \ + version="${RELEASE_VERSION}-debug" \ + author="Eduardo Silva " \ + org.opencontainers.image.description="Fluent Bit debug container image" \ + org.opencontainers.image.title="Fluent Bit Debug" \ + org.opencontainers.image.licenses="Apache-2.0" \ + org.opencontainers.image.vendor="Fluent Organization" \ + org.opencontainers.image.version="${RELEASE_VERSION}-debug" \ + org.opencontainers.image.source="https://github.com/fluent/fluent-bit" \ + org.opencontainers.image.documentation="https://docs.fluentbit.io/" \ + org.opencontainers.image.authors="Eduardo Silva " + +COPY --from=qemu-arm32 /usr/bin/qemu-arm-static /usr/bin/ +COPY --from=qemu-arm64 /usr/bin/qemu-aarch64-static /usr/bin/ +ENV DEBIAN_FRONTEND=noninteractive + +# hadolint ignore=DL3008 +RUN echo "deb http://deb.debian.org/debian bookworm-backports main" >> /etc/apt/sources.list && \ + apt-get update && \ + apt-get install -y --no-install-recommends \ + libssl3 \ + libcurl4 \ + libnghttp2-14 \ + librtmp1 \ + libssh2-1 \ + libpsl5 \ + libbrotli1 \ + libsasl2-2 \ + pkg-config \ + libpq5 \ + libsystemd0/bookworm-backports \ + zlib1g \ + ca-certificates \ + libatomic1 \ + libgcrypt20 \ + libyaml-0-2 \ + bash gdb valgrind build-essential \ + git bash-completion vim tmux jq \ + dnsutils iputils-ping iputils-arping iputils-tracepath iputils-clockdiff \ + tcpdump curl nmap tcpflow iftop \ + net-tools mtr netcat-openbsd bridge-utils iperf ngrep \ + openssl \ + htop atop strace iotop sysstat ncdu logrotate hdparm pciutils psmisc tree pv \ + make tar flex bison \ + libssl-dev libsasl2-dev libsystemd-dev/bookworm-backports zlib1g-dev libpq-dev libyaml-dev postgresql-server-dev-all \ + && apt-get satisfy -y cmake "cmake (<< 4.0)" \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + +RUN rm -f /usr/bin/qemu-*-static +COPY --from=builder /fluent-bit /fluent-bit + +EXPOSE 2020 + +# No entry point so we can just shell in +CMD ["/fluent-bit/bin/fluent-bit", "-c", "/fluent-bit/etc/fluent-bit.conf"] + + From 7af3c5feeeaeec438bf3d27931f9a2d5bd41ad17 Mon Sep 17 00:00:00 2001 From: Patrick Stephens Date: Sun, 3 May 2026 12:35:05 +0100 Subject: [PATCH 129/213] [upstream] in_elasticsearch: fixing missing config parameter Upstream-Ref: https://github.com/fluent/fluent-bit/commit/3c4f8f42af92b4d6c6ee4eb29d155fa279b95c92 Cherry-picked from Fluent Bit v4.2.4 --- source/plugins/in_elasticsearch/in_elasticsearch.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/plugins/in_elasticsearch/in_elasticsearch.c b/source/plugins/in_elasticsearch/in_elasticsearch.c index db09aa28..34daecc1 100644 --- a/source/plugins/in_elasticsearch/in_elasticsearch.c +++ b/source/plugins/in_elasticsearch/in_elasticsearch.c @@ -237,7 +237,7 @@ static struct flb_config_map config_map[] = { { FLB_CONFIG_MAP_BOOL, "http2", "true", 0, FLB_TRUE, offsetof(struct flb_in_elasticsearch, enable_http2), - NULL + "Enable HTTP/2 support" }, { From 7d215b4e91ae78d595dc4d2c93920296c931b21f Mon Sep 17 00:00:00 2001 From: Patrick Stephens Date: Sun, 3 May 2026 12:35:05 +0100 Subject: [PATCH 130/213] [upstream] in_forward: improve configuration parameter Upstream-Ref: https://github.com/fluent/fluent-bit/commit/c88c5455537868939e977d45334b4650e86adbfa Cherry-picked from Fluent Bit v4.2.4 --- source/plugins/in_forward/fw.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/source/plugins/in_forward/fw.c b/source/plugins/in_forward/fw.c index d33b36fb..504f1c7a 100644 --- a/source/plugins/in_forward/fw.c +++ b/source/plugins/in_forward/fw.c @@ -481,12 +481,12 @@ static struct flb_config_map config_map[] = { { FLB_CONFIG_MAP_STR, "shared_key", NULL, 0, FLB_TRUE, offsetof(struct flb_in_fw_config, shared_key), - "Shared key for authentication" + "Shared key for secure forward authentication." }, { FLB_CONFIG_MAP_STR, "self_hostname", NULL, 0, FLB_FALSE, 0, - "Hostname" + "Hostname used in the handshake process for secure forward authentication." }, { FLB_CONFIG_MAP_STR, "security.users", NULL, @@ -501,7 +501,7 @@ static struct flb_config_map config_map[] = { { FLB_CONFIG_MAP_STR, "unix_perm", (char *)NULL, 0, FLB_TRUE, offsetof(struct flb_in_fw_config, unix_perm_str), - "Set the permissions for the UNIX socket" + "Set the permissions for the UNIX socket." }, { FLB_CONFIG_MAP_SIZE, "buffer_chunk_size", FLB_IN_FW_CHUNK_SIZE, @@ -516,7 +516,7 @@ static struct flb_config_map config_map[] = { { FLB_CONFIG_MAP_BOOL, "empty_shared_key", "false", 0, FLB_TRUE, offsetof(struct flb_in_fw_config, empty_shared_key), - "Set an empty shared key for authentication" + "Enable an empty string as the shared key for authentication." }, {0} }; From ec58a567438cc20ca57b667b05069fd2d26de636 Mon Sep 17 00:00:00 2001 From: Patrick Stephens Date: Sun, 3 May 2026 12:35:05 +0100 Subject: [PATCH 131/213] [upstream] build: prevent the toolchain from emitting an Upstream-Ref: https://github.com/fluent/fluent-bit/commit/f2dd991366b19c1ad504f6a92dd41a175c6f1110 Cherry-picked from Fluent Bit v4.2.4 --- source/CMakeLists.txt | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/source/CMakeLists.txt b/source/CMakeLists.txt index af51a0cc..3a6d5b11 100644 --- a/source/CMakeLists.txt +++ b/source/CMakeLists.txt @@ -83,6 +83,14 @@ if (MSVC) add_compile_options(/MT) else() set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wall") + if (CMAKE_SYSTEM_NAME STREQUAL "Linux") + # Prevent the toolchain from emitting an executable stack on Linux targets, + # which triggers kernel warnings (e.g. "started with executable stack") and + # weakens security hardening. The linker flag is not supported on macOS. + add_compile_options(-Wa,--noexecstack) + set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,-z,noexecstack") + set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -Wl,-z,noexecstack") + endif() # The following flags are to enhance security, but it may impact performance, # we disable them by default. endif() From d157b0dc47ed73c5c77817e0c4b5d7bf596597e8 Mon Sep 17 00:00:00 2001 From: Patrick Stephens Date: Sun, 3 May 2026 12:35:06 +0100 Subject: [PATCH 132/213] [upstream] in_forward: fix segfault and double-free in trace Upstream-Ref: https://github.com/fluent/fluent-bit/commit/8cc6da67f2c0925db95d129be131566a197ff4aa Cherry-picked from Fluent Bit v4.2.4 --- source/plugins/in_forward/fw_prot.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/source/plugins/in_forward/fw_prot.c b/source/plugins/in_forward/fw_prot.c index 3fb40918..03de8191 100644 --- a/source/plugins/in_forward/fw_prot.c +++ b/source/plugins/in_forward/fw_prot.c @@ -1146,8 +1146,8 @@ static int append_log(struct flb_input_instance *ins, struct fw_conn *conn, else if (event_type == FLB_EVENT_TYPE_TRACES) { off = 0; ret = ctr_decode_msgpack_create(&ctr, (char *) data, len, &off); - if (ret == -1) { - flb_error("could not decode trace message. ret=%d", ret); + if (ret != CTR_DECODE_MSGPACK_SUCCESS) { + flb_plg_error(ins, "could not decode trace message. ret=%d", ret); return -1; } @@ -1159,7 +1159,7 @@ static int append_log(struct flb_input_instance *ins, struct fw_conn *conn, ctr_decode_msgpack_destroy(ctr); return -1; } - ctr_decode_msgpack_destroy(ctr); + /* Note: flb_input_trace_append takes ownership of ctr and destroys it on success */ } return 0; From afc8ad0cc66c82e634c3b0bb2dcc2fd1c3ebfbd1 Mon Sep 17 00:00:00 2001 From: Patrick Stephens Date: Sun, 3 May 2026 12:35:06 +0100 Subject: [PATCH 133/213] [upstream] in_node_exporter_metrics: Increase buffer size to Upstream-Ref: https://github.com/fluent/fluent-bit/commit/7ded9ae04e1693718edb8bc88e8b8d154c77d314 Cherry-picked from Fluent Bit v4.2.4 --- source/plugins/in_node_exporter_metrics/ne_utils.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/plugins/in_node_exporter_metrics/ne_utils.c b/source/plugins/in_node_exporter_metrics/ne_utils.c index 0d2a6574..1bfe7425 100644 --- a/source/plugins/in_node_exporter_metrics/ne_utils.c +++ b/source/plugins/in_node_exporter_metrics/ne_utils.c @@ -153,7 +153,7 @@ int ne_utils_file_read_lines(const char *mount, const char *path, struct mk_list int len; int ret; FILE *f; - char line[512]; + char line[2048]; char real_path[2048]; mk_list_init(list); From b99c00a69ba39759dede1a153b176aaf2e171000 Mon Sep 17 00:00:00 2001 From: Patrick Stephens Date: Sun, 3 May 2026 12:35:06 +0100 Subject: [PATCH 134/213] [upstream] in_http: added missing config parameter descriptions. Upstream-Ref: https://github.com/fluent/fluent-bit/commit/12b191b5647e14d9adae28436ad4a50a3ea9dc43 Cherry-picked from Fluent Bit v4.2.4 --- source/plugins/in_http/http.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/source/plugins/in_http/http.c b/source/plugins/in_http/http.c index ff9687ed..f220f121 100644 --- a/source/plugins/in_http/http.c +++ b/source/plugins/in_http/http.c @@ -204,39 +204,39 @@ static struct flb_config_map config_map[] = { { FLB_CONFIG_MAP_BOOL, "http2", "true", 0, FLB_TRUE, offsetof(struct flb_http, enable_http2), - NULL + "Enable HTTP/2 support." }, { FLB_CONFIG_MAP_SIZE, "buffer_max_size", HTTP_BUFFER_MAX_SIZE, 0, FLB_TRUE, offsetof(struct flb_http, buffer_max_size), - "" + "Set the maximum buffer size to store incoming data." }, { FLB_CONFIG_MAP_SIZE, "buffer_chunk_size", HTTP_BUFFER_CHUNK_SIZE, 0, FLB_TRUE, offsetof(struct flb_http, buffer_chunk_size), - "" + "Set the initial buffer size to store incoming data." }, { FLB_CONFIG_MAP_SLIST_1, "success_header", NULL, FLB_CONFIG_MAP_MULT, FLB_TRUE, offsetof(struct flb_http, success_headers), - "Add an HTTP header key/value pair on success. Multiple headers can be set" + "Add an HTTP header key/value pair on success. Multiple headers can be set." }, { FLB_CONFIG_MAP_STR, "tag_key", NULL, 0, FLB_TRUE, offsetof(struct flb_http, tag_key), - "" + "Specify a key name for extracting the tag from incoming request data." }, + { FLB_CONFIG_MAP_INT, "successful_response_code", "201", 0, FLB_TRUE, offsetof(struct flb_http, successful_response_code), "Set successful response code. 200, 201 and 204 are supported." }, - /* EOF */ {0} }; From cc8fbd910190078b798de82f090d07511c3ee191 Mon Sep 17 00:00:00 2001 From: Patrick Stephens Date: Sun, 3 May 2026 12:35:06 +0100 Subject: [PATCH 135/213] [upstream] out_flowcounter: Fix incorrect bytes calculation Upstream-Ref: https://github.com/fluent/fluent-bit/commit/3a501b239fffffe199d769aaf36bb5d30fad2367 Cherry-picked from Fluent Bit v4.2.4 --- source/plugins/out_flowcounter/out_flowcounter.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/source/plugins/out_flowcounter/out_flowcounter.c b/source/plugins/out_flowcounter/out_flowcounter.c index 37965ddb..0fe0307a 100644 --- a/source/plugins/out_flowcounter/out_flowcounter.c +++ b/source/plugins/out_flowcounter/out_flowcounter.c @@ -216,6 +216,9 @@ static void out_fcount_flush(struct flb_event_chunk *event_chunk, while ((ret = flb_log_event_decoder_next( &log_decoder, &log_event)) == FLB_EVENT_DECODER_SUCCESS) { + /* bytes consumed in the event stream so far */ + off = log_decoder.offset; + if (ctx->event_based) { flb_time_copy(&tm, &log_event.timestamp); } @@ -225,6 +228,7 @@ static void out_fcount_flush(struct flb_event_chunk *event_chunk, t = tm.tm.tv_sec; if (time_is_valid(t, ctx) == FLB_FALSE) { flb_plg_warn(ctx->ins, "out of range. Skip the record"); + last_off = off; continue; } From 9b762d7321dac8134b1c1a36018d9866156fd160 Mon Sep 17 00:00:00 2001 From: Patrick Stephens Date: Sun, 3 May 2026 12:35:06 +0100 Subject: [PATCH 136/213] [upstream] in_ebpf: add missing config option descriptions Upstream-Ref: https://github.com/fluent/fluent-bit/commit/ee1d69dd4efb7f769d7d23208bd325cff7ddd6cc Cherry-picked from Fluent Bit v4.2.4 --- source/plugins/in_ebpf/in_ebpf.c | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/source/plugins/in_ebpf/in_ebpf.c b/source/plugins/in_ebpf/in_ebpf.c index 4bcb25f0..403f6df0 100644 --- a/source/plugins/in_ebpf/in_ebpf.c +++ b/source/plugins/in_ebpf/in_ebpf.c @@ -231,11 +231,22 @@ static int in_ebpf_exit(void *in_context, struct flb_config *config) { } static struct flb_config_map config_map[] = { - {FLB_CONFIG_MAP_STR, "ringbuf_map_name", FLB_IN_EBPF_DEFAULT_RINGBUF_MAP_NAME, 0, FLB_TRUE, - offsetof(struct flb_in_ebpf_context, ringbuf_map_name)}, - {FLB_CONFIG_MAP_INT, "poll_ms", FLB_IN_EBPF_DEFAULT_POLL_MS, 0, FLB_TRUE, - offsetof(struct flb_in_ebpf_context, poll_ms)}, - {FLB_CONFIG_MAP_STR, "Trace", NULL, FLB_CONFIG_MAP_MULT, FLB_FALSE, 0}, + { + FLB_CONFIG_MAP_STR, "ringbuf_map_name", FLB_IN_EBPF_DEFAULT_RINGBUF_MAP_NAME, + 0, FLB_TRUE, offsetof(struct flb_in_ebpf_context, ringbuf_map_name), + "Set the name of the eBPF ring buffer map to read events from" + }, + { + FLB_CONFIG_MAP_INT, "poll_ms", FLB_IN_EBPF_DEFAULT_POLL_MS, + 0, FLB_TRUE, offsetof(struct flb_in_ebpf_context, poll_ms), + "Set the polling interval in milliseconds for collecting events" + }, + { + FLB_CONFIG_MAP_STR, "Trace", NULL, + FLB_CONFIG_MAP_MULT, FLB_FALSE, 0, + "Set the eBPF trace to enable (for example, bind, malloc, signal). Can be set multiple times" + }, + /* EOF */ {0} }; From 4c75247116c457d586103f57502bbe3e49612af0 Mon Sep 17 00:00:00 2001 From: Patrick Stephens Date: Sun, 3 May 2026 12:35:07 +0100 Subject: [PATCH 137/213] [upstream] in_gpu_metrics: updated config parameters Upstream-Ref: https://github.com/fluent/fluent-bit/commit/275ef3bb68471e18829542eb510860843a868d5a Cherry-picked from Fluent Bit v4.2.4 --- source/plugins/in_gpu_metrics/gpu_metrics.c | 34 +++++++++++---------- 1 file changed, 18 insertions(+), 16 deletions(-) diff --git a/source/plugins/in_gpu_metrics/gpu_metrics.c b/source/plugins/in_gpu_metrics/gpu_metrics.c index e434258e..56cdda4b 100644 --- a/source/plugins/in_gpu_metrics/gpu_metrics.c +++ b/source/plugins/in_gpu_metrics/gpu_metrics.c @@ -168,34 +168,36 @@ static int in_gpu_exit(void *data, struct flb_config *config) static struct flb_config_map config_map[] = { { - FLB_CONFIG_MAP_TIME, "scrape_interval", "5", - 0, FLB_TRUE, offsetof(struct in_gpu_metrics, scrape_interval), - "Scrape interval for GPU metrics" - }, - { - FLB_CONFIG_MAP_STR, "path_sysfs", "/sys", - 0, FLB_TRUE, offsetof(struct in_gpu_metrics, path_sysfs), - "Path to sysfs" + FLB_CONFIG_MAP_STR, "cards_exclude", "", + 0, FLB_TRUE, offsetof(struct in_gpu_metrics, cards_exclude), + "Exclude GPU cards by ID. Accepts '*' for all, comma-separated IDs " + "(e.g., '0,1,2'), or ranges (e.g., '0-2')." }, { FLB_CONFIG_MAP_STR, "cards_include", "*", 0, FLB_TRUE, offsetof(struct in_gpu_metrics, cards_include), - "Cards to include" - }, - { - FLB_CONFIG_MAP_STR, "cards_exclude", "", - 0, FLB_TRUE, offsetof(struct in_gpu_metrics, cards_exclude), - "Cards to exclude" + "Include GPU cards by ID. Accepts '*' for all, comma-separated IDs " + "(e.g., '0,1,2'), or ranges (e.g., '0-2')." }, { FLB_CONFIG_MAP_BOOL, "enable_power", "true", 0, FLB_TRUE, offsetof(struct in_gpu_metrics, enable_power), - "Enable power metrics" + "Enable collection of GPU power consumption metrics (gpu_power_watts)." }, { FLB_CONFIG_MAP_BOOL, "enable_temperature", "true", 0, FLB_TRUE, offsetof(struct in_gpu_metrics, enable_temperature), - "Enable temperature metrics" + "Enable collection of GPU temperature metrics (gpu_temperature_celsius)." + }, + { + FLB_CONFIG_MAP_STR, "path_sysfs", "/sys", + 0, FLB_TRUE, offsetof(struct in_gpu_metrics, path_sysfs), + "sysfs mount point." + }, + { + FLB_CONFIG_MAP_TIME, "scrape_interval", "5", + 0, FLB_TRUE, offsetof(struct in_gpu_metrics, scrape_interval), + "Scrape interval to collect GPU metrics." }, {0} }; From 02e3c779b48796f0a16572389e7de9c07ade3e0d Mon Sep 17 00:00:00 2001 From: Patrick Stephens Date: Sun, 3 May 2026 12:35:07 +0100 Subject: [PATCH 138/213] [upstream] build: Tweak linker flags for zstd Upstream-Ref: https://github.com/fluent/fluent-bit/commit/6d6372b9ffd322c16ae5ac40ea938b7938675be9 Cherry-picked from Fluent Bit v4.2.4 --- source/CMakeLists.txt | 30 ++++++++++++++++++++++++------ 1 file changed, 24 insertions(+), 6 deletions(-) diff --git a/source/CMakeLists.txt b/source/CMakeLists.txt index 3a6d5b11..a162ac24 100644 --- a/source/CMakeLists.txt +++ b/source/CMakeLists.txt @@ -596,14 +596,32 @@ add_subdirectory(${FLB_PATH_LIB_MINIZ} EXCLUDE_FROM_ALL) # Zstd (zstd) if(FLB_PREFER_SYSTEM_LIB_ZSTD) - find_package(PkgConfig) - pkg_check_modules(LIBZSTD libzstd>=1.4.8) + find_package(ZSTD 1.4.8 QUIET) + if(ZSTD_FOUND) + set(LIBZSTD_FOUND TRUE) + set(LIBZSTD_LIBRARIES ZSTD::ZSTD) + else() + find_package(PkgConfig) + pkg_check_modules(LIBZSTD libzstd>=1.4.8) + if(LIBZSTD_FOUND) + include_directories(${LIBZSTD_INCLUDE_DIRS}) + link_directories(${LIBZSTD_LIBRARY_DIRS}) + add_library(ZSTD::ZSTD INTERFACE IMPORTED) + set_property(TARGET ZSTD::ZSTD PROPERTY + INTERFACE_LINK_LIBRARIES "${LIBZSTD_LIBRARIES}") + set(LIBZSTD_LIBRARIES ZSTD::ZSTD) + endif() + endif() endif() -if(LIBZSTD_FOUND) - include_directories(${LIBZSTD_INCLUDE_DIRS}) - link_directories(${LIBZSTD_LIBRARY_DIRS}) -else() + +if(NOT LIBZSTD_FOUND) include(cmake/zstd.cmake) + + if(NOT MSVC) + target_compile_options(libzstd_static PRIVATE -fvisibility=hidden) + endif() + + set(LIBZSTD_LIBRARIES libzstd_static) endif() # ring buffer library From 1269546238e738b9f36ffbeab690e0e2bc343e57 Mon Sep 17 00:00:00 2001 From: Patrick Stephens Date: Sun, 3 May 2026 12:35:07 +0100 Subject: [PATCH 139/213] [upstream] build: Use bundled namespace instead of the simple Upstream-Ref: https://github.com/fluent/fluent-bit/commit/5a96ef544ff52e6e2be7535dce7ddfcbad11c705 Cherry-picked from Fluent Bit v4.2.4 --- source/CMakeLists.txt | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/source/CMakeLists.txt b/source/CMakeLists.txt index a162ac24..44f73585 100644 --- a/source/CMakeLists.txt +++ b/source/CMakeLists.txt @@ -599,17 +599,17 @@ if(FLB_PREFER_SYSTEM_LIB_ZSTD) find_package(ZSTD 1.4.8 QUIET) if(ZSTD_FOUND) set(LIBZSTD_FOUND TRUE) - set(LIBZSTD_LIBRARIES ZSTD::ZSTD) + set(LIBZSTD_LIBRARIES BUNDLED::ZSTD) else() find_package(PkgConfig) pkg_check_modules(LIBZSTD libzstd>=1.4.8) if(LIBZSTD_FOUND) include_directories(${LIBZSTD_INCLUDE_DIRS}) link_directories(${LIBZSTD_LIBRARY_DIRS}) - add_library(ZSTD::ZSTD INTERFACE IMPORTED) - set_property(TARGET ZSTD::ZSTD PROPERTY + add_library(BUNDLED::ZSTD INTERFACE IMPORTED) + set_property(TARGET BUNDLED::ZSTD PROPERTY INTERFACE_LINK_LIBRARIES "${LIBZSTD_LIBRARIES}") - set(LIBZSTD_LIBRARIES ZSTD::ZSTD) + set(LIBZSTD_LIBRARIES BUNDLED::ZSTD) endif() endif() endif() From 54d0d3db8e14f5afa3cf5c524ceab5586e9b9850 Mon Sep 17 00:00:00 2001 From: Patrick Stephens Date: Sun, 3 May 2026 12:35:07 +0100 Subject: [PATCH 140/213] [upstream] build: Address fallbacking issue of finding libzstd Upstream-Ref: https://github.com/fluent/fluent-bit/commit/eb9f9bdb098f1c84dd9889dedbea33a7f5143f79 Cherry-picked from Fluent Bit v4.2.4 --- source/CMakeLists.txt | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/source/CMakeLists.txt b/source/CMakeLists.txt index 44f73585..1ea772b8 100644 --- a/source/CMakeLists.txt +++ b/source/CMakeLists.txt @@ -599,8 +599,28 @@ if(FLB_PREFER_SYSTEM_LIB_ZSTD) find_package(ZSTD 1.4.8 QUIET) if(ZSTD_FOUND) set(LIBZSTD_FOUND TRUE) - set(LIBZSTD_LIBRARIES BUNDLED::ZSTD) - else() + # Use the target provided by find_package(ZSTD) + # Modern FindZSTD provides ZSTD::libzstd_static or ZSTD::libzstd_shared + if(TARGET ZSTD::libzstd_static) + set(LIBZSTD_LIBRARIES ZSTD::libzstd_static) + elseif(TARGET ZSTD::libzstd_shared) + set(LIBZSTD_LIBRARIES ZSTD::libzstd_shared) + elseif(TARGET ZSTD::ZSTD) + set(LIBZSTD_LIBRARIES ZSTD::ZSTD) + else() + # Fallback: create a wrapper if no target is provided + add_library(BUNDLED::ZSTD INTERFACE IMPORTED) + if(ZSTD_LIBRARIES) + set_property(TARGET BUNDLED::ZSTD PROPERTY + INTERFACE_LINK_LIBRARIES "${ZSTD_LIBRARIES}") + endif() + if(ZSTD_INCLUDE_DIRS) + set_property(TARGET BUNDLED::ZSTD PROPERTY + INTERFACE_INCLUDE_DIRECTORIES "${ZSTD_INCLUDE_DIRS}") + endif() + set(LIBZSTD_LIBRARIES BUNDLED::ZSTD) + endif() + else() find_package(PkgConfig) pkg_check_modules(LIBZSTD libzstd>=1.4.8) if(LIBZSTD_FOUND) From a9aff7178e2ff13add1836a1153f5e1c3109a5bc Mon Sep 17 00:00:00 2001 From: Patrick Stephens Date: Sun, 3 May 2026 12:35:07 +0100 Subject: [PATCH 141/213] [upstream] in_winevtlog: Plug glitches of time offset for Upstream-Ref: https://github.com/fluent/fluent-bit/commit/282081f42c919f2acdbf5973a11e44213525e06a Cherry-picked from Fluent Bit v4.2.4 --- source/plugins/in_winevtlog/pack.c | 80 +++++++++++++++++++++++------- 1 file changed, 63 insertions(+), 17 deletions(-) diff --git a/source/plugins/in_winevtlog/pack.c b/source/plugins/in_winevtlog/pack.c index ecffc648..da69f724 100644 --- a/source/plugins/in_winevtlog/pack.c +++ b/source/plugins/in_winevtlog/pack.c @@ -23,6 +23,7 @@ #include #include #include +#include #include #include #include @@ -265,12 +266,18 @@ static int pack_systemtime(struct winevtlog_config *ctx, SYSTEMTIME *st) static int pack_filetime(struct winevtlog_config *ctx, ULONGLONG filetime) { - LARGE_INTEGER timestamp; + FILETIME ft; + SYSTEMTIME st_utc; + SYSTEMTIME st_local; + DYNAMIC_TIME_ZONE_INFORMATION dtzi; CHAR buf[64]; - size_t len = 0; - FILETIME ft, ft_local; - SYSTEMTIME st; + size_t len; + LONG bias_minutes; + int offset_hours; + int offset_minutes; + char offset_sign; _locale_t locale; + DWORD tz_id; _tzset(); @@ -278,26 +285,65 @@ static int pack_filetime(struct winevtlog_config *ctx, ULONGLONG filetime) if (locale == NULL) { return -1; } - timestamp.QuadPart = filetime; - ft.dwHighDateTime = timestamp.HighPart; - ft.dwLowDateTime = timestamp.LowPart; - FileTimeToLocalFileTime(&ft, &ft_local); - if (FileTimeToSystemTime(&ft_local, &st)) { - struct tm tm = {st.wSecond, st.wMinute, st.wHour, st.wDay, st.wMonth-1, st.wYear-1900, st.wDayOfWeek, 0, -1}; - len = _strftime_l(buf, 64, FORMAT_ISO8601, &tm, locale); - if (len == 0) { - flb_errno(); - _free_locale(locale); - return -1; - } + + ft.dwHighDateTime = (DWORD)(filetime >> 32); + ft.dwLowDateTime = (DWORD)(filetime & 0xFFFFFFFF); + + if (!FileTimeToSystemTime(&ft, &st_utc)) { + _free_locale(locale); + return -1; + } + + tz_id = GetDynamicTimeZoneInformation(&dtzi); + + if (!SystemTimeToTzSpecificLocalTimeEx(&dtzi, &st_utc, &st_local)) { _free_locale(locale); + return -1; + } - flb_log_event_encoder_append_body_string(ctx->log_encoder, buf, len); + /* Determine bias (minutes) */ + bias_minutes = dtzi.Bias; + + if (tz_id == TIME_ZONE_ID_DAYLIGHT) { + bias_minutes += dtzi.DaylightBias; + } + else if (tz_id == TIME_ZONE_ID_STANDARD) { + bias_minutes += dtzi.StandardBias; + } + + /* Windows bias: minutes to add to local to get UTC + ISO8601 offset: inverse sign */ + if (bias_minutes > 0) { + offset_sign = '-'; } else { + offset_sign = '+'; + bias_minutes = -bias_minutes; + } + + offset_hours = bias_minutes / 60; + offset_minutes = bias_minutes % 60; + + len = _snprintf_s(buf, sizeof(buf), _TRUNCATE, + "%04d-%02d-%02d %02d:%02d:%02d %c%02d%02d", + st_local.wYear, + st_local.wMonth, + st_local.wDay, + st_local.wHour, + st_local.wMinute, + st_local.wSecond, + offset_sign, + offset_hours, + offset_minutes); + + if (len <= 0) { + _free_locale(locale); return -1; } + _free_locale(locale); + + flb_log_event_encoder_append_body_string(ctx->log_encoder, buf, len); return 0; } From f00f1d1302d10c25d46696c67505b9c235ba60ed Mon Sep 17 00:00:00 2001 From: Patrick Stephens Date: Sun, 3 May 2026 12:35:07 +0100 Subject: [PATCH 142/213] [upstream] in_winevtlog: Remove needless locale related lines Upstream-Ref: https://github.com/fluent/fluent-bit/commit/909b1a917b4f46c40657a73821541fa03581a823 Cherry-picked from Fluent Bit v4.2.4 --- source/plugins/in_winevtlog/pack.c | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/source/plugins/in_winevtlog/pack.c b/source/plugins/in_winevtlog/pack.c index da69f724..688f3ae1 100644 --- a/source/plugins/in_winevtlog/pack.c +++ b/source/plugins/in_winevtlog/pack.c @@ -276,28 +276,20 @@ static int pack_filetime(struct winevtlog_config *ctx, ULONGLONG filetime) int offset_hours; int offset_minutes; char offset_sign; - _locale_t locale; DWORD tz_id; _tzset(); - locale = _get_current_locale(); - if (locale == NULL) { - return -1; - } - ft.dwHighDateTime = (DWORD)(filetime >> 32); ft.dwLowDateTime = (DWORD)(filetime & 0xFFFFFFFF); if (!FileTimeToSystemTime(&ft, &st_utc)) { - _free_locale(locale); return -1; } tz_id = GetDynamicTimeZoneInformation(&dtzi); if (!SystemTimeToTzSpecificLocalTimeEx(&dtzi, &st_utc, &st_local)) { - _free_locale(locale); return -1; } @@ -337,12 +329,9 @@ static int pack_filetime(struct winevtlog_config *ctx, ULONGLONG filetime) offset_minutes); if (len <= 0) { - _free_locale(locale); return -1; } - _free_locale(locale); - flb_log_event_encoder_append_body_string(ctx->log_encoder, buf, len); return 0; } From e3f2918b33c74cd775d189fcd033e6c3c74b35f5 Mon Sep 17 00:00:00 2001 From: Patrick Stephens Date: Sun, 3 May 2026 12:35:08 +0100 Subject: [PATCH 143/213] [upstream] out_cloudwatch_logs: increase MAX_EVENT_LEN to 1MB Upstream-Ref: https://github.com/fluent/fluent-bit/commit/c994082fdb800a3d4dcd940973b886ea60f35a5e Cherry-picked from Fluent Bit v4.2.4 --- .../out_cloudwatch_logs/cloudwatch_api.h | 20 ++- source/tests/runtime/out_cloudwatch.c | 149 ++++++++++++++++++ 2 files changed, 164 insertions(+), 5 deletions(-) diff --git a/source/plugins/out_cloudwatch_logs/cloudwatch_api.h b/source/plugins/out_cloudwatch_logs/cloudwatch_api.h index 837e4b85..5c262eae 100644 --- a/source/plugins/out_cloudwatch_logs/cloudwatch_api.h +++ b/source/plugins/out_cloudwatch_logs/cloudwatch_api.h @@ -21,9 +21,10 @@ #define FLB_OUT_CLOUDWATCH_API /* - * The CloudWatch API documents that the maximum payload is 1,048,576 bytes - * For reasons that are under investigation, using that number in this plugin - * leads to API errors. No issues have been seen setting it to 1,000,000 bytes. + * The CloudWatch API documents that the maximum payload is 1,048,576 bytes. + * This is the total size limit for the entire PutLogEvents request payload buffer. + * Individual events are capped at MAX_EVENT_LEN (1,000,000 bytes) as a conservative + * safety margin to account for JSON encoding overhead and per-event metadata. */ #define PUT_LOG_EVENTS_PAYLOAD_SIZE 1048576 #define MAX_EVENTS_PER_PUT 10000 @@ -43,8 +44,17 @@ /* Maximum number of character limits including both the Attributes key and its value */ #define ATTRIBUTES_MAX_LEN 300 -/* 256KiB minus 26 bytes for the event */ -#define MAX_EVENT_LEN 262118 +/* + * https://docs.aws.amazon.com/AmazonCloudWatchLogs/latest/APIReference/API_PutLogEvents.html + * AWS CloudWatch's documented maximum event size is 1,048,576 bytes (1 MiB), + * including JSON encoding overhead (structure, escaping, etc.). + * + * Setting MAX_EVENT_LEN to 1,000,000 bytes (1 MB) provides a ~4.6% safety margin + * to account for JSON encoding overhead and ensure reliable operation. + * Testing confirmed messages up to 1,048,546 bytes (encoding to 1,048,586 bytes) + * succeed, though we use a conservative limit for production safety. + */ +#define MAX_EVENT_LEN 1000000 /* Prefix used for entity fields only */ #define AWS_ENTITY_PREFIX "aws_entity" diff --git a/source/tests/runtime/out_cloudwatch.c b/source/tests/runtime/out_cloudwatch.c index 35d785f4..41fc4fa9 100644 --- a/source/tests/runtime/out_cloudwatch.c +++ b/source/tests/runtime/out_cloudwatch.c @@ -5,10 +5,17 @@ /* Test data */ #include "data/td/json_td.h" /* JSON_TD */ +/* CloudWatch API constants */ +#include "../../plugins/out_cloudwatch_logs/cloudwatch_api.h" + #define ERROR_ALREADY_EXISTS "{\"__type\":\"ResourceAlreadyExistsException\"}" /* not a real error code, but tests that the code can respond to any error */ #define ERROR_UNKNOWN "{\"__type\":\"UNKNOWN\"}" +/* JSON structure constants for test message generation */ +static const char *TEST_JSON_PREFIX = "{\"message\":\""; +static const char *TEST_JSON_SUFFIX = "\"}"; + /* It writes a big JSON message (copied from TD test) */ void flb_test_cloudwatch_success(void) { @@ -393,6 +400,145 @@ void flb_test_cloudwatch_error_put_retention_policy(void) flb_destroy(ctx); } +/* Helper function to create a large JSON message of specified size */ +static char* create_large_json_message(size_t target_size) +{ + size_t prefix_len = strlen(TEST_JSON_PREFIX); + size_t suffix_len = strlen(TEST_JSON_SUFFIX); + size_t overhead = prefix_len + suffix_len; + size_t data_size; + char *json; + + /* Reject target_size too small for valid JSON structure */ + if (target_size < overhead + 1) { + return NULL; + } + + json = flb_malloc(target_size + 1); + if (!json) { + return NULL; + } + + /* Build JSON: prefix + data + suffix */ + memcpy(json, TEST_JSON_PREFIX, prefix_len); + data_size = target_size - overhead; + + /* Fill with 'A' characters */ + memset(json + prefix_len, 'A', data_size); + + /* Close JSON object */ + memcpy(json + prefix_len + data_size, TEST_JSON_SUFFIX, suffix_len); + json[target_size] = '\0'; + + /* Caller must free */ + return json; +} + +/* Helper to setup and run a CloudWatch test with custom JSON data */ +static void run_cloudwatch_test_with_data(char *data, size_t data_len) +{ + int ret; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + setenv("FLB_CLOUDWATCH_PLUGIN_UNDER_TEST", "true", 1); + + ctx = flb_create(); + TEST_CHECK(ctx != NULL); + + in_ffd = flb_input(ctx, (char *) "lib", NULL); + TEST_CHECK(in_ffd >= 0); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + out_ffd = flb_output(ctx, (char *) "cloudwatch_logs", NULL); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd, "match", "test", NULL); + flb_output_set(ctx, out_ffd, "region", "us-west-2", NULL); + flb_output_set(ctx, out_ffd, "log_group_name", "fluent", NULL); + flb_output_set(ctx, out_ffd, "log_stream_prefix", "from-fluent-", NULL); + flb_output_set(ctx, out_ffd, "auto_create_group", "On", NULL); + flb_output_set(ctx, out_ffd, "net.keepalive", "Off", NULL); + flb_output_set(ctx, out_ffd, "Retry_Limit", "1", NULL); + + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + if (data) { + flb_lib_push(ctx, in_ffd, data, data_len); + } + + sleep(2); + flb_stop(ctx); + flb_destroy(ctx); +} + +/* Test event size at maximum allowed limit (should succeed without truncation) */ +void flb_test_cloudwatch_event_size_at_limit(void) +{ + char *large_json; + + /* Create message at MAX_EVENT_LEN */ + large_json = create_large_json_message(MAX_EVENT_LEN); + TEST_CHECK(large_json != NULL); + + if (large_json) { + run_cloudwatch_test_with_data(large_json, strlen(large_json)); + flb_free(large_json); + } +} + +/* Test event size exceeding limit (should be truncated to MAX_EVENT_LEN) */ +void flb_test_cloudwatch_event_size_over_limit(void) +{ + char *large_json; + + /* Create message exceeding MAX_EVENT_LEN by 1 byte to test truncation */ + large_json = create_large_json_message(MAX_EVENT_LEN + 1); + TEST_CHECK(large_json != NULL); + + if (large_json) { + run_cloudwatch_test_with_data(large_json, strlen(large_json)); + flb_free(large_json); + } +} + +/* Test event with trailing backslash at truncation boundary */ +void flb_test_cloudwatch_event_truncation_with_backslash(void) +{ + char *large_json; + size_t prefix_len = strlen(TEST_JSON_PREFIX); + size_t suffix_len = strlen(TEST_JSON_SUFFIX); + size_t total_len; + size_t data_len; + size_t i; + + /* Create base message exceeding MAX_EVENT_LEN */ + large_json = create_large_json_message(MAX_EVENT_LEN + 100); + TEST_CHECK(large_json != NULL); + + if (large_json) { + total_len = strlen(large_json); + data_len = total_len - prefix_len - suffix_len; + + /* Replace pairs of characters with valid escape sequence "\\" */ + for (i = 98; i < data_len - 1; i += 100) { + large_json[prefix_len + i] = '\\'; + large_json[prefix_len + i + 1] = '\\'; + } + + size_t boundary = MAX_EVENT_LEN - 1; /* index in full JSON string */ + /* Ensure a backslash is at the exact truncation boundary */ + if (boundary + 1 < total_len - suffix_len) { + large_json[boundary] = '\\'; + large_json[boundary + 1] = '\\'; + } + + run_cloudwatch_test_with_data(large_json, strlen(large_json)); + flb_free(large_json); + } +} + /* Test list */ TEST_LIST = { {"success", flb_test_cloudwatch_success }, @@ -405,5 +551,8 @@ TEST_LIST = { {"put_retention_policy_success", flb_test_cloudwatch_put_retention_policy_success }, {"already_exists_create_group_put_retention_policy", flb_test_cloudwatch_already_exists_create_group_put_retention_policy }, {"error_put_retention_policy", flb_test_cloudwatch_error_put_retention_policy }, + {"event_size_at_limit", flb_test_cloudwatch_event_size_at_limit }, + {"event_size_over_limit", flb_test_cloudwatch_event_size_over_limit }, + {"event_truncation_with_backslash", flb_test_cloudwatch_event_truncation_with_backslash }, {NULL, NULL} }; From 9a8d44555455256ced577eaa49f4230a57b0e53b Mon Sep 17 00:00:00 2001 From: Patrick Stephens Date: Sun, 3 May 2026 12:35:08 +0100 Subject: [PATCH 144/213] [upstream] aws: Implement simple_aggregation operation Upstream-Ref: https://github.com/fluent/fluent-bit/commit/3da0766fece3abc14ee92be1c5bf476ec7fe2d61 Cherry-picked from Fluent Bit v4.2.4 --- .../fluent-bit/aws/flb_aws_aggregation.h | 90 +++++++ source/src/aws/CMakeLists.txt | 1 + source/src/aws/flb_aws_aggregation.c | 234 ++++++++++++++++++ 3 files changed, 325 insertions(+) create mode 100644 source/include/fluent-bit/aws/flb_aws_aggregation.h create mode 100644 source/src/aws/flb_aws_aggregation.c diff --git a/source/include/fluent-bit/aws/flb_aws_aggregation.h b/source/include/fluent-bit/aws/flb_aws_aggregation.h new file mode 100644 index 00000000..c9a64f23 --- /dev/null +++ b/source/include/fluent-bit/aws/flb_aws_aggregation.h @@ -0,0 +1,90 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2025 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_AWS_AGGREGATION_H +#define FLB_AWS_AGGREGATION_H + +#include +#include + +/* Aggregation buffer structure */ +struct flb_aws_agg_buffer { + char *agg_buf; /* aggregated records buffer */ + size_t agg_buf_size; /* total size of aggregation buffer */ + size_t agg_buf_offset; /* current offset in aggregation buffer */ +}; + +/* Initialize aggregation buffer + * Returns: + * 0 = success + * -1 = error + */ +int flb_aws_aggregation_init(struct flb_aws_agg_buffer *buf, size_t max_record_size); + +/* Destroy aggregation buffer */ +void flb_aws_aggregation_destroy(struct flb_aws_agg_buffer *buf); + +/* Try to add event data to aggregation buffer + * Returns: + * 0 = success, event added to aggregation buffer + * 1 = buffer full, caller should finalize and retry + */ +int flb_aws_aggregation_add(struct flb_aws_agg_buffer *buf, + const char *data, size_t data_len, + size_t max_record_size); + +/* Finalize aggregated record + * Returns: + * 0 = success + * -1 = error (no data to finalize) + * + * Output is written to buf->agg_buf and the size is returned via out_size parameter + */ +int flb_aws_aggregation_finalize(struct flb_aws_agg_buffer *buf, + int add_final_newline, + size_t *out_size); + +/* Reset aggregation buffer for reuse */ +void flb_aws_aggregation_reset(struct flb_aws_agg_buffer *buf); + +/* Process event with simple aggregation + * Converts msgpack to JSON, optionally adds log_key and time_key, + * then adds to aggregation buffer + * + * Returns: + * -1 = failure, record not added + * 0 = success, record added + * 1 = buffer full, caller should finalize and retry + * 2 = record could not be processed, discard it + */ +int flb_aws_aggregation_process_event(struct flb_aws_agg_buffer *agg_buf, + char *tmp_buf, + size_t tmp_buf_size, + size_t *tmp_buf_offset, + const msgpack_object *obj, + struct flb_time *tms, + struct flb_config *config, + struct flb_output_instance *ins, + const char *stream_name, + const char *log_key, + const char *time_key, + const char *time_key_format, + size_t max_event_size); + +#endif diff --git a/source/src/aws/CMakeLists.txt b/source/src/aws/CMakeLists.txt index 941e811b..a8d1bdf7 100644 --- a/source/src/aws/CMakeLists.txt +++ b/source/src/aws/CMakeLists.txt @@ -15,6 +15,7 @@ set(src "flb_aws_imds.c" "flb_aws_credentials_http.c" "flb_aws_credentials_profile.c" + "flb_aws_aggregation.c" ) message(STATUS "=== AWS Credentials ===") diff --git a/source/src/aws/flb_aws_aggregation.c b/source/src/aws/flb_aws_aggregation.c new file mode 100644 index 00000000..c27b8d4d --- /dev/null +++ b/source/src/aws/flb_aws_aggregation.c @@ -0,0 +1,234 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2025 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include +#include + +#include +#include + +int flb_aws_aggregation_init(struct flb_aws_agg_buffer *buf, size_t max_record_size) +{ + if (!buf) { + return -1; + } + + buf->agg_buf = flb_malloc(max_record_size); + if (!buf->agg_buf) { + flb_errno(); + return -1; + } + + buf->agg_buf_size = max_record_size; + buf->agg_buf_offset = 0; + + return 0; +} + +void flb_aws_aggregation_destroy(struct flb_aws_agg_buffer *buf) +{ + if (buf && buf->agg_buf) { + flb_free(buf->agg_buf); + buf->agg_buf = NULL; + buf->agg_buf_size = 0; + buf->agg_buf_offset = 0; + } +} + +int flb_aws_aggregation_add(struct flb_aws_agg_buffer *buf, + const char *data, size_t data_len, + size_t max_record_size) +{ + if (!buf || !data || data_len == 0) { + return -1; + } + + /* Check if adding this data would exceed the max record size */ + if (buf->agg_buf_offset + data_len > max_record_size) { + /* Buffer full, caller should finalize and retry */ + return 1; + } + + /* Add data to aggregation buffer */ + memcpy(buf->agg_buf + buf->agg_buf_offset, data, data_len); + buf->agg_buf_offset += data_len; + + return 0; +} + +int flb_aws_aggregation_finalize(struct flb_aws_agg_buffer *buf, + int add_final_newline, + size_t *out_size) +{ + if (!buf || !out_size) { + return -1; + } + + /* Check if there's any data to finalize */ + if (buf->agg_buf_offset == 0) { + return -1; + } + + /* Add final newline if requested (for Firehose) */ + if (add_final_newline && buf->agg_buf_offset < buf->agg_buf_size) { + buf->agg_buf[buf->agg_buf_offset] = '\n'; + buf->agg_buf_offset++; + } + + *out_size = buf->agg_buf_offset; + return 0; +} + +void flb_aws_aggregation_reset(struct flb_aws_agg_buffer *buf) +{ + if (buf) { + buf->agg_buf_offset = 0; + } +} + +/* + * Process event with simple aggregation + * Shared implementation for Kinesis Streams and Firehose + */ +int flb_aws_aggregation_process_event(struct flb_aws_agg_buffer *agg_buf, + char *tmp_buf, + size_t tmp_buf_size, + size_t *tmp_buf_offset, + const msgpack_object *obj, + struct flb_time *tms, + struct flb_config *config, + struct flb_output_instance *ins, + const char *stream_name, + const char *log_key, + const char *time_key, + const char *time_key_format, + size_t max_event_size) +{ + size_t written = 0; + int ret; + char *tmp_buf_ptr; + char *time_key_ptr; + struct tm time_stamp; + struct tm *tmp; + size_t len; + size_t tmp_size; + char *out_buf; + + tmp_buf_ptr = tmp_buf + *tmp_buf_offset; + ret = flb_msgpack_to_json(tmp_buf_ptr, + tmp_buf_size - *tmp_buf_offset, + obj, config->json_escape_unicode); + if (ret <= 0) { + return 1; + } + written = (size_t) ret; + + /* Discard empty messages */ + if (written <= 2) { + flb_plg_debug(ins, "Found empty log message, %s", stream_name); + return 2; + } + + if (log_key) { + written -= 2; + tmp_buf_ptr++; + (*tmp_buf_offset)++; + } + + if ((written + 1) >= max_event_size) { + flb_plg_warn(ins, "[size=%zu] Discarding record which is larger than " + "max size allowed, %s", written + 1, stream_name); + return 2; + } + + if (time_key) { + tmp = gmtime_r(&tms->tm.tv_sec, &time_stamp); + if (!tmp) { + flb_plg_error(ins, "Could not create time stamp for %lu unix " + "seconds, discarding record, %s", tms->tm.tv_sec, stream_name); + return 2; + } + + len = flb_aws_strftime_precision(&out_buf, time_key_format, tms); + tmp_size = (tmp_buf_size - *tmp_buf_offset) - written; + if (len > tmp_size) { + flb_free(out_buf); + return 1; + } + + if (len == 0) { + flb_plg_error(ins, "Failed to add time_key %s to record, %s", + time_key, stream_name); + flb_free(out_buf); + return 2; + } + else { + time_key_ptr = tmp_buf_ptr + written - 1; + memcpy(time_key_ptr, ",", 1); + time_key_ptr++; + memcpy(time_key_ptr, "\"", 1); + time_key_ptr++; + memcpy(time_key_ptr, time_key, strlen(time_key)); + time_key_ptr += strlen(time_key); + memcpy(time_key_ptr, "\":\"", 3); + time_key_ptr += 3; + + memcpy(time_key_ptr, out_buf, len); + flb_free(out_buf); + time_key_ptr += len; + memcpy(time_key_ptr, "\"}", 2); + time_key_ptr += 2; + written = (time_key_ptr - tmp_buf_ptr); + } + } + + if ((written + 1) >= max_event_size) { + flb_plg_warn(ins, "[size=%zu] Discarding record which is larger than " + "max size allowed, %s", written + 1, stream_name); + return 2; + } + + /* Append newline */ + tmp_size = (tmp_buf_size - *tmp_buf_offset) - written; + if (tmp_size <= 1) { + return 1; + } + + memcpy(tmp_buf_ptr + written, "\n", 1); + written++; + + /* Try to add to aggregation buffer */ + tmp_buf_ptr = tmp_buf + *tmp_buf_offset; + ret = flb_aws_aggregation_add(agg_buf, tmp_buf_ptr, written, max_event_size); + + if (ret == 1) { + return 1; + } + else if (ret < 0) { + flb_plg_error(ins, "Failed to add record to aggregation buffer"); + return -1; + } + + *tmp_buf_offset += written; + return 0; +} From 458759f069a49a8824e077a70594676e5b83c0b2 Mon Sep 17 00:00:00 2001 From: Patrick Stephens Date: Sun, 3 May 2026 12:35:08 +0100 Subject: [PATCH 145/213] [upstream] tests: aws_aggregation Upstream-Ref: https://github.com/fluent/fluent-bit/commit/f6c1c69fdfa2115f547af29cde02d47605874505 Cherry-picked from Fluent Bit v4.2.4 --- source/tests/internal/CMakeLists.txt | 1 + source/tests/internal/aws_aggregation.c | 544 ++++++++++++++++++++++++ 2 files changed, 545 insertions(+) create mode 100644 source/tests/internal/aws_aggregation.c diff --git a/source/tests/internal/CMakeLists.txt b/source/tests/internal/CMakeLists.txt index fe6c1dcf..d9b91ddf 100644 --- a/source/tests/internal/CMakeLists.txt +++ b/source/tests/internal/CMakeLists.txt @@ -144,6 +144,7 @@ if(FLB_AWS) ${UNIT_TESTS_FILES} aws_util.c aws_compress.c + aws_aggregation.c aws_credentials.c aws_credentials_ec2.c aws_credentials_sts.c diff --git a/source/tests/internal/aws_aggregation.c b/source/tests/internal/aws_aggregation.c new file mode 100644 index 00000000..a29ca5ec --- /dev/null +++ b/source/tests/internal/aws_aggregation.c @@ -0,0 +1,544 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2025 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include "flb_tests_internal.h" + +#include + +#define MAX_RECORD_SIZE 1024000 + +/* Test: Initialize and destroy aggregation buffer */ +void test_aws_aggregation_init_destroy() +{ + struct flb_aws_agg_buffer buf; + int ret; + + /* Test successful initialization */ + ret = flb_aws_aggregation_init(&buf, MAX_RECORD_SIZE); + TEST_CHECK(ret == 0); + TEST_CHECK(buf.agg_buf != NULL); + TEST_CHECK(buf.agg_buf_size == MAX_RECORD_SIZE); + TEST_CHECK(buf.agg_buf_offset == 0); + + /* Test destroy */ + flb_aws_aggregation_destroy(&buf); + TEST_CHECK(buf.agg_buf == NULL); + TEST_CHECK(buf.agg_buf_size == 0); + TEST_CHECK(buf.agg_buf_offset == 0); +} + +/* Test: Add single record to buffer */ +void test_aws_aggregation_add_single() +{ + struct flb_aws_agg_buffer buf; + int ret; + const char *data = "{\"message\":\"test\"}\n"; + size_t data_len = strlen(data); + + ret = flb_aws_aggregation_init(&buf, MAX_RECORD_SIZE); + TEST_CHECK(ret == 0); + + /* Add single record */ + ret = flb_aws_aggregation_add(&buf, data, data_len, MAX_RECORD_SIZE); + TEST_CHECK(ret == 0); + TEST_CHECK(buf.agg_buf_offset == data_len); + + /* Verify content */ + TEST_CHECK(memcmp(buf.agg_buf, data, data_len) == 0); + + flb_aws_aggregation_destroy(&buf); +} + +/* Test: Add multiple records to buffer */ +void test_aws_aggregation_add_multiple() +{ + struct flb_aws_agg_buffer buf; + int ret; + const char *data1 = "{\"message\":\"test1\"}\n"; + const char *data2 = "{\"message\":\"test2\"}\n"; + const char *data3 = "{\"message\":\"test3\"}\n"; + size_t len1 = strlen(data1); + size_t len2 = strlen(data2); + size_t len3 = strlen(data3); + + ret = flb_aws_aggregation_init(&buf, MAX_RECORD_SIZE); + TEST_CHECK(ret == 0); + + /* Add first record */ + ret = flb_aws_aggregation_add(&buf, data1, len1, MAX_RECORD_SIZE); + TEST_CHECK(ret == 0); + TEST_CHECK(buf.agg_buf_offset == len1); + + /* Add second record */ + ret = flb_aws_aggregation_add(&buf, data2, len2, MAX_RECORD_SIZE); + TEST_CHECK(ret == 0); + TEST_CHECK(buf.agg_buf_offset == len1 + len2); + + /* Add third record */ + ret = flb_aws_aggregation_add(&buf, data3, len3, MAX_RECORD_SIZE); + TEST_CHECK(ret == 0); + TEST_CHECK(buf.agg_buf_offset == len1 + len2 + len3); + + /* Verify all content is concatenated */ + TEST_CHECK(memcmp(buf.agg_buf, data1, len1) == 0); + TEST_CHECK(memcmp(buf.agg_buf + len1, data2, len2) == 0); + TEST_CHECK(memcmp(buf.agg_buf + len1 + len2, data3, len3) == 0); + + flb_aws_aggregation_destroy(&buf); +} + +/* Test: Buffer full detection */ +void test_aws_aggregation_buffer_full() +{ + struct flb_aws_agg_buffer buf; + int ret; + size_t small_size = 100; + char data[150]; + + /* Initialize with small buffer */ + ret = flb_aws_aggregation_init(&buf, small_size); + TEST_CHECK(ret == 0); + + /* Create data larger than buffer */ + memset(data, 'A', sizeof(data) - 1); + data[sizeof(data) - 1] = '\0'; + + /* Try to add data that exceeds buffer size */ + ret = flb_aws_aggregation_add(&buf, data, sizeof(data) - 1, small_size); + TEST_CHECK(ret == 1); /* Should return 1 (buffer full) */ + + flb_aws_aggregation_destroy(&buf); +} + +/* Test: Buffer full with multiple adds */ +void test_aws_aggregation_buffer_full_multiple() +{ + struct flb_aws_agg_buffer buf; + int ret; + size_t small_size = 100; + const char *data = "0123456789"; /* 10 bytes */ + size_t data_len = strlen(data); + int i; + + ret = flb_aws_aggregation_init(&buf, small_size); + TEST_CHECK(ret == 0); + + /* Add records until buffer is full */ + for (i = 0; i < 9; i++) { + ret = flb_aws_aggregation_add(&buf, data, data_len, small_size); + TEST_CHECK(ret == 0); /* Should succeed */ + } + + /* This should fill the buffer (90 bytes used) */ + TEST_CHECK(buf.agg_buf_offset == 90); + + /* Try to add one more (would be 100 bytes, at limit) */ + ret = flb_aws_aggregation_add(&buf, data, data_len, small_size); + TEST_CHECK(ret == 0); /* Should succeed, exactly at limit */ + + /* Now buffer is full, next add should fail */ + ret = flb_aws_aggregation_add(&buf, data, data_len, small_size); + TEST_CHECK(ret == 1); /* Should return 1 (buffer full) */ + + flb_aws_aggregation_destroy(&buf); +} + +/* Test: Finalize with both modes (with and without newline) */ +void test_aws_aggregation_finalize() +{ + struct flb_aws_agg_buffer buf; + int ret; + const char *data = "{\"message\":\"test\"}\n"; + size_t data_len = strlen(data); + size_t out_size; + + /* Test without newline (Kinesis Streams mode) */ + ret = flb_aws_aggregation_init(&buf, MAX_RECORD_SIZE); + TEST_CHECK(ret == 0); + + ret = flb_aws_aggregation_add(&buf, data, data_len, MAX_RECORD_SIZE); + TEST_CHECK(ret == 0); + + ret = flb_aws_aggregation_finalize(&buf, 0, &out_size); + TEST_CHECK(ret == 0); + TEST_CHECK(out_size == data_len); + + flb_aws_aggregation_destroy(&buf); + + /* Test with newline (Firehose mode) */ + ret = flb_aws_aggregation_init(&buf, MAX_RECORD_SIZE); + TEST_CHECK(ret == 0); + + ret = flb_aws_aggregation_add(&buf, data, data_len, MAX_RECORD_SIZE); + TEST_CHECK(ret == 0); + + ret = flb_aws_aggregation_finalize(&buf, 1, &out_size); + TEST_CHECK(ret == 0); + TEST_CHECK(out_size == data_len + 1); + TEST_CHECK(buf.agg_buf[data_len] == '\n'); + + flb_aws_aggregation_destroy(&buf); +} + +/* Test: Finalize empty buffer */ +void test_aws_aggregation_finalize_empty() +{ + struct flb_aws_agg_buffer buf; + int ret; + size_t out_size; + + ret = flb_aws_aggregation_init(&buf, MAX_RECORD_SIZE); + TEST_CHECK(ret == 0); + + /* Try to finalize empty buffer */ + ret = flb_aws_aggregation_finalize(&buf, 1, &out_size); + TEST_CHECK(ret == -1); /* Should fail */ + + flb_aws_aggregation_destroy(&buf); +} + +/* Test: Reset and reuse buffer (complete cycle) */ +void test_aws_aggregation_reset_reuse() +{ + struct flb_aws_agg_buffer buf; + int ret; + const char *data1 = "{\"message\":\"test1\"}\n"; + const char *data2 = "{\"message\":\"test2\"}\n"; + size_t len1 = strlen(data1); + size_t len2 = strlen(data2); + size_t out_size; + + ret = flb_aws_aggregation_init(&buf, MAX_RECORD_SIZE); + TEST_CHECK(ret == 0); + + /* First batch */ + ret = flb_aws_aggregation_add(&buf, data1, len1, MAX_RECORD_SIZE); + TEST_CHECK(ret == 0); + + ret = flb_aws_aggregation_finalize(&buf, 1, &out_size); + TEST_CHECK(ret == 0); + + /* Reset for reuse */ + flb_aws_aggregation_reset(&buf); + TEST_CHECK(buf.agg_buf_offset == 0); + + /* Second batch */ + ret = flb_aws_aggregation_add(&buf, data2, len2, MAX_RECORD_SIZE); + TEST_CHECK(ret == 0); + TEST_CHECK(buf.agg_buf_offset == len2); + + ret = flb_aws_aggregation_finalize(&buf, 1, &out_size); + TEST_CHECK(ret == 0); + TEST_CHECK(out_size == len2 + 1); + + flb_aws_aggregation_destroy(&buf); +} + +/* Test: Large aggregation (many small records) */ +void test_aws_aggregation_large() +{ + struct flb_aws_agg_buffer buf; + int ret; + const char *data = "{\"msg\":\"x\"}\n"; /* 12 bytes */ + size_t data_len = strlen(data); + int i; + int count = 1000; + + ret = flb_aws_aggregation_init(&buf, MAX_RECORD_SIZE); + TEST_CHECK(ret == 0); + + /* Add 1000 small records */ + for (i = 0; i < count; i++) { + ret = flb_aws_aggregation_add(&buf, data, data_len, MAX_RECORD_SIZE); + TEST_CHECK(ret == 0); + } + + TEST_CHECK(buf.agg_buf_offset == data_len * count); + + flb_aws_aggregation_destroy(&buf); +} + +/* Test: NULL parameter handling */ +void test_aws_aggregation_null_params() +{ + struct flb_aws_agg_buffer buf; + int ret; + size_t out_size; + + /* Test init with NULL */ + ret = flb_aws_aggregation_init(NULL, MAX_RECORD_SIZE); + TEST_CHECK(ret == -1); + + /* Initialize properly for other tests */ + ret = flb_aws_aggregation_init(&buf, MAX_RECORD_SIZE); + TEST_CHECK(ret == 0); + + /* Test add with NULL buffer */ + ret = flb_aws_aggregation_add(NULL, "data", 4, MAX_RECORD_SIZE); + TEST_CHECK(ret == -1); + + /* Test add with NULL data */ + ret = flb_aws_aggregation_add(&buf, NULL, 4, MAX_RECORD_SIZE); + TEST_CHECK(ret == -1); + + /* Test add with zero length */ + ret = flb_aws_aggregation_add(&buf, "data", 0, MAX_RECORD_SIZE); + TEST_CHECK(ret == -1); + + /* Test finalize with NULL buffer */ + ret = flb_aws_aggregation_finalize(NULL, 1, &out_size); + TEST_CHECK(ret == -1); + + /* Test finalize with NULL out_size */ + ret = flb_aws_aggregation_finalize(&buf, 1, NULL); + TEST_CHECK(ret == -1); + + /* Test destroy with NULL (should not crash) */ + flb_aws_aggregation_destroy(NULL); + + /* Test reset with NULL (should not crash) */ + flb_aws_aggregation_reset(NULL); + + flb_aws_aggregation_destroy(&buf); +} + +/* Test: Exact boundary conditions */ +void test_aws_aggregation_boundary() +{ + struct flb_aws_agg_buffer buf; + int ret; + size_t exact_size = 50; + char data[50]; + size_t out_size; + + /* Initialize with exact size */ + ret = flb_aws_aggregation_init(&buf, exact_size); + TEST_CHECK(ret == 0); + + /* Fill exactly to the boundary */ + memset(data, 'X', exact_size); + ret = flb_aws_aggregation_add(&buf, data, exact_size, exact_size); + TEST_CHECK(ret == 0); + TEST_CHECK(buf.agg_buf_offset == exact_size); + + /* Try to add one more byte - should fail */ + ret = flb_aws_aggregation_add(&buf, "Y", 1, exact_size); + TEST_CHECK(ret == 1); + + /* Finalize without newline should work */ + ret = flb_aws_aggregation_finalize(&buf, 0, &out_size); + TEST_CHECK(ret == 0); + TEST_CHECK(out_size == exact_size); + + flb_aws_aggregation_destroy(&buf); +} + +/* Test: Finalize with newline at boundary */ +void test_aws_aggregation_finalize_boundary() +{ + struct flb_aws_agg_buffer buf; + int ret; + size_t size = 100; + char data[99]; + size_t out_size; + + ret = flb_aws_aggregation_init(&buf, size); + TEST_CHECK(ret == 0); + + /* Fill to size-1 to leave room for newline */ + memset(data, 'A', 99); + ret = flb_aws_aggregation_add(&buf, data, 99, size); + TEST_CHECK(ret == 0); + + /* Finalize with newline should work */ + ret = flb_aws_aggregation_finalize(&buf, 1, &out_size); + TEST_CHECK(ret == 0); + TEST_CHECK(out_size == 100); + TEST_CHECK(buf.agg_buf[99] == '\n'); + + flb_aws_aggregation_destroy(&buf); +} + +/* Test: Multiple reset cycles */ +void test_aws_aggregation_multiple_resets() +{ + struct flb_aws_agg_buffer buf; + int ret; + const char *data = "test_data\n"; + size_t data_len = strlen(data); + int i; + + ret = flb_aws_aggregation_init(&buf, MAX_RECORD_SIZE); + TEST_CHECK(ret == 0); + + /* Perform multiple add/reset cycles */ + for (i = 0; i < 10; i++) { + ret = flb_aws_aggregation_add(&buf, data, data_len, MAX_RECORD_SIZE); + TEST_CHECK(ret == 0); + TEST_CHECK(buf.agg_buf_offset == data_len); + + flb_aws_aggregation_reset(&buf); + TEST_CHECK(buf.agg_buf_offset == 0); + } + + flb_aws_aggregation_destroy(&buf); +} + +/* Test: Very small buffer size */ +void test_aws_aggregation_tiny_buffer() +{ + struct flb_aws_agg_buffer buf; + int ret; + const char *data = "AB"; + size_t out_size; + + /* Initialize with very small buffer */ + ret = flb_aws_aggregation_init(&buf, 10); + TEST_CHECK(ret == 0); + + /* Add small data */ + ret = flb_aws_aggregation_add(&buf, data, 2, 10); + TEST_CHECK(ret == 0); + + /* Add more small data */ + ret = flb_aws_aggregation_add(&buf, data, 2, 10); + TEST_CHECK(ret == 0); + + /* Should have 4 bytes */ + TEST_CHECK(buf.agg_buf_offset == 4); + + /* Finalize */ + ret = flb_aws_aggregation_finalize(&buf, 0, &out_size); + TEST_CHECK(ret == 0); + TEST_CHECK(out_size == 4); + + flb_aws_aggregation_destroy(&buf); +} + +/* Test: Sequential finalize without reset */ +void test_aws_aggregation_double_finalize() +{ + struct flb_aws_agg_buffer buf; + int ret; + const char *data = "test\n"; + size_t data_len = strlen(data); + size_t out_size1, out_size2; + + ret = flb_aws_aggregation_init(&buf, MAX_RECORD_SIZE); + TEST_CHECK(ret == 0); + + ret = flb_aws_aggregation_add(&buf, data, data_len, MAX_RECORD_SIZE); + TEST_CHECK(ret == 0); + + /* First finalize */ + ret = flb_aws_aggregation_finalize(&buf, 0, &out_size1); + TEST_CHECK(ret == 0); + TEST_CHECK(out_size1 == data_len); + + /* Second finalize without reset - should still work */ + ret = flb_aws_aggregation_finalize(&buf, 0, &out_size2); + TEST_CHECK(ret == 0); + TEST_CHECK(out_size2 == out_size1); + + flb_aws_aggregation_destroy(&buf); +} + +/* Test: Add after finalize without reset */ +void test_aws_aggregation_add_after_finalize() +{ + struct flb_aws_agg_buffer buf; + int ret; + const char *data1 = "first\n"; + const char *data2 = "second\n"; + size_t len1 = strlen(data1); + size_t len2 = strlen(data2); + size_t out_size; + + ret = flb_aws_aggregation_init(&buf, MAX_RECORD_SIZE); + TEST_CHECK(ret == 0); + + /* Add first data */ + ret = flb_aws_aggregation_add(&buf, data1, len1, MAX_RECORD_SIZE); + TEST_CHECK(ret == 0); + + /* Finalize */ + ret = flb_aws_aggregation_finalize(&buf, 0, &out_size); + TEST_CHECK(ret == 0); + + /* Add more data without reset - should append */ + ret = flb_aws_aggregation_add(&buf, data2, len2, MAX_RECORD_SIZE); + TEST_CHECK(ret == 0); + TEST_CHECK(buf.agg_buf_offset == len1 + len2); + + flb_aws_aggregation_destroy(&buf); +} + +/* Test: Alternating add and finalize patterns */ +void test_aws_aggregation_alternating_pattern() +{ + struct flb_aws_agg_buffer buf; + int ret; + const char *data = "X"; + size_t out_size; + int i; + + ret = flb_aws_aggregation_init(&buf, MAX_RECORD_SIZE); + TEST_CHECK(ret == 0); + + /* Add one byte, finalize, reset - repeat */ + for (i = 0; i < 5; i++) { + ret = flb_aws_aggregation_add(&buf, data, 1, MAX_RECORD_SIZE); + TEST_CHECK(ret == 0); + TEST_CHECK(buf.agg_buf_offset == 1); + + ret = flb_aws_aggregation_finalize(&buf, 0, &out_size); + TEST_CHECK(ret == 0); + TEST_CHECK(out_size == 1); + + flb_aws_aggregation_reset(&buf); + TEST_CHECK(buf.agg_buf_offset == 0); + } + + flb_aws_aggregation_destroy(&buf); +} + +/* Test list */ +TEST_LIST = { + {"aws_aggregation_init_destroy", test_aws_aggregation_init_destroy}, + {"aws_aggregation_add_single", test_aws_aggregation_add_single}, + {"aws_aggregation_add_multiple", test_aws_aggregation_add_multiple}, + {"aws_aggregation_buffer_full", test_aws_aggregation_buffer_full}, + {"aws_aggregation_buffer_full_multiple", test_aws_aggregation_buffer_full_multiple}, + {"aws_aggregation_finalize", test_aws_aggregation_finalize}, + {"aws_aggregation_finalize_empty", test_aws_aggregation_finalize_empty}, + {"aws_aggregation_reset_reuse", test_aws_aggregation_reset_reuse}, + {"aws_aggregation_large", test_aws_aggregation_large}, + {"aws_aggregation_null_params", test_aws_aggregation_null_params}, + {"aws_aggregation_boundary", test_aws_aggregation_boundary}, + {"aws_aggregation_finalize_boundary", test_aws_aggregation_finalize_boundary}, + {"aws_aggregation_multiple_resets", test_aws_aggregation_multiple_resets}, + {"aws_aggregation_tiny_buffer", test_aws_aggregation_tiny_buffer}, + {"aws_aggregation_double_finalize", test_aws_aggregation_double_finalize}, + {"aws_aggregation_add_after_finalize", test_aws_aggregation_add_after_finalize}, + {"aws_aggregation_alternating_pattern", test_aws_aggregation_alternating_pattern}, + {NULL, NULL} +}; From 90383a2838d592acb26cb4e18f63cf72dafea9c4 Mon Sep 17 00:00:00 2001 From: Patrick Stephens Date: Sun, 3 May 2026 12:35:08 +0100 Subject: [PATCH 146/213] [upstream] out_kinesis_firehose: Add simple_aggregation Upstream-Ref: https://github.com/fluent/fluent-bit/commit/eb3916481e60aea7377546ca923110b852d03b91 Cherry-picked from Fluent Bit v4.2.4 --- .../plugins/out_kinesis_firehose/firehose.c | 25 ++- .../plugins/out_kinesis_firehose/firehose.h | 6 + .../out_kinesis_firehose/firehose_api.c | 174 +++++++++++++++++- 3 files changed, 201 insertions(+), 4 deletions(-) diff --git a/source/plugins/out_kinesis_firehose/firehose.c b/source/plugins/out_kinesis_firehose/firehose.c index 0b998599..d068279e 100644 --- a/source/plugins/out_kinesis_firehose/firehose.c +++ b/source/plugins/out_kinesis_firehose/firehose.c @@ -314,10 +314,10 @@ static int cb_firehose_init(struct flb_output_instance *ins, return -1; } -struct flush *new_flush_buffer() +struct flush *new_flush_buffer(struct flb_firehose *ctx) { struct flush *buf; - + int ret; buf = flb_calloc(1, sizeof(struct flush)); if (!buf) { @@ -341,6 +341,18 @@ struct flush *new_flush_buffer() } buf->events_capacity = MAX_EVENTS_PER_PUT; + /* Initialize aggregation buffer if simple_aggregation is enabled */ + buf->agg_buf_initialized = FLB_FALSE; + if (ctx->simple_aggregation) { + ret = flb_aws_aggregation_init(&buf->agg_buf, MAX_EVENT_SIZE); + if (ret < 0) { + flb_plg_error(ctx->ins, "Failed to initialize aggregation buffer"); + flush_destroy(buf); + return NULL; + } + buf->agg_buf_initialized = FLB_TRUE; + } + return buf; } @@ -356,7 +368,7 @@ static void cb_firehose_flush(struct flb_event_chunk *event_chunk, (void) i_ins; (void) config; - buf = new_flush_buffer(); + buf = new_flush_buffer(ctx); if (!buf) { flb_plg_error(ctx->ins, "Failed to construct flush buffer"); FLB_OUTPUT_RETURN(FLB_RETRY); @@ -508,6 +520,13 @@ static struct flb_config_map config_map[] = { "AWS Profile name. AWS Profiles can be configured with AWS CLI and are usually stored in " "$HOME/.aws/ directory." }, + + { + FLB_CONFIG_MAP_BOOL, "simple_aggregation", "false", + 0, FLB_TRUE, offsetof(struct flb_firehose, simple_aggregation), + "Enable simple aggregation to combine multiple records into single API calls. " + "This reduces the number of requests and can improve throughput." + }, /* EOF */ {0} }; diff --git a/source/plugins/out_kinesis_firehose/firehose.h b/source/plugins/out_kinesis_firehose/firehose.h index 6d27bf78..ac25701e 100644 --- a/source/plugins/out_kinesis_firehose/firehose.h +++ b/source/plugins/out_kinesis_firehose/firehose.h @@ -26,6 +26,7 @@ #include #include #include +#include #define DEFAULT_TIME_KEY_FORMAT "%Y-%m-%dT%H:%M:%S" @@ -56,6 +57,10 @@ struct flush { char *event_buf; size_t event_buf_size; + /* aggregation buffer for simple_aggregation mode */ + struct flb_aws_agg_buffer agg_buf; + int agg_buf_initialized; + int records_sent; int records_processed; }; @@ -94,6 +99,7 @@ struct flb_firehose { int custom_endpoint; int retry_requests; int compression; + int simple_aggregation; /* must be freed on shutdown if custom_endpoint is not set */ char *endpoint; diff --git a/source/plugins/out_kinesis_firehose/firehose_api.c b/source/plugins/out_kinesis_firehose/firehose_api.c index a87f2008..0fd3d5c7 100644 --- a/source/plugins/out_kinesis_firehose/firehose_api.c +++ b/source/plugins/out_kinesis_firehose/firehose_api.c @@ -38,6 +38,7 @@ #include #include +#include #include #include @@ -53,6 +54,9 @@ #define ERR_CODE_SERVICE_UNAVAILABLE "ServiceUnavailableException" +/* Forward declarations */ +static int send_log_events(struct flb_firehose *ctx, struct flush *buf); + static struct flb_aws_header put_record_batch_header = { .key = "X-Amz-Target", .key_len = 12, @@ -141,6 +145,29 @@ static int end_put_payload(struct flb_firehose *ctx, struct flush *buf, } +/* + * Process event with simple aggregation (Firehose version) + * Uses shared aggregation implementation + */ +static int process_event_simple_aggregation(struct flb_firehose *ctx, struct flush *buf, + const msgpack_object *obj, struct flb_time *tms, + struct flb_config *config) +{ + return flb_aws_aggregation_process_event(&buf->agg_buf, + buf->tmp_buf, + buf->tmp_buf_size, + &buf->tmp_buf_offset, + obj, + tms, + config, + ctx->ins, + ctx->delivery_stream, + ctx->log_key, + ctx->time_key, + ctx->time_key_format, + MAX_EVENT_SIZE); +} + /* * Processes the msgpack object * -1 = failure, record not added @@ -356,6 +383,102 @@ static void reset_flush_buf(struct flb_firehose *ctx, struct flush *buf) { buf->data_size += strlen(ctx->delivery_stream); } +/* Finalize and send aggregated record */ +static int send_aggregated_record(struct flb_firehose *ctx, struct flush *buf) { + int ret; + size_t agg_size; + size_t b64_len; + struct firehose_event *event; + void *compressed_tmp_buf; + size_t compressed_size; + + /* Finalize the aggregated record */ + ret = flb_aws_aggregation_finalize(&buf->agg_buf, 1, &agg_size); + if (ret < 0) { + /* No data to finalize */ + return 0; + } + + /* Handle compression if enabled */ + if (ctx->compression != FLB_AWS_COMPRESS_NONE) { + ret = flb_aws_compression_b64_truncate_compress(ctx->compression, + MAX_B64_EVENT_SIZE, + buf->agg_buf.agg_buf, + agg_size, + &compressed_tmp_buf, + &compressed_size); + if (ret == -1) { + flb_plg_error(ctx->ins, "Unable to compress aggregated record, discarding, %s", + ctx->delivery_stream); + flb_aws_aggregation_reset(&buf->agg_buf); + return 0; + } + + /* Ensure event_buf is large enough */ + if (buf->event_buf == NULL || buf->event_buf_size < compressed_size) { + flb_free(buf->event_buf); + buf->event_buf = compressed_tmp_buf; + buf->event_buf_size = compressed_size; + compressed_tmp_buf = NULL; + } else { + memcpy(buf->event_buf, compressed_tmp_buf, compressed_size); + flb_free(compressed_tmp_buf); + } + agg_size = compressed_size; + } + else { + /* Base64 encode the aggregated record */ + size_t size = (agg_size * 1.5) + 4; + if (buf->event_buf == NULL || buf->event_buf_size < size) { + flb_free(buf->event_buf); + buf->event_buf = flb_malloc(size); + buf->event_buf_size = size; + if (buf->event_buf == NULL) { + flb_errno(); + return -1; + } + } + + ret = flb_base64_encode((unsigned char *) buf->event_buf, size, &b64_len, + (unsigned char *) buf->agg_buf.agg_buf, agg_size); + if (ret != 0) { + flb_errno(); + return -1; + } + agg_size = b64_len; + } + + /* Copy to tmp_buf for sending */ + if (buf->tmp_buf_size < agg_size) { + flb_plg_error(ctx->ins, "Aggregated record too large for buffer"); + flb_aws_aggregation_reset(&buf->agg_buf); + return 0; + } + + memcpy(buf->tmp_buf, buf->event_buf, agg_size); + + /* Create event record */ + event = &buf->events[0]; + event->json = buf->tmp_buf; + event->len = agg_size; + event->timestamp.tv_sec = 0; + event->timestamp.tv_nsec = 0; + buf->event_index = 1; + + /* Calculate data_size for the payload */ + buf->data_size = PUT_RECORD_BATCH_HEADER_LEN + PUT_RECORD_BATCH_FOOTER_LEN; + buf->data_size += strlen(ctx->delivery_stream); + buf->data_size += agg_size + PUT_RECORD_BATCH_PER_RECORD_LEN; + + /* Send the aggregated record */ + ret = send_log_events(ctx, buf); + + /* Reset aggregation buffer */ + flb_aws_aggregation_reset(&buf->agg_buf); + + return ret; +} + /* constructs a put payload, and then sends */ static int send_log_events(struct flb_firehose *ctx, struct flush *buf) { int ret; @@ -445,6 +568,46 @@ static int add_event(struct flb_firehose *ctx, struct flush *buf, reset_flush_buf(ctx, buf); } + /* Use simple aggregation if enabled */ + if (ctx->simple_aggregation) { +retry_add_event_agg: + retry_add = FLB_FALSE; + ret = process_event_simple_aggregation(ctx, buf, obj, tms, config); + if (ret < 0) { + return -1; + } + else if (ret == 1) { + /* Buffer full - check if buffer was empty before sending (record too large) */ + if (buf->agg_buf.agg_buf_offset == 0) { + flb_plg_warn(ctx->ins, "Discarding unprocessable record (too large for aggregation buffer), %s", + ctx->delivery_stream); + reset_flush_buf(ctx, buf); + return 0; + } + + /* Send aggregated record and retry */ + ret = send_aggregated_record(ctx, buf); + reset_flush_buf(ctx, buf); + if (ret < 0) { + return -1; + } + retry_add = FLB_TRUE; + } + else if (ret == 2) { + /* Discard this record */ + flb_plg_warn(ctx->ins, "Discarding large or unprocessable record, %s", + ctx->delivery_stream); + return 0; + } + + if (retry_add == FLB_TRUE) { + goto retry_add_event_agg; + } + + return 0; + } + + /* Normal processing without aggregation */ retry_add_event: retry_add = FLB_FALSE; ret = process_event(ctx, buf, obj, tms, config); @@ -603,7 +766,13 @@ int process_and_send_records(struct flb_firehose *ctx, struct flush *buf, flb_log_event_decoder_destroy(&log_decoder); /* send any remaining events */ - ret = send_log_events(ctx, buf); + if (ctx->simple_aggregation) { + /* Send any remaining aggregated data */ + ret = send_aggregated_record(ctx, buf); + } + else { + ret = send_log_events(ctx, buf); + } reset_flush_buf(ctx, buf); if (ret < 0) { @@ -953,6 +1122,9 @@ int put_record_batch(struct flb_firehose *ctx, struct flush *buf, void flush_destroy(struct flush *buf) { if (buf) { + if (buf->agg_buf_initialized) { + flb_aws_aggregation_destroy(&buf->agg_buf); + } flb_free(buf->tmp_buf); flb_free(buf->out_buf); flb_free(buf->events); From da65eddc6714efa763c0626b8660b7bb652bb77e Mon Sep 17 00:00:00 2001 From: Patrick Stephens Date: Sun, 3 May 2026 12:35:08 +0100 Subject: [PATCH 147/213] [upstream] tests: Add firehose test cases for simple_aggregation Upstream-Ref: https://github.com/fluent/fluent-bit/commit/94c4b5080f876e462f1b85ac7921c6f448ae969c Cherry-picked from Fluent Bit v4.2.4 --- source/tests/runtime/out_firehose.c | 344 ++++++++++++++++++++++++++++ 1 file changed, 344 insertions(+) diff --git a/source/tests/runtime/out_firehose.c b/source/tests/runtime/out_firehose.c index cead6b3c..e6963707 100644 --- a/source/tests/runtime/out_firehose.c +++ b/source/tests/runtime/out_firehose.c @@ -189,6 +189,341 @@ void flb_test_firehose_nonsense_error(void) } +void flb_test_firehose_simple_aggregation(void) +{ + int ret; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + setenv("FLB_FIREHOSE_PLUGIN_UNDER_TEST", "true", 1); + + ctx = flb_create(); + + in_ffd = flb_input(ctx, (char *) "lib", NULL); + TEST_CHECK(in_ffd >= 0); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + out_ffd = flb_output(ctx, (char *) "kinesis_firehose", NULL); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd, "match", "*", NULL); + flb_output_set(ctx, out_ffd, "region", "us-west-2", NULL); + flb_output_set(ctx, out_ffd, "delivery_stream", "fluent", NULL); + flb_output_set(ctx, out_ffd, "simple_aggregation", "On", NULL); + flb_output_set(ctx, out_ffd, "Retry_Limit", "1", NULL); + + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + /* Push multiple small records */ + flb_lib_push(ctx, in_ffd, (char *) "[1, {\"message\":\"test1\"}]", 25); + flb_lib_push(ctx, in_ffd, (char *) "[1, {\"message\":\"test2\"}]", 25); + flb_lib_push(ctx, in_ffd, (char *) "[1, {\"message\":\"test3\"}]", 25); + + sleep(2); + flb_stop(ctx); + flb_destroy(ctx); +} + +void flb_test_firehose_aggregation_with_time_key(void) +{ + int ret; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + setenv("FLB_FIREHOSE_PLUGIN_UNDER_TEST", "true", 1); + + ctx = flb_create(); + + in_ffd = flb_input(ctx, (char *) "lib", NULL); + TEST_CHECK(in_ffd >= 0); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + out_ffd = flb_output(ctx, (char *) "kinesis_firehose", NULL); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd, "match", "*", NULL); + flb_output_set(ctx, out_ffd, "region", "us-west-2", NULL); + flb_output_set(ctx, out_ffd, "delivery_stream", "fluent", NULL); + flb_output_set(ctx, out_ffd, "simple_aggregation", "On", NULL); + flb_output_set(ctx, out_ffd, "time_key", "timestamp", NULL); + flb_output_set(ctx, out_ffd, "Retry_Limit", "1", NULL); + + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + /* Push records with time_key enabled */ + flb_lib_push(ctx, in_ffd, (char *) "[1, {\"message\":\"with_time1\"}]", 30); + flb_lib_push(ctx, in_ffd, (char *) "[1, {\"message\":\"with_time2\"}]", 30); + + sleep(2); + flb_stop(ctx); + flb_destroy(ctx); +} + +void flb_test_firehose_aggregation_with_log_key(void) +{ + int ret; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + const char *record = "[1, {\"message\":\"with_log_key\"}]"; + + setenv("FLB_FIREHOSE_PLUGIN_UNDER_TEST", "true", 1); + + ctx = flb_create(); + + in_ffd = flb_input(ctx, (char *) "lib", NULL); + TEST_CHECK(in_ffd >= 0); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + out_ffd = flb_output(ctx, (char *) "kinesis_firehose", NULL); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd, "match", "*", NULL); + flb_output_set(ctx, out_ffd, "region", "us-west-2", NULL); + flb_output_set(ctx, out_ffd, "delivery_stream", "fluent", NULL); + flb_output_set(ctx, out_ffd, "simple_aggregation", "On", NULL); + flb_output_set(ctx, out_ffd, "log_key", "log", NULL); + flb_output_set(ctx, out_ffd, "Retry_Limit", "1", NULL); + + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + /* Push records with log_key enabled */ + flb_lib_push(ctx, in_ffd, (char *) record, strlen(record)); + + sleep(2); + flb_stop(ctx); + flb_destroy(ctx); +} + +void flb_test_firehose_aggregation_many_records(void) +{ + int ret; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + int i; + char record[100]; + + setenv("FLB_FIREHOSE_PLUGIN_UNDER_TEST", "true", 1); + + ctx = flb_create(); + + in_ffd = flb_input(ctx, (char *) "lib", NULL); + TEST_CHECK(in_ffd >= 0); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + out_ffd = flb_output(ctx, (char *) "kinesis_firehose", NULL); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd, "match", "*", NULL); + flb_output_set(ctx, out_ffd, "region", "us-west-2", NULL); + flb_output_set(ctx, out_ffd, "delivery_stream", "fluent", NULL); + flb_output_set(ctx, out_ffd, "simple_aggregation", "On", NULL); + flb_output_set(ctx, out_ffd, "Retry_Limit", "1", NULL); + + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + /* Push many small records to test aggregation efficiency */ + for (i = 0; i < 50; i++) { + ret = snprintf(record, sizeof(record), "[1, {\"id\":%d,\"msg\":\"test\"}]", i); + TEST_CHECK(ret < sizeof(record)); + flb_lib_push(ctx, in_ffd, record, strlen(record)); + } + + sleep(3); + flb_stop(ctx); + flb_destroy(ctx); +} + +void flb_test_firehose_aggregation_with_compression(void) +{ + int ret; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + const char *record1 = "[1, {\"message\":\"compressed1\"}]"; + const char *record2 = "[1, {\"message\":\"compressed2\"}]"; + + setenv("FLB_FIREHOSE_PLUGIN_UNDER_TEST", "true", 1); + + ctx = flb_create(); + + in_ffd = flb_input(ctx, (char *) "lib", NULL); + TEST_CHECK(in_ffd >= 0); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + out_ffd = flb_output(ctx, (char *) "kinesis_firehose", NULL); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd, "match", "*", NULL); + flb_output_set(ctx, out_ffd, "region", "us-west-2", NULL); + flb_output_set(ctx, out_ffd, "delivery_stream", "fluent", NULL); + flb_output_set(ctx, out_ffd, "simple_aggregation", "On", NULL); + flb_output_set(ctx, out_ffd, "compression", "gzip", NULL); + flb_output_set(ctx, out_ffd, "Retry_Limit", "1", NULL); + + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + /* Push records with compression enabled */ + flb_lib_push(ctx, in_ffd, (char *) record1, strlen(record1)); + flb_lib_push(ctx, in_ffd, (char *) record2, strlen(record2)); + + sleep(2); + flb_stop(ctx); + flb_destroy(ctx); +} + +void flb_test_firehose_aggregation_combined_params(void) +{ + int ret; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + const char *record = "[1, {\"message\":\"combined_test\"}]"; + + setenv("FLB_FIREHOSE_PLUGIN_UNDER_TEST", "true", 1); + + ctx = flb_create(); + + in_ffd = flb_input(ctx, (char *) "lib", NULL); + TEST_CHECK(in_ffd >= 0); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + out_ffd = flb_output(ctx, (char *) "kinesis_firehose", NULL); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd, "match", "*", NULL); + flb_output_set(ctx, out_ffd, "region", "us-west-2", NULL); + flb_output_set(ctx, out_ffd, "delivery_stream", "fluent", NULL); + flb_output_set(ctx, out_ffd, "simple_aggregation", "On", NULL); + flb_output_set(ctx, out_ffd, "time_key", "timestamp", NULL); + flb_output_set(ctx, out_ffd, "time_key_format", "%Y-%m-%d %H:%M:%S", NULL); + flb_output_set(ctx, out_ffd, "compression", "gzip", NULL); + flb_output_set(ctx, out_ffd, "Retry_Limit", "1", NULL); + + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + /* Test with all features combined */ + flb_lib_push(ctx, in_ffd, (char *) record, strlen(record)); + + sleep(2); + flb_stop(ctx); + flb_destroy(ctx); +} + +void flb_test_firehose_aggregation_empty_records(void) +{ + int ret; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + setenv("FLB_FIREHOSE_PLUGIN_UNDER_TEST", "true", 1); + + ctx = flb_create(); + + in_ffd = flb_input(ctx, (char *) "lib", NULL); + TEST_CHECK(in_ffd >= 0); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + out_ffd = flb_output(ctx, (char *) "kinesis_firehose", NULL); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd, "match", "*", NULL); + flb_output_set(ctx, out_ffd, "region", "us-west-2", NULL); + flb_output_set(ctx, out_ffd, "delivery_stream", "fluent", NULL); + flb_output_set(ctx, out_ffd, "simple_aggregation", "On", NULL); + flb_output_set(ctx, out_ffd, "Retry_Limit", "1", NULL); + + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + /* Push empty and minimal records */ + flb_lib_push(ctx, in_ffd, (char *) "[1, {}]", 7); + flb_lib_push(ctx, in_ffd, (char *) "[1, {\"a\":\"\"}]", 13); + + sleep(2); + flb_stop(ctx); + flb_destroy(ctx); +} + +void flb_test_firehose_aggregation_error_handling(void) +{ + int ret; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + const char *record1 = "[1, {\"message\":\"error_test1\"}]"; + const char *record2 = "[1, {\"message\":\"error_test2\"}]"; + + setenv("FLB_FIREHOSE_PLUGIN_UNDER_TEST", "true", 1); + setenv("TEST_PUT_RECORD_BATCH_ERROR", ERROR_THROUGHPUT, 1); + + ctx = flb_create(); + + in_ffd = flb_input(ctx, (char *) "lib", NULL); + TEST_CHECK(in_ffd >= 0); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + out_ffd = flb_output(ctx, (char *) "kinesis_firehose", NULL); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd, "match", "*", NULL); + flb_output_set(ctx, out_ffd, "region", "us-west-2", NULL); + flb_output_set(ctx, out_ffd, "delivery_stream", "fluent", NULL); + flb_output_set(ctx, out_ffd, "simple_aggregation", "On", NULL); + flb_output_set(ctx, out_ffd, "Retry_Limit", "1", NULL); + + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + /* Test error handling with aggregation enabled */ + flb_lib_push(ctx, in_ffd, (char *) record1, strlen(record1)); + flb_lib_push(ctx, in_ffd, (char *) record2, strlen(record2)); + + sleep(2); + flb_stop(ctx); + flb_destroy(ctx); + unsetenv("TEST_PUT_RECORD_BATCH_ERROR"); +} + +void flb_test_firehose_aggregation_custom_time_format(void) +{ + int ret; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + const char *record = "[1, {\"message\":\"unix_time\"}]"; + + setenv("FLB_FIREHOSE_PLUGIN_UNDER_TEST", "true", 1); + + ctx = flb_create(); + + in_ffd = flb_input(ctx, (char *) "lib", NULL); + TEST_CHECK(in_ffd >= 0); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + out_ffd = flb_output(ctx, (char *) "kinesis_firehose", NULL); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd, "match", "*", NULL); + flb_output_set(ctx, out_ffd, "region", "us-west-2", NULL); + flb_output_set(ctx, out_ffd, "delivery_stream", "fluent", NULL); + flb_output_set(ctx, out_ffd, "simple_aggregation", "On", NULL); + flb_output_set(ctx, out_ffd, "time_key", "ts", NULL); + flb_output_set(ctx, out_ffd, "time_key_format", "%s", NULL); + flb_output_set(ctx, out_ffd, "Retry_Limit", "1", NULL); + + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + /* Test with Unix timestamp format */ + flb_lib_push(ctx, in_ffd, (char *) record, strlen(record)); + + sleep(2); + flb_stop(ctx); + flb_destroy(ctx); +} + /* Test list */ TEST_LIST = { {"success", flb_test_firehose_success }, @@ -196,5 +531,14 @@ TEST_LIST = { {"throughput_error", flb_test_firehose_throughput_error }, {"unknown_error", flb_test_firehose_error_unknown }, {"nonsense_error", flb_test_firehose_nonsense_error }, + {"simple_aggregation", flb_test_firehose_simple_aggregation }, + {"aggregation_with_time_key", flb_test_firehose_aggregation_with_time_key }, + {"aggregation_with_log_key", flb_test_firehose_aggregation_with_log_key }, + {"aggregation_many_records", flb_test_firehose_aggregation_many_records }, + {"aggregation_with_compression", flb_test_firehose_aggregation_with_compression }, + {"aggregation_combined_params", flb_test_firehose_aggregation_combined_params }, + {"aggregation_empty_records", flb_test_firehose_aggregation_empty_records }, + {"aggregation_error_handling", flb_test_firehose_aggregation_error_handling }, + {"aggregation_custom_time_format", flb_test_firehose_aggregation_custom_time_format }, {NULL, NULL} }; From 28d666e55597dc780346a74f421e5986397d1f06 Mon Sep 17 00:00:00 2001 From: Patrick Stephens Date: Sun, 3 May 2026 12:35:08 +0100 Subject: [PATCH 148/213] [upstream] out_kinesis_streams: Add simple_aggregation operation Upstream-Ref: https://github.com/fluent/fluent-bit/commit/976938a13a930a91192d4d6fe8abf4602edd71b1 Cherry-picked from Fluent Bit v4.2.4 --- source/plugins/out_kinesis_streams/kinesis.c | 25 ++- source/plugins/out_kinesis_streams/kinesis.h | 6 + .../plugins/out_kinesis_streams/kinesis_api.c | 151 ++++++++++++++++-- 3 files changed, 169 insertions(+), 13 deletions(-) diff --git a/source/plugins/out_kinesis_streams/kinesis.c b/source/plugins/out_kinesis_streams/kinesis.c index 03556752..55a0cf41 100644 --- a/source/plugins/out_kinesis_streams/kinesis.c +++ b/source/plugins/out_kinesis_streams/kinesis.c @@ -308,10 +308,10 @@ static int cb_kinesis_init(struct flb_output_instance *ins, return -1; } -static struct flush *new_flush_buffer(const char *tag, int tag_len) +static struct flush *new_flush_buffer(struct flb_kinesis *ctx, const char *tag, int tag_len) { struct flush *buf; - + int ret; buf = flb_calloc(1, sizeof(struct flush)); if (!buf) { @@ -338,6 +338,18 @@ static struct flush *new_flush_buffer(const char *tag, int tag_len) buf->tag = tag; buf->tag_len = tag_len; + /* Initialize aggregation buffer if simple_aggregation is enabled */ + buf->agg_buf_initialized = FLB_FALSE; + if (ctx->simple_aggregation) { + ret = flb_aws_aggregation_init(&buf->agg_buf, MAX_EVENT_SIZE); + if (ret < 0) { + flb_plg_error(ctx->ins, "Failed to initialize aggregation buffer"); + kinesis_flush_destroy(buf); + return NULL; + } + buf->agg_buf_initialized = FLB_TRUE; + } + return buf; } @@ -353,7 +365,7 @@ static void cb_kinesis_flush(struct flb_event_chunk *event_chunk, (void) i_ins; (void) config; - buf = new_flush_buffer(event_chunk->tag, flb_sds_len(event_chunk->tag)); + buf = new_flush_buffer(ctx, event_chunk->tag, flb_sds_len(event_chunk->tag)); if (!buf) { flb_plg_error(ctx->ins, "Failed to construct flush buffer"); FLB_OUTPUT_RETURN(FLB_RETRY); @@ -503,6 +515,13 @@ static struct flb_config_map config_map[] = { "$HOME/.aws/ directory." }, + { + FLB_CONFIG_MAP_BOOL, "simple_aggregation", "false", + 0, FLB_TRUE, offsetof(struct flb_kinesis, simple_aggregation), + "Enable simple aggregation to combine multiple records into single API calls. " + "This reduces the number of requests and can improve throughput." + }, + /* EOF */ {0} }; diff --git a/source/plugins/out_kinesis_streams/kinesis.h b/source/plugins/out_kinesis_streams/kinesis.h index a731d358..0faccf49 100644 --- a/source/plugins/out_kinesis_streams/kinesis.h +++ b/source/plugins/out_kinesis_streams/kinesis.h @@ -26,6 +26,7 @@ #include #include #include +#include #define DEFAULT_TIME_KEY_FORMAT "%Y-%m-%dT%H:%M:%S" @@ -56,6 +57,10 @@ struct flush { char *event_buf; size_t event_buf_size; + /* aggregation buffer for simple_aggregation mode */ + struct flb_aws_agg_buffer agg_buf; + int agg_buf_initialized; + int records_sent; int records_processed; @@ -92,6 +97,7 @@ struct flb_kinesis { const char *log_key; const char *external_id; int retry_requests; + int simple_aggregation; char *sts_endpoint; int custom_endpoint; uint16_t port; diff --git a/source/plugins/out_kinesis_streams/kinesis_api.c b/source/plugins/out_kinesis_streams/kinesis_api.c index 3f6d0f93..e3e8b502 100644 --- a/source/plugins/out_kinesis_streams/kinesis_api.c +++ b/source/plugins/out_kinesis_streams/kinesis_api.c @@ -36,6 +36,7 @@ #include #include #include +#include #include #include @@ -51,6 +52,9 @@ #define ERR_CODE_EXCEEDED_THROUGHPUT "ProvisionedThroughputExceededException" +/* Forward declarations */ +static int send_log_events(struct flb_kinesis *ctx, struct flush *buf); + static struct flb_aws_header put_records_target_header = { .key = "X-Amz-Target", .key_len = 12, @@ -202,6 +206,29 @@ static int end_put_payload(struct flb_kinesis *ctx, struct flush *buf, } +/* + * Process event with simple aggregation (Kinesis Streams version) + * Uses shared aggregation implementation + */ +static int process_event_simple_aggregation(struct flb_kinesis *ctx, struct flush *buf, + const msgpack_object *obj, struct flb_time *tms, + struct flb_config *config) +{ + return flb_aws_aggregation_process_event(&buf->agg_buf, + buf->tmp_buf, + buf->tmp_buf_size, + &buf->tmp_buf_offset, + obj, + tms, + config, + ctx->ins, + ctx->stream_name, + ctx->log_key, + ctx->time_key, + ctx->time_key_format, + MAX_EVENT_SIZE); +} + /* * Processes the msgpack object * -1 = failure, record not added @@ -391,6 +418,70 @@ static void reset_flush_buf(struct flb_kinesis *ctx, struct flush *buf) { buf->data_size += strlen(ctx->stream_name); } +/* Finalize and send aggregated record (Kinesis Streams version - no final newline) */ +static int send_aggregated_record(struct flb_kinesis *ctx, struct flush *buf) { + int ret; + size_t agg_size; + size_t b64_len; + struct kinesis_event *event; + + /* Finalize without final newline (Kinesis Streams doesn't need it) */ + ret = flb_aws_aggregation_finalize(&buf->agg_buf, 0, &agg_size); + if (ret < 0) { + return 0; + } + + /* Base64 encode the aggregated record */ + size_t size = (agg_size * 1.5) + 4; + if (buf->event_buf == NULL || buf->event_buf_size < size) { + flb_free(buf->event_buf); + buf->event_buf = flb_malloc(size); + buf->event_buf_size = size; + if (buf->event_buf == NULL) { + flb_errno(); + return -1; + } + } + + ret = flb_base64_encode((unsigned char *) buf->event_buf, size, &b64_len, + (unsigned char *) buf->agg_buf.agg_buf, agg_size); + if (ret != 0) { + flb_errno(); + return -1; + } + agg_size = b64_len; + + /* Copy to tmp_buf */ + if (buf->tmp_buf_size < agg_size) { + flb_plg_error(ctx->ins, "Aggregated record too large for buffer"); + flb_aws_aggregation_reset(&buf->agg_buf); + return 0; + } + + memcpy(buf->tmp_buf, buf->event_buf, agg_size); + + /* Create event record */ + event = &buf->events[0]; + event->json = buf->tmp_buf; + event->len = agg_size; + event->timestamp.tv_sec = 0; + event->timestamp.tv_nsec = 0; + buf->event_index = 1; + + /* Calculate data_size for the payload */ + buf->data_size = PUT_RECORDS_HEADER_LEN + PUT_RECORDS_FOOTER_LEN; + buf->data_size += strlen(ctx->stream_name); + buf->data_size += agg_size + PUT_RECORDS_PER_RECORD_LEN; + + /* Send the aggregated record */ + ret = send_log_events(ctx, buf); + + /* Reset aggregation buffer */ + flb_aws_aggregation_reset(&buf->agg_buf); + + return ret; +} + /* constructs a put payload, and then sends */ static int send_log_events(struct flb_kinesis *ctx, struct flush *buf) { int ret; @@ -476,10 +567,48 @@ static int add_event(struct flb_kinesis *ctx, struct flush *buf, size_t event_bytes = 0; if (buf->event_index == 0) { - /* init */ reset_flush_buf(ctx, buf); } + /* Use simple aggregation if enabled */ + if (ctx->simple_aggregation) { +retry_add_event_agg: + retry_add = FLB_FALSE; + ret = process_event_simple_aggregation(ctx, buf, obj, tms, config); + if (ret < 0) { + return -1; + } + else if (ret == 1) { + /* Buffer full - check if buffer was empty before sending (record too large) */ + if (buf->agg_buf.agg_buf_offset == 0) { + flb_plg_warn(ctx->ins, "Discarding unprocessable record (too large for aggregation buffer), %s", + ctx->stream_name); + reset_flush_buf(ctx, buf); + return 0; + } + + /* Send aggregated record and retry */ + ret = send_aggregated_record(ctx, buf); + reset_flush_buf(ctx, buf); + if (ret < 0) { + return -1; + } + retry_add = FLB_TRUE; + } + else if (ret == 2) { + flb_plg_warn(ctx->ins, "Discarding large or unprocessable record, %s", + ctx->stream_name); + return 0; + } + + if (retry_add == FLB_TRUE) { + goto retry_add_event_agg; + } + + return 0; + } + + /* Normal processing without aggregation */ retry_add_event: retry_add = FLB_FALSE; ret = process_event(ctx, buf, obj, tms, config); @@ -488,16 +617,13 @@ static int add_event(struct flb_kinesis *ctx, struct flush *buf, } else if (ret == 1) { if (buf->event_index <= 0) { - /* somehow the record was larger than our entire request buffer */ flb_plg_warn(ctx->ins, "Discarding massive log record, %s", ctx->stream_name); - return 0; /* discard this record and return to caller */ + return 0; } - /* send logs and then retry the add */ retry_add = FLB_TRUE; goto send; } else if (ret == 2) { - /* discard this record and return to caller */ flb_plg_warn(ctx->ins, "Discarding large or unprocessable record, %s", ctx->stream_name); return 0; @@ -508,17 +634,14 @@ static int add_event(struct flb_kinesis *ctx, struct flush *buf, if ((buf->data_size + event_bytes) > PUT_RECORDS_PAYLOAD_SIZE) { if (buf->event_index <= 0) { - /* somehow the record was larger than our entire request buffer */ flb_plg_warn(ctx->ins, "[size=%zu] Discarding massive log record, %s", event_bytes, ctx->stream_name); - return 0; /* discard this record and return to caller */ + return 0; } - /* do not send this event */ retry_add = FLB_TRUE; goto send; } - /* send is not needed yet, return to caller */ buf->data_size += event_bytes; buf->event_index++; @@ -633,7 +756,12 @@ int process_and_send_to_kinesis(struct flb_kinesis *ctx, struct flush *buf, flb_log_event_decoder_destroy(&log_decoder); /* send any remaining events */ - ret = send_log_events(ctx, buf); + if (ctx->simple_aggregation) { + ret = send_aggregated_record(ctx, buf); + } + else { + ret = send_log_events(ctx, buf); + } reset_flush_buf(ctx, buf); if (ret < 0) { return -1; @@ -981,6 +1109,9 @@ int put_records(struct flb_kinesis *ctx, struct flush *buf, void kinesis_flush_destroy(struct flush *buf) { if (buf) { + if (buf->agg_buf_initialized) { + flb_aws_aggregation_destroy(&buf->agg_buf); + } flb_free(buf->tmp_buf); flb_free(buf->out_buf); flb_free(buf->events); From 0dc27b1dcf9a20b10acdac4ed90b63f6c6c5951d Mon Sep 17 00:00:00 2001 From: Patrick Stephens Date: Sun, 3 May 2026 12:35:09 +0100 Subject: [PATCH 149/213] [upstream] tests: Add kinesis test cases for simple_aggregation Upstream-Ref: https://github.com/fluent/fluent-bit/commit/0572b9ccc2485a7d29a798e488b291478f34b9ab Cherry-picked from Fluent Bit v4.2.4 --- source/tests/runtime/out_kinesis.c | 152 +++++++++++++++++++++++++++++ 1 file changed, 152 insertions(+) diff --git a/source/tests/runtime/out_kinesis.c b/source/tests/runtime/out_kinesis.c index fa66a7ef..1b6f74e8 100644 --- a/source/tests/runtime/out_kinesis.c +++ b/source/tests/runtime/out_kinesis.c @@ -295,6 +295,154 @@ void flb_test_kinesis_invalid_port(void) flb_destroy(ctx); } +void flb_test_kinesis_simple_aggregation(void) +{ + int ret; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + setenv("FLB_KINESIS_PLUGIN_UNDER_TEST", "true", 1); + + ctx = flb_create(); + + in_ffd = flb_input(ctx, (char *) "lib", NULL); + TEST_CHECK(in_ffd >= 0); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + out_ffd = flb_output(ctx, (char *) "kinesis_streams", NULL); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd, "match", "*", NULL); + flb_output_set(ctx, out_ffd, "region", "us-west-2", NULL); + flb_output_set(ctx, out_ffd, "stream", "fluent", NULL); + flb_output_set(ctx, out_ffd, "simple_aggregation", "On", NULL); + flb_output_set(ctx, out_ffd, "Retry_Limit", "1", NULL); + + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + /* Push multiple small records */ + flb_lib_push(ctx, in_ffd, (char *) "[1, {\"message\":\"test1\"}]", 25); + flb_lib_push(ctx, in_ffd, (char *) "[1, {\"message\":\"test2\"}]", 25); + flb_lib_push(ctx, in_ffd, (char *) "[1, {\"message\":\"test3\"}]", 25); + + sleep(2); + flb_stop(ctx); + flb_destroy(ctx); +} + +void flb_test_kinesis_aggregation_with_time_key(void) +{ + int ret; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + setenv("FLB_KINESIS_PLUGIN_UNDER_TEST", "true", 1); + + ctx = flb_create(); + + in_ffd = flb_input(ctx, (char *) "lib", NULL); + TEST_CHECK(in_ffd >= 0); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + out_ffd = flb_output(ctx, (char *) "kinesis_streams", NULL); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd, "match", "*", NULL); + flb_output_set(ctx, out_ffd, "region", "us-west-2", NULL); + flb_output_set(ctx, out_ffd, "stream", "fluent", NULL); + flb_output_set(ctx, out_ffd, "simple_aggregation", "On", NULL); + flb_output_set(ctx, out_ffd, "time_key", "timestamp", NULL); + flb_output_set(ctx, out_ffd, "Retry_Limit", "1", NULL); + + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + /* Push records with time_key enabled */ + flb_lib_push(ctx, in_ffd, (char *) "[1, {\"message\":\"with_time1\"}]", 30); + flb_lib_push(ctx, in_ffd, (char *) "[1, {\"message\":\"with_time2\"}]", 30); + + sleep(2); + flb_stop(ctx); + flb_destroy(ctx); +} + +void flb_test_kinesis_aggregation_with_log_key(void) +{ + int ret; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + const char *record = "[1, {\"message\":\"with_log_key\"}]"; + + setenv("FLB_KINESIS_PLUGIN_UNDER_TEST", "true", 1); + + ctx = flb_create(); + + in_ffd = flb_input(ctx, (char *) "lib", NULL); + TEST_CHECK(in_ffd >= 0); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + out_ffd = flb_output(ctx, (char *) "kinesis_streams", NULL); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd, "match", "*", NULL); + flb_output_set(ctx, out_ffd, "region", "us-west-2", NULL); + flb_output_set(ctx, out_ffd, "stream", "fluent", NULL); + flb_output_set(ctx, out_ffd, "simple_aggregation", "On", NULL); + flb_output_set(ctx, out_ffd, "log_key", "log", NULL); + flb_output_set(ctx, out_ffd, "Retry_Limit", "1", NULL); + + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + /* Push records with log_key enabled */ + flb_lib_push(ctx, in_ffd, (char *) record, strlen(record)); + + sleep(2); + flb_stop(ctx); + flb_destroy(ctx); +} + +void flb_test_kinesis_aggregation_many_records(void) +{ + int ret; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + int i; + char record[100]; + + setenv("FLB_KINESIS_PLUGIN_UNDER_TEST", "true", 1); + + ctx = flb_create(); + + in_ffd = flb_input(ctx, (char *) "lib", NULL); + TEST_CHECK(in_ffd >= 0); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + out_ffd = flb_output(ctx, (char *) "kinesis_streams", NULL); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd, "match", "*", NULL); + flb_output_set(ctx, out_ffd, "region", "us-west-2", NULL); + flb_output_set(ctx, out_ffd, "stream", "fluent", NULL); + flb_output_set(ctx, out_ffd, "simple_aggregation", "On", NULL); + flb_output_set(ctx, out_ffd, "Retry_Limit", "1", NULL); + + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + /* Push many small records to test aggregation efficiency */ + for (i = 0; i < 50; i++) { + ret = snprintf(record, sizeof(record), "[1, {\"id\":%d,\"msg\":\"test\"}]", i); + TEST_CHECK(ret < sizeof(record)); + flb_lib_push(ctx, in_ffd, record, strlen(record)); + } + + sleep(3); + flb_stop(ctx); + flb_destroy(ctx); +} + /* Test list */ TEST_LIST = { {"success", flb_test_firehose_success }, @@ -305,5 +453,9 @@ TEST_LIST = { {"default_port", flb_test_kinesis_default_port }, {"custom_port", flb_test_kinesis_custom_port }, {"invalid_port", flb_test_kinesis_invalid_port }, + {"simple_aggregation", flb_test_kinesis_simple_aggregation }, + {"aggregation_with_time_key", flb_test_kinesis_aggregation_with_time_key }, + {"aggregation_with_log_key", flb_test_kinesis_aggregation_with_log_key }, + {"aggregation_many_records", flb_test_kinesis_aggregation_many_records }, {NULL, NULL} }; From 979c064a0d13b5ab8303df767f355770ae07c2b2 Mon Sep 17 00:00:00 2001 From: Patrick Stephens Date: Sun, 3 May 2026 12:35:09 +0100 Subject: [PATCH 150/213] [upstream] http_client: add ipv6 bracket if missing from host Upstream-Ref: https://github.com/fluent/fluent-bit/commit/1d384687a7de8d455c89d1b7e4adeb80495f0dfc Cherry-picked from Fluent Bit v4.2.4 --- source/src/flb_http_client.c | 56 ++++++++++++++++++++++++++++++++++-- 1 file changed, 53 insertions(+), 3 deletions(-) diff --git a/source/src/flb_http_client.c b/source/src/flb_http_client.c index 810e172f..e1bd048a 100644 --- a/source/src/flb_http_client.c +++ b/source/src/flb_http_client.c @@ -33,7 +33,13 @@ #define _GNU_SOURCE #include +#ifdef FLB_SYSTEM_WINDOWS +#include +#include +#endif + #include +#include #include #include #include @@ -617,11 +623,55 @@ static int add_host_and_content_length(struct flb_http_client *c) out_port = c->port; } - if (c->flags & FLB_IO_TLS && out_port == 443) { - tmp = flb_sds_copy(host, out_host, strlen(out_host)); + /* Check if out_host is an unbracketed IPv6 address */ + struct in6_addr addr; + char *zone_id; + char addr_buf[INET6_ADDRSTRLEN]; + int is_ipv6 = 0; + int is_https_default_port; + const char *host_for_header; + + if (out_host && out_host[0] != '[') { + /* Strip zone ID if present (e.g., fe80::1%eth0 -> fe80::1) */ + zone_id = strchr(out_host, '%'); + if (zone_id) { + len = zone_id - out_host; + if (len < INET6_ADDRSTRLEN) { + memcpy(addr_buf, out_host, len); + addr_buf[len] = '\0'; + is_ipv6 = (inet_pton(AF_INET6, addr_buf, &addr) == 1); + } + } + else { + is_ipv6 = (inet_pton(AF_INET6, out_host, &addr) == 1); + } + } + + /* Use stripped address (without zone ID) for Host header if zone ID was present */ + host_for_header = (is_ipv6 && zone_id) ? addr_buf : out_host; + + /* Check if connection uses TLS and port is 443 (HTTPS default) */ + is_https_default_port = flb_stream_get_flag_status(&u->base, FLB_IO_TLS) && out_port == 443; + + if (is_https_default_port) { + if (is_ipv6) { + /* IPv6 address needs brackets for RFC compliance */ + tmp = flb_sds_printf(&host, "[%s]", host_for_header); + } + else { + /* HTTPS on default port 443 - omit port from Host header */ + tmp = flb_sds_copy(host, out_host, strlen(out_host)); + } } else { - tmp = flb_sds_printf(&host, "%s:%i", out_host, out_port); + if (is_ipv6) { + /* IPv6 address needs brackets when combined with port */ + tmp = flb_sds_printf(&host, "[%s]:%i", host_for_header, out_port); + } + else { + /* IPv4 address, domain name, or already bracketed IPv6 */ + tmp = flb_sds_printf(&host, "%s:%i", out_host, out_port); + } } if (!tmp) { From f9cd0ce9004014282a535754fbeaa2237ec24fd7 Mon Sep 17 00:00:00 2001 From: Patrick Stephens Date: Sun, 3 May 2026 12:35:09 +0100 Subject: [PATCH 151/213] [upstream] tests: expand http_client tests for host headers Upstream-Ref: https://github.com/fluent/fluent-bit/commit/1348732ba53ec9be3e2ad2681f0af719e2d486f4 Cherry-picked from Fluent Bit v4.2.4 --- source/tests/internal/http_client.c | 187 ++++++++++++++++++++++++++++ 1 file changed, 187 insertions(+) diff --git a/source/tests/internal/http_client.c b/source/tests/internal/http_client.c index 45e77e1b..6e04150c 100644 --- a/source/tests/internal/http_client.c +++ b/source/tests/internal/http_client.c @@ -466,6 +466,178 @@ void test_http_add_proxy_auth_header() test_ctx_destroy(ctx); } +/* Helper function to verify Host header value */ +static void check_host_header(struct flb_http_client *c, const char *expected) +{ + flb_sds_t ret_str = flb_http_get_header(c, "Host", 4); + if (!TEST_CHECK(ret_str != NULL)) { + TEST_MSG("flb_http_get_header failed"); + exit(EXIT_FAILURE); + } + + if (!TEST_CHECK(flb_sds_cmp(ret_str, expected, strlen(expected)) == 0)) { + TEST_MSG("strcmp failed. got=%s expect=%s", ret_str, expected); + } + + flb_sds_destroy(ret_str); +} + +/* Helper to test basic host header formatting */ +static void test_host_header_format(const char *host, int port, const char *expected) +{ + struct test_ctx *ctx = test_ctx_create(); + if (!TEST_CHECK(ctx != NULL)) { + exit(EXIT_FAILURE); + } + + struct flb_http_client *c = flb_http_client(ctx->u_conn, FLB_HTTP_GET, "/", + NULL, 0, host, port, NULL, 0); + if (!TEST_CHECK(c != NULL)) { + TEST_MSG("flb_http_client failed"); + test_ctx_destroy(ctx); + exit(EXIT_FAILURE); + } + + check_host_header(c, expected); + flb_http_client_destroy(c); + test_ctx_destroy(ctx); +} + +/* Helper to test TLS host header formatting */ +static void test_tls_host_header_format(const char *host, int port, const char *expected) +{ + struct test_ctx *ctx = test_ctx_create(); + if (!TEST_CHECK(ctx != NULL)) { + exit(EXIT_FAILURE); + } + + struct flb_upstream *u_tls = flb_upstream_create(ctx->config, host, port, FLB_IO_TLS, NULL); + if (!TEST_CHECK(u_tls != NULL)) { + TEST_MSG("flb_upstream_create failed"); + test_ctx_destroy(ctx); + exit(EXIT_FAILURE); + } + + struct flb_connection *u_conn_tls = flb_calloc(1, sizeof(struct flb_connection)); + if (!TEST_CHECK(u_conn_tls != NULL)) { + TEST_MSG("flb_calloc failed"); + flb_upstream_destroy(u_tls); + test_ctx_destroy(ctx); + exit(EXIT_FAILURE); + } + u_conn_tls->upstream = u_tls; + + struct flb_http_client *c = flb_http_client(u_conn_tls, FLB_HTTP_GET, "/", + NULL, 0, host, port, NULL, 0); + if (!TEST_CHECK(c != NULL)) { + TEST_MSG("flb_http_client failed"); + flb_free(u_conn_tls); + flb_upstream_destroy(u_tls); + test_ctx_destroy(ctx); + exit(EXIT_FAILURE); + } + + check_host_header(c, expected); + flb_http_client_destroy(c); + flb_free(u_conn_tls); + flb_upstream_destroy(u_tls); + test_ctx_destroy(ctx); +} + +void test_http_ipv6_host_header() +{ + test_host_header_format("::1", 8080, "[::1]:8080"); +} + +void test_http_ipv6_bracketed_host_header() +{ + test_host_header_format("[::1]", 8080, "[::1]:8080"); +} + +void test_http_ipv4_host_header() +{ + test_host_header_format("192.168.1.1", 8080, "192.168.1.1:8080"); +} + +void test_http_domain_host_header() +{ + test_host_header_format("example.com", 8080, "example.com:8080"); +} + +void test_https_default_port_host_header() +{ + test_tls_host_header_format("example.com", 443, "example.com"); +} + +/* Test various IPv6 address formats */ +void test_ipv6_formats_host_header() +{ + size_t index; + struct { + const char *input; + const char *expected; + } test_cases[] = { + {"2001:db8::1", "[2001:db8::1]:8080"}, + {"2001:0db8:0000:0000:0000:0000:0000:0001", "[2001:0db8:0000:0000:0000:0000:0000:0001]:8080"}, + {"::ffff:192.0.2.1", "[::ffff:192.0.2.1]:8080"}, + {"fe80::1", "[fe80::1]:8080"}, + {"::1", "[::1]:8080"}, + {"::", "[::]:8080"}, + {NULL, NULL} + }; + + for (index = 0; test_cases[index].input != NULL; index++) { + test_host_header_format(test_cases[index].input, 8080, test_cases[index].expected); + } +} + +void test_http_port_80_host_header() +{ + test_host_header_format("example.com", 80, "example.com:80"); +} + +void test_port_443_without_tls_host_header() +{ + test_host_header_format("example.com", 443, "example.com:443"); +} + +void test_ipv6_zone_id_host_header() +{ + test_host_header_format("fe80::1%eth0", 8080, "[fe80::1]:8080"); +} + +void test_https_non_standard_port_host_header() +{ + test_tls_host_header_format("example.com", 8443, "example.com:8443"); +} + +void test_ipv6_bracketed_zone_id_host_header() +{ + /* Already bracketed input - zone ID detection only works on unbracketed addresses, + * so this passes through as-is. In practice, bracketed input shouldn't have zone IDs. */ + test_host_header_format("[fe80::1%eth0]", 8080, "[fe80::1%eth0]:8080"); +} + +void test_https_ipv6_default_port_host_header() +{ + test_tls_host_header_format("::1", 443, "[::1]"); +} + +void test_https_ipv6_non_standard_port_host_header() +{ + test_tls_host_header_format("::1", 8443, "[::1]:8443"); +} + +void test_https_ipv6_zone_id_default_port_host_header() +{ + test_tls_host_header_format("fe80::1%eth0", 443, "[fe80::1]"); +} + +void test_https_ipv6_zone_id_non_standard_port_host_header() +{ + test_tls_host_header_format("fe80::1%eth0", 8443, "[fe80::1]:8443"); +} + TEST_LIST = { { "http_buffer_increase" , test_http_buffer_increase}, { "add_get_header" , test_http_add_get_header}, @@ -474,5 +646,20 @@ TEST_LIST = { { "encoding_gzip" , test_http_encoding_gzip}, { "add_basic_auth_header" , test_http_add_basic_auth_header}, { "add_proxy_auth_header" , test_http_add_proxy_auth_header}, + { "ipv6_host_header" , test_http_ipv6_host_header}, + { "ipv6_bracketed_host_header", test_http_ipv6_bracketed_host_header}, + { "ipv4_host_header" , test_http_ipv4_host_header}, + { "domain_host_header" , test_http_domain_host_header}, + { "https_default_port_host_header", test_https_default_port_host_header}, + { "ipv6_formats_host_header", test_ipv6_formats_host_header}, + { "http_port_80_host_header", test_http_port_80_host_header}, + { "port_443_without_tls_host_header", test_port_443_without_tls_host_header}, + { "ipv6_zone_id_host_header", test_ipv6_zone_id_host_header}, + { "https_non_standard_port_host_header", test_https_non_standard_port_host_header}, + { "ipv6_bracketed_zone_id_host_header", test_ipv6_bracketed_zone_id_host_header}, + { "https_ipv6_default_port_host_header", test_https_ipv6_default_port_host_header}, + { "https_ipv6_non_standard_port_host_header", test_https_ipv6_non_standard_port_host_header}, + { "https_ipv6_zone_id_default_port_host_header", test_https_ipv6_zone_id_default_port_host_header}, + { "https_ipv6_zone_id_non_standard_port_host_header", test_https_ipv6_zone_id_non_standard_port_host_header}, { 0 } }; From 874adc2a77400eeae1f0e2ac813eb10028b35efc Mon Sep 17 00:00:00 2001 From: Patrick Stephens Date: Sun, 3 May 2026 12:35:09 +0100 Subject: [PATCH 152/213] [upstream] network: Add ws2tcpip.h to avoid missing func defs Upstream-Ref: https://github.com/fluent/fluent-bit/commit/f760d58249ba3ac6f82ec9acd4a1139d6a284e10 Cherry-picked from Fluent Bit v4.2.4 --- source/src/flb_network.c | 1 + 1 file changed, 1 insertion(+) diff --git a/source/src/flb_network.c b/source/src/flb_network.c index 5d6937ca..c5777d4f 100644 --- a/source/src/flb_network.c +++ b/source/src/flb_network.c @@ -30,6 +30,7 @@ #ifdef FLB_SYSTEM_WINDOWS #define poll WSAPoll #include +#include #else #include #endif From e3d67fbd767a33c05fb5ae5534891da0b86950fd Mon Sep 17 00:00:00 2001 From: Patrick Stephens Date: Sun, 3 May 2026 12:35:09 +0100 Subject: [PATCH 153/213] [upstream] utils: fix support for ipv6 addresses Upstream-Ref: https://github.com/fluent/fluent-bit/commit/a959af8c7a457cf0470f09c2756ffefcb2a144e7 Cherry-picked from Fluent Bit v4.2.4 --- source/src/flb_utils.c | 243 ++++++++++++++++++++++++++++++++++------- 1 file changed, 205 insertions(+), 38 deletions(-) diff --git a/source/src/flb_utils.c b/source/src/flb_utils.c index 84304dc5..31ee44e6 100644 --- a/source/src/flb_utils.c +++ b/source/src/flb_utils.c @@ -1429,13 +1429,103 @@ static char *flb_utils_copy_host_sds(const char *string, int pos_init, int pos_e if (string[pos_end-1] != ']') { return NULL; } - return flb_sds_create_len(string + pos_init + 1, pos_end - 1); + return flb_sds_create_len(string + pos_init + 1, pos_end - pos_init - 2); } else { - return flb_sds_create_len(string + pos_init, pos_end); + return flb_sds_create_len(string + pos_init, pos_end - pos_init); } } +/* Validate IPv6 bracket syntax in URL host part */ +static int validate_ipv6_brackets(const char *p, const char **out_bracket) +{ + const char *host_end; + const char *bracket = NULL; + const char *closing; + const char *query_or_fragment; + + /* Only inspect the host portion (up to the first '/', '?', or '#') */ + host_end = strchr(p, '/'); + query_or_fragment = strpbrk(p, "?#"); + + /* Use the earliest delimiter found */ + if (query_or_fragment && (!host_end || query_or_fragment < host_end)) { + host_end = query_or_fragment; + } + + if (!host_end) { + host_end = p + strlen(p); + } + + if (p[0] == '[') { + closing = memchr(p, ']', host_end - p); + if (!closing || closing == p + 1) { + /* Missing closing bracket or empty brackets [] */ + return -1; + } + bracket = closing; + } + else { + /* Non-bracketed hosts must not contain ']' before the first '/' */ + closing = memchr(p, ']', host_end - p); + if (closing) { + return -1; + } + } + + if (out_bracket) { + *out_bracket = bracket; + } + return 0; +} + +/* Helper to create URI with prepended '/' if it starts with '?' or '#' */ +static char *create_uri_with_slash(const char *uri_part) +{ + char *uri; + size_t uri_part_len; + + if (!uri_part || *uri_part == '\0') { + return flb_strdup("/"); + } + + /* If URI starts with '?' or '#', prepend '/' */ + if (*uri_part == '?' || *uri_part == '#') { + uri_part_len = strlen(uri_part); + /* Allocate space for '/' + uri_part + '\0' */ + uri = flb_malloc(uri_part_len + 2); + if (!uri) { + return NULL; + } + uri[0] = '/'; + /* +1 to include '\0' */ + memcpy(uri + 1, uri_part, uri_part_len + 1); + return uri; + } + + /* URI already starts with '/' or is a normal path */ + return flb_strdup(uri_part); +} + +/* SDS version: Helper to create URI with prepended '/' if it starts with '?' or '#' */ +static flb_sds_t create_uri_with_slash_sds(const char *uri_part) +{ + char *result; + flb_sds_t uri; + + /* Use the regular version to create the string */ + result = create_uri_with_slash(uri_part); + if (!result) { + return NULL; + } + + /* Convert to SDS */ + uri = flb_sds_create(result); + flb_free(result); + + return uri; +} + int flb_utils_url_split(const char *in_url, char **out_protocol, char **out_host, char **out_port, char **out_uri) { @@ -1446,6 +1536,7 @@ int flb_utils_url_split(const char *in_url, char **out_protocol, char *p; char *tmp; char *sep; + const char *bracket = NULL; /* Protocol */ p = strstr(in_url, "://"); @@ -1465,17 +1556,34 @@ int flb_utils_url_split(const char *in_url, char **out_protocol, /* Advance position after protocol */ p += 3; - /* Check for first '/' */ + /* Validate IPv6 brackets */ sep = strchr(p, '/'); - tmp = strchr(p, ':'); + if (validate_ipv6_brackets(p, &bracket) < 0) { + flb_errno(); + goto error; + } - /* Validate port separator is found before the first slash */ - if (sep && tmp) { - if (tmp > sep) { - tmp = NULL; - } + /* Compute end of host segment (before '/', '?', or '#') */ + const char *host_end = sep; + const char *qf = strpbrk(p, "?#"); + + if (!host_end || (qf && qf < host_end)) { + host_end = qf; + } + if (!host_end) { + host_end = p + strlen(p); + } + + if (bracket) { + /* For bracketed IPv6, only ports after ']' and before URI delimiters are valid */ + tmp = memchr(bracket, ':', host_end - bracket); + } + else { + /* Non-IPv6: limit ':' search to the host portion */ + tmp = memchr(p, ':', host_end - p); } + /* Extract host if port separator was found */ if (tmp) { host = flb_copy_host(p, 0, tmp - p); if (!host) { @@ -1483,30 +1591,46 @@ int flb_utils_url_split(const char *in_url, char **out_protocol, goto error; } p = tmp + 1; + } - /* Look for an optional URI */ - tmp = strchr(p, '/'); + /* Find URI delimiter (/, ?, or #) */ + tmp = strpbrk(p, "/?#"); + + if (!host) { + /* No port: extract host */ if (tmp) { - port = mk_string_copy_substr(p, 0, tmp - p); - uri = flb_strdup(tmp); + host = flb_copy_host(p, 0, tmp - p); } else { - port = flb_strdup(p); - uri = flb_strdup("/"); + host = flb_copy_host(p, 0, strlen(p)); + } + if (!host) { + flb_errno(); + goto error; } } else { - tmp = strchr(p, '/'); + /* Port exists: extract port */ if (tmp) { - host = flb_copy_host(p, 0, tmp - p); - uri = flb_strdup(tmp); + port = mk_string_copy_substr(p, 0, tmp - p); } else { - host = flb_copy_host(p, 0, strlen(p)); - uri = flb_strdup("/"); + port = flb_strdup(p); } } + /* Extract URI */ + if (tmp) { + uri = create_uri_with_slash(tmp); + if (!uri) { + flb_errno(); + goto error; + } + } + else { + uri = flb_strdup("/"); + } + if (!port) { if (strcmp(protocol, "http") == 0) { port = flb_strdup("80"); @@ -1527,6 +1651,15 @@ int flb_utils_url_split(const char *in_url, char **out_protocol, if (protocol) { flb_free(protocol); } + if (host) { + flb_free(host); + } + if (port) { + flb_free(port); + } + if (uri) { + flb_free(uri); + } return -1; } @@ -1542,6 +1675,7 @@ int flb_utils_url_split_sds(const flb_sds_t in_url, flb_sds_t *out_protocol, char *p = NULL; char *tmp = NULL; char *sep = NULL; + const char *bracket = NULL; /* Protocol */ p = strstr(in_url, "://"); @@ -1561,17 +1695,34 @@ int flb_utils_url_split_sds(const flb_sds_t in_url, flb_sds_t *out_protocol, /* Advance position after protocol */ p += 3; - /* Check for first '/' */ + /* Validate IPv6 brackets */ sep = strchr(p, '/'); - tmp = strchr(p, ':'); + if (validate_ipv6_brackets(p, &bracket) < 0) { + flb_errno(); + goto error; + } - /* Validate port separator is found before the first slash */ - if (sep && tmp) { - if (tmp > sep) { - tmp = NULL; - } + /* Compute end of host segment (before '/', '?', or '#') */ + const char *host_end = sep; + const char *qf = strpbrk(p, "?#"); + + if (!host_end || (qf && qf < host_end)) { + host_end = qf; + } + if (!host_end) { + host_end = p + strlen(p); } + if (bracket) { + /* For bracketed IPv6, only ports after ']' and before URI delimiters are valid */ + tmp = memchr(bracket, ':', host_end - bracket); + } + else { + /* Non-IPv6: limit ':' search to the host portion */ + tmp = memchr(p, ':', host_end - p); + } + + /* Extract host if port separator was found */ if (tmp) { host = flb_utils_copy_host_sds(p, 0, tmp - p); if (!host) { @@ -1579,29 +1730,45 @@ int flb_utils_url_split_sds(const flb_sds_t in_url, flb_sds_t *out_protocol, goto error; } p = tmp + 1; + } - /* Look for an optional URI */ - tmp = strchr(p, '/'); + /* Find URI delimiter (/, ?, or #) */ + tmp = strpbrk(p, "/?#"); + + if (!host) { + /* No port: extract host */ if (tmp) { - port = flb_sds_create_len(p, tmp - p); - uri = flb_sds_create(tmp); + host = flb_utils_copy_host_sds(p, 0, tmp - p); } else { - port = flb_sds_create_len(p, strlen(p)); - uri = flb_sds_create("/"); + host = flb_utils_copy_host_sds(p, 0, strlen(p)); + } + if (!host) { + flb_errno(); + goto error; } } else { - tmp = strchr(p, '/'); + /* Port exists: extract port */ if (tmp) { - host = flb_utils_copy_host_sds(p, 0, tmp - p); - uri = flb_sds_create(tmp); + port = flb_sds_create_len(p, tmp - p); } else { - host = flb_utils_copy_host_sds(p, 0, strlen(p)); - uri = flb_sds_create("/"); + port = flb_sds_create_len(p, strlen(p)); + } + } + + /* Extract URI */ + if (tmp) { + uri = create_uri_with_slash_sds(tmp); + if (!uri) { + flb_errno(); + goto error; } } + else { + uri = flb_sds_create("/"); + } if (!port) { if (strcmp(protocol, "http") == 0) { From 34c8a5864df23966f91594928544c0b8e29dad9d Mon Sep 17 00:00:00 2001 From: Patrick Stephens Date: Sun, 3 May 2026 12:35:09 +0100 Subject: [PATCH 154/213] [upstream] tests: add IPv4 and IPv6 tests for utils Upstream-Ref: https://github.com/fluent/fluent-bit/commit/167647382e9d2bbc54871e0c111aac60ac84a240 Cherry-picked from Fluent Bit v4.2.4 --- source/tests/internal/utils.c | 43 +++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/source/tests/internal/utils.c b/source/tests/internal/utils.c index 82ca93dc..680b093f 100644 --- a/source/tests/internal/utils.c +++ b/source/tests/internal/utils.c @@ -36,6 +36,49 @@ struct url_check url_checks[] = { {0, "https://fluentbit.io:1234/", "https", "fluentbit.io", "1234", "/"}, {0, "https://fluentbit.io:1234/v", "https", "fluentbit.io", "1234", "/v"}, {-1, "://", NULL, NULL, NULL, NULL}, + // IPv6 tests + {0, "https://[::1]/something", "https", "::1", "443", "/something"}, + {0, "http://[::1]/something", "http", "::1", "80", "/something"}, + {0, "https://[::1]", "https", "::1", "443", "/"}, + {0, "https://[::1]:1234/something", "https", "::1", "1234", "/something"}, + {0, "http://[::1]:1234", "http", "::1", "1234", "/"}, + {0, "http://[::1]:1234/", "http", "::1", "1234", "/"}, + {0, "http://[::1]:1234/v", "http", "::1", "1234", "/v"}, + {0, "https://[2001:db8::1]", "https", "2001:db8::1", "443", "/"}, + {0, "https://[2001:db8::1]:1234/something", "https", "2001:db8::1", "1234", "/something"}, + {0, "http://[2001:0db8:0000:0000:0000:0000:0000:0001]:1234/something", "http", "2001:0db8:0000:0000:0000:0000:0000:0001", "1234", "/something"}, + {0, "https://[::192.9.5.5]:1234/v", "https", "::192.9.5.5", "1234", "/v"}, + {0, "https://[::1]/path?query=[value]", "https", "::1", "443", "/path?query=[value]"}, + /* Query string with brackets (no path) */ + {0, "https://example.com?query=[1]", "https", "example.com", "443", "/?query=[1]"}, + {0, "http://example.com?query=[value]&other=[2]", "http", "example.com", "80", "/?query=[value]&other=[2]"}, + {0, "https://[::1]?query=[value]", "https", "::1", "443", "/?query=[value]"}, + {0, "https://[2001:db8::1]:8080?query=[1]", "https", "2001:db8::1", "8080", "/?query=[1]"}, + /* Fragment with brackets */ + {0, "https://example.com#fragment=[1]", "https", "example.com", "443", "/#fragment=[1]"}, + {0, "https://[::1]#fragment=[value]", "https", "::1", "443", "/#fragment=[value]"}, + /* Query and fragment with brackets */ + {0, "https://example.com?query=[1]#fragment=[2]", "https", "example.com", "443", "/?query=[1]#fragment=[2]"}, + /* Port with query/fragment (non-IPv6) */ + {0, "https://example.com:8080?query=[1]", "https", "example.com", "8080", "/?query=[1]"}, + {0, "http://example.com:9000#fragment=[1]", "http", "example.com", "9000", "/#fragment=[1]"}, + /* Empty query/fragment */ + {0, "https://example.com?", "https", "example.com", "443", "/?"}, + {0, "https://example.com#", "https", "example.com", "443", "/#"}, + {0, "https://[::1]?", "https", "::1", "443", "/?"}, + /* IPv6 edge cases - malformed brackets */ + {-1, "http://[::1:8080/path", NULL, NULL, NULL, NULL}, /* missing closing bracket */ + {-1, "http://::1]:8080/path", NULL, NULL, NULL, NULL}, /* missing opening bracket in host */ + {-1, "http://[]:8080/path", NULL, NULL, NULL, NULL}, /* empty brackets */ + {-1, "http://host]name.com/path", NULL, NULL, NULL, NULL}, /* closing bracket in hostname without opening */ + {-1, "http://host]name.com?query=1", NULL, NULL, NULL, NULL}, /* closing bracket in hostname without opening (query) */ + /* Colons in query/fragment should not be treated as port separators */ + {0, "https://example.com?q=a:b", "https", "example.com", "443", "/?q=a:b"}, + {0, "https://example.com/path?time=12:30:45", "https", "example.com", "443", "/path?time=12:30:45"}, + {0, "http://example.com#section:subsection", "http", "example.com", "80", "/#section:subsection"}, + {0, "https://example.com?q=a:b#frag:ment", "https", "example.com", "443", "/?q=a:b#frag:ment"}, + {0, "https://[::1]?time=12:30", "https", "::1", "443", "/?time=12:30"}, + {0, "http://example.com:8080?q=a:b:c", "http", "example.com", "8080", "/?q=a:b:c"} }; void test_url_split_sds() From 4c7664541978d7e3b23501e764ff357261986dc0 Mon Sep 17 00:00:00 2001 From: Patrick Stephens Date: Sun, 3 May 2026 12:35:10 +0100 Subject: [PATCH 155/213] [upstream] lib: sqlite: upgrade to v3.51.1 Upstream-Ref: https://github.com/fluent/fluent-bit/commit/e9d8b5d28c3b33868a4de97c1b996386d496323d Cherry-picked from Fluent Bit v4.2.4 --- source/cmake/libraries.cmake | 2 +- .../CMakeLists.txt | 0 .../sqlite3.c | 28185 +++++++++++----- .../sqlite3.h | 1205 +- 4 files changed, 20050 insertions(+), 9342 deletions(-) rename source/lib/{sqlite-amalgamation-3450200 => sqlite-amalgamation-3510100}/CMakeLists.txt (100%) rename source/lib/{sqlite-amalgamation-3450200 => sqlite-amalgamation-3510100}/sqlite3.c (92%) rename source/lib/{sqlite-amalgamation-3450200 => sqlite-amalgamation-3510100}/sqlite3.h (92%) diff --git a/source/cmake/libraries.cmake b/source/cmake/libraries.cmake index 4836a58c..2433106f 100644 --- a/source/cmake/libraries.cmake +++ b/source/cmake/libraries.cmake @@ -14,7 +14,7 @@ set(FLB_PATH_LIB_CHUNKIO "lib/chunkio") set(FLB_PATH_LIB_LUAJIT "lib/luajit-04dca791") set(FLB_PATH_LIB_MONKEY "lib/monkey") set(FLB_PATH_LIB_JSMN "lib/jsmn") -set(FLB_PATH_LIB_SQLITE "lib/sqlite-amalgamation-3450200") +set(FLB_PATH_LIB_SQLITE "lib/sqlite-amalgamation-3510100") set(FLB_PATH_LIB_JANSSON "lib/jansson-e23f558") set(FLB_PATH_LIB_ONIGMO "lib/onigmo") set(FLB_PATH_LIB_MPACK "lib/mpack-amalgamation-1.1.1") diff --git a/source/lib/sqlite-amalgamation-3450200/CMakeLists.txt b/source/lib/sqlite-amalgamation-3510100/CMakeLists.txt similarity index 100% rename from source/lib/sqlite-amalgamation-3450200/CMakeLists.txt rename to source/lib/sqlite-amalgamation-3510100/CMakeLists.txt diff --git a/source/lib/sqlite-amalgamation-3450200/sqlite3.c b/source/lib/sqlite-amalgamation-3510100/sqlite3.c similarity index 92% rename from source/lib/sqlite-amalgamation-3450200/sqlite3.c rename to source/lib/sqlite-amalgamation-3510100/sqlite3.c index 55ca3094..912ac269 100644 --- a/source/lib/sqlite-amalgamation-3450200/sqlite3.c +++ b/source/lib/sqlite-amalgamation-3510100/sqlite3.c @@ -1,6 +1,6 @@ /****************************************************************************** ** This file is an amalgamation of many separate C source files from SQLite -** version 3.45.2. By combining all the individual C code files into this +** version 3.51.1. By combining all the individual C code files into this ** single large file, the entire code can be compiled as a single translation ** unit. This allows many compilers to do optimizations that would not be ** possible if the files were compiled separately. Performance improvements @@ -18,8 +18,11 @@ ** separate file. This file contains only code for the core SQLite library. ** ** The content in this amalgamation comes from Fossil check-in -** d8cd6d49b46a395b13955387d05e9e1a2a47. +** 281fc0e9afc38674b9b0991943b9e9d1e64c with changes in files: +** +** */ +#ifndef SQLITE_AMALGAMATION #define SQLITE_CORE 1 #define SQLITE_AMALGAMATION 1 #ifndef SQLITE_PRIVATE @@ -167,7 +170,9 @@ #define HAVE_UTIME 1 #else /* This is not VxWorks. */ -#define OS_VXWORKS 0 +#ifndef OS_VXWORKS +# define OS_VXWORKS 0 +#endif #define HAVE_FCHOWN 1 #define HAVE_READLINK 1 #define HAVE_LSTAT 1 @@ -256,10 +261,13 @@ /* ** Macro to disable warnings about missing "break" at the end of a "case". */ -#if GCC_VERSION>=7000000 -# define deliberate_fall_through __attribute__((fallthrough)); -#else -# define deliberate_fall_through +#if defined(__has_attribute) +# if __has_attribute(fallthrough) +# define deliberate_fall_through __attribute__((fallthrough)); +# endif +#endif +#if !defined(deliberate_fall_through) +# define deliberate_fall_through #endif /* @@ -446,7 +454,7 @@ extern "C" { ** ** Since [version 3.6.18] ([dateof:3.6.18]), ** SQLite source code has been stored in the -** Fossil configuration management +** Fossil configuration management ** system. ^The SQLITE_SOURCE_ID macro evaluates to ** a string which identifies a particular check-in of SQLite ** within its configuration management system. ^The SQLITE_SOURCE_ID @@ -459,9 +467,12 @@ extern "C" { ** [sqlite3_libversion_number()], [sqlite3_sourceid()], ** [sqlite_version()] and [sqlite_source_id()]. */ -#define SQLITE_VERSION "3.45.2" -#define SQLITE_VERSION_NUMBER 3045002 -#define SQLITE_SOURCE_ID "2024-03-12 11:06:23 d8cd6d49b46a395b13955387d05e9e1a2a47e54fb99f3c9b59835bbefad6af77" +#define SQLITE_VERSION "3.51.1" +#define SQLITE_VERSION_NUMBER 3051001 +#define SQLITE_SOURCE_ID "2025-11-28 17:28:25 281fc0e9afc38674b9b0991943b9e9d1e64c6cbdb133d35f6f5c87ff6af38a88" +#define SQLITE_SCM_BRANCH "branch-3.51" +#define SQLITE_SCM_TAGS "release version-3.51.1" +#define SQLITE_SCM_DATETIME "2025-11-28T17:28:25.933Z" /* ** CAPI3REF: Run-Time Library Version Numbers @@ -481,9 +492,9 @@ extern "C" { ** assert( strcmp(sqlite3_libversion(),SQLITE_VERSION)==0 ); ** )^ ** -** ^The sqlite3_version[] string constant contains the text of [SQLITE_VERSION] -** macro. ^The sqlite3_libversion() function returns a pointer to the -** to the sqlite3_version[] string constant. The sqlite3_libversion() +** ^The sqlite3_version[] string constant contains the text of the +** [SQLITE_VERSION] macro. ^The sqlite3_libversion() function returns a +** pointer to the sqlite3_version[] string constant. The sqlite3_libversion() ** function is provided for use in DLLs since DLL users usually do not have ** direct access to string constants within the DLL. ^The ** sqlite3_libversion_number() function returns an integer equal to @@ -683,7 +694,7 @@ typedef int (*sqlite3_callback)(void*,int,char**, char**); ** without having to use a lot of C code. ** ** ^The sqlite3_exec() interface runs zero or more UTF-8 encoded, -** semicolon-separate SQL statements passed into its 2nd argument, +** semicolon-separated SQL statements passed into its 2nd argument, ** in the context of the [database connection] passed in as its 1st ** argument. ^If the callback function of the 3rd argument to ** sqlite3_exec() is not NULL, then it is invoked for each result row @@ -716,7 +727,7 @@ typedef int (*sqlite3_callback)(void*,int,char**, char**); ** result row is NULL then the corresponding string pointer for the ** sqlite3_exec() callback is a NULL pointer. ^The 4th argument to the ** sqlite3_exec() callback is an array of pointers to strings where each -** entry represents the name of corresponding result column as obtained +** entry represents the name of a corresponding result column as obtained ** from [sqlite3_column_name()]. ** ** ^If the 2nd parameter to sqlite3_exec() is a NULL pointer, a pointer @@ -810,6 +821,9 @@ SQLITE_API int sqlite3_exec( #define SQLITE_ERROR_MISSING_COLLSEQ (SQLITE_ERROR | (1<<8)) #define SQLITE_ERROR_RETRY (SQLITE_ERROR | (2<<8)) #define SQLITE_ERROR_SNAPSHOT (SQLITE_ERROR | (3<<8)) +#define SQLITE_ERROR_RESERVESIZE (SQLITE_ERROR | (4<<8)) +#define SQLITE_ERROR_KEY (SQLITE_ERROR | (5<<8)) +#define SQLITE_ERROR_UNABLE (SQLITE_ERROR | (6<<8)) #define SQLITE_IOERR_READ (SQLITE_IOERR | (1<<8)) #define SQLITE_IOERR_SHORT_READ (SQLITE_IOERR | (2<<8)) #define SQLITE_IOERR_WRITE (SQLITE_IOERR | (3<<8)) @@ -844,6 +858,8 @@ SQLITE_API int sqlite3_exec( #define SQLITE_IOERR_DATA (SQLITE_IOERR | (32<<8)) #define SQLITE_IOERR_CORRUPTFS (SQLITE_IOERR | (33<<8)) #define SQLITE_IOERR_IN_PAGE (SQLITE_IOERR | (34<<8)) +#define SQLITE_IOERR_BADKEY (SQLITE_IOERR | (35<<8)) +#define SQLITE_IOERR_CODEC (SQLITE_IOERR | (36<<8)) #define SQLITE_LOCKED_SHAREDCACHE (SQLITE_LOCKED | (1<<8)) #define SQLITE_LOCKED_VTAB (SQLITE_LOCKED | (2<<8)) #define SQLITE_BUSY_RECOVERY (SQLITE_BUSY | (1<<8)) @@ -902,7 +918,7 @@ SQLITE_API int sqlite3_exec( ** Note in particular that passing the SQLITE_OPEN_EXCLUSIVE flag into ** [sqlite3_open_v2()] does *not* cause the underlying database file ** to be opened using O_EXCL. Passing SQLITE_OPEN_EXCLUSIVE into -** [sqlite3_open_v2()] has historically be a no-op and might become an +** [sqlite3_open_v2()] has historically been a no-op and might become an ** error in future versions of SQLite. */ #define SQLITE_OPEN_READONLY 0x00000001 /* Ok for sqlite3_open_v2() */ @@ -965,6 +981,13 @@ SQLITE_API int sqlite3_exec( ** filesystem supports doing multiple write operations atomically when those ** write operations are bracketed by [SQLITE_FCNTL_BEGIN_ATOMIC_WRITE] and ** [SQLITE_FCNTL_COMMIT_ATOMIC_WRITE]. +** +** The SQLITE_IOCAP_SUBPAGE_READ property means that it is ok to read +** from the database file in amounts that are not a multiple of the +** page size and that do not begin at a page boundary. Without this +** property, SQLite is careful to only do full-page reads and write +** on aligned pages, with the one exception that it will do a sub-page +** read of the first page to access the database header. */ #define SQLITE_IOCAP_ATOMIC 0x00000001 #define SQLITE_IOCAP_ATOMIC512 0x00000002 @@ -981,6 +1004,7 @@ SQLITE_API int sqlite3_exec( #define SQLITE_IOCAP_POWERSAFE_OVERWRITE 0x00001000 #define SQLITE_IOCAP_IMMUTABLE 0x00002000 #define SQLITE_IOCAP_BATCH_ATOMIC 0x00004000 +#define SQLITE_IOCAP_SUBPAGE_READ 0x00008000 /* ** CAPI3REF: File Locking Levels @@ -988,7 +1012,7 @@ SQLITE_API int sqlite3_exec( ** SQLite uses one of these integer values as the second ** argument to calls it makes to the xLock() and xUnlock() methods ** of an [sqlite3_io_methods] object. These values are ordered from -** lest restrictive to most restrictive. +** least restrictive to most restrictive. ** ** The argument to xLock() is always SHARED or higher. The argument to ** xUnlock is either SHARED or NONE. @@ -1077,16 +1101,16 @@ struct sqlite3_file { ** ** xLock() upgrades the database file lock. In other words, xLock() moves the ** database file lock in the direction NONE toward EXCLUSIVE. The argument to -** xLock() is always on of SHARED, RESERVED, PENDING, or EXCLUSIVE, never +** xLock() is always one of SHARED, RESERVED, PENDING, or EXCLUSIVE, never ** SQLITE_LOCK_NONE. If the database file lock is already at or above the ** requested lock, then the call to xLock() is a no-op. ** xUnlock() downgrades the database file lock to either SHARED or NONE. -* If the lock is already at or below the requested lock state, then the call +** If the lock is already at or below the requested lock state, then the call ** to xUnlock() is a no-op. ** The xCheckReservedLock() method checks whether any database connection, ** either in this process or in some other process, is holding a RESERVED, -** PENDING, or EXCLUSIVE lock on the file. It returns true -** if such a lock exists and false otherwise. +** PENDING, or EXCLUSIVE lock on the file. It returns, via its output +** pointer parameter, true if such a lock exists and false otherwise. ** ** The xFileControl() method is a generic interface that allows custom ** VFS implementations to directly control an open file using the @@ -1127,6 +1151,7 @@ struct sqlite3_file { **
  • [SQLITE_IOCAP_POWERSAFE_OVERWRITE] **
  • [SQLITE_IOCAP_IMMUTABLE] **
  • [SQLITE_IOCAP_BATCH_ATOMIC] +**
  • [SQLITE_IOCAP_SUBPAGE_READ] ** ** ** The SQLITE_IOCAP_ATOMIC property means that all writes of @@ -1228,7 +1253,7 @@ struct sqlite3_io_methods { ** connection. See also [SQLITE_FCNTL_FILE_POINTER]. ** **
  • [[SQLITE_FCNTL_SYNC_OMITTED]] -** No longer in use. +** The SQLITE_FCNTL_SYNC_OMITTED file-control is no longer used. ** **
  • [[SQLITE_FCNTL_SYNC]] ** The [SQLITE_FCNTL_SYNC] opcode is generated internally by SQLite and @@ -1303,7 +1328,7 @@ struct sqlite3_io_methods { ** **
  • [[SQLITE_FCNTL_VFSNAME]] ** ^The [SQLITE_FCNTL_VFSNAME] opcode can be used to obtain the names of -** all [VFSes] in the VFS stack. The names are of all VFS shims and the +** all [VFSes] in the VFS stack. The names of all VFS shims and the ** final bottom-level VFS are written into memory obtained from ** [sqlite3_malloc()] and the result is stored in the char* variable ** that the fourth parameter of [sqlite3_file_control()] points to. @@ -1317,7 +1342,7 @@ struct sqlite3_io_methods { ** ^The [SQLITE_FCNTL_VFS_POINTER] opcode finds a pointer to the top-level ** [VFSes] currently in use. ^(The argument X in ** sqlite3_file_control(db,SQLITE_FCNTL_VFS_POINTER,X) must be -** of type "[sqlite3_vfs] **". This opcodes will set *X +** of type "[sqlite3_vfs] **". This opcode will set *X ** to a pointer to the top-level VFS.)^ ** ^When there are multiple VFS shims in the stack, this opcode finds the ** upper-most shim only. @@ -1404,6 +1429,11 @@ struct sqlite3_io_methods { ** pointed to by the pArg argument. This capability is used during testing ** and only needs to be supported when SQLITE_TEST is defined. ** +**
  • [[SQLITE_FCNTL_NULL_IO]] +** The [SQLITE_FCNTL_NULL_IO] opcode sets the low-level file descriptor +** or file handle for the [sqlite3_file] object such that it will no longer +** read or write to the database file. +** **
  • [[SQLITE_FCNTL_WAL_BLOCK]] ** The [SQLITE_FCNTL_WAL_BLOCK] is a signal to the VFS layer that it might ** be advantageous to block on the next WAL lock if the lock is not immediately @@ -1462,6 +1492,12 @@ struct sqlite3_io_methods { ** the value that M is to be set to. Before returning, the 32-bit signed ** integer is overwritten with the previous value of M. ** +**
  • [[SQLITE_FCNTL_BLOCK_ON_CONNECT]] +** The [SQLITE_FCNTL_BLOCK_ON_CONNECT] opcode is used to configure the +** VFS to block when taking a SHARED lock to connect to a wal mode database. +** This is used to implement the functionality associated with +** SQLITE_SETLK_BLOCK_ON_CONNECT. +** **
  • [[SQLITE_FCNTL_DATA_VERSION]] ** The [SQLITE_FCNTL_DATA_VERSION] opcode is used to detect changes to ** a database file. The argument is a pointer to a 32-bit unsigned integer. @@ -1496,7 +1532,7 @@ struct sqlite3_io_methods { **
  • [[SQLITE_FCNTL_EXTERNAL_READER]] ** The EXPERIMENTAL [SQLITE_FCNTL_EXTERNAL_READER] opcode is used to detect ** whether or not there is a database client in another process with a wal-mode -** transaction open on the database or not. It is only available on unix.The +** transaction open on the database or not. It is only available on unix. The ** (void*) argument passed with this file-control should be a pointer to a ** value of type (int). The integer value is set to 1 if the database is a wal ** mode database and there exists at least one client in another process that @@ -1514,6 +1550,15 @@ struct sqlite3_io_methods { ** database is not a temp db, then the [SQLITE_FCNTL_RESET_CACHE] file-control ** purges the contents of the in-memory page cache. If there is an open ** transaction, or if the db is a temp-db, this opcode is a no-op, not an error. +** +**
  • [[SQLITE_FCNTL_FILESTAT]] +** The [SQLITE_FCNTL_FILESTAT] opcode returns low-level diagnostic information +** about the [sqlite3_file] objects used access the database and journal files +** for the given schema. The fourth parameter to [sqlite3_file_control()] +** should be an initialized [sqlite3_str] pointer. JSON text describing +** various aspects of the sqlite3_file object is appended to the sqlite3_str. +** The SQLITE_FCNTL_FILESTAT opcode is usually a no-op, unless compile-time +** options are used to enable it. ** */ #define SQLITE_FCNTL_LOCKSTATE 1 @@ -1557,6 +1602,9 @@ struct sqlite3_io_methods { #define SQLITE_FCNTL_EXTERNAL_READER 40 #define SQLITE_FCNTL_CKSM_FILE 41 #define SQLITE_FCNTL_RESET_CACHE 42 +#define SQLITE_FCNTL_NULL_IO 43 +#define SQLITE_FCNTL_BLOCK_ON_CONNECT 44 +#define SQLITE_FCNTL_FILESTAT 45 /* deprecated names */ #define SQLITE_GET_LOCKPROXYFILE SQLITE_FCNTL_GET_LOCKPROXYFILE @@ -1919,7 +1967,7 @@ struct sqlite3_vfs { ** SQLite interfaces so that an application usually does not need to ** invoke sqlite3_initialize() directly. For example, [sqlite3_open()] ** calls sqlite3_initialize() so the SQLite library will be automatically -** initialized when [sqlite3_open()] is called if it has not be initialized +** initialized when [sqlite3_open()] is called if it has not been initialized ** already. ^However, if SQLite is compiled with the [SQLITE_OMIT_AUTOINIT] ** compile-time option, then the automatic calls to sqlite3_initialize() ** are omitted and the application must call sqlite3_initialize() directly @@ -2176,21 +2224,21 @@ struct sqlite3_mem_methods { ** The [sqlite3_mem_methods] ** structure is filled with the currently defined memory allocation routines.)^ ** This option can be used to overload the default memory allocation -** routines with a wrapper that simulations memory allocation failure or +** routines with a wrapper that simulates memory allocation failure or ** tracks memory usage, for example. ** ** [[SQLITE_CONFIG_SMALL_MALLOC]]
    SQLITE_CONFIG_SMALL_MALLOC
    -**
    ^The SQLITE_CONFIG_SMALL_MALLOC option takes single argument of +**
    ^The SQLITE_CONFIG_SMALL_MALLOC option takes a single argument of ** type int, interpreted as a boolean, which if true provides a hint to ** SQLite that it should avoid large memory allocations if possible. ** SQLite will run faster if it is free to make large memory allocations, -** but some application might prefer to run slower in exchange for +** but some applications might prefer to run slower in exchange for ** guarantees about memory fragmentation that are possible if large ** allocations are avoided. This hint is normally off. **
    ** ** [[SQLITE_CONFIG_MEMSTATUS]]
    SQLITE_CONFIG_MEMSTATUS
    -**
    ^The SQLITE_CONFIG_MEMSTATUS option takes single argument of type int, +**
    ^The SQLITE_CONFIG_MEMSTATUS option takes a single argument of type int, ** interpreted as a boolean, which enables or disables the collection of ** memory allocation statistics. ^(When memory allocation statistics are ** disabled, the following SQLite interfaces become non-operational: @@ -2235,7 +2283,7 @@ struct sqlite3_mem_methods { ** ^If pMem is NULL and N is non-zero, then each database connection ** does an initial bulk allocation for page cache memory ** from [sqlite3_malloc()] sufficient for N cache lines if N is positive or -** of -1024*N bytes if N is negative, . ^If additional +** of -1024*N bytes if N is negative. ^If additional ** page cache memory is needed beyond what is provided by the initial ** allocation, then SQLite goes to [sqlite3_malloc()] separately for each ** additional cache line.
    @@ -2264,7 +2312,7 @@ struct sqlite3_mem_methods { **
    ^(The SQLITE_CONFIG_MUTEX option takes a single argument which is a ** pointer to an instance of the [sqlite3_mutex_methods] structure. ** The argument specifies alternative low-level mutex routines to be used -** in place the mutex routines built into SQLite.)^ ^SQLite makes a copy of +** in place of the mutex routines built into SQLite.)^ ^SQLite makes a copy of ** the content of the [sqlite3_mutex_methods] structure before the call to ** [sqlite3_config()] returns. ^If SQLite is compiled with ** the [SQLITE_THREADSAFE | SQLITE_THREADSAFE=0] compile-time option then @@ -2287,13 +2335,16 @@ struct sqlite3_mem_methods { ** ** [[SQLITE_CONFIG_LOOKASIDE]]
    SQLITE_CONFIG_LOOKASIDE
    **
    ^(The SQLITE_CONFIG_LOOKASIDE option takes two arguments that determine -** the default size of lookaside memory on each [database connection]. +** the default size of [lookaside memory] on each [database connection]. ** The first argument is the -** size of each lookaside buffer slot and the second is the number of -** slots allocated to each database connection.)^ ^(SQLITE_CONFIG_LOOKASIDE -** sets the default lookaside size. The [SQLITE_DBCONFIG_LOOKASIDE] -** option to [sqlite3_db_config()] can be used to change the lookaside -** configuration on individual connections.)^
    +** size of each lookaside buffer slot ("sz") and the second is the number of +** slots allocated to each database connection ("cnt").)^ +** ^(SQLITE_CONFIG_LOOKASIDE sets the default lookaside size. +** The [SQLITE_DBCONFIG_LOOKASIDE] option to [sqlite3_db_config()] can +** be used to change the lookaside configuration on individual connections.)^ +** The [-DSQLITE_DEFAULT_LOOKASIDE] option can be used to change the +** default lookaside configuration at compile-time. +** ** ** [[SQLITE_CONFIG_PCACHE2]]
    SQLITE_CONFIG_PCACHE2
    **
    ^(The SQLITE_CONFIG_PCACHE2 option takes a single argument which is @@ -2303,7 +2354,7 @@ struct sqlite3_mem_methods { ** ** [[SQLITE_CONFIG_GETPCACHE2]]
    SQLITE_CONFIG_GETPCACHE2
    **
    ^(The SQLITE_CONFIG_GETPCACHE2 option takes a single argument which -** is a pointer to an [sqlite3_pcache_methods2] object. SQLite copies of +** is a pointer to an [sqlite3_pcache_methods2] object. SQLite copies off ** the current page cache implementation into that object.)^
    ** ** [[SQLITE_CONFIG_LOG]]
    SQLITE_CONFIG_LOG
    @@ -2320,7 +2371,7 @@ struct sqlite3_mem_methods { ** the logger function is a copy of the first parameter to the corresponding ** [sqlite3_log()] call and is intended to be a [result code] or an ** [extended result code]. ^The third parameter passed to the logger is -** log message after formatting via [sqlite3_snprintf()]. +** a log message after formatting via [sqlite3_snprintf()]. ** The SQLite logging interface is not reentrant; the logger function ** supplied by the application must not invoke any SQLite interface. ** In a multi-threaded application, the application-defined logger @@ -2456,6 +2507,22 @@ struct sqlite3_mem_methods { ** configuration setting is never used, then the default maximum is determined ** by the [SQLITE_MEMDB_DEFAULT_MAXSIZE] compile-time option. If that ** compile-time option is not set, then the default maximum is 1073741824. +** +** [[SQLITE_CONFIG_ROWID_IN_VIEW]] +**
    SQLITE_CONFIG_ROWID_IN_VIEW +**
    The SQLITE_CONFIG_ROWID_IN_VIEW option enables or disables the ability +** for VIEWs to have a ROWID. The capability can only be enabled if SQLite is +** compiled with -DSQLITE_ALLOW_ROWID_IN_VIEW, in which case the capability +** defaults to on. This configuration option queries the current setting or +** changes the setting to off or on. The argument is a pointer to an integer. +** If that integer initially holds a value of 1, then the ability for VIEWs to +** have ROWIDs is activated. If the integer initially holds zero, then the +** ability is deactivated. Any other initial value for the integer leaves the +** setting unchanged. After changes, if any, the integer is written with +** a 1 or 0, if the ability for VIEWs to have ROWIDs is on or off. If SQLite +** is compiled without -DSQLITE_ALLOW_ROWID_IN_VIEW (which is the usual and +** recommended case) then the integer is always filled with zero, regardless +** if its initial value. ** */ #define SQLITE_CONFIG_SINGLETHREAD 1 /* nil */ @@ -2487,12 +2554,21 @@ struct sqlite3_mem_methods { #define SQLITE_CONFIG_SMALL_MALLOC 27 /* boolean */ #define SQLITE_CONFIG_SORTERREF_SIZE 28 /* int nByte */ #define SQLITE_CONFIG_MEMDB_MAXSIZE 29 /* sqlite3_int64 */ +#define SQLITE_CONFIG_ROWID_IN_VIEW 30 /* int* */ /* ** CAPI3REF: Database Connection Configuration Options ** ** These constants are the available integer configuration options that -** can be passed as the second argument to the [sqlite3_db_config()] interface. +** can be passed as the second parameter to the [sqlite3_db_config()] interface. +** +** The [sqlite3_db_config()] interface is a var-args function. It takes a +** variable number of parameters, though always at least two. The number of +** parameters passed into sqlite3_db_config() depends on which of these +** constants is given as the second parameter. This documentation page +** refers to parameters beyond the second as "arguments". Thus, when this +** page says "the N-th argument" it means "the N-th parameter past the +** configuration option" or "the (N+2)-th parameter to sqlite3_db_config()". ** ** New configuration options may be added in future releases of SQLite. ** Existing configuration options might be discontinued. Applications @@ -2504,31 +2580,57 @@ struct sqlite3_mem_methods { **
    ** [[SQLITE_DBCONFIG_LOOKASIDE]] **
    SQLITE_DBCONFIG_LOOKASIDE
    -**
    ^This option takes three additional arguments that determine the -** [lookaside memory allocator] configuration for the [database connection]. -** ^The first argument (the third parameter to [sqlite3_db_config()] is a +**
    The SQLITE_DBCONFIG_LOOKASIDE option is used to adjust the +** configuration of the [lookaside memory allocator] within a database +** connection. +** The arguments to the SQLITE_DBCONFIG_LOOKASIDE option are not +** in the [DBCONFIG arguments|usual format]. +** The SQLITE_DBCONFIG_LOOKASIDE option takes three arguments, not two, +** so that a call to [sqlite3_db_config()] that uses SQLITE_DBCONFIG_LOOKASIDE +** should have a total of five parameters. +**
      +**
    1. The first argument ("buf") is a ** pointer to a memory buffer to use for lookaside memory. -** ^The first argument after the SQLITE_DBCONFIG_LOOKASIDE verb -** may be NULL in which case SQLite will allocate the -** lookaside buffer itself using [sqlite3_malloc()]. ^The second argument is the -** size of each lookaside buffer slot. ^The third argument is the number of -** slots. The size of the buffer in the first argument must be greater than -** or equal to the product of the second and third arguments. The buffer -** must be aligned to an 8-byte boundary. ^If the second argument to -** SQLITE_DBCONFIG_LOOKASIDE is not a multiple of 8, it is internally -** rounded down to the next smaller multiple of 8. ^(The lookaside memory +** The first argument may be NULL in which case SQLite will allocate the +** lookaside buffer itself using [sqlite3_malloc()]. +**

    2. The second argument ("sz") is the +** size of each lookaside buffer slot. Lookaside is disabled if "sz" +** is less than 8. The "sz" argument should be a multiple of 8 less than +** 65536. If "sz" does not meet this constraint, it is reduced in size until +** it does. +**

    3. The third argument ("cnt") is the number of slots. Lookaside is disabled +** if "cnt"is less than 1. The "cnt" value will be reduced, if necessary, so +** that the product of "sz" and "cnt" does not exceed 2,147,418,112. The "cnt" +** parameter is usually chosen so that the product of "sz" and "cnt" is less +** than 1,000,000. +**

    +**

    If the "buf" argument is not NULL, then it must +** point to a memory buffer with a size that is greater than +** or equal to the product of "sz" and "cnt". +** The buffer must be aligned to an 8-byte boundary. +** The lookaside memory ** configuration for a database connection can only be changed when that ** connection is not currently using lookaside memory, or in other words -** when the "current value" returned by -** [sqlite3_db_status](D,[SQLITE_DBSTATUS_LOOKASIDE_USED],...) is zero. +** when the value returned by [SQLITE_DBSTATUS_LOOKASIDE_USED] is zero. ** Any attempt to change the lookaside memory configuration when lookaside ** memory is in use leaves the configuration unchanged and returns -** [SQLITE_BUSY].)^

    +** [SQLITE_BUSY]. +** If the "buf" argument is NULL and an attempt +** to allocate memory based on "sz" and "cnt" fails, then +** lookaside is silently disabled. +**

    +** The [SQLITE_CONFIG_LOOKASIDE] configuration option can be used to set the +** default lookaside configuration at initialization. The +** [-DSQLITE_DEFAULT_LOOKASIDE] option can be used to set the default lookaside +** configuration at compile-time. Typical values for lookaside are 1200 for +** "sz" and 40 to 100 for "cnt". +**

    ** ** [[SQLITE_DBCONFIG_ENABLE_FKEY]] **
    SQLITE_DBCONFIG_ENABLE_FKEY
    **
    ^This option is used to enable or disable the enforcement of -** [foreign key constraints]. There should be two additional arguments. +** [foreign key constraints]. This is the same setting that is +** enabled or disabled by the [PRAGMA foreign_keys] statement. ** The first argument is an integer which is 0 to disable FK enforcement, ** positive to enable FK enforcement or negative to leave FK enforcement ** unchanged. The second parameter is a pointer to an integer into which @@ -2550,13 +2652,13 @@ struct sqlite3_mem_methods { **

    Originally this option disabled all triggers. ^(However, since ** SQLite version 3.35.0, TEMP triggers are still allowed even if ** this option is off. So, in other words, this option now only disables -** triggers in the main database schema or in the schemas of ATTACH-ed +** triggers in the main database schema or in the schemas of [ATTACH]-ed ** databases.)^

    ** ** [[SQLITE_DBCONFIG_ENABLE_VIEW]] **
    SQLITE_DBCONFIG_ENABLE_VIEW
    **
    ^This option is used to enable or disable [CREATE VIEW | views]. -** There should be two additional arguments. +** There must be two additional arguments. ** The first argument is an integer which is 0 to disable views, ** positive to enable views or negative to leave the setting unchanged. ** The second parameter is a pointer to an integer into which @@ -2572,17 +2674,20 @@ struct sqlite3_mem_methods { ** ** [[SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER]] **
    SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER
    -**
    ^This option is used to enable or disable the -** [fts3_tokenizer()] function which is part of the -** [FTS3] full-text search engine extension. -** There should be two additional arguments. -** The first argument is an integer which is 0 to disable fts3_tokenizer() or -** positive to enable fts3_tokenizer() or negative to leave the setting -** unchanged. -** The second parameter is a pointer to an integer into which -** is written 0 or 1 to indicate whether fts3_tokenizer is disabled or enabled -** following this call. The second parameter may be a NULL pointer, in -** which case the new setting is not reported back.
    +**
    ^This option is used to enable or disable using the +** [fts3_tokenizer()] function - part of the [FTS3] full-text search engine +** extension - without using bound parameters as the parameters. Doing so +** is disabled by default. There must be two additional arguments. The first +** argument is an integer. If it is passed 0, then using fts3_tokenizer() +** without bound parameters is disabled. If it is passed a positive value, +** then calling fts3_tokenizer without bound parameters is enabled. If it +** is passed a negative value, this setting is not modified - this can be +** used to query for the current setting. The second parameter is a pointer +** to an integer into which is written 0 or 1 to indicate the current value +** of this setting (after it is modified, if applicable). The second +** parameter may be a NULL pointer, in which case the value of the setting +** is not reported back. Refer to [FTS3] documentation for further details. +**
    ** ** [[SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION]] **
    SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION
    @@ -2590,12 +2695,12 @@ struct sqlite3_mem_methods { ** interface independently of the [load_extension()] SQL function. ** The [sqlite3_enable_load_extension()] API enables or disables both the ** C-API [sqlite3_load_extension()] and the SQL function [load_extension()]. -** There should be two additional arguments. +** There must be two additional arguments. ** When the first argument to this interface is 1, then only the C-API is ** enabled and the SQL function remains disabled. If the first argument to ** this interface is 0, then both the C-API and the SQL function are disabled. -** If the first argument is -1, then no changes are made to state of either the -** C-API or the SQL function. +** If the first argument is -1, then no changes are made to the state of either +** the C-API or the SQL function. ** The second parameter is a pointer to an integer into which ** is written 0 or 1 to indicate whether [sqlite3_load_extension()] interface ** is disabled or enabled following this call. The second parameter may @@ -2604,23 +2709,30 @@ struct sqlite3_mem_methods { ** ** [[SQLITE_DBCONFIG_MAINDBNAME]]
    SQLITE_DBCONFIG_MAINDBNAME
    **
    ^This option is used to change the name of the "main" database -** schema. ^The sole argument is a pointer to a constant UTF8 string -** which will become the new schema name in place of "main". ^SQLite -** does not make a copy of the new main schema name string, so the application -** must ensure that the argument passed into this DBCONFIG option is unchanged -** until after the database connection closes. +** schema. This option does not follow the +** [DBCONFIG arguments|usual SQLITE_DBCONFIG argument format]. +** This option takes exactly one additional argument so that the +** [sqlite3_db_config()] call has a total of three parameters. The +** extra argument must be a pointer to a constant UTF8 string which +** will become the new schema name in place of "main". ^SQLite does +** not make a copy of the new main schema name string, so the application +** must ensure that the argument passed into SQLITE_DBCONFIG MAINDBNAME +** is unchanged until after the database connection closes. **
    ** ** [[SQLITE_DBCONFIG_NO_CKPT_ON_CLOSE]] **
    SQLITE_DBCONFIG_NO_CKPT_ON_CLOSE
    -**
    Usually, when a database in wal mode is closed or detached from a -** database handle, SQLite checks if this will mean that there are now no -** connections at all to the database. If so, it performs a checkpoint -** operation before closing the connection. This option may be used to -** override this behavior. The first parameter passed to this operation -** is an integer - positive to disable checkpoints-on-close, or zero (the -** default) to enable them, and negative to leave the setting unchanged. -** The second parameter is a pointer to an integer +**
    Usually, when a database in [WAL mode] is closed or detached from a +** database handle, SQLite checks if if there are other connections to the +** same database, and if there are no other database connection (if the +** connection being closed is the last open connection to the database), +** then SQLite performs a [checkpoint] before closing the connection and +** deletes the WAL file. The SQLITE_DBCONFIG_NO_CKPT_ON_CLOSE option can +** be used to override that behavior. The first argument passed to this +** operation (the third parameter to [sqlite3_db_config()]) is an integer +** which is positive to disable checkpoints-on-close, or zero (the default) +** to enable them, and negative to leave the setting unchanged. +** The second argument (the fourth parameter) is a pointer to an integer ** into which is written 0 or 1 to indicate whether checkpoints-on-close ** have been disabled - 0 if they are not disabled, 1 if they are. **
    @@ -2706,7 +2818,7 @@ struct sqlite3_mem_methods { ** [[SQLITE_DBCONFIG_LEGACY_ALTER_TABLE]] **
    SQLITE_DBCONFIG_LEGACY_ALTER_TABLE
    **
    The SQLITE_DBCONFIG_LEGACY_ALTER_TABLE option activates or deactivates -** the legacy behavior of the [ALTER TABLE RENAME] command such it +** the legacy behavior of the [ALTER TABLE RENAME] command such that it ** behaves as it did prior to [version 3.24.0] (2018-06-04). See the ** "Compatibility Notice" on the [ALTER TABLE RENAME documentation] for ** additional information. This feature can also be turned on and off @@ -2755,7 +2867,7 @@ struct sqlite3_mem_methods { **
    SQLITE_DBCONFIG_LEGACY_FILE_FORMAT
    **
    The SQLITE_DBCONFIG_LEGACY_FILE_FORMAT option activates or deactivates ** the legacy file format flag. When activated, this flag causes all newly -** created database file to have a schema format version number (the 4-byte +** created database files to have a schema format version number (the 4-byte ** integer found at offset 44 into the database header) of 1. This in turn ** means that the resulting database file will be readable and writable by ** any SQLite version back to 3.0.0 ([dateof:3.0.0]). Without this setting, @@ -2781,8 +2893,8 @@ struct sqlite3_mem_methods { ** statistics. For statistics to be collected, the flag must be set on ** the database handle both when the SQL statement is prepared and when it ** is stepped. The flag is set (collection of statistics is enabled) -** by default. This option takes two arguments: an integer and a pointer to -** an integer.. The first argument is 1, 0, or -1 to enable, disable, or +** by default.

    This option takes two arguments: an integer and a pointer to +** an integer. The first argument is 1, 0, or -1 to enable, disable, or ** leave unchanged the statement scanstatus option. If the second argument ** is not NULL, then the value of the statement scanstatus setting after ** processing the first argument is written into the integer that the second @@ -2795,7 +2907,7 @@ struct sqlite3_mem_methods { ** in which tables and indexes are scanned so that the scans start at the end ** and work toward the beginning rather than starting at the beginning and ** working toward the end. Setting SQLITE_DBCONFIG_REVERSE_SCANORDER is the -** same as setting [PRAGMA reverse_unordered_selects]. This option takes +** same as setting [PRAGMA reverse_unordered_selects].

    This option takes ** two arguments which are an integer and a pointer to an integer. The first ** argument is 1, 0, or -1 to enable, disable, or leave unchanged the ** reverse scan order flag, respectively. If the second argument is not NULL, @@ -2804,7 +2916,76 @@ struct sqlite3_mem_methods { ** first argument. **

    ** +** [[SQLITE_DBCONFIG_ENABLE_ATTACH_CREATE]] +**
    SQLITE_DBCONFIG_ENABLE_ATTACH_CREATE
    +**
    The SQLITE_DBCONFIG_ENABLE_ATTACH_CREATE option enables or disables +** the ability of the [ATTACH DATABASE] SQL command to create a new database +** file if the database filed named in the ATTACH command does not already +** exist. This ability of ATTACH to create a new database is enabled by +** default. Applications can disable or reenable the ability for ATTACH to +** create new database files using this DBCONFIG option.

    +** This option takes two arguments which are an integer and a pointer +** to an integer. The first argument is 1, 0, or -1 to enable, disable, or +** leave unchanged the attach-create flag, respectively. If the second +** argument is not NULL, then 0 or 1 is written into the integer that the +** second argument points to depending on if the attach-create flag is set +** after processing the first argument. +**

    +** +** [[SQLITE_DBCONFIG_ENABLE_ATTACH_WRITE]] +**
    SQLITE_DBCONFIG_ENABLE_ATTACH_WRITE
    +**
    The SQLITE_DBCONFIG_ENABLE_ATTACH_WRITE option enables or disables the +** ability of the [ATTACH DATABASE] SQL command to open a database for writing. +** This capability is enabled by default. Applications can disable or +** reenable this capability using the current DBCONFIG option. If +** this capability is disabled, the [ATTACH] command will still work, +** but the database will be opened read-only. If this option is disabled, +** then the ability to create a new database using [ATTACH] is also disabled, +** regardless of the value of the [SQLITE_DBCONFIG_ENABLE_ATTACH_CREATE] +** option.

    +** This option takes two arguments which are an integer and a pointer +** to an integer. The first argument is 1, 0, or -1 to enable, disable, or +** leave unchanged the ability to ATTACH another database for writing, +** respectively. If the second argument is not NULL, then 0 or 1 is written +** into the integer to which the second argument points, depending on whether +** the ability to ATTACH a read/write database is enabled or disabled +** after processing the first argument. +**

    +** +** [[SQLITE_DBCONFIG_ENABLE_COMMENTS]] +**
    SQLITE_DBCONFIG_ENABLE_COMMENTS
    +**
    The SQLITE_DBCONFIG_ENABLE_COMMENTS option enables or disables the +** ability to include comments in SQL text. Comments are enabled by default. +** An application can disable or reenable comments in SQL text using this +** DBCONFIG option.

    +** This option takes two arguments which are an integer and a pointer +** to an integer. The first argument is 1, 0, or -1 to enable, disable, or +** leave unchanged the ability to use comments in SQL text, +** respectively. If the second argument is not NULL, then 0 or 1 is written +** into the integer that the second argument points to depending on if +** comments are allowed in SQL text after processing the first argument. +**

    +** ** +** +** [[DBCONFIG arguments]]

    Arguments To SQLITE_DBCONFIG Options

    +** +**

    Most of the SQLITE_DBCONFIG options take two arguments, so that the +** overall call to [sqlite3_db_config()] has a total of four parameters. +** The first argument (the third parameter to sqlite3_db_config()) is an integer. +** The second argument is a pointer to an integer. If the first argument is 1, +** then the option becomes enabled. If the first integer argument is 0, then the +** option is disabled. If the first argument is -1, then the option setting +** is unchanged. The second argument, the pointer to an integer, may be NULL. +** If the second argument is not NULL, then a value of 0 or 1 is written into +** the integer to which the second argument points, depending on whether the +** setting is disabled or enabled after applying any changes specified by +** the first argument. +** +**

    While most SQLITE_DBCONFIG options use the argument format +** described in the previous paragraph, the [SQLITE_DBCONFIG_MAINDBNAME] +** and [SQLITE_DBCONFIG_LOOKASIDE] options are different. See the +** documentation of those exceptional options for details. */ #define SQLITE_DBCONFIG_MAINDBNAME 1000 /* const char* */ #define SQLITE_DBCONFIG_LOOKASIDE 1001 /* void* int int */ @@ -2826,7 +3007,10 @@ struct sqlite3_mem_methods { #define SQLITE_DBCONFIG_TRUSTED_SCHEMA 1017 /* int int* */ #define SQLITE_DBCONFIG_STMT_SCANSTATUS 1018 /* int int* */ #define SQLITE_DBCONFIG_REVERSE_SCANORDER 1019 /* int int* */ -#define SQLITE_DBCONFIG_MAX 1019 /* Largest DBCONFIG */ +#define SQLITE_DBCONFIG_ENABLE_ATTACH_CREATE 1020 /* int int* */ +#define SQLITE_DBCONFIG_ENABLE_ATTACH_WRITE 1021 /* int int* */ +#define SQLITE_DBCONFIG_ENABLE_COMMENTS 1022 /* int int* */ +#define SQLITE_DBCONFIG_MAX 1022 /* Largest DBCONFIG */ /* ** CAPI3REF: Enable Or Disable Extended Result Codes @@ -2918,10 +3102,14 @@ SQLITE_API void sqlite3_set_last_insert_rowid(sqlite3*,sqlite3_int64); ** deleted by the most recently completed INSERT, UPDATE or DELETE ** statement on the database connection specified by the only parameter. ** The two functions are identical except for the type of the return value -** and that if the number of rows modified by the most recent INSERT, UPDATE +** and that if the number of rows modified by the most recent INSERT, UPDATE, ** or DELETE is greater than the maximum value supported by type "int", then ** the return value of sqlite3_changes() is undefined. ^Executing any other ** type of SQL statement does not modify the value returned by these functions. +** For the purposes of this interface, a CREATE TABLE AS SELECT statement +** does not count as an INSERT, UPDATE or DELETE statement and hence the rows +** added to the new table by the CREATE TABLE AS SELECT statement are not +** counted. ** ** ^Only changes made directly by the INSERT, UPDATE or DELETE statement are ** considered - auxiliary changes caused by [CREATE TRIGGER | triggers], @@ -3074,7 +3262,7 @@ SQLITE_API int sqlite3_is_interrupted(sqlite3*); ** ^These routines return 0 if the statement is incomplete. ^If a ** memory allocation fails, then SQLITE_NOMEM is returned. ** -** ^These routines do not parse the SQL statements thus +** ^These routines do not parse the SQL statements and thus ** will not detect syntactically incorrect SQL. ** ** ^(If SQLite has not been initialized using [sqlite3_initialize()] prior @@ -3176,6 +3364,44 @@ SQLITE_API int sqlite3_busy_handler(sqlite3*,int(*)(void*,int),void*); */ SQLITE_API int sqlite3_busy_timeout(sqlite3*, int ms); +/* +** CAPI3REF: Set the Setlk Timeout +** METHOD: sqlite3 +** +** This routine is only useful in SQLITE_ENABLE_SETLK_TIMEOUT builds. If +** the VFS supports blocking locks, it sets the timeout in ms used by +** eligible locks taken on wal mode databases by the specified database +** handle. In non-SQLITE_ENABLE_SETLK_TIMEOUT builds, or if the VFS does +** not support blocking locks, this function is a no-op. +** +** Passing 0 to this function disables blocking locks altogether. Passing +** -1 to this function requests that the VFS blocks for a long time - +** indefinitely if possible. The results of passing any other negative value +** are undefined. +** +** Internally, each SQLite database handle stores two timeout values - the +** busy-timeout (used for rollback mode databases, or if the VFS does not +** support blocking locks) and the setlk-timeout (used for blocking locks +** on wal-mode databases). The sqlite3_busy_timeout() method sets both +** values, this function sets only the setlk-timeout value. Therefore, +** to configure separate busy-timeout and setlk-timeout values for a single +** database handle, call sqlite3_busy_timeout() followed by this function. +** +** Whenever the number of connections to a wal mode database falls from +** 1 to 0, the last connection takes an exclusive lock on the database, +** then checkpoints and deletes the wal file. While it is doing this, any +** new connection that tries to read from the database fails with an +** SQLITE_BUSY error. Or, if the SQLITE_SETLK_BLOCK_ON_CONNECT flag is +** passed to this API, the new connection blocks until the exclusive lock +** has been released. +*/ +SQLITE_API int sqlite3_setlk_timeout(sqlite3*, int ms, int flags); + +/* +** CAPI3REF: Flags for sqlite3_setlk_timeout() +*/ +#define SQLITE_SETLK_BLOCK_ON_CONNECT 0x01 + /* ** CAPI3REF: Convenience Routines For Running Queries ** METHOD: sqlite3 @@ -3183,7 +3409,7 @@ SQLITE_API int sqlite3_busy_timeout(sqlite3*, int ms); ** This is a legacy interface that is preserved for backwards compatibility. ** Use of this interface is not recommended. ** -** Definition: A result table is memory data structure created by the +** Definition: A result table is a memory data structure created by the ** [sqlite3_get_table()] interface. A result table records the ** complete query results from one or more queries. ** @@ -3326,7 +3552,7 @@ SQLITE_API char *sqlite3_vsnprintf(int,char*,const char*, va_list); ** ^Calling sqlite3_free() with a pointer previously returned ** by sqlite3_malloc() or sqlite3_realloc() releases that memory so ** that it might be reused. ^The sqlite3_free() routine is -** a no-op if is called with a NULL pointer. Passing a NULL pointer +** a no-op if it is called with a NULL pointer. Passing a NULL pointer ** to sqlite3_free() is harmless. After being freed, memory ** should neither be read nor written. Even reading previously freed ** memory might result in a segmentation fault or other severe error. @@ -3344,13 +3570,13 @@ SQLITE_API char *sqlite3_vsnprintf(int,char*,const char*, va_list); ** sqlite3_free(X). ** ^sqlite3_realloc(X,N) returns a pointer to a memory allocation ** of at least N bytes in size or NULL if insufficient memory is available. -** ^If M is the size of the prior allocation, then min(N,M) bytes -** of the prior allocation are copied into the beginning of buffer returned +** ^If M is the size of the prior allocation, then min(N,M) bytes of the +** prior allocation are copied into the beginning of the buffer returned ** by sqlite3_realloc(X,N) and the prior allocation is freed. ** ^If sqlite3_realloc(X,N) returns NULL and N is positive, then the ** prior allocation is not freed. ** -** ^The sqlite3_realloc64(X,N) interfaces works the same as +** ^The sqlite3_realloc64(X,N) interface works the same as ** sqlite3_realloc(X,N) except that N is a 64-bit unsigned integer instead ** of a 32-bit signed integer. ** @@ -3400,7 +3626,7 @@ SQLITE_API sqlite3_uint64 sqlite3_msize(void*); ** was last reset. ^The values returned by [sqlite3_memory_used()] and ** [sqlite3_memory_highwater()] include any overhead ** added by SQLite in its implementation of [sqlite3_malloc()], -** but not overhead added by the any underlying system library +** but not overhead added by any underlying system library ** routines that [sqlite3_malloc()] may call. ** ** ^The memory high-water mark is reset to the current value of @@ -3601,8 +3827,8 @@ SQLITE_API int sqlite3_set_authorizer( #define SQLITE_RECURSIVE 33 /* NULL NULL */ /* -** CAPI3REF: Tracing And Profiling Functions -** METHOD: sqlite3 +** CAPI3REF: Deprecated Tracing And Profiling Functions +** DEPRECATED ** ** These routines are deprecated. Use the [sqlite3_trace_v2()] interface ** instead of the routines described here. @@ -3852,7 +4078,7 @@ SQLITE_API void sqlite3_progress_handler(sqlite3*, int, int(*)(void*), void*); ** there is no harm in trying.) ** ** ^(

    [SQLITE_OPEN_SHAREDCACHE]
    -**
    The database is opened [shared cache] enabled, overriding +**
    The database is opened with [shared cache] enabled, overriding ** the default shared cache setting provided by ** [sqlite3_enable_shared_cache()].)^ ** The [use of shared cache mode is discouraged] and hence shared cache @@ -3860,14 +4086,14 @@ SQLITE_API void sqlite3_progress_handler(sqlite3*, int, int(*)(void*), void*); ** this option is a no-op. ** ** ^(
    [SQLITE_OPEN_PRIVATECACHE]
    -**
    The database is opened [shared cache] disabled, overriding +**
    The database is opened with [shared cache] disabled, overriding ** the default shared cache setting provided by ** [sqlite3_enable_shared_cache()].)^ ** ** [[OPEN_EXRESCODE]] ^(
    [SQLITE_OPEN_EXRESCODE]
    **
    The database connection comes up in "extended result code mode". -** In other words, the database behaves has if -** [sqlite3_extended_result_codes(db,1)] where called on the database +** In other words, the database behaves as if +** [sqlite3_extended_result_codes(db,1)] were called on the database ** connection as soon as the connection is created. In addition to setting ** the extended result code mode, this flag also causes [sqlite3_open_v2()] ** to return an extended result code.
    @@ -4195,7 +4421,7 @@ SQLITE_API sqlite3_file *sqlite3_database_file_object(const char*); ** ** The sqlite3_create_filename(D,J,W,N,P) allocates memory to hold a version of ** database filename D with corresponding journal file J and WAL file W and -** with N URI parameters key/values pairs in the array P. The result from +** an array P of N URI Key/Value pairs. The result from ** sqlite3_create_filename(D,J,W,N,P) is a pointer to a database filename that ** is safe to pass to routines like: **
      @@ -4278,7 +4504,7 @@ SQLITE_API void sqlite3_free_filename(sqlite3_filename); ** subsequent calls to other SQLite interface functions.)^ ** ** ^The sqlite3_errstr(E) interface returns the English-language text -** that describes the [result code] E, as UTF-8, or NULL if E is not an +** that describes the [result code] E, as UTF-8, or NULL if E is not a ** result code for which a text error message is available. ** ^(Memory to hold the error message string is managed internally ** and must not be freed by the application)^. @@ -4286,7 +4512,7 @@ SQLITE_API void sqlite3_free_filename(sqlite3_filename); ** ^If the most recent error references a specific token in the input ** SQL, the sqlite3_error_offset() interface returns the byte offset ** of the start of that token. ^The byte offset returned by -** sqlite3_error_offset() assumes that the input SQL is UTF8. +** sqlite3_error_offset() assumes that the input SQL is UTF-8. ** ^If the most recent error does not reference a specific token in the input ** SQL, then the sqlite3_error_offset() function returns -1. ** @@ -4311,6 +4537,34 @@ SQLITE_API const void *sqlite3_errmsg16(sqlite3*); SQLITE_API const char *sqlite3_errstr(int); SQLITE_API int sqlite3_error_offset(sqlite3 *db); +/* +** CAPI3REF: Set Error Codes And Message +** METHOD: sqlite3 +** +** Set the error code of the database handle passed as the first argument +** to errcode, and the error message to a copy of nul-terminated string +** zErrMsg. If zErrMsg is passed NULL, then the error message is set to +** the default message associated with the supplied error code. Subsequent +** calls to [sqlite3_errcode()] and [sqlite3_errmsg()] and similar will +** return the values set by this routine in place of what was previously +** set by SQLite itself. +** +** This function returns SQLITE_OK if the error code and error message are +** successfully set, SQLITE_NOMEM if an OOM occurs, and SQLITE_MISUSE if +** the database handle is NULL or invalid. +** +** The error code and message set by this routine remains in effect until +** they are changed, either by another call to this routine or until they are +** changed to by SQLite itself to reflect the result of some subsquent +** API call. +** +** This function is intended for use by SQLite extensions or wrappers. The +** idea is that an extension or wrapper can use this routine to set error +** messages and error codes and thus behave more like a core SQLite +** feature from the point of view of an application. +*/ +SQLITE_API int sqlite3_set_errmsg(sqlite3 *db, int errcode, const char *zErrMsg); + /* ** CAPI3REF: Prepared Statement Object ** KEYWORDS: {prepared statement} {prepared statements} @@ -4385,8 +4639,8 @@ SQLITE_API int sqlite3_limit(sqlite3*, int id, int newVal); ** ** These constants define various performance limits ** that can be lowered at run-time using [sqlite3_limit()]. -** The synopsis of the meanings of the various limits is shown below. -** Additional information is available at [limits | Limits in SQLite]. +** A concise description of these limits follows, and additional information +** is available at [limits | Limits in SQLite]. ** **
      ** [[SQLITE_LIMIT_LENGTH]] ^(
      SQLITE_LIMIT_LENGTH
      @@ -4451,7 +4705,7 @@ SQLITE_API int sqlite3_limit(sqlite3*, int id, int newVal); /* ** CAPI3REF: Prepare Flags ** -** These constants define various flags that can be passed into +** These constants define various flags that can be passed into the ** "prepFlags" parameter of the [sqlite3_prepare_v3()] and ** [sqlite3_prepare16_v3()] interfaces. ** @@ -4481,11 +4735,22 @@ SQLITE_API int sqlite3_limit(sqlite3*, int id, int newVal); **
      The SQLITE_PREPARE_NO_VTAB flag causes the SQL compiler ** to return an error (error code SQLITE_ERROR) if the statement uses ** any virtual tables. +** +** [[SQLITE_PREPARE_DONT_LOG]]
      SQLITE_PREPARE_DONT_LOG
      +**
      The SQLITE_PREPARE_DONT_LOG flag prevents SQL compiler +** errors from being sent to the error log defined by +** [SQLITE_CONFIG_LOG]. This can be used, for example, to do test +** compiles to see if some SQL syntax is well-formed, without generating +** messages on the global error log when it is not. If the test compile +** fails, the sqlite3_prepare_v3() call returns the same error indications +** with or without this flag; it just omits the call to [sqlite3_log()] that +** logs the error. **
      */ #define SQLITE_PREPARE_PERSISTENT 0x01 #define SQLITE_PREPARE_NORMALIZE 0x02 #define SQLITE_PREPARE_NO_VTAB 0x04 +#define SQLITE_PREPARE_DONT_LOG 0x10 /* ** CAPI3REF: Compiling An SQL Statement @@ -4518,13 +4783,17 @@ SQLITE_API int sqlite3_limit(sqlite3*, int id, int newVal); ** and sqlite3_prepare16_v3() use UTF-16. ** ** ^If the nByte argument is negative, then zSql is read up to the -** first zero terminator. ^If nByte is positive, then it is the -** number of bytes read from zSql. ^If nByte is zero, then no prepared +** first zero terminator. ^If nByte is positive, then it is the maximum +** number of bytes read from zSql. When nByte is positive, zSql is read +** up to the first zero terminator or until the nByte bytes have been read, +** whichever comes first. ^If nByte is zero, then no prepared ** statement is generated. ** If the caller knows that the supplied string is nul-terminated, then ** there is a small performance advantage to passing an nByte parameter that ** is the number of bytes in the input string including ** the nul-terminator. +** Note that nByte measures the length of the input in bytes, not +** characters, even for the UTF-16 interfaces. ** ** ^If pzTail is not NULL then *pzTail is made to point to the first byte ** past the end of the first SQL statement in zSql. These routines only @@ -4657,7 +4926,7 @@ SQLITE_API int sqlite3_prepare16_v3( ** ** ^The sqlite3_expanded_sql() interface returns NULL if insufficient memory ** is available to hold the result, or if the result would exceed the -** the maximum string length determined by the [SQLITE_LIMIT_LENGTH]. +** maximum string length determined by the [SQLITE_LIMIT_LENGTH]. ** ** ^The [SQLITE_TRACE_SIZE_LIMIT] compile-time option limits the size of ** bound parameter expansions. ^The [SQLITE_OMIT_TRACE] compile-time @@ -4845,7 +5114,7 @@ typedef struct sqlite3_value sqlite3_value; ** ** The context in which an SQL function executes is stored in an ** sqlite3_context object. ^A pointer to an sqlite3_context object -** is always first parameter to [application-defined SQL functions]. +** is always the first parameter to [application-defined SQL functions]. ** The application-defined SQL function implementation will pass this ** pointer through into calls to [sqlite3_result_int | sqlite3_result()], ** [sqlite3_aggregate_context()], [sqlite3_user_data()], @@ -4861,7 +5130,7 @@ typedef struct sqlite3_context sqlite3_context; ** METHOD: sqlite3_stmt ** ** ^(In the SQL statement text input to [sqlite3_prepare_v2()] and its variants, -** literals may be replaced by a [parameter] that matches one of following +** literals may be replaced by a [parameter] that matches one of the following ** templates: ** **
        @@ -4906,7 +5175,7 @@ typedef struct sqlite3_context sqlite3_context; ** ** [[byte-order determination rules]] ^The byte-order of ** UTF16 input text is determined by the byte-order mark (BOM, U+FEFF) -** found in first character, which is removed, or in the absence of a BOM +** found in the first character, which is removed, or in the absence of a BOM ** the byte order is the native byte order of the host ** machine for sqlite3_bind_text16() or the byte order specified in ** the 6th parameter for sqlite3_bind_text64().)^ @@ -4926,7 +5195,7 @@ typedef struct sqlite3_context sqlite3_context; ** or sqlite3_bind_text16() or sqlite3_bind_text64() then ** that parameter must be the byte offset ** where the NUL terminator would occur assuming the string were NUL -** terminated. If any NUL characters occurs at byte offsets less than +** terminated. If any NUL characters occur at byte offsets less than ** the value of the fourth parameter then the resulting string value will ** contain embedded NULs. The result of expressions involving strings ** with embedded NULs is undefined. @@ -4969,9 +5238,11 @@ typedef struct sqlite3_context sqlite3_context; ** associated with the pointer P of type T. ^D is either a NULL pointer or ** a pointer to a destructor function for P. ^SQLite will invoke the ** destructor D with a single argument of P when it is finished using -** P. The T parameter should be a static string, preferably a string -** literal. The sqlite3_bind_pointer() routine is part of the -** [pointer passing interface] added for SQLite 3.20.0. +** P, even if the call to sqlite3_bind_pointer() fails. Due to a +** historical design quirk, results are undefined if D is +** SQLITE_TRANSIENT. The T parameter should be a static string, +** preferably a string literal. The sqlite3_bind_pointer() routine is +** part of the [pointer passing interface] added for SQLite 3.20.0. ** ** ^If any of the sqlite3_bind_*() routines are called with a NULL pointer ** for the [prepared statement] or with a prepared statement for which @@ -5138,7 +5409,7 @@ SQLITE_API const void *sqlite3_column_name16(sqlite3_stmt*, int N); ** METHOD: sqlite3_stmt ** ** ^These routines provide a means to determine the database, table, and -** table column that is the origin of a particular result column in +** table column that is the origin of a particular result column in a ** [SELECT] statement. ** ^The name of the database or table or column can be returned as ** either a UTF-8 or UTF-16 string. ^The _database_ routines return @@ -5276,7 +5547,7 @@ SQLITE_API const void *sqlite3_column_decltype16(sqlite3_stmt*,int); ** other than [SQLITE_ROW] before any subsequent invocation of ** sqlite3_step(). Failure to reset the prepared statement using ** [sqlite3_reset()] would result in an [SQLITE_MISUSE] return from -** sqlite3_step(). But after [version 3.6.23.1] ([dateof:3.6.23.1], +** sqlite3_step(). But after [version 3.6.23.1] ([dateof:3.6.23.1]), ** sqlite3_step() began ** calling [sqlite3_reset()] automatically in this circumstance rather ** than returning [SQLITE_MISUSE]. This is not considered a compatibility @@ -5582,7 +5853,7 @@ SQLITE_API int sqlite3_column_type(sqlite3_stmt*, int iCol); ** ** ^The sqlite3_finalize() function is called to delete a [prepared statement]. ** ^If the most recent evaluation of the statement encountered no errors -** or if the statement is never been evaluated, then sqlite3_finalize() returns +** or if the statement has never been evaluated, then sqlite3_finalize() returns ** SQLITE_OK. ^If the most recent evaluation of statement S failed, then ** sqlite3_finalize(S) returns the appropriate [error code] or ** [extended error code]. @@ -5707,8 +5978,8 @@ SQLITE_API int sqlite3_reset(sqlite3_stmt *pStmt); ** ** For best security, the [SQLITE_DIRECTONLY] flag is recommended for ** all application-defined SQL functions that do not need to be -** used inside of triggers, view, CHECK constraints, or other elements of -** the database schema. This flags is especially recommended for SQL +** used inside of triggers, views, CHECK constraints, or other elements of +** the database schema. This flag is especially recommended for SQL ** functions that have side effects or reveal internal application state. ** Without this flag, an attacker might be able to modify the schema of ** a database file to include invocations of the function with parameters @@ -5739,7 +6010,7 @@ SQLITE_API int sqlite3_reset(sqlite3_stmt *pStmt); ** [user-defined window functions|available here]. ** ** ^(If the final parameter to sqlite3_create_function_v2() or -** sqlite3_create_window_function() is not NULL, then it is destructor for +** sqlite3_create_window_function() is not NULL, then it is the destructor for ** the application data pointer. The destructor is invoked when the function ** is deleted, either by being overloaded or when the database connection ** closes.)^ ^The destructor is also invoked if the call to @@ -5814,7 +6085,7 @@ SQLITE_API int sqlite3_create_window_function( /* ** CAPI3REF: Text Encodings ** -** These constant define integer codes that represent the various +** These constants define integer codes that represent the various ** text encodings supported by SQLite. */ #define SQLITE_UTF8 1 /* IMP: R-37514-35566 */ @@ -5895,7 +6166,7 @@ SQLITE_API int sqlite3_create_window_function( ** This flag instructs SQLite to omit some corner-case optimizations that ** might disrupt the operation of the [sqlite3_value_subtype()] function, ** causing it to return zero rather than the correct subtype(). -** SQL functions that invokes [sqlite3_value_subtype()] should have this +** All SQL functions that invoke [sqlite3_value_subtype()] should have this ** property. If the SQLITE_SUBTYPE property is omitted, then the return ** value from [sqlite3_value_subtype()] might sometimes be zero even though ** a non-zero subtype was specified by the function argument expression. @@ -5906,11 +6177,20 @@ SQLITE_API int sqlite3_create_window_function( ** result. ** Every function that invokes [sqlite3_result_subtype()] should have this ** property. If it does not, then the call to [sqlite3_result_subtype()] -** might become a no-op if the function is used as term in an +** might become a no-op if the function is used as a term in an ** [expression index]. On the other hand, SQL functions that never invoke ** [sqlite3_result_subtype()] should avoid setting this property, as the ** purpose of this property is to disable certain optimizations that are ** incompatible with subtypes. +** +** [[SQLITE_SELFORDER1]]
        SQLITE_SELFORDER1
        +** The SQLITE_SELFORDER1 flag indicates that the function is an aggregate +** that internally orders the values provided to the first argument. The +** ordered-set aggregate SQL notation with a single ORDER BY term can be +** used to invoke this function. If the ordered-set aggregate notation is +** used on a function that lacks this flag, then an error is raised. Note +** that the ordered-set aggregate syntax is only available if SQLite is +** built using the -DSQLITE_ENABLE_ORDERED_SET_AGGREGATES compile-time option. **
        ** */ @@ -5919,6 +6199,7 @@ SQLITE_API int sqlite3_create_window_function( #define SQLITE_SUBTYPE 0x000100000 #define SQLITE_INNOCUOUS 0x000200000 #define SQLITE_RESULT_SUBTYPE 0x001000000 +#define SQLITE_SELFORDER1 0x002000000 /* ** CAPI3REF: Deprecated Functions @@ -6023,7 +6304,7 @@ SQLITE_API SQLITE_DEPRECATED int sqlite3_memory_alarm(void(*)(void*,sqlite3_int6 ** sqlite3_value_nochange(X) interface returns true if and only if ** the column corresponding to X is unchanged by the UPDATE operation ** that the xUpdate method call was invoked to implement and if -** and the prior [xColumn] method call that was invoked to extracted +** the prior [xColumn] method call that was invoked to extract ** the value for that column returned without setting a result (probably ** because it queried [sqlite3_vtab_nochange()] and found that the column ** was unchanging). ^Within an [xUpdate] method, any value for which @@ -6116,7 +6397,7 @@ SQLITE_API int sqlite3_value_encoding(sqlite3_value*); ** one SQL function to another. Use the [sqlite3_result_subtype()] ** routine to set the subtype for the return value of an SQL function. ** -** Every [application-defined SQL function] that invoke this interface +** Every [application-defined SQL function] that invokes this interface ** should include the [SQLITE_SUBTYPE] property in the text ** encoding argument when the function is [sqlite3_create_function|registered]. ** If the [SQLITE_SUBTYPE] property is omitted, then sqlite3_value_subtype() @@ -6129,7 +6410,7 @@ SQLITE_API unsigned int sqlite3_value_subtype(sqlite3_value*); ** METHOD: sqlite3_value ** ** ^The sqlite3_value_dup(V) interface makes a copy of the [sqlite3_value] -** object D and returns a pointer to that copy. ^The [sqlite3_value] returned +** object V and returns a pointer to that copy. ^The [sqlite3_value] returned ** is a [protected sqlite3_value] object even if the input is not. ** ^The sqlite3_value_dup(V) interface returns NULL if V is NULL or if a ** memory allocation fails. ^If V is a [pointer value], then the result @@ -6167,7 +6448,7 @@ SQLITE_API void sqlite3_value_free(sqlite3_value*); ** allocation error occurs. ** ** ^(The amount of space allocated by sqlite3_aggregate_context(C,N) is -** determined by the N parameter on first successful call. Changing the +** determined by the N parameter on the first successful call. Changing the ** value of N in any subsequent call to sqlite3_aggregate_context() within ** the same aggregate function instance will not resize the memory ** allocation.)^ Within the xFinal callback, it is customary to set @@ -6296,6 +6577,7 @@ SQLITE_API void sqlite3_set_auxdata(sqlite3_context*, int N, void*, void (*)(voi ** or a NULL pointer if there were no prior calls to ** sqlite3_set_clientdata() with the same values of D and N. ** Names are compared using strcmp() and are thus case sensitive. +** It returns 0 on success and SQLITE_NOMEM on allocation failure. ** ** If P and X are both non-NULL, then the destructor X is invoked with ** argument P on the first of the following occurrences: @@ -6329,7 +6611,7 @@ SQLITE_API void sqlite3_set_auxdata(sqlite3_context*, int N, void*, void (*)(voi ** ** Security Warning: These interfaces should not be exposed in scripting ** languages or in other circumstances where it might be possible for an -** an attacker to invoke them. Any agent that can invoke these interfaces +** attacker to invoke them. Any agent that can invoke these interfaces ** can probably also take control of the process. ** ** Database connection client data is only available for SQLite @@ -6443,7 +6725,7 @@ typedef void (*sqlite3_destructor_type)(void*); ** pointed to by the 2nd parameter are taken as the application-defined ** function result. If the 3rd parameter is non-negative, then it ** must be the byte offset into the string where the NUL terminator would -** appear if the string where NUL terminated. If any NUL characters occur +** appear if the string were NUL terminated. If any NUL characters occur ** in the string at a byte offset that is less than the value of the 3rd ** parameter, then the resulting string will contain embedded NULs and the ** result of expressions operating on strings with embedded NULs is undefined. @@ -6501,7 +6783,7 @@ typedef void (*sqlite3_destructor_type)(void*); ** string and preferably a string literal. The sqlite3_result_pointer() ** routine is part of the [pointer passing interface] added for SQLite 3.20.0. ** -** If these routines are called from within the different thread +** If these routines are called from within a different thread ** than the one containing the application-defined function that received ** the [sqlite3_context] pointer, the results are undefined. */ @@ -6907,7 +7189,7 @@ SQLITE_API sqlite3 *sqlite3_db_handle(sqlite3_stmt*); ** METHOD: sqlite3 ** ** ^The sqlite3_db_name(D,N) interface returns a pointer to the schema name -** for the N-th database on database connection D, or a NULL pointer of N is +** for the N-th database on database connection D, or a NULL pointer if N is ** out of range. An N value of 0 means the main database file. An N of 1 is ** the "temp" schema. Larger values of N correspond to various ATTACH-ed ** databases. @@ -7002,7 +7284,7 @@ SQLITE_API int sqlite3_txn_state(sqlite3*,const char *zSchema); **
        The SQLITE_TXN_READ state means that the database is currently ** in a read transaction. Content has been read from the database file ** but nothing in the database file has changed. The transaction state -** will advanced to SQLITE_TXN_WRITE if any changes occur and there are +** will be advanced to SQLITE_TXN_WRITE if any changes occur and there are ** no other conflicting concurrent write transactions. The transaction ** state will revert to SQLITE_TXN_NONE following a [ROLLBACK] or ** [COMMIT].
        @@ -7011,7 +7293,7 @@ SQLITE_API int sqlite3_txn_state(sqlite3*,const char *zSchema); **
        The SQLITE_TXN_WRITE state means that the database is currently ** in a write transaction. Content has been written to the database file ** but has not yet committed. The transaction state will change to -** to SQLITE_TXN_NONE at the next [ROLLBACK] or [COMMIT].
        +** SQLITE_TXN_NONE at the next [ROLLBACK] or [COMMIT]. */ #define SQLITE_TXN_NONE 0 #define SQLITE_TXN_READ 1 @@ -7162,6 +7444,8 @@ SQLITE_API int sqlite3_autovacuum_pages( ** ** ^The second argument is a pointer to the function to invoke when a ** row is updated, inserted or deleted in a rowid table. +** ^The update hook is disabled by invoking sqlite3_update_hook() +** with a NULL pointer as the second parameter. ** ^The first argument to the callback is a copy of the third argument ** to sqlite3_update_hook(). ** ^The second callback argument is one of [SQLITE_INSERT], [SQLITE_DELETE], @@ -7183,6 +7467,12 @@ SQLITE_API int sqlite3_autovacuum_pages( ** The exceptions defined in this paragraph might change in a future ** release of SQLite. ** +** Whether the update hook is invoked before or after the +** corresponding change is currently unspecified and may differ +** depending on the type of change. Do not rely on the order of the +** hook call with regards to the final result of the operation which +** triggers the hook. +** ** The update hook implementation must not do anything that will modify ** the database connection that invoked the update hook. Any actions ** to modify the database connection must be deferred until after the @@ -7284,7 +7574,7 @@ SQLITE_API int sqlite3_db_release_memory(sqlite3*); ** CAPI3REF: Impose A Limit On Heap Size ** ** These interfaces impose limits on the amount of heap memory that will be -** by all database connections within a single process. +** used by all database connections within a single process. ** ** ^The sqlite3_soft_heap_limit64() interface sets and/or queries the ** soft limit on the amount of heap memory that may be allocated by SQLite. @@ -7342,7 +7632,7 @@ SQLITE_API int sqlite3_db_release_memory(sqlite3*); **
      )^ ** ** The circumstances under which SQLite will enforce the heap limits may -** changes in future releases of SQLite. +** change in future releases of SQLite. */ SQLITE_API sqlite3_int64 sqlite3_soft_heap_limit64(sqlite3_int64 N); SQLITE_API sqlite3_int64 sqlite3_hard_heap_limit64(sqlite3_int64 N); @@ -7457,8 +7747,8 @@ SQLITE_API int sqlite3_table_column_metadata( ** ^The entry point is zProc. ** ^(zProc may be 0, in which case SQLite will try to come up with an ** entry point name on its own. It first tries "sqlite3_extension_init". -** If that does not work, it constructs a name "sqlite3_X_init" where the -** X is consists of the lower-case equivalent of all ASCII alphabetic +** If that does not work, it constructs a name "sqlite3_X_init" where +** X consists of the lower-case equivalent of all ASCII alphabetic ** characters in the filename from the last "/" to the first following ** "." and omitting any initial "lib".)^ ** ^The sqlite3_load_extension() interface returns @@ -7529,7 +7819,7 @@ SQLITE_API int sqlite3_enable_load_extension(sqlite3 *db, int onoff); ** ^(Even though the function prototype shows that xEntryPoint() takes ** no arguments and returns void, SQLite invokes xEntryPoint() with three ** arguments and expects an integer result as if the signature of the -** entry point where as follows: +** entry point were as follows: ** **
       **    int xEntryPoint(
      @@ -7693,7 +7983,7 @@ struct sqlite3_module {
       ** virtual table and might not be checked again by the byte code.)^ ^(The
       ** aConstraintUsage[].omit flag is an optimization hint. When the omit flag
       ** is left in its default setting of false, the constraint will always be
      -** checked separately in byte code.  If the omit flag is change to true, then
      +** checked separately in byte code.  If the omit flag is changed to true, then
       ** the constraint may or may not be checked in byte code.  In other words,
       ** when the omit flag is true there is no guarantee that the constraint will
       ** not be checked again using byte code.)^
      @@ -7717,9 +8007,11 @@ struct sqlite3_module {
       ** will be returned by the strategy.
       **
       ** The xBestIndex method may optionally populate the idxFlags field with a
      -** mask of SQLITE_INDEX_SCAN_* flags. Currently there is only one such flag -
      -** SQLITE_INDEX_SCAN_UNIQUE. If the xBestIndex method sets this flag, SQLite
      -** assumes that the strategy may visit at most one row.
      +** mask of SQLITE_INDEX_SCAN_* flags. One such flag is
      +** [SQLITE_INDEX_SCAN_HEX], which if set causes the [EXPLAIN QUERY PLAN]
      +** output to show the idxNum as hex instead of as decimal.  Another flag is
      +** SQLITE_INDEX_SCAN_UNIQUE, which if set indicates that the query plan will
      +** return at most one row.
       **
       ** Additionally, if xBestIndex sets the SQLITE_INDEX_SCAN_UNIQUE flag, then
       ** SQLite also assumes that if a call to the xUpdate() method is made as
      @@ -7783,7 +8075,9 @@ struct sqlite3_index_info {
       ** [sqlite3_index_info].idxFlags field to some combination of
       ** these bits.
       */
      -#define SQLITE_INDEX_SCAN_UNIQUE      1     /* Scan visits at most 1 row */
      +#define SQLITE_INDEX_SCAN_UNIQUE 0x00000001 /* Scan visits at most 1 row */
      +#define SQLITE_INDEX_SCAN_HEX    0x00000002 /* Display idxNum as hex */
      +                                            /* in EXPLAIN QUERY PLAN */
       
       /*
       ** CAPI3REF: Virtual Table Constraint Operator Codes
      @@ -7856,7 +8150,7 @@ struct sqlite3_index_info {
       ** the implementation of the [virtual table module].   ^The fourth
       ** parameter is an arbitrary client data pointer that is passed through
       ** into the [xCreate] and [xConnect] methods of the virtual table module
      -** when a new virtual table is be being created or reinitialized.
      +** when a new virtual table is being created or reinitialized.
       **
       ** ^The sqlite3_create_module_v2() interface has a fifth parameter which
       ** is a pointer to a destructor for the pClientData.  ^SQLite will
      @@ -8021,7 +8315,7 @@ typedef struct sqlite3_blob sqlite3_blob;
       ** in *ppBlob. Otherwise an [error code] is returned and, unless the error
       ** code is SQLITE_MISUSE, *ppBlob is set to NULL.)^ ^This means that, provided
       ** the API is not misused, it is always safe to call [sqlite3_blob_close()]
      -** on *ppBlob after this function it returns.
      +** on *ppBlob after this function returns.
       **
       ** This function fails with SQLITE_ERROR if any of the following are true:
       ** 
        @@ -8141,7 +8435,7 @@ SQLITE_API int sqlite3_blob_close(sqlite3_blob *); ** ** ^Returns the size in bytes of the BLOB accessible via the ** successfully opened [BLOB handle] in its only argument. ^The -** incremental blob I/O routines can only read or overwriting existing +** incremental blob I/O routines can only read or overwrite existing ** blob content; they cannot change the size of a blob. ** ** This routine only works on a [BLOB handle] which has been created @@ -8291,7 +8585,7 @@ SQLITE_API int sqlite3_vfs_unregister(sqlite3_vfs*); ** ^The sqlite3_mutex_alloc() routine allocates a new ** mutex and returns a pointer to it. ^The sqlite3_mutex_alloc() ** routine returns NULL if it is unable to allocate the requested -** mutex. The argument to sqlite3_mutex_alloc() must one of these +** mutex. The argument to sqlite3_mutex_alloc() must be one of these ** integer constants: ** **
          @@ -8524,7 +8818,7 @@ SQLITE_API int sqlite3_mutex_notheld(sqlite3_mutex*); ** CAPI3REF: Retrieve the mutex for a database connection ** METHOD: sqlite3 ** -** ^This interface returns a pointer the [sqlite3_mutex] object that +** ^This interface returns a pointer to the [sqlite3_mutex] object that ** serializes access to the [database connection] given in the argument ** when the [threading mode] is Serialized. ** ^If the [threading mode] is Single-thread or Multi-thread then this @@ -8620,6 +8914,7 @@ SQLITE_API int sqlite3_test_control(int op, ...); #define SQLITE_TESTCTRL_JSON_SELFCHECK 14 #define SQLITE_TESTCTRL_OPTIMIZATIONS 15 #define SQLITE_TESTCTRL_ISKEYWORD 16 /* NOT USED */ +#define SQLITE_TESTCTRL_GETOPT 16 #define SQLITE_TESTCTRL_SCRATCHMALLOC 17 /* NOT USED */ #define SQLITE_TESTCTRL_INTERNAL_FUNCTIONS 17 #define SQLITE_TESTCTRL_LOCALTIME_FAULT 18 @@ -8639,21 +8934,21 @@ SQLITE_API int sqlite3_test_control(int op, ...); #define SQLITE_TESTCTRL_TRACEFLAGS 31 #define SQLITE_TESTCTRL_TUNE 32 #define SQLITE_TESTCTRL_LOGEST 33 -#define SQLITE_TESTCTRL_USELONGDOUBLE 34 +#define SQLITE_TESTCTRL_USELONGDOUBLE 34 /* NOT USED */ #define SQLITE_TESTCTRL_LAST 34 /* Largest TESTCTRL */ /* ** CAPI3REF: SQL Keyword Checking ** ** These routines provide access to the set of SQL language keywords -** recognized by SQLite. Applications can uses these routines to determine +** recognized by SQLite. Applications can use these routines to determine ** whether or not a specific identifier needs to be escaped (for example, ** by enclosing in double-quotes) so as not to confuse the parser. ** ** The sqlite3_keyword_count() interface returns the number of distinct ** keywords understood by SQLite. ** -** The sqlite3_keyword_name(N,Z,L) interface finds the N-th keyword and +** The sqlite3_keyword_name(N,Z,L) interface finds the 0-based N-th keyword and ** makes *Z point to that keyword expressed as UTF8 and writes the number ** of bytes in the keyword into *L. The string that *Z points to is not ** zero-terminated. The sqlite3_keyword_name(N,Z,L) routine returns @@ -8814,7 +9109,7 @@ SQLITE_API void sqlite3_str_reset(sqlite3_str*); ** content of the dynamic string under construction in X. The value ** returned by [sqlite3_str_value(X)] is managed by the sqlite3_str object X ** and might be freed or altered by any subsequent method on the same -** [sqlite3_str] object. Applications must not used the pointer returned +** [sqlite3_str] object. Applications must not use the pointer returned by ** [sqlite3_str_value(X)] after any subsequent method call on the same ** object. ^Applications may change the content of the string returned ** by [sqlite3_str_value(X)] as long as they do not write into any bytes @@ -8900,7 +9195,7 @@ SQLITE_API int sqlite3_status64( ** allocation which could not be satisfied by the [SQLITE_CONFIG_PAGECACHE] ** buffer and where forced to overflow to [sqlite3_malloc()]. The ** returned value includes allocations that overflowed because they -** where too large (they were larger than the "sz" parameter to +** were too large (they were larger than the "sz" parameter to ** [SQLITE_CONFIG_PAGECACHE]) and allocations that overflowed because ** no space was left in the page cache.)^ ** @@ -8959,9 +9254,18 @@ SQLITE_API int sqlite3_status64( ** ^The sqlite3_db_status() routine returns SQLITE_OK on success and a ** non-zero [error code] on failure. ** +** ^The sqlite3_db_status64(D,O,C,H,R) routine works exactly the same +** way as sqlite3_db_status(D,O,C,H,R) routine except that the C and H +** parameters are pointer to 64-bit integers (type: sqlite3_int64) instead +** of pointers to 32-bit integers, which allows larger status values +** to be returned. If a status value exceeds 2,147,483,647 then +** sqlite3_db_status() will truncate the value whereas sqlite3_db_status64() +** will return the full value. +** ** See also: [sqlite3_status()] and [sqlite3_stmt_status()]. */ SQLITE_API int sqlite3_db_status(sqlite3*, int op, int *pCur, int *pHiwtr, int resetFlg); +SQLITE_API int sqlite3_db_status64(sqlite3*,int,sqlite3_int64*,sqlite3_int64*,int); /* ** CAPI3REF: Status Parameters for database connections @@ -8984,28 +9288,29 @@ SQLITE_API int sqlite3_db_status(sqlite3*, int op, int *pCur, int *pHiwtr, int r ** [[SQLITE_DBSTATUS_LOOKASIDE_HIT]] ^(
          SQLITE_DBSTATUS_LOOKASIDE_HIT
          **
          This parameter returns the number of malloc attempts that were ** satisfied using lookaside memory. Only the high-water value is meaningful; -** the current value is always zero.)^ +** the current value is always zero.
          )^ ** ** [[SQLITE_DBSTATUS_LOOKASIDE_MISS_SIZE]] ** ^(
          SQLITE_DBSTATUS_LOOKASIDE_MISS_SIZE
          -**
          This parameter returns the number malloc attempts that might have +**
          This parameter returns the number of malloc attempts that might have ** been satisfied using lookaside memory but failed due to the amount of ** memory requested being larger than the lookaside slot size. ** Only the high-water value is meaningful; -** the current value is always zero.)^ +** the current value is always zero.
          )^ ** ** [[SQLITE_DBSTATUS_LOOKASIDE_MISS_FULL]] ** ^(
          SQLITE_DBSTATUS_LOOKASIDE_MISS_FULL
          -**
          This parameter returns the number malloc attempts that might have +**
          This parameter returns the number of malloc attempts that might have ** been satisfied using lookaside memory but failed due to all lookaside ** memory already being in use. ** Only the high-water value is meaningful; -** the current value is always zero.)^ +** the current value is always zero.
          )^ ** ** [[SQLITE_DBSTATUS_CACHE_USED]] ^(
          SQLITE_DBSTATUS_CACHE_USED
          **
          This parameter returns the approximate number of bytes of heap ** memory used by all pager caches associated with the database connection.)^ ** ^The highwater mark associated with SQLITE_DBSTATUS_CACHE_USED is always 0. +**
          ** ** [[SQLITE_DBSTATUS_CACHE_USED_SHARED]] ** ^(
          SQLITE_DBSTATUS_CACHE_USED_SHARED
          @@ -9014,10 +9319,10 @@ SQLITE_API int sqlite3_db_status(sqlite3*, int op, int *pCur, int *pHiwtr, int r ** memory used by that pager cache is divided evenly between the attached ** connections.)^ In other words, if none of the pager caches associated ** with the database connection are shared, this request returns the same -** value as DBSTATUS_CACHE_USED. Or, if one or more or the pager caches are +** value as DBSTATUS_CACHE_USED. Or, if one or more of the pager caches are ** shared, the value returned by this call will be smaller than that returned ** by DBSTATUS_CACHE_USED. ^The highwater mark associated with -** SQLITE_DBSTATUS_CACHE_USED_SHARED is always 0. +** SQLITE_DBSTATUS_CACHE_USED_SHARED is always 0. ** ** [[SQLITE_DBSTATUS_SCHEMA_USED]] ^(
          SQLITE_DBSTATUS_SCHEMA_USED
          **
          This parameter returns the approximate number of bytes of heap @@ -9027,6 +9332,7 @@ SQLITE_API int sqlite3_db_status(sqlite3*, int op, int *pCur, int *pHiwtr, int r ** schema memory is shared with other database connections due to ** [shared cache mode] being enabled. ** ^The highwater mark associated with SQLITE_DBSTATUS_SCHEMA_USED is always 0. +**
          ** ** [[SQLITE_DBSTATUS_STMT_USED]] ^(
          SQLITE_DBSTATUS_STMT_USED
          **
          This parameter returns the approximate number of bytes of heap @@ -9056,6 +9362,10 @@ SQLITE_API int sqlite3_db_status(sqlite3*, int op, int *pCur, int *pHiwtr, int r ** If an IO or other error occurs while writing a page to disk, the effect ** on subsequent SQLITE_DBSTATUS_CACHE_WRITE requests is undefined.)^ ^The ** highwater mark associated with SQLITE_DBSTATUS_CACHE_WRITE is always 0. +**

          +** ^(There is overlap between the quantities measured by this parameter +** (SQLITE_DBSTATUS_CACHE_WRITE) and SQLITE_DBSTATUS_TEMPBUF_SPILL. +** Resetting one will reduce the other.)^ **

          ** ** [[SQLITE_DBSTATUS_CACHE_SPILL]] ^(
          SQLITE_DBSTATUS_CACHE_SPILL
          @@ -9063,7 +9373,7 @@ SQLITE_API int sqlite3_db_status(sqlite3*, int op, int *pCur, int *pHiwtr, int r ** been written to disk in the middle of a transaction due to the page ** cache overflowing. Transactions are more efficient if they are written ** to disk all at once. When pages spill mid-transaction, that introduces -** additional overhead. This parameter can be used help identify +** additional overhead. This parameter can be used to help identify ** inefficiencies that can be resolved by increasing the cache size. ** ** @@ -9071,6 +9381,18 @@ SQLITE_API int sqlite3_db_status(sqlite3*, int op, int *pCur, int *pHiwtr, int r **
          This parameter returns zero for the current value if and only if ** all foreign key constraints (deferred or immediate) have been ** resolved.)^ ^The highwater mark is always 0. +** +** [[SQLITE_DBSTATUS_TEMPBUF_SPILL] ^(
          SQLITE_DBSTATUS_TEMPBUF_SPILL
          +**
          ^(This parameter returns the number of bytes written to temporary +** files on disk that could have been kept in memory had sufficient memory +** been available. This value includes writes to intermediate tables that +** are part of complex queries, external sorts that spill to disk, and +** writes to TEMP tables.)^ +** ^The highwater mark is always 0. +**

          +** ^(There is overlap between the quantities measured by this parameter +** (SQLITE_DBSTATUS_TEMPBUF_SPILL) and SQLITE_DBSTATUS_CACHE_WRITE. +** Resetting one will reduce the other.)^ **

          ** */ @@ -9087,7 +9409,8 @@ SQLITE_API int sqlite3_db_status(sqlite3*, int op, int *pCur, int *pHiwtr, int r #define SQLITE_DBSTATUS_DEFERRED_FKS 10 #define SQLITE_DBSTATUS_CACHE_USED_SHARED 11 #define SQLITE_DBSTATUS_CACHE_SPILL 12 -#define SQLITE_DBSTATUS_MAX 12 /* Largest defined DBSTATUS */ +#define SQLITE_DBSTATUS_TEMPBUF_SPILL 13 +#define SQLITE_DBSTATUS_MAX 13 /* Largest defined DBSTATUS */ /* @@ -9134,13 +9457,13 @@ SQLITE_API int sqlite3_stmt_status(sqlite3_stmt*, int op,int resetFlg); ** [[SQLITE_STMTSTATUS_SORT]]
          SQLITE_STMTSTATUS_SORT
          **
          ^This is the number of sort operations that have occurred. ** A non-zero value in this counter may indicate an opportunity to -** improvement performance through careful use of indices.
          +** improve performance through careful use of indices. ** ** [[SQLITE_STMTSTATUS_AUTOINDEX]]
          SQLITE_STMTSTATUS_AUTOINDEX
          **
          ^This is the number of rows inserted into transient indices that ** were created automatically in order to help joins run faster. ** A non-zero value in this counter may indicate an opportunity to -** improvement performance by adding permanent indices that do not +** improve performance by adding permanent indices that do not ** need to be reinitialized each time the statement is run.
          ** ** [[SQLITE_STMTSTATUS_VM_STEP]]
          SQLITE_STMTSTATUS_VM_STEP
          @@ -9149,19 +9472,19 @@ SQLITE_API int sqlite3_stmt_status(sqlite3_stmt*, int op,int resetFlg); ** to 2147483647. The number of virtual machine operations can be ** used as a proxy for the total work done by the prepared statement. ** If the number of virtual machine operations exceeds 2147483647 -** then the value returned by this statement status code is undefined. +** then the value returned by this statement status code is undefined. ** ** [[SQLITE_STMTSTATUS_REPREPARE]]
          SQLITE_STMTSTATUS_REPREPARE
          **
          ^This is the number of times that the prepare statement has been ** automatically regenerated due to schema changes or changes to -** [bound parameters] that might affect the query plan. +** [bound parameters] that might affect the query plan.
          ** ** [[SQLITE_STMTSTATUS_RUN]]
          SQLITE_STMTSTATUS_RUN
          **
          ^This is the number of times that the prepared statement has ** been run. A single "run" for the purposes of this counter is one ** or more calls to [sqlite3_step()] followed by a call to [sqlite3_reset()]. ** The counter is incremented on the first [sqlite3_step()] call of each -** cycle. +** cycle.
          ** ** [[SQLITE_STMTSTATUS_FILTER_MISS]] ** [[SQLITE_STMTSTATUS_FILTER HIT]] @@ -9171,7 +9494,7 @@ SQLITE_API int sqlite3_stmt_status(sqlite3_stmt*, int op,int resetFlg); ** step was bypassed because a Bloom filter returned not-found. The ** corresponding SQLITE_STMTSTATUS_FILTER_MISS value is the number of ** times that the Bloom filter returned a find, and thus the join step -** had to be processed as normal. +** had to be processed as normal. ** ** [[SQLITE_STMTSTATUS_MEMUSED]]
          SQLITE_STMTSTATUS_MEMUSED
          **
          ^This is the approximate number of bytes of heap memory @@ -9276,9 +9599,9 @@ struct sqlite3_pcache_page { ** SQLite will typically create one cache instance for each open database file, ** though this is not guaranteed. ^The ** first parameter, szPage, is the size in bytes of the pages that must -** be allocated by the cache. ^szPage will always a power of two. ^The +** be allocated by the cache. ^szPage will always be a power of two. ^The ** second parameter szExtra is a number of bytes of extra storage -** associated with each page cache entry. ^The szExtra parameter will +** associated with each page cache entry. ^The szExtra parameter will be ** a number less than 250. SQLite will use the ** extra szExtra bytes on each page to store metadata about the underlying ** database page on disk. The value passed into szExtra depends @@ -9286,17 +9609,17 @@ struct sqlite3_pcache_page { ** ^The third argument to xCreate(), bPurgeable, is true if the cache being ** created will be used to cache database pages of a file stored on disk, or ** false if it is used for an in-memory database. The cache implementation -** does not have to do anything special based with the value of bPurgeable; +** does not have to do anything special based upon the value of bPurgeable; ** it is purely advisory. ^On a cache where bPurgeable is false, SQLite will ** never invoke xUnpin() except to deliberately delete a page. ** ^In other words, calls to xUnpin() on a cache with bPurgeable set to ** false will always have the "discard" flag set to true. -** ^Hence, a cache created with bPurgeable false will +** ^Hence, a cache created with bPurgeable set to false will ** never contain any unpinned pages. ** ** [[the xCachesize() page cache method]] ** ^(The xCachesize() method may be called at any time by SQLite to set the -** suggested maximum cache-size (number of pages stored by) the cache +** suggested maximum cache-size (number of pages stored) for the cache ** instance passed as the first argument. This is the value configured using ** the SQLite "[PRAGMA cache_size]" command.)^ As with the bPurgeable ** parameter, the implementation is not required to do anything with this @@ -9323,12 +9646,12 @@ struct sqlite3_pcache_page { ** implementation must return a pointer to the page buffer with its content ** intact. If the requested page is not already in the cache, then the ** cache implementation should use the value of the createFlag -** parameter to help it determined what action to take: +** parameter to help it determine what action to take: ** ** ** - + - +
          createFlag Behavior when page is not already in cache **
          0 Do not allocate a new page. Return NULL. -**
          1 Allocate a new page if it easy and convenient to do so. +**
          1 Allocate a new page if it is easy and convenient to do so. ** Otherwise return NULL. **
          2 Make every effort to allocate a new page. Only return ** NULL if allocating a new page is effectively impossible. @@ -9345,7 +9668,7 @@ struct sqlite3_pcache_page { ** as its second argument. If the third parameter, discard, is non-zero, ** then the page must be evicted from the cache. ** ^If the discard parameter is -** zero, then the page may be discarded or retained at the discretion of +** zero, then the page may be discarded or retained at the discretion of the ** page cache implementation. ^The page cache implementation ** may choose to evict unpinned pages at any time. ** @@ -9363,7 +9686,7 @@ struct sqlite3_pcache_page { ** When SQLite calls the xTruncate() method, the cache must discard all ** existing cache entries with page numbers (keys) greater than or equal ** to the value of the iLimit parameter passed to xTruncate(). If any -** of these pages are pinned, they are implicitly unpinned, meaning that +** of these pages are pinned, they become implicitly unpinned, meaning that ** they can be safely discarded. ** ** [[the xDestroy() page cache method]] @@ -9543,7 +9866,7 @@ typedef struct sqlite3_backup sqlite3_backup; ** external process or via a database connection other than the one being ** used by the backup operation, then the backup will be automatically ** restarted by the next call to sqlite3_backup_step(). ^If the source -** database is modified by the using the same database connection as is used +** database is modified by using the same database connection as is used ** by the backup operation, then the backup database is automatically ** updated at the same time. ** @@ -9560,7 +9883,7 @@ typedef struct sqlite3_backup sqlite3_backup; ** and may not be used following a call to sqlite3_backup_finish(). ** ** ^The value returned by sqlite3_backup_finish is [SQLITE_OK] if no -** sqlite3_backup_step() errors occurred, regardless or whether or not +** sqlite3_backup_step() errors occurred, regardless of whether or not ** sqlite3_backup_step() completed. ** ^If an out-of-memory condition or IO error occurred during any prior ** sqlite3_backup_step() call on the same [sqlite3_backup] object, then @@ -9615,6 +9938,16 @@ typedef struct sqlite3_backup sqlite3_backup; ** APIs are not strictly speaking threadsafe. If they are invoked at the ** same time as another thread is invoking sqlite3_backup_step() it is ** possible that they return invalid values. +** +** Alternatives To Using The Backup API +** +** Other techniques for safely creating a consistent backup of an SQLite +** database include: +** +**
            +**
          • The [VACUUM INTO] command. +**
          • The [sqlite3_rsync] utility program. +**
          */ SQLITE_API sqlite3_backup *sqlite3_backup_init( sqlite3 *pDest, /* Destination database handle */ @@ -9652,7 +9985,7 @@ SQLITE_API int sqlite3_backup_pagecount(sqlite3_backup *p); ** application receives an SQLITE_LOCKED error, it may call the ** sqlite3_unlock_notify() method with the blocked connection handle as ** the first argument to register for a callback that will be invoked -** when the blocking connections current transaction is concluded. ^The +** when the blocking connection's current transaction is concluded. ^The ** callback is invoked from within the [sqlite3_step] or [sqlite3_close] ** call that concludes the blocking connection's transaction. ** @@ -9672,7 +10005,7 @@ SQLITE_API int sqlite3_backup_pagecount(sqlite3_backup *p); ** blocked connection already has a registered unlock-notify callback, ** then the new callback replaces the old.)^ ^If sqlite3_unlock_notify() is ** called with a NULL pointer as its second argument, then any existing -** unlock-notify callback is canceled. ^The blocked connections +** unlock-notify callback is canceled. ^The blocked connection's ** unlock-notify callback may also be canceled by closing the blocked ** connection using [sqlite3_close()]. ** @@ -9842,7 +10175,7 @@ SQLITE_API void sqlite3_log(int iErrCode, const char *zFormat, ...); ** is the number of pages currently in the write-ahead log file, ** including those that were just committed. ** -** The callback function should normally return [SQLITE_OK]. ^If an error +** ^The callback function should normally return [SQLITE_OK]. ^If an error ** code is returned, that error will propagate back up through the ** SQLite code base to cause the statement that provoked the callback ** to report an error, though the commit will have still occurred. If the @@ -9850,13 +10183,26 @@ SQLITE_API void sqlite3_log(int iErrCode, const char *zFormat, ...); ** that does not correspond to any valid SQLite error code, the results ** are undefined. ** -** A single database handle may have at most a single write-ahead log callback -** registered at one time. ^Calling [sqlite3_wal_hook()] replaces any -** previously registered write-ahead log callback. ^The return value is -** a copy of the third parameter from the previous call, if any, or 0. -** ^Note that the [sqlite3_wal_autocheckpoint()] interface and the -** [wal_autocheckpoint pragma] both invoke [sqlite3_wal_hook()] and will -** overwrite any prior [sqlite3_wal_hook()] settings. +** ^A single database handle may have at most a single write-ahead log +** callback registered at one time. ^Calling [sqlite3_wal_hook()] +** replaces the default behavior or previously registered write-ahead +** log callback. +** +** ^The return value is a copy of the third parameter from the +** previous call, if any, or 0. +** +** ^The [sqlite3_wal_autocheckpoint()] interface and the +** [wal_autocheckpoint pragma] both invoke [sqlite3_wal_hook()] and +** will overwrite any prior [sqlite3_wal_hook()] settings. +** +** ^If a write-ahead log callback is set using this function then +** [sqlite3_wal_checkpoint_v2()] or [PRAGMA wal_checkpoint] +** should be invoked periodically to keep the write-ahead log file +** from growing without bound. +** +** ^Passing a NULL pointer for the callback disables automatic +** checkpointing entirely. To re-enable the default behavior, call +** sqlite3_wal_autocheckpoint(db,1000) or use [PRAGMA wal_checkpoint]. */ SQLITE_API void *sqlite3_wal_hook( sqlite3*, @@ -9873,7 +10219,7 @@ SQLITE_API void *sqlite3_wal_hook( ** to automatically [checkpoint] ** after committing a transaction if there are N or ** more frames in the [write-ahead log] file. ^Passing zero or -** a negative value as the nFrame parameter disables automatic +** a negative value as the N parameter disables automatic ** checkpoints entirely. ** ** ^The callback registered by this function replaces any existing callback @@ -9889,9 +10235,10 @@ SQLITE_API void *sqlite3_wal_hook( ** ** ^Every new [database connection] defaults to having the auto-checkpoint ** enabled with a threshold of 1000 or [SQLITE_DEFAULT_WAL_AUTOCHECKPOINT] -** pages. The use of this interface -** is only necessary if the default setting is found to be suboptimal -** for a particular application. +** pages. +** +** ^The use of this interface is only necessary if the default setting +** is found to be suboptimal for a particular application. */ SQLITE_API int sqlite3_wal_autocheckpoint(sqlite3 *db, int N); @@ -9956,6 +10303,11 @@ SQLITE_API int sqlite3_wal_checkpoint(sqlite3 *db, const char *zDb); ** ^This mode works the same way as SQLITE_CHECKPOINT_RESTART with the ** addition that it also truncates the log file to zero bytes just prior ** to a successful return. +** +**
          SQLITE_CHECKPOINT_NOOP
          +** ^This mode always checkpoints zero frames. The only reason to invoke +** a NOOP checkpoint is to access the values returned by +** sqlite3_wal_checkpoint_v2() via output parameters *pnLog and *pnCkpt. ** ** ** ^If pnLog is not NULL, then *pnLog is set to the total number of frames in @@ -10026,6 +10378,7 @@ SQLITE_API int sqlite3_wal_checkpoint_v2( ** See the [sqlite3_wal_checkpoint_v2()] documentation for details on the ** meaning of each of these checkpoint modes. */ +#define SQLITE_CHECKPOINT_NOOP -1 /* Do no work at all */ #define SQLITE_CHECKPOINT_PASSIVE 0 /* Do as much as possible w/o blocking */ #define SQLITE_CHECKPOINT_FULL 1 /* Wait for writers, then checkpoint */ #define SQLITE_CHECKPOINT_RESTART 2 /* Like FULL but wait for readers */ @@ -10070,7 +10423,7 @@ SQLITE_API int sqlite3_vtab_config(sqlite3*, int op, ...); ** support constraints. In this configuration (which is the default) if ** a call to the [xUpdate] method returns [SQLITE_CONSTRAINT], then the entire ** statement is rolled back as if [ON CONFLICT | OR ABORT] had been -** specified as part of the users SQL statement, regardless of the actual +** specified as part of the user's SQL statement, regardless of the actual ** ON CONFLICT mode specified. ** ** If X is non-zero, then the virtual table implementation guarantees @@ -10104,7 +10457,7 @@ SQLITE_API int sqlite3_vtab_config(sqlite3*, int op, ...); ** [[SQLITE_VTAB_INNOCUOUS]]
          SQLITE_VTAB_INNOCUOUS
          **
          Calls of the form ** [sqlite3_vtab_config](db,SQLITE_VTAB_INNOCUOUS) from within the -** the [xConnect] or [xCreate] methods of a [virtual table] implementation +** [xConnect] or [xCreate] methods of a [virtual table] implementation ** identify that virtual table as being safe to use from within triggers ** and views. Conceptually, the SQLITE_VTAB_INNOCUOUS tag means that the ** virtual table can do no serious harm even if it is controlled by a @@ -10232,26 +10585,47 @@ SQLITE_API const char *sqlite3_vtab_collation(sqlite3_index_info*,int); **
        • ** ^(If the sqlite3_vtab_distinct() interface returns 2, that means ** that the query planner does not need the rows returned in any particular -** order, as long as rows with the same values in all "aOrderBy" columns -** are adjacent.)^ ^(Furthermore, only a single row for each particular -** combination of values in the columns identified by the "aOrderBy" field -** needs to be returned.)^ ^It is always ok for two or more rows with the same -** values in all "aOrderBy" columns to be returned, as long as all such rows -** are adjacent. ^The virtual table may, if it chooses, omit extra rows -** that have the same value for all columns identified by "aOrderBy". -** ^However omitting the extra rows is optional. +** order, as long as rows with the same values in all columns identified +** by "aOrderBy" are adjacent.)^ ^(Furthermore, when two or more rows +** contain the same values for all columns identified by "colUsed", all but +** one such row may optionally be omitted from the result.)^ +** The virtual table is not required to omit rows that are duplicates +** over the "colUsed" columns, but if the virtual table can do that without +** too much extra effort, it could potentially help the query to run faster. ** This mode is used for a DISTINCT query. **

        • -** ^(If the sqlite3_vtab_distinct() interface returns 3, that means -** that the query planner needs only distinct rows but it does need the -** rows to be sorted.)^ ^The virtual table implementation is free to omit -** rows that are identical in all aOrderBy columns, if it wants to, but -** it is not required to omit any rows. This mode is used for queries +** ^(If the sqlite3_vtab_distinct() interface returns 3, that means the +** virtual table must return rows in the order defined by "aOrderBy" as +** if the sqlite3_vtab_distinct() interface had returned 0. However if +** two or more rows in the result have the same values for all columns +** identified by "colUsed", then all but one such row may optionally be +** omitted.)^ Like when the return value is 2, the virtual table +** is not required to omit rows that are duplicates over the "colUsed" +** columns, but if the virtual table can do that without +** too much extra effort, it could potentially help the query to run faster. +** This mode is used for queries ** that have both DISTINCT and ORDER BY clauses. ** ** +**

          The following table summarizes the conditions under which the +** virtual table is allowed to set the "orderByConsumed" flag based on +** the value returned by sqlite3_vtab_distinct(). This table is a +** restatement of the previous four paragraphs: +** +** +** +**
          sqlite3_vtab_distinct() return value +** Rows are returned in aOrderBy order +** Rows with the same value in all aOrderBy columns are adjacent +** Duplicates over all colUsed columns may be omitted +**
          0yesyesno +**
          1noyesno +**
          2noyesyes +**
          3yesyesyes +**
          +** ** ^For the purposes of comparing virtual table output values to see if the -** values are same value for sorting purposes, two NULL values are considered +** values are the same value for sorting purposes, two NULL values are considered ** to be the same. In other words, the comparison operator is "IS" ** (or "IS NOT DISTINCT FROM") and not "==". ** @@ -10261,7 +10635,7 @@ SQLITE_API const char *sqlite3_vtab_collation(sqlite3_index_info*,int); ** ** ^A virtual table implementation is always free to return rows in any order ** it wants, as long as the "orderByConsumed" flag is not set. ^When the -** the "orderByConsumed" flag is unset, the query planner will add extra +** "orderByConsumed" flag is unset, the query planner will add extra ** [bytecode] to ensure that the final results returned by the SQL query are ** ordered correctly. The use of the "orderByConsumed" flag and the ** sqlite3_vtab_distinct() interface is merely an optimization. ^Careful @@ -10358,7 +10732,7 @@ SQLITE_API int sqlite3_vtab_in(sqlite3_index_info*, int iCons, int bHandle); ** sqlite3_vtab_in_next(X,P) should be one of the parameters to the ** xFilter method which invokes these routines, and specifically ** a parameter that was previously selected for all-at-once IN constraint -** processing use the [sqlite3_vtab_in()] interface in the +** processing using the [sqlite3_vtab_in()] interface in the ** [xBestIndex|xBestIndex method]. ^(If the X parameter is not ** an xFilter argument that was selected for all-at-once IN constraint ** processing, then these routines return [SQLITE_ERROR].)^ @@ -10373,7 +10747,7 @@ SQLITE_API int sqlite3_vtab_in(sqlite3_index_info*, int iCons, int bHandle); **   ){ **   // do something with pVal **   } -**   if( rc!=SQLITE_OK ){ +**   if( rc!=SQLITE_DONE ){ **   // an error has occurred **   } ** )^ @@ -10413,7 +10787,7 @@ SQLITE_API int sqlite3_vtab_in_next(sqlite3_value *pVal, sqlite3_value **ppOut); ** and only if *V is set to a value. ^The sqlite3_vtab_rhs_value(P,J,V) ** inteface returns SQLITE_NOTFOUND if the right-hand side of the J-th ** constraint is not available. ^The sqlite3_vtab_rhs_value() interface -** can return an result code other than SQLITE_OK or SQLITE_NOTFOUND if +** can return a result code other than SQLITE_OK or SQLITE_NOTFOUND if ** something goes wrong. ** ** The sqlite3_vtab_rhs_value() interface is usually only successful if @@ -10441,8 +10815,8 @@ SQLITE_API int sqlite3_vtab_rhs_value(sqlite3_index_info*, int, sqlite3_value ** ** KEYWORDS: {conflict resolution mode} ** ** These constants are returned by [sqlite3_vtab_on_conflict()] to -** inform a [virtual table] implementation what the [ON CONFLICT] mode -** is for the SQL statement being evaluated. +** inform a [virtual table] implementation of the [ON CONFLICT] mode +** for the SQL statement being evaluated. ** ** Note that the [SQLITE_IGNORE] constant is also used as a potential ** return value from the [sqlite3_set_authorizer()] callback and that @@ -10482,39 +10856,39 @@ SQLITE_API int sqlite3_vtab_rhs_value(sqlite3_index_info*, int, sqlite3_value ** ** [[SQLITE_SCANSTAT_EST]]

          SQLITE_SCANSTAT_EST
          **
          ^The "double" variable pointed to by the V parameter will be set to the ** query planner's estimate for the average number of rows output from each -** iteration of the X-th loop. If the query planner's estimates was accurate, +** iteration of the X-th loop. If the query planner's estimate was accurate, ** then this value will approximate the quotient NVISIT/NLOOP and the ** product of this value for all prior loops with the same SELECTID will -** be the NLOOP value for the current loop. +** be the NLOOP value for the current loop.
          ** ** [[SQLITE_SCANSTAT_NAME]]
          SQLITE_SCANSTAT_NAME
          **
          ^The "const char *" variable pointed to by the V parameter will be set ** to a zero-terminated UTF-8 string containing the name of the index or table -** used for the X-th loop. +** used for the X-th loop.
          ** ** [[SQLITE_SCANSTAT_EXPLAIN]]
          SQLITE_SCANSTAT_EXPLAIN
          **
          ^The "const char *" variable pointed to by the V parameter will be set ** to a zero-terminated UTF-8 string containing the [EXPLAIN QUERY PLAN] -** description for the X-th loop. +** description for the X-th loop.
          ** ** [[SQLITE_SCANSTAT_SELECTID]]
          SQLITE_SCANSTAT_SELECTID
          **
          ^The "int" variable pointed to by the V parameter will be set to the ** id for the X-th query plan element. The id value is unique within the ** statement. The select-id is the same value as is output in the first -** column of an [EXPLAIN QUERY PLAN] query. +** column of an [EXPLAIN QUERY PLAN] query.
          ** ** [[SQLITE_SCANSTAT_PARENTID]]
          SQLITE_SCANSTAT_PARENTID
          **
          The "int" variable pointed to by the V parameter will be set to the -** the id of the parent of the current query element, if applicable, or +** id of the parent of the current query element, if applicable, or ** to zero if the query element has no parent. This is the same value as -** returned in the second column of an [EXPLAIN QUERY PLAN] query. +** returned in the second column of an [EXPLAIN QUERY PLAN] query.
          ** ** [[SQLITE_SCANSTAT_NCYCLE]]
          SQLITE_SCANSTAT_NCYCLE
          **
          The sqlite3_int64 output value is set to the number of cycles, ** according to the processor time-stamp counter, that elapsed while the ** query element was being processed. This value is not available for ** all query elements - if it is unavailable the output variable is -** set to -1. +** set to -1.
          ** */ #define SQLITE_SCANSTAT_NLOOP 0 @@ -10555,8 +10929,8 @@ SQLITE_API int sqlite3_vtab_rhs_value(sqlite3_index_info*, int, sqlite3_value ** ** sqlite3_stmt_scanstatus_v2() with a zeroed flags parameter. ** ** Parameter "idx" identifies the specific query element to retrieve statistics -** for. Query elements are numbered starting from zero. A value of -1 may be -** to query for statistics regarding the entire query. ^If idx is out of range +** for. Query elements are numbered starting from zero. A value of -1 may +** retrieve statistics for the entire query. ^If idx is out of range ** - less than -1 or greater than or equal to the total number of query ** elements used to implement the statement - a non-zero value is returned and ** the variable that pOut points to is unchanged. @@ -10599,7 +10973,7 @@ SQLITE_API void sqlite3_stmt_scanstatus_reset(sqlite3_stmt*); ** METHOD: sqlite3 ** ** ^If a write-transaction is open on [database connection] D when the -** [sqlite3_db_cacheflush(D)] interface invoked, any dirty +** [sqlite3_db_cacheflush(D)] interface is invoked, any dirty ** pages in the pager-cache that are not currently in use are written out ** to disk. A dirty page may be in use if a database cursor created by an ** active SQL statement is reading from it, or if it is page 1 of a database @@ -10713,8 +11087,8 @@ SQLITE_API int sqlite3_db_cacheflush(sqlite3*); ** triggers; and so forth. ** ** When the [sqlite3_blob_write()] API is used to update a blob column, -** the pre-update hook is invoked with SQLITE_DELETE. This is because the -** in this case the new values are not available. In this case, when a +** the pre-update hook is invoked with SQLITE_DELETE, because +** the new values are not yet available. In this case, when a ** callback made with op==SQLITE_DELETE is actually a write using the ** sqlite3_blob_write() API, the [sqlite3_preupdate_blobwrite()] returns ** the index of the column being written. In other cases, where the @@ -10793,6 +11167,14 @@ typedef struct sqlite3_snapshot { ** If there is not already a read-transaction open on schema S when ** this function is called, one is opened automatically. ** +** If a read-transaction is opened by this function, then it is guaranteed +** that the returned snapshot object may not be invalidated by a database +** writer or checkpointer until after the read-transaction is closed. This +** is not guaranteed if a read-transaction is already open when this +** function is called. In that case, any subsequent write or checkpoint +** operation on the database may invalidate the returned snapshot handle, +** even while the read-transaction remains open. +** ** The following must be true for this function to succeed. If any of ** the following statements are false when sqlite3_snapshot_get() is ** called, SQLITE_ERROR is returned. The final value of *P is undefined @@ -10824,7 +11206,7 @@ typedef struct sqlite3_snapshot { ** The [sqlite3_snapshot_get()] interface is only available when the ** [SQLITE_ENABLE_SNAPSHOT] compile-time option is used. */ -SQLITE_API SQLITE_EXPERIMENTAL int sqlite3_snapshot_get( +SQLITE_API int sqlite3_snapshot_get( sqlite3 *db, const char *zSchema, sqlite3_snapshot **ppSnapshot @@ -10873,7 +11255,7 @@ SQLITE_API SQLITE_EXPERIMENTAL int sqlite3_snapshot_get( ** The [sqlite3_snapshot_open()] interface is only available when the ** [SQLITE_ENABLE_SNAPSHOT] compile-time option is used. */ -SQLITE_API SQLITE_EXPERIMENTAL int sqlite3_snapshot_open( +SQLITE_API int sqlite3_snapshot_open( sqlite3 *db, const char *zSchema, sqlite3_snapshot *pSnapshot @@ -10890,7 +11272,7 @@ SQLITE_API SQLITE_EXPERIMENTAL int sqlite3_snapshot_open( ** The [sqlite3_snapshot_free()] interface is only available when the ** [SQLITE_ENABLE_SNAPSHOT] compile-time option is used. */ -SQLITE_API SQLITE_EXPERIMENTAL void sqlite3_snapshot_free(sqlite3_snapshot*); +SQLITE_API void sqlite3_snapshot_free(sqlite3_snapshot*); /* ** CAPI3REF: Compare the ages of two snapshot handles. @@ -10917,7 +11299,7 @@ SQLITE_API SQLITE_EXPERIMENTAL void sqlite3_snapshot_free(sqlite3_snapshot*); ** This interface is only available if SQLite is compiled with the ** [SQLITE_ENABLE_SNAPSHOT] option. */ -SQLITE_API SQLITE_EXPERIMENTAL int sqlite3_snapshot_cmp( +SQLITE_API int sqlite3_snapshot_cmp( sqlite3_snapshot *p1, sqlite3_snapshot *p2 ); @@ -10945,20 +11327,21 @@ SQLITE_API SQLITE_EXPERIMENTAL int sqlite3_snapshot_cmp( ** This interface is only available if SQLite is compiled with the ** [SQLITE_ENABLE_SNAPSHOT] option. */ -SQLITE_API SQLITE_EXPERIMENTAL int sqlite3_snapshot_recover(sqlite3 *db, const char *zDb); +SQLITE_API int sqlite3_snapshot_recover(sqlite3 *db, const char *zDb); /* ** CAPI3REF: Serialize a database ** -** The sqlite3_serialize(D,S,P,F) interface returns a pointer to memory -** that is a serialization of the S database on [database connection] D. +** The sqlite3_serialize(D,S,P,F) interface returns a pointer to +** memory that is a serialization of the S database on +** [database connection] D. If S is a NULL pointer, the main database is used. ** If P is not a NULL pointer, then the size of the database in bytes ** is written into *P. ** ** For an ordinary on-disk database file, the serialization is just a ** copy of the disk file. For an in-memory database or a "TEMP" database, ** the serialization is the same sequence of bytes which would be written -** to disk if that database where backed up to disk. +** to disk if that database were backed up to disk. ** ** The usual case is that sqlite3_serialize() copies the serialization of ** the database into memory obtained from [sqlite3_malloc64()] and returns @@ -10967,7 +11350,7 @@ SQLITE_API SQLITE_EXPERIMENTAL int sqlite3_snapshot_recover(sqlite3 *db, const c ** contains the SQLITE_SERIALIZE_NOCOPY bit, then no memory allocations ** are made, and the sqlite3_serialize() function will return a pointer ** to the contiguous memory representation of the database that SQLite -** is currently using for that database, or NULL if the no such contiguous +** is currently using for that database, or NULL if no such contiguous ** memory representation of the database exists. A contiguous memory ** representation of the database will usually only exist if there has ** been a prior call to [sqlite3_deserialize(D,S,...)] with the same @@ -11018,12 +11401,13 @@ SQLITE_API unsigned char *sqlite3_serialize( ** ** The sqlite3_deserialize(D,S,P,N,M,F) interface causes the ** [database connection] D to disconnect from database S and then -** reopen S as an in-memory database based on the serialization contained -** in P. The serialized database P is N bytes in size. M is the size of -** the buffer P, which might be larger than N. If M is larger than N, and -** the SQLITE_DESERIALIZE_READONLY bit is not set in F, then SQLite is -** permitted to add content to the in-memory database as long as the total -** size does not exceed M bytes. +** reopen S as an in-memory database based on the serialization +** contained in P. If S is a NULL pointer, the main database is +** used. The serialized database P is N bytes in size. M is the size +** of the buffer P, which might be larger than N. If M is larger than +** N, and the SQLITE_DESERIALIZE_READONLY bit is not set in F, then +** SQLite is permitted to add content to the in-memory database as +** long as the total size does not exceed M bytes. ** ** If the SQLITE_DESERIALIZE_FREEONCLOSE bit is set in F, then SQLite will ** invoke sqlite3_free() on the serialization buffer when the database @@ -11038,7 +11422,7 @@ SQLITE_API unsigned char *sqlite3_serialize( ** database is currently in a read transaction or is involved in a backup ** operation. ** -** It is not possible to deserialized into the TEMP database. If the +** It is not possible to deserialize into the TEMP database. If the ** S argument to sqlite3_deserialize(D,S,P,N,M,F) is "temp" then the ** function returns SQLITE_ERROR. ** @@ -11060,7 +11444,7 @@ SQLITE_API int sqlite3_deserialize( sqlite3 *db, /* The database connection */ const char *zSchema, /* Which DB to reopen with the deserialization */ unsigned char *pData, /* The serialized database content */ - sqlite3_int64 szDb, /* Number bytes in the deserialization */ + sqlite3_int64 szDb, /* Number of bytes in the deserialization */ sqlite3_int64 szBuf, /* Total size of buffer pData[] */ unsigned mFlags /* Zero or more SQLITE_DESERIALIZE_* flags */ ); @@ -11068,7 +11452,7 @@ SQLITE_API int sqlite3_deserialize( /* ** CAPI3REF: Flags for sqlite3_deserialize() ** -** The following are allowed values for 6th argument (the F argument) to +** The following are allowed values for the 6th argument (the F argument) to ** the [sqlite3_deserialize(D,S,P,N,M,F)] interface. ** ** The SQLITE_DESERIALIZE_FREEONCLOSE means that the database serialization @@ -11090,6 +11474,54 @@ SQLITE_API int sqlite3_deserialize( #define SQLITE_DESERIALIZE_RESIZEABLE 2 /* Resize using sqlite3_realloc64() */ #define SQLITE_DESERIALIZE_READONLY 4 /* Database is read-only */ +/* +** CAPI3REF: Bind array values to the CARRAY table-valued function +** +** The sqlite3_carray_bind(S,I,P,N,F,X) interface binds an array value to +** one of the first argument of the [carray() table-valued function]. The +** S parameter is a pointer to the [prepared statement] that uses the carray() +** functions. I is the parameter index to be bound. P is a pointer to the +** array to be bound, and N is the number of eements in the array. The +** F argument is one of constants [SQLITE_CARRAY_INT32], [SQLITE_CARRAY_INT64], +** [SQLITE_CARRAY_DOUBLE], [SQLITE_CARRAY_TEXT], or [SQLITE_CARRAY_BLOB] to +** indicate the datatype of the array being bound. The X argument is not a +** NULL pointer, then SQLite will invoke the function X on the P parameter +** after it has finished using P, even if the call to +** sqlite3_carray_bind() fails. The special-case finalizer +** SQLITE_TRANSIENT has no effect here. +*/ +SQLITE_API int sqlite3_carray_bind( + sqlite3_stmt *pStmt, /* Statement to be bound */ + int i, /* Parameter index */ + void *aData, /* Pointer to array data */ + int nData, /* Number of data elements */ + int mFlags, /* CARRAY flags */ + void (*xDel)(void*) /* Destructor for aData */ +); + +/* +** CAPI3REF: Datatypes for the CARRAY table-valued function +** +** The fifth argument to the [sqlite3_carray_bind()] interface musts be +** one of the following constants, to specify the datatype of the array +** that is being bound into the [carray table-valued function]. +*/ +#define SQLITE_CARRAY_INT32 0 /* Data is 32-bit signed integers */ +#define SQLITE_CARRAY_INT64 1 /* Data is 64-bit signed integers */ +#define SQLITE_CARRAY_DOUBLE 2 /* Data is doubles */ +#define SQLITE_CARRAY_TEXT 3 /* Data is char* */ +#define SQLITE_CARRAY_BLOB 4 /* Data is struct iovec */ + +/* +** Versions of the above #defines that omit the initial SQLITE_, for +** legacy compatibility. +*/ +#define CARRAY_INT32 0 /* Data is 32-bit signed integers */ +#define CARRAY_INT64 1 /* Data is 64-bit signed integers */ +#define CARRAY_DOUBLE 2 /* Data is doubles */ +#define CARRAY_TEXT 3 /* Data is char* */ +#define CARRAY_BLOB 4 /* Data is struct iovec */ + /* ** Undo the hack that converts floating point types to integer for ** builds on processors without floating point support. @@ -11101,8 +11533,6 @@ SQLITE_API int sqlite3_deserialize( #if defined(__wasi__) # undef SQLITE_WASI # define SQLITE_WASI 1 -# undef SQLITE_OMIT_WAL -# define SQLITE_OMIT_WAL 1/* because it requires shared memory APIs */ # ifndef SQLITE_OMIT_LOAD_EXTENSION # define SQLITE_OMIT_LOAD_EXTENSION # endif @@ -11114,7 +11544,7 @@ SQLITE_API int sqlite3_deserialize( #if 0 } /* End of the 'extern "C"' block */ #endif -#endif /* SQLITE3_H */ +/* #endif for SQLITE3_H will be added by mksqlite3.tcl */ /******** Begin file sqlite3rtree.h *********/ /* @@ -11595,9 +12025,10 @@ SQLITE_API void sqlite3session_table_filter( ** is inserted while a session object is enabled, then later deleted while ** the same session object is disabled, no INSERT record will appear in the ** changeset, even though the delete took place while the session was disabled. -** Or, if one field of a row is updated while a session is disabled, and -** another field of the same row is updated while the session is enabled, the -** resulting changeset will contain an UPDATE change that updates both fields. +** Or, if one field of a row is updated while a session is enabled, and +** then another field of the same row is updated while the session is disabled, +** the resulting changeset will contain an UPDATE change that updates both +** fields. */ SQLITE_API int sqlite3session_changeset( sqlite3_session *pSession, /* Session object */ @@ -11669,8 +12100,9 @@ SQLITE_API sqlite3_int64 sqlite3session_changeset_size(sqlite3_session *pSession ** database zFrom the contents of the two compatible tables would be ** identical. ** -** It an error if database zFrom does not exist or does not contain the -** required compatible table. +** Unless the call to this function is a no-op as described above, it is an +** error if database zFrom does not exist or does not contain the required +** compatible table. ** ** If the operation is successful, SQLITE_OK is returned. Otherwise, an SQLite ** error code. In this case, if argument pzErrMsg is not NULL, *pzErrMsg @@ -11805,7 +12237,7 @@ SQLITE_API int sqlite3changeset_start_v2( ** The following flags may passed via the 4th parameter to ** [sqlite3changeset_start_v2] and [sqlite3changeset_start_v2_strm]: ** -**
          SQLITE_CHANGESETAPPLY_INVERT
          +**
          SQLITE_CHANGESETSTART_INVERT
          ** Invert the changeset while iterating through it. This is equivalent to ** inverting a changeset using sqlite3changeset_invert() before applying it. ** It is an error to specify this flag with a patchset. @@ -12120,19 +12552,6 @@ SQLITE_API int sqlite3changeset_concat( void **ppOut /* OUT: Buffer containing output changeset */ ); - -/* -** CAPI3REF: Upgrade the Schema of a Changeset/Patchset -*/ -SQLITE_API int sqlite3changeset_upgrade( - sqlite3 *db, - const char *zDb, - int nIn, const void *pIn, /* Input changeset */ - int *pnOut, void **ppOut /* OUT: Inverse of input */ -); - - - /* ** CAPI3REF: Changegroup Handle ** @@ -12294,6 +12713,30 @@ SQLITE_API int sqlite3changegroup_schema(sqlite3_changegroup*, sqlite3*, const c */ SQLITE_API int sqlite3changegroup_add(sqlite3_changegroup*, int nData, void *pData); +/* +** CAPI3REF: Add A Single Change To A Changegroup +** METHOD: sqlite3_changegroup +** +** This function adds the single change currently indicated by the iterator +** passed as the second argument to the changegroup object. The rules for +** adding the change are just as described for [sqlite3changegroup_add()]. +** +** If the change is successfully added to the changegroup, SQLITE_OK is +** returned. Otherwise, an SQLite error code is returned. +** +** The iterator must point to a valid entry when this function is called. +** If it does not, SQLITE_ERROR is returned and no change is added to the +** changegroup. Additionally, the iterator must not have been opened with +** the SQLITE_CHANGESETAPPLY_INVERT flag. In this case SQLITE_ERROR is also +** returned. +*/ +SQLITE_API int sqlite3changegroup_add_change( + sqlite3_changegroup*, + sqlite3_changeset_iter* +); + + + /* ** CAPI3REF: Obtain A Composite Changeset From A Changegroup ** METHOD: sqlite3_changegroup @@ -12338,14 +12781,32 @@ SQLITE_API void sqlite3changegroup_delete(sqlite3_changegroup*); ** update the "main" database attached to handle db with the changes found in ** the changeset passed via the second and third arguments. ** +** All changes made by these functions are enclosed in a savepoint transaction. +** If any other error (aside from a constraint failure when attempting to +** write to the target database) occurs, then the savepoint transaction is +** rolled back, restoring the target database to its original state, and an +** SQLite error code returned. Additionally, starting with version 3.51.0, +** an error code and error message that may be accessed using the +** [sqlite3_errcode()] and [sqlite3_errmsg()] APIs are left in the database +** handle. +** ** The fourth argument (xFilter) passed to these functions is the "filter -** callback". If it is not NULL, then for each table affected by at least one -** change in the changeset, the filter callback is invoked with -** the table name as the second argument, and a copy of the context pointer -** passed as the sixth argument as the first. If the "filter callback" -** returns zero, then no attempt is made to apply any changes to the table. -** Otherwise, if the return value is non-zero or the xFilter argument to -** is NULL, all changes related to the table are attempted. +** callback". This may be passed NULL, in which case all changes in the +** changeset are applied to the database. For sqlite3changeset_apply() and +** sqlite3_changeset_apply_v2(), if it is not NULL, then it is invoked once +** for each table affected by at least one change in the changeset. In this +** case the table name is passed as the second argument, and a copy of +** the context pointer passed as the sixth argument to apply() or apply_v2() +** as the first. If the "filter callback" returns zero, then no attempt is +** made to apply any changes to the table. Otherwise, if the return value is +** non-zero, all changes related to the table are attempted. +** +** For sqlite3_changeset_apply_v3(), the xFilter callback is invoked once +** per change. The second argument in this case is an sqlite3_changeset_iter +** that may be queried using the usual APIs for the details of the current +** change. If the "filter callback" returns zero in this case, then no attempt +** is made to apply the current change. If it returns non-zero, the change +** is applied. ** ** For each table that is not excluded by the filter callback, this function ** tests that the target database contains a compatible table. A table is @@ -12366,11 +12827,11 @@ SQLITE_API void sqlite3changegroup_delete(sqlite3_changegroup*); ** one such warning is issued for each table in the changeset. ** ** For each change for which there is a compatible table, an attempt is made -** to modify the table contents according to the UPDATE, INSERT or DELETE -** change. If a change cannot be applied cleanly, the conflict handler -** function passed as the fifth argument to sqlite3changeset_apply() may be -** invoked. A description of exactly when the conflict handler is invoked for -** each type of change is below. +** to modify the table contents according to each UPDATE, INSERT or DELETE +** change that is not excluded by a filter callback. If a change cannot be +** applied cleanly, the conflict handler function passed as the fifth argument +** to sqlite3changeset_apply() may be invoked. A description of exactly when +** the conflict handler is invoked for each type of change is below. ** ** Unlike the xFilter argument, xConflict may not be passed NULL. The results ** of passing anything other than a valid function pointer as the xConflict @@ -12466,12 +12927,6 @@ SQLITE_API void sqlite3changegroup_delete(sqlite3_changegroup*); ** This can be used to further customize the application's conflict ** resolution strategy. ** -** All changes made by these functions are enclosed in a savepoint transaction. -** If any other error (aside from a constraint failure when attempting to -** write to the target database) occurs, then the savepoint transaction is -** rolled back, restoring the target database to its original state, and an -** SQLite error code returned. -** ** If the output parameters (ppRebase) and (pnRebase) are non-NULL and ** the input is a changeset (not a patchset), then sqlite3changeset_apply_v2() ** may set (*ppRebase) to point to a "rebase" that may be used with the @@ -12521,6 +12976,23 @@ SQLITE_API int sqlite3changeset_apply_v2( void **ppRebase, int *pnRebase, /* OUT: Rebase data */ int flags /* SESSION_CHANGESETAPPLY_* flags */ ); +SQLITE_API int sqlite3changeset_apply_v3( + sqlite3 *db, /* Apply change to "main" db of this handle */ + int nChangeset, /* Size of changeset in bytes */ + void *pChangeset, /* Changeset blob */ + int(*xFilter)( + void *pCtx, /* Copy of sixth arg to _apply() */ + sqlite3_changeset_iter *p /* Handle describing change */ + ), + int(*xConflict)( + void *pCtx, /* Copy of sixth arg to _apply() */ + int eConflict, /* DATA, MISSING, CONFLICT, CONSTRAINT */ + sqlite3_changeset_iter *p /* Handle describing change and conflict */ + ), + void *pCtx, /* First argument passed to xConflict */ + void **ppRebase, int *pnRebase, /* OUT: Rebase data */ + int flags /* SESSION_CHANGESETAPPLY_* flags */ +); /* ** CAPI3REF: Flags for sqlite3changeset_apply_v2 @@ -12940,6 +13412,23 @@ SQLITE_API int sqlite3changeset_apply_v2_strm( void **ppRebase, int *pnRebase, int flags ); +SQLITE_API int sqlite3changeset_apply_v3_strm( + sqlite3 *db, /* Apply change to "main" db of this handle */ + int (*xInput)(void *pIn, void *pData, int *pnData), /* Input function */ + void *pIn, /* First arg for xInput */ + int(*xFilter)( + void *pCtx, /* Copy of sixth arg to _apply() */ + sqlite3_changeset_iter *p + ), + int(*xConflict)( + void *pCtx, /* Copy of sixth arg to _apply() */ + int eConflict, /* DATA, MISSING, CONFLICT, CONSTRAINT */ + sqlite3_changeset_iter *p /* Handle describing change and conflict */ + ), + void *pCtx, /* First argument passed to xConflict */ + void **ppRebase, int *pnRebase, + int flags +); SQLITE_API int sqlite3changeset_concat_strm( int (*xInputA)(void *pIn, void *pData, int *pnData), void *pInA, @@ -13098,8 +13587,8 @@ struct Fts5PhraseIter { ** EXTENSION API FUNCTIONS ** ** xUserData(pFts): -** Return a copy of the context pointer the extension function was -** registered with. +** Return a copy of the pUserData pointer passed to the xCreateFunction() +** API when the extension function was registered. ** ** xColumnTotalSize(pFts, iCol, pnToken): ** If parameter iCol is less than zero, set output variable *pnToken @@ -13281,6 +13770,10 @@ struct Fts5PhraseIter { ** (i.e. if it is a contentless table), then this API always iterates ** through an empty set (all calls to xPhraseFirst() set iCol to -1). ** +** In all cases, matches are visited in (column ASC, offset ASC) order. +** i.e. all those in column 0, sorted by offset, followed by those in +** column 1, etc. +** ** xPhraseNext() ** See xPhraseFirst above. ** @@ -13337,19 +13830,57 @@ struct Fts5PhraseIter { ** value returned by xInstCount(), SQLITE_RANGE is returned. Otherwise, ** output variable (*ppToken) is set to point to a buffer containing the ** matching document token, and (*pnToken) to the size of that buffer in -** bytes. This API is not available if the specified token matches a -** prefix query term. In that case both output variables are always set -** to 0. +** bytes. ** ** The output text is not a copy of the document text that was tokenized. ** It is the output of the tokenizer module. For tokendata=1 tables, this ** includes any embedded 0x00 and trailing data. ** +** This API may be slow in some cases if the token identified by parameters +** iIdx and iToken matched a prefix token in the query. In most cases, the +** first call to this API for each prefix token in the query is forced +** to scan the portion of the full-text index that matches the prefix +** token to collect the extra data required by this API. If the prefix +** token matches a large number of token instances in the document set, +** this may be a performance problem. +** +** If the user knows in advance that a query may use this API for a +** prefix token, FTS5 may be configured to collect all required data as part +** of the initial querying of the full-text index, avoiding the second scan +** entirely. This also causes prefix queries that do not use this API to +** run more slowly and use more memory. FTS5 may be configured in this way +** either on a per-table basis using the [FTS5 insttoken | 'insttoken'] +** option, or on a per-query basis using the +** [fts5_insttoken | fts5_insttoken()] user function. +** ** This API can be quite slow if used with an FTS5 table created with the ** "detail=none" or "detail=column" option. +** +** xColumnLocale(pFts5, iIdx, pzLocale, pnLocale) +** If parameter iCol is less than zero, or greater than or equal to the +** number of columns in the table, SQLITE_RANGE is returned. +** +** Otherwise, this function attempts to retrieve the locale associated +** with column iCol of the current row. Usually, there is no associated +** locale, and output parameters (*pzLocale) and (*pnLocale) are set +** to NULL and 0, respectively. However, if the fts5_locale() function +** was used to associate a locale with the value when it was inserted +** into the fts5 table, then (*pzLocale) is set to point to a nul-terminated +** buffer containing the name of the locale in utf-8 encoding. (*pnLocale) +** is set to the size in bytes of the buffer, not including the +** nul-terminator. +** +** If successful, SQLITE_OK is returned. Or, if an error occurs, an +** SQLite error code is returned. The final value of the output parameters +** is undefined in this case. +** +** xTokenize_v2: +** Tokenize text using the tokenizer belonging to the FTS5 table. This +** API is the same as the xTokenize() API, except that it allows a tokenizer +** locale to be specified. */ struct Fts5ExtensionApi { - int iVersion; /* Currently always set to 3 */ + int iVersion; /* Currently always set to 4 */ void *(*xUserData)(Fts5Context*); @@ -13391,6 +13922,15 @@ struct Fts5ExtensionApi { const char **ppToken, int *pnToken ); int (*xInstToken)(Fts5Context*, int iIdx, int iToken, const char**, int*); + + /* Below this point are iVersion>=4 only */ + int (*xColumnLocale)(Fts5Context*, int iCol, const char **pz, int *pn); + int (*xTokenize_v2)(Fts5Context*, + const char *pText, int nText, /* Text to tokenize */ + const char *pLocale, int nLocale, /* Locale to pass to tokenizer */ + void *pCtx, /* Context passed to xToken() */ + int (*xToken)(void*, int, const char*, int, int, int) /* Callback */ + ); }; /* @@ -13411,7 +13951,7 @@ struct Fts5ExtensionApi { ** A tokenizer instance is required to actually tokenize text. ** ** The first argument passed to this function is a copy of the (void*) -** pointer provided by the application when the fts5_tokenizer object +** pointer provided by the application when the fts5_tokenizer_v2 object ** was registered with FTS5 (the third argument to xCreateTokenizer()). ** The second and third arguments are an array of nul-terminated strings ** containing the tokenizer arguments, if any, specified following the @@ -13435,7 +13975,7 @@ struct Fts5ExtensionApi { ** argument passed to this function is a pointer to an Fts5Tokenizer object ** returned by an earlier call to xCreate(). ** -** The second argument indicates the reason that FTS5 is requesting +** The third argument indicates the reason that FTS5 is requesting ** tokenization of the supplied text. This is always one of the following ** four values: ** @@ -13459,6 +13999,13 @@ struct Fts5ExtensionApi { ** on a columnsize=0 database. ** ** +** The sixth and seventh arguments passed to xTokenize() - pLocale and +** nLocale - are a pointer to a buffer containing the locale to use for +** tokenization (e.g. "en_US") and its size in bytes, respectively. The +** pLocale buffer is not nul-terminated. pLocale may be passed NULL (in +** which case nLocale is always 0) to indicate that the tokenizer should +** use its default locale. +** ** For each token in the input string, the supplied callback xToken() must ** be invoked. The first argument to it should be a copy of the pointer ** passed as the second argument to xTokenize(). The third and fourth @@ -13482,6 +14029,30 @@ struct Fts5ExtensionApi { ** may abandon the tokenization and return any error code other than ** SQLITE_OK or SQLITE_DONE. ** +** If the tokenizer is registered using an fts5_tokenizer_v2 object, +** then the xTokenize() method has two additional arguments - pLocale +** and nLocale. These specify the locale that the tokenizer should use +** for the current request. If pLocale and nLocale are both 0, then the +** tokenizer should use its default locale. Otherwise, pLocale points to +** an nLocale byte buffer containing the name of the locale to use as utf-8 +** text. pLocale is not nul-terminated. +** +** FTS5_TOKENIZER +** +** There is also an fts5_tokenizer object. This is an older, deprecated, +** version of fts5_tokenizer_v2. It is similar except that: +** +**
            +**
          • There is no "iVersion" field, and +**
          • The xTokenize() method does not take a locale argument. +**
          +** +** Legacy fts5_tokenizer tokenizers must be registered using the +** legacy xCreateTokenizer() function, instead of xCreateTokenizer_v2(). +** +** Tokenizer implementations registered using either API may be retrieved +** using both xFindTokenizer() and xFindTokenizer_v2(). +** ** SYNONYM SUPPORT ** ** Custom tokenizers may also support synonyms. Consider a case in which a @@ -13590,6 +14161,33 @@ struct Fts5ExtensionApi { ** inefficient. */ typedef struct Fts5Tokenizer Fts5Tokenizer; +typedef struct fts5_tokenizer_v2 fts5_tokenizer_v2; +struct fts5_tokenizer_v2 { + int iVersion; /* Currently always 2 */ + + int (*xCreate)(void*, const char **azArg, int nArg, Fts5Tokenizer **ppOut); + void (*xDelete)(Fts5Tokenizer*); + int (*xTokenize)(Fts5Tokenizer*, + void *pCtx, + int flags, /* Mask of FTS5_TOKENIZE_* flags */ + const char *pText, int nText, + const char *pLocale, int nLocale, + int (*xToken)( + void *pCtx, /* Copy of 2nd argument to xTokenize() */ + int tflags, /* Mask of FTS5_TOKEN_* flags */ + const char *pToken, /* Pointer to buffer containing token */ + int nToken, /* Size of token in bytes */ + int iStart, /* Byte offset of token within input text */ + int iEnd /* Byte offset of end of token within input text */ + ) + ); +}; + +/* +** New code should use the fts5_tokenizer_v2 type to define tokenizer +** implementations. The following type is included for legacy applications +** that still use it. +*/ typedef struct fts5_tokenizer fts5_tokenizer; struct fts5_tokenizer { int (*xCreate)(void*, const char **azArg, int nArg, Fts5Tokenizer **ppOut); @@ -13609,6 +14207,7 @@ struct fts5_tokenizer { ); }; + /* Flags that may be passed as the third argument to xTokenize() */ #define FTS5_TOKENIZE_QUERY 0x0001 #define FTS5_TOKENIZE_PREFIX 0x0002 @@ -13628,7 +14227,7 @@ struct fts5_tokenizer { */ typedef struct fts5_api fts5_api; struct fts5_api { - int iVersion; /* Currently always set to 2 */ + int iVersion; /* Currently always set to 3 */ /* Create a new tokenizer */ int (*xCreateTokenizer)( @@ -13655,6 +14254,25 @@ struct fts5_api { fts5_extension_function xFunction, void (*xDestroy)(void*) ); + + /* APIs below this point are only available if iVersion>=3 */ + + /* Create a new tokenizer */ + int (*xCreateTokenizer_v2)( + fts5_api *pApi, + const char *zName, + void *pUserData, + fts5_tokenizer_v2 *pTokenizer, + void (*xDestroy)(void*) + ); + + /* Find an existing tokenizer */ + int (*xFindTokenizer_v2)( + fts5_api *pApi, + const char *zName, + void **ppUserData, + fts5_tokenizer_v2 **ppTokenizer + ); }; /* @@ -13668,6 +14286,7 @@ struct fts5_api { #endif /* _FTS5_H */ /******** End of fts5.h *********/ +#endif /* SQLITE3_H */ /************** End of sqlite3.h *********************************************/ /************** Continuing where we left off in sqliteInt.h ******************/ @@ -13713,6 +14332,7 @@ struct fts5_api { #ifndef SQLITE_MAX_LENGTH # define SQLITE_MAX_LENGTH 1000000000 #endif +#define SQLITE_MIN_LENGTH 30 /* Minimum value for the length limit */ /* ** This is the maximum number of @@ -13725,14 +14345,22 @@ struct fts5_api { ** * Terms in the GROUP BY or ORDER BY clauses of a SELECT statement. ** * Terms in the VALUES clause of an INSERT statement ** -** The hard upper limit here is 32676. Most database people will +** The hard upper limit here is 32767. Most database people will ** tell you that in a well-normalized database, you usually should ** not have more than a dozen or so columns in any table. And if ** that is the case, there is no point in having more than a few ** dozen values in any of the other situations described above. +** +** An index can only have SQLITE_MAX_COLUMN columns from the user +** point of view, but the underlying b-tree that implements the index +** might have up to twice as many columns in a WITHOUT ROWID table, +** since must also store the primary key at the end. Hence the +** column count for Index is u16 instead of i16. */ -#ifndef SQLITE_MAX_COLUMN +#if !defined(SQLITE_MAX_COLUMN) # define SQLITE_MAX_COLUMN 2000 +#elif SQLITE_MAX_COLUMN>32767 +# error SQLITE_MAX_COLUMN may not exceed 32767 #endif /* @@ -13778,9 +14406,13 @@ struct fts5_api { /* ** The maximum number of arguments to an SQL function. +** +** This value has a hard upper limit of 32767 due to storage +** constraints (it needs to fit inside a i16). We keep it +** lower than that to prevent abuse. */ #ifndef SQLITE_MAX_FUNCTION_ARG -# define SQLITE_MAX_FUNCTION_ARG 127 +# define SQLITE_MAX_FUNCTION_ARG 1000 #endif /* @@ -13873,7 +14505,7 @@ struct fts5_api { ** Maximum number of pages in one database file. ** ** This is really just the default value for the max_page_count pragma. -** This value can be lowered (or raised) at run-time using that the +** This value can be lowered (or raised) at run-time using the ** max_page_count macro. */ #ifndef SQLITE_MAX_PAGE_COUNT @@ -14297,6 +14929,8 @@ struct fts5_api { # define SQLITE_OMIT_ALTERTABLE #endif +#define SQLITE_DIGIT_SEPARATOR '_' + /* ** Return true (non-zero) if the input is an integer that is too large ** to fit in 32-bits. This macro is used inside of various testcase() @@ -14378,6 +15012,7 @@ struct HashElem { HashElem *next, *prev; /* Next and previous elements in the table */ void *data; /* Data associated with this element */ const char *pKey; /* Key associated with this element */ + unsigned int h; /* hash for pKey */ }; /* @@ -14462,135 +15097,135 @@ SQLITE_PRIVATE void sqlite3HashClear(Hash*); #define TK_OR 43 #define TK_AND 44 #define TK_IS 45 -#define TK_MATCH 46 -#define TK_LIKE_KW 47 -#define TK_BETWEEN 48 -#define TK_IN 49 -#define TK_ISNULL 50 -#define TK_NOTNULL 51 -#define TK_NE 52 -#define TK_EQ 53 -#define TK_GT 54 -#define TK_LE 55 -#define TK_LT 56 -#define TK_GE 57 -#define TK_ESCAPE 58 -#define TK_ID 59 -#define TK_COLUMNKW 60 -#define TK_DO 61 -#define TK_FOR 62 -#define TK_IGNORE 63 -#define TK_INITIALLY 64 -#define TK_INSTEAD 65 -#define TK_NO 66 -#define TK_KEY 67 -#define TK_OF 68 -#define TK_OFFSET 69 -#define TK_PRAGMA 70 -#define TK_RAISE 71 -#define TK_RECURSIVE 72 -#define TK_REPLACE 73 -#define TK_RESTRICT 74 -#define TK_ROW 75 -#define TK_ROWS 76 -#define TK_TRIGGER 77 -#define TK_VACUUM 78 -#define TK_VIEW 79 -#define TK_VIRTUAL 80 -#define TK_WITH 81 -#define TK_NULLS 82 -#define TK_FIRST 83 -#define TK_LAST 84 -#define TK_CURRENT 85 -#define TK_FOLLOWING 86 -#define TK_PARTITION 87 -#define TK_PRECEDING 88 -#define TK_RANGE 89 -#define TK_UNBOUNDED 90 -#define TK_EXCLUDE 91 -#define TK_GROUPS 92 -#define TK_OTHERS 93 -#define TK_TIES 94 -#define TK_GENERATED 95 -#define TK_ALWAYS 96 -#define TK_MATERIALIZED 97 -#define TK_REINDEX 98 -#define TK_RENAME 99 -#define TK_CTIME_KW 100 -#define TK_ANY 101 -#define TK_BITAND 102 -#define TK_BITOR 103 -#define TK_LSHIFT 104 -#define TK_RSHIFT 105 -#define TK_PLUS 106 -#define TK_MINUS 107 -#define TK_STAR 108 -#define TK_SLASH 109 -#define TK_REM 110 -#define TK_CONCAT 111 -#define TK_PTR 112 -#define TK_COLLATE 113 -#define TK_BITNOT 114 -#define TK_ON 115 -#define TK_INDEXED 116 -#define TK_STRING 117 -#define TK_JOIN_KW 118 -#define TK_CONSTRAINT 119 -#define TK_DEFAULT 120 -#define TK_NULL 121 -#define TK_PRIMARY 122 -#define TK_UNIQUE 123 -#define TK_CHECK 124 -#define TK_REFERENCES 125 -#define TK_AUTOINCR 126 -#define TK_INSERT 127 -#define TK_DELETE 128 -#define TK_UPDATE 129 -#define TK_SET 130 -#define TK_DEFERRABLE 131 -#define TK_FOREIGN 132 -#define TK_DROP 133 -#define TK_UNION 134 -#define TK_ALL 135 -#define TK_EXCEPT 136 -#define TK_INTERSECT 137 -#define TK_SELECT 138 -#define TK_VALUES 139 -#define TK_DISTINCT 140 -#define TK_DOT 141 -#define TK_FROM 142 -#define TK_JOIN 143 -#define TK_USING 144 -#define TK_ORDER 145 -#define TK_GROUP 146 -#define TK_HAVING 147 -#define TK_LIMIT 148 -#define TK_WHERE 149 -#define TK_RETURNING 150 -#define TK_INTO 151 -#define TK_NOTHING 152 -#define TK_FLOAT 153 -#define TK_BLOB 154 -#define TK_INTEGER 155 -#define TK_VARIABLE 156 -#define TK_CASE 157 -#define TK_WHEN 158 -#define TK_THEN 159 -#define TK_ELSE 160 -#define TK_INDEX 161 -#define TK_ALTER 162 -#define TK_ADD 163 -#define TK_WINDOW 164 -#define TK_OVER 165 -#define TK_FILTER 166 -#define TK_COLUMN 167 -#define TK_AGG_FUNCTION 168 -#define TK_AGG_COLUMN 169 -#define TK_TRUEFALSE 170 -#define TK_ISNOT 171 +#define TK_ISNOT 46 +#define TK_MATCH 47 +#define TK_LIKE_KW 48 +#define TK_BETWEEN 49 +#define TK_IN 50 +#define TK_ISNULL 51 +#define TK_NOTNULL 52 +#define TK_NE 53 +#define TK_EQ 54 +#define TK_GT 55 +#define TK_LE 56 +#define TK_LT 57 +#define TK_GE 58 +#define TK_ESCAPE 59 +#define TK_ID 60 +#define TK_COLUMNKW 61 +#define TK_DO 62 +#define TK_FOR 63 +#define TK_IGNORE 64 +#define TK_INITIALLY 65 +#define TK_INSTEAD 66 +#define TK_NO 67 +#define TK_KEY 68 +#define TK_OF 69 +#define TK_OFFSET 70 +#define TK_PRAGMA 71 +#define TK_RAISE 72 +#define TK_RECURSIVE 73 +#define TK_REPLACE 74 +#define TK_RESTRICT 75 +#define TK_ROW 76 +#define TK_ROWS 77 +#define TK_TRIGGER 78 +#define TK_VACUUM 79 +#define TK_VIEW 80 +#define TK_VIRTUAL 81 +#define TK_WITH 82 +#define TK_NULLS 83 +#define TK_FIRST 84 +#define TK_LAST 85 +#define TK_CURRENT 86 +#define TK_FOLLOWING 87 +#define TK_PARTITION 88 +#define TK_PRECEDING 89 +#define TK_RANGE 90 +#define TK_UNBOUNDED 91 +#define TK_EXCLUDE 92 +#define TK_GROUPS 93 +#define TK_OTHERS 94 +#define TK_TIES 95 +#define TK_GENERATED 96 +#define TK_ALWAYS 97 +#define TK_MATERIALIZED 98 +#define TK_REINDEX 99 +#define TK_RENAME 100 +#define TK_CTIME_KW 101 +#define TK_ANY 102 +#define TK_BITAND 103 +#define TK_BITOR 104 +#define TK_LSHIFT 105 +#define TK_RSHIFT 106 +#define TK_PLUS 107 +#define TK_MINUS 108 +#define TK_STAR 109 +#define TK_SLASH 110 +#define TK_REM 111 +#define TK_CONCAT 112 +#define TK_PTR 113 +#define TK_COLLATE 114 +#define TK_BITNOT 115 +#define TK_ON 116 +#define TK_INDEXED 117 +#define TK_STRING 118 +#define TK_JOIN_KW 119 +#define TK_CONSTRAINT 120 +#define TK_DEFAULT 121 +#define TK_NULL 122 +#define TK_PRIMARY 123 +#define TK_UNIQUE 124 +#define TK_CHECK 125 +#define TK_REFERENCES 126 +#define TK_AUTOINCR 127 +#define TK_INSERT 128 +#define TK_DELETE 129 +#define TK_UPDATE 130 +#define TK_SET 131 +#define TK_DEFERRABLE 132 +#define TK_FOREIGN 133 +#define TK_DROP 134 +#define TK_UNION 135 +#define TK_ALL 136 +#define TK_EXCEPT 137 +#define TK_INTERSECT 138 +#define TK_SELECT 139 +#define TK_VALUES 140 +#define TK_DISTINCT 141 +#define TK_DOT 142 +#define TK_FROM 143 +#define TK_JOIN 144 +#define TK_USING 145 +#define TK_ORDER 146 +#define TK_GROUP 147 +#define TK_HAVING 148 +#define TK_LIMIT 149 +#define TK_WHERE 150 +#define TK_RETURNING 151 +#define TK_INTO 152 +#define TK_NOTHING 153 +#define TK_FLOAT 154 +#define TK_BLOB 155 +#define TK_INTEGER 156 +#define TK_VARIABLE 157 +#define TK_CASE 158 +#define TK_WHEN 159 +#define TK_THEN 160 +#define TK_ELSE 161 +#define TK_INDEX 162 +#define TK_ALTER 163 +#define TK_ADD 164 +#define TK_WINDOW 165 +#define TK_OVER 166 +#define TK_FILTER 167 +#define TK_COLUMN 168 +#define TK_AGG_FUNCTION 169 +#define TK_AGG_COLUMN 170 +#define TK_TRUEFALSE 171 #define TK_FUNCTION 172 -#define TK_UMINUS 173 -#define TK_UPLUS 174 +#define TK_UPLUS 173 +#define TK_UMINUS 174 #define TK_TRUTH 175 #define TK_REGISTER 176 #define TK_VECTOR 177 @@ -14599,8 +15234,10 @@ SQLITE_PRIVATE void sqlite3HashClear(Hash*); #define TK_ASTERISK 180 #define TK_SPAN 181 #define TK_ERROR 182 -#define TK_SPACE 183 -#define TK_ILLEGAL 184 +#define TK_QNUMBER 183 +#define TK_SPACE 184 +#define TK_COMMENT 185 +#define TK_ILLEGAL 186 /************** End of parse.h ***********************************************/ /************** Continuing where we left off in sqliteInt.h ******************/ @@ -14609,6 +15246,7 @@ SQLITE_PRIVATE void sqlite3HashClear(Hash*); #include #include #include +#include /* ** Use a macro to replace memcpy() if compiled with SQLITE_INLINE_MEMCPY. @@ -14629,7 +15267,8 @@ SQLITE_PRIVATE void sqlite3HashClear(Hash*); #ifdef SQLITE_OMIT_FLOATING_POINT # define double sqlite_int64 # define float sqlite_int64 -# define LONGDOUBLE_TYPE sqlite_int64 +# define fabs(X) ((X)<0?-(X):(X)) +# define sqlite3IsOverflow(X) 0 # ifndef SQLITE_BIG_DBL # define SQLITE_BIG_DBL (((sqlite3_int64)1)<<50) # endif @@ -14734,7 +15373,17 @@ SQLITE_PRIVATE void sqlite3HashClear(Hash*); ** ourselves. */ #ifndef offsetof -#define offsetof(STRUCTURE,FIELD) ((int)((char*)&((STRUCTURE*)0)->FIELD)) +# define offsetof(ST,M) ((size_t)((char*)&((ST*)0)->M - (char*)0)) +#endif + +/* +** Work around C99 "flex-array" syntax for pre-C99 compilers, so as +** to avoid complaints from -fsanitize=strict-bounds. +*/ +#if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) +# define FLEXARRAY +#else +# define FLEXARRAY 1 #endif /* @@ -14804,9 +15453,6 @@ SQLITE_PRIVATE void sqlite3HashClear(Hash*); # define INT8_TYPE signed char # endif #endif -#ifndef LONGDOUBLE_TYPE -# define LONGDOUBLE_TYPE long double -#endif typedef sqlite_int64 i64; /* 8-byte signed integer */ typedef sqlite_uint64 u64; /* 8-byte unsigned integer */ typedef UINT32_TYPE u32; /* 4-byte unsigned integer */ @@ -14815,6 +15461,11 @@ typedef INT16_TYPE i16; /* 2-byte signed integer */ typedef UINT8_TYPE u8; /* 1-byte unsigned integer */ typedef INT8_TYPE i8; /* 1-byte signed integer */ +/* A bitfield type for use inside of structures. Always follow with :N where +** N is the number of bits. +*/ +typedef unsigned bft; /* Bit Field Type */ + /* ** SQLITE_MAX_U32 is a u64 constant that is the maximum u64 value ** that can be stored in a u32 without loss of data. The value @@ -14853,6 +15504,8 @@ typedef u64 tRowcnt; ** 0.5 -> -10 0.1 -> -33 0.0625 -> -40 */ typedef INT16_TYPE LogEst; +#define LOGEST_MIN (-32768) +#define LOGEST_MAX (32767) /* ** Set the SQLITE_PTRSIZE macro to the number of bytes in a pointer @@ -14862,7 +15515,7 @@ typedef INT16_TYPE LogEst; # define SQLITE_PTRSIZE __SIZEOF_POINTER__ # elif defined(i386) || defined(__i386__) || defined(_M_IX86) || \ defined(_M_ARM) || defined(__arm__) || defined(__x86) || \ - (defined(__APPLE__) && defined(__POWERPC__)) || \ + (defined(__APPLE__) && defined(__ppc__)) || \ (defined(__TOS_AIX__) && !defined(__64BIT__)) # define SQLITE_PTRSIZE 4 # else @@ -14981,6 +15634,14 @@ typedef INT16_TYPE LogEst; #define LARGEST_UINT64 (0xffffffff|(((u64)0xffffffff)<<32)) #define SMALLEST_INT64 (((i64)-1) - LARGEST_INT64) +/* +** Macro SMXV(n) return the maximum value that can be held in variable n, +** assuming n is a signed integer type. UMXV(n) is similar for unsigned +** integer types. +*/ +#define SMXV(n) ((((i64)1)<<(sizeof(n)*8-1))-1) +#define UMXV(n) ((((i64)1)<<(sizeof(n)*8))-1) + /* ** Round up a number to the next larger multiple of 8. This is used ** to force 8-byte alignment on 64-bit architectures. @@ -15100,6 +15761,8 @@ SQLITE_PRIVATE u32 sqlite3TreeTrace; ** 0x00020000 Transform DISTINCT into GROUP BY ** 0x00040000 SELECT tree dump after all code has been generated ** 0x00080000 NOT NULL strength reduction +** 0x00100000 Pointers are all shown as zero +** 0x00200000 EXISTS-to-JOIN optimization */ /* @@ -15123,14 +15786,14 @@ SQLITE_PRIVATE u32 sqlite3WhereTrace; ** 0xFFFF---- Low-level debug messages ** ** 0x00000001 Code generation -** 0x00000002 Solver +** 0x00000002 Solver (Use 0x40000 for less detail) ** 0x00000004 Solver costs ** 0x00000008 WhereLoop inserts ** ** 0x00000010 Display sqlite3_index_info xBestIndex calls ** 0x00000020 Range an equality scan metrics ** 0x00000040 IN operator decisions -** 0x00000080 WhereLoop cost adjustements +** 0x00000080 WhereLoop cost adjustments ** 0x00000100 ** 0x00000200 Covering index decisions ** 0x00000400 OR optimization @@ -15142,6 +15805,9 @@ SQLITE_PRIVATE u32 sqlite3WhereTrace; ** ** 0x00010000 Show more detail when printing WHERE terms ** 0x00020000 Show WHERE terms returned from whereScanNext() +** 0x00040000 Solver overview messages +** 0x00080000 Star-query heuristic +** 0x00100000 Pointers are all shown as zero */ @@ -15214,7 +15880,7 @@ struct BusyHandler { ** pointer will work here as long as it is distinct from SQLITE_STATIC ** and SQLITE_TRANSIENT. */ -#define SQLITE_DYNAMIC ((sqlite3_destructor_type)sqlite3OomClear) +#define SQLITE_DYNAMIC ((sqlite3_destructor_type)sqlite3RowSetClear) /* ** When SQLITE_OMIT_WSD is defined, it means that the target platform does @@ -15306,6 +15972,7 @@ typedef struct Savepoint Savepoint; typedef struct Select Select; typedef struct SQLiteThread SQLiteThread; typedef struct SelectDest SelectDest; +typedef struct Subquery Subquery; typedef struct SrcItem SrcItem; typedef struct SrcList SrcList; typedef struct sqlite3_str StrAccum; /* Internal alias for sqlite3_str */ @@ -15434,8 +16101,8 @@ typedef int VList; ** must provide its own VFS implementation together with sqlite3_os_init() ** and sqlite3_os_end() routines. */ -#if !defined(SQLITE_OS_KV) && !defined(SQLITE_OS_OTHER) && \ - !defined(SQLITE_OS_UNIX) && !defined(SQLITE_OS_WIN) +#if SQLITE_OS_KV+1<=1 && SQLITE_OS_OTHER+1<=1 && \ + SQLITE_OS_WIN+1<=1 && SQLITE_OS_UNIX+1<=1 # if defined(_WIN32) || defined(WIN32) || defined(__CYGWIN__) || \ defined(__MINGW32__) || defined(__BORLANDC__) # define SQLITE_OS_WIN 1 @@ -15779,6 +16446,22 @@ typedef struct PgHdr DbPage; #define PAGER_JOURNALMODE_MEMORY 4 /* In-memory journal file */ #define PAGER_JOURNALMODE_WAL 5 /* Use write-ahead logging */ +#define isWalMode(x) ((x)==PAGER_JOURNALMODE_WAL) + +/* +** The argument to this macro is a file descriptor (type sqlite3_file*). +** Return 0 if it is not open, or non-zero (but not 1) if it is. +** +** This is so that expressions can be written as: +** +** if( isOpen(pPager->jfd) ){ ... +** +** instead of +** +** if( pPager->jfd->pMethods ){ ... +*/ +#define isOpen(pFd) ((pFd)->pMethods!=0) + /* ** Flags that make up the mask passed to sqlite3PagerGet(). */ @@ -16188,6 +16871,9 @@ SQLITE_PRIVATE int sqlite3BtreeCursor( ); SQLITE_PRIVATE BtCursor *sqlite3BtreeFakeValidCursor(void); SQLITE_PRIVATE int sqlite3BtreeCursorSize(void); +#ifdef SQLITE_DEBUG +SQLITE_PRIVATE int sqlite3BtreeClosesWithCursor(Btree*,BtCursor*); +#endif SQLITE_PRIVATE void sqlite3BtreeCursorZero(BtCursor*); SQLITE_PRIVATE void sqlite3BtreeCursorHintFlags(BtCursor*, unsigned); #ifdef SQLITE_ENABLE_CURSOR_HINTS @@ -16262,6 +16948,7 @@ struct BtreePayload { SQLITE_PRIVATE int sqlite3BtreeInsert(BtCursor*, const BtreePayload *pPayload, int flags, int seekResult); SQLITE_PRIVATE int sqlite3BtreeFirst(BtCursor*, int *pRes); +SQLITE_PRIVATE int sqlite3BtreeIsEmpty(BtCursor *pCur, int *pRes); SQLITE_PRIVATE int sqlite3BtreeLast(BtCursor*, int *pRes); SQLITE_PRIVATE int sqlite3BtreeNext(BtCursor*, int flags); SQLITE_PRIVATE int sqlite3BtreeEof(BtCursor*); @@ -16279,6 +16966,7 @@ SQLITE_PRIVATE int sqlite3BtreeIntegrityCheck( sqlite3 *db, /* Database connection that is running the check */ Btree *p, /* The btree to be checked */ Pgno *aRoot, /* An array of root pages numbers for individual trees */ + sqlite3_value *aCnt, /* OUT: entry counts for each btree in aRoot[] */ int nRoot, /* Number of entries in aRoot[] */ int mxErr, /* Stop reporting errors after this many */ int *pnErr, /* OUT: Write number of errors seen to this variable */ @@ -16405,6 +17093,20 @@ typedef struct Vdbe Vdbe; */ typedef struct sqlite3_value Mem; typedef struct SubProgram SubProgram; +typedef struct SubrtnSig SubrtnSig; + +/* +** A signature for a reusable subroutine that materializes the RHS of +** an IN operator. +*/ +struct SubrtnSig { + int selId; /* SELECT-id for the SELECT statement on the RHS */ + u8 bComplete; /* True if fully coded and available for reusable */ + char *zAff; /* Affinity of the overall IN expression */ + int iTable; /* Ephemeral table generated by the subroutine */ + int iAddr; /* Subroutine entry address */ + int regReturn; /* Register used to hold return address */ +}; /* ** A single instruction of the virtual machine has an opcode @@ -16433,6 +17135,7 @@ struct VdbeOp { u32 *ai; /* Used when p4type is P4_INTARRAY */ SubProgram *pProgram; /* Used when p4type is P4_SUBPROGRAM */ Table *pTab; /* Used when p4type is P4_TABLE */ + SubrtnSig *pSubrtnSig; /* Used when p4type is P4_SUBRTNSIG */ #ifdef SQLITE_ENABLE_CURSOR_HINTS Expr *pExpr; /* Used when p4type is P4_EXPR */ #endif @@ -16500,6 +17203,7 @@ typedef struct VdbeOpList VdbeOpList; #define P4_INTARRAY (-14) /* P4 is a vector of 32-bit integers */ #define P4_FUNCCTX (-15) /* P4 is a pointer to an sqlite3_context object */ #define P4_TABLEREF (-16) /* Like P4_TABLE, but reference counted */ +#define P4_SUBRTNSIG (-17) /* P4 is a SubrtnSig pointer */ /* Error message codes for OP_Halt */ #define P5_ConstraintNotNull 1 @@ -16549,12 +17253,12 @@ typedef struct VdbeOpList VdbeOpList; #define OP_Vacuum 5 #define OP_VFilter 6 /* jump, synopsis: iplan=r[P3] zplan='P4' */ #define OP_VUpdate 7 /* synopsis: data=r[P3@P2] */ -#define OP_Init 8 /* jump, synopsis: Start at P2 */ +#define OP_Init 8 /* jump0, synopsis: Start at P2 */ #define OP_Goto 9 /* jump */ #define OP_Gosub 10 /* jump */ -#define OP_InitCoroutine 11 /* jump */ -#define OP_Yield 12 /* jump */ -#define OP_MustBeInt 13 /* jump */ +#define OP_InitCoroutine 11 /* jump0 */ +#define OP_Yield 12 /* jump0 */ +#define OP_MustBeInt 13 /* jump0 */ #define OP_Jump 14 /* jump */ #define OP_Once 15 /* jump */ #define OP_If 16 /* jump */ @@ -16562,175 +17266,176 @@ typedef struct VdbeOpList VdbeOpList; #define OP_IsType 18 /* jump, synopsis: if typeof(P1.P3) in P5 goto P2 */ #define OP_Not 19 /* same as TK_NOT, synopsis: r[P2]= !r[P1] */ #define OP_IfNullRow 20 /* jump, synopsis: if P1.nullRow then r[P3]=NULL, goto P2 */ -#define OP_SeekLT 21 /* jump, synopsis: key=r[P3@P4] */ -#define OP_SeekLE 22 /* jump, synopsis: key=r[P3@P4] */ -#define OP_SeekGE 23 /* jump, synopsis: key=r[P3@P4] */ -#define OP_SeekGT 24 /* jump, synopsis: key=r[P3@P4] */ +#define OP_SeekLT 21 /* jump0, synopsis: key=r[P3@P4] */ +#define OP_SeekLE 22 /* jump0, synopsis: key=r[P3@P4] */ +#define OP_SeekGE 23 /* jump0, synopsis: key=r[P3@P4] */ +#define OP_SeekGT 24 /* jump0, synopsis: key=r[P3@P4] */ #define OP_IfNotOpen 25 /* jump, synopsis: if( !csr[P1] ) goto P2 */ #define OP_IfNoHope 26 /* jump, synopsis: key=r[P3@P4] */ #define OP_NoConflict 27 /* jump, synopsis: key=r[P3@P4] */ #define OP_NotFound 28 /* jump, synopsis: key=r[P3@P4] */ #define OP_Found 29 /* jump, synopsis: key=r[P3@P4] */ -#define OP_SeekRowid 30 /* jump, synopsis: intkey=r[P3] */ +#define OP_SeekRowid 30 /* jump0, synopsis: intkey=r[P3] */ #define OP_NotExists 31 /* jump, synopsis: intkey=r[P3] */ -#define OP_Last 32 /* jump */ -#define OP_IfSmaller 33 /* jump */ +#define OP_Last 32 /* jump0 */ +#define OP_IfSizeBetween 33 /* jump */ #define OP_SorterSort 34 /* jump */ #define OP_Sort 35 /* jump */ -#define OP_Rewind 36 /* jump */ -#define OP_SorterNext 37 /* jump */ -#define OP_Prev 38 /* jump */ -#define OP_Next 39 /* jump */ -#define OP_IdxLE 40 /* jump, synopsis: key=r[P3@P4] */ -#define OP_IdxGT 41 /* jump, synopsis: key=r[P3@P4] */ -#define OP_IdxLT 42 /* jump, synopsis: key=r[P3@P4] */ +#define OP_Rewind 36 /* jump0 */ +#define OP_IfEmpty 37 /* jump, synopsis: if( empty(P1) ) goto P2 */ +#define OP_SorterNext 38 /* jump */ +#define OP_Prev 39 /* jump */ +#define OP_Next 40 /* jump */ +#define OP_IdxLE 41 /* jump, synopsis: key=r[P3@P4] */ +#define OP_IdxGT 42 /* jump, synopsis: key=r[P3@P4] */ #define OP_Or 43 /* same as TK_OR, synopsis: r[P3]=(r[P1] || r[P2]) */ #define OP_And 44 /* same as TK_AND, synopsis: r[P3]=(r[P1] && r[P2]) */ -#define OP_IdxGE 45 /* jump, synopsis: key=r[P3@P4] */ -#define OP_RowSetRead 46 /* jump, synopsis: r[P3]=rowset(P1) */ -#define OP_RowSetTest 47 /* jump, synopsis: if r[P3] in rowset(P1) goto P2 */ -#define OP_Program 48 /* jump */ -#define OP_FkIfZero 49 /* jump, synopsis: if fkctr[P1]==0 goto P2 */ -#define OP_IsNull 50 /* jump, same as TK_ISNULL, synopsis: if r[P1]==NULL goto P2 */ -#define OP_NotNull 51 /* jump, same as TK_NOTNULL, synopsis: if r[P1]!=NULL goto P2 */ -#define OP_Ne 52 /* jump, same as TK_NE, synopsis: IF r[P3]!=r[P1] */ -#define OP_Eq 53 /* jump, same as TK_EQ, synopsis: IF r[P3]==r[P1] */ -#define OP_Gt 54 /* jump, same as TK_GT, synopsis: IF r[P3]>r[P1] */ -#define OP_Le 55 /* jump, same as TK_LE, synopsis: IF r[P3]<=r[P1] */ -#define OP_Lt 56 /* jump, same as TK_LT, synopsis: IF r[P3]=r[P1] */ -#define OP_ElseEq 58 /* jump, same as TK_ESCAPE */ -#define OP_IfPos 59 /* jump, synopsis: if r[P1]>0 then r[P1]-=P3, goto P2 */ -#define OP_IfNotZero 60 /* jump, synopsis: if r[P1]!=0 then r[P1]--, goto P2 */ -#define OP_DecrJumpZero 61 /* jump, synopsis: if (--r[P1])==0 goto P2 */ -#define OP_IncrVacuum 62 /* jump */ -#define OP_VNext 63 /* jump */ -#define OP_Filter 64 /* jump, synopsis: if key(P3@P4) not in filter(P1) goto P2 */ -#define OP_PureFunc 65 /* synopsis: r[P3]=func(r[P2@NP]) */ -#define OP_Function 66 /* synopsis: r[P3]=func(r[P2@NP]) */ -#define OP_Return 67 -#define OP_EndCoroutine 68 -#define OP_HaltIfNull 69 /* synopsis: if r[P3]=null halt */ -#define OP_Halt 70 -#define OP_Integer 71 /* synopsis: r[P2]=P1 */ -#define OP_Int64 72 /* synopsis: r[P2]=P4 */ -#define OP_String 73 /* synopsis: r[P2]='P4' (len=P1) */ -#define OP_BeginSubrtn 74 /* synopsis: r[P2]=NULL */ -#define OP_Null 75 /* synopsis: r[P2..P3]=NULL */ -#define OP_SoftNull 76 /* synopsis: r[P1]=NULL */ -#define OP_Blob 77 /* synopsis: r[P2]=P4 (len=P1) */ -#define OP_Variable 78 /* synopsis: r[P2]=parameter(P1,P4) */ -#define OP_Move 79 /* synopsis: r[P2@P3]=r[P1@P3] */ -#define OP_Copy 80 /* synopsis: r[P2@P3+1]=r[P1@P3+1] */ -#define OP_SCopy 81 /* synopsis: r[P2]=r[P1] */ -#define OP_IntCopy 82 /* synopsis: r[P2]=r[P1] */ -#define OP_FkCheck 83 -#define OP_ResultRow 84 /* synopsis: output=r[P1@P2] */ -#define OP_CollSeq 85 -#define OP_AddImm 86 /* synopsis: r[P1]=r[P1]+P2 */ -#define OP_RealAffinity 87 -#define OP_Cast 88 /* synopsis: affinity(r[P1]) */ -#define OP_Permutation 89 -#define OP_Compare 90 /* synopsis: r[P1@P3] <-> r[P2@P3] */ -#define OP_IsTrue 91 /* synopsis: r[P2] = coalesce(r[P1]==TRUE,P3) ^ P4 */ -#define OP_ZeroOrNull 92 /* synopsis: r[P2] = 0 OR NULL */ -#define OP_Offset 93 /* synopsis: r[P3] = sqlite_offset(P1) */ -#define OP_Column 94 /* synopsis: r[P3]=PX cursor P1 column P2 */ -#define OP_TypeCheck 95 /* synopsis: typecheck(r[P1@P2]) */ -#define OP_Affinity 96 /* synopsis: affinity(r[P1@P2]) */ -#define OP_MakeRecord 97 /* synopsis: r[P3]=mkrec(r[P1@P2]) */ -#define OP_Count 98 /* synopsis: r[P2]=count() */ -#define OP_ReadCookie 99 -#define OP_SetCookie 100 -#define OP_ReopenIdx 101 /* synopsis: root=P2 iDb=P3 */ -#define OP_BitAnd 102 /* same as TK_BITAND, synopsis: r[P3]=r[P1]&r[P2] */ -#define OP_BitOr 103 /* same as TK_BITOR, synopsis: r[P3]=r[P1]|r[P2] */ -#define OP_ShiftLeft 104 /* same as TK_LSHIFT, synopsis: r[P3]=r[P2]<>r[P1] */ -#define OP_Add 106 /* same as TK_PLUS, synopsis: r[P3]=r[P1]+r[P2] */ -#define OP_Subtract 107 /* same as TK_MINUS, synopsis: r[P3]=r[P2]-r[P1] */ -#define OP_Multiply 108 /* same as TK_STAR, synopsis: r[P3]=r[P1]*r[P2] */ -#define OP_Divide 109 /* same as TK_SLASH, synopsis: r[P3]=r[P2]/r[P1] */ -#define OP_Remainder 110 /* same as TK_REM, synopsis: r[P3]=r[P2]%r[P1] */ -#define OP_Concat 111 /* same as TK_CONCAT, synopsis: r[P3]=r[P2]+r[P1] */ -#define OP_OpenRead 112 /* synopsis: root=P2 iDb=P3 */ -#define OP_OpenWrite 113 /* synopsis: root=P2 iDb=P3 */ -#define OP_BitNot 114 /* same as TK_BITNOT, synopsis: r[P2]= ~r[P1] */ -#define OP_OpenDup 115 -#define OP_OpenAutoindex 116 /* synopsis: nColumn=P2 */ -#define OP_String8 117 /* same as TK_STRING, synopsis: r[P2]='P4' */ -#define OP_OpenEphemeral 118 /* synopsis: nColumn=P2 */ -#define OP_SorterOpen 119 -#define OP_SequenceTest 120 /* synopsis: if( cursor[P1].ctr++ ) pc = P2 */ -#define OP_OpenPseudo 121 /* synopsis: P3 columns in r[P2] */ -#define OP_Close 122 -#define OP_ColumnsUsed 123 -#define OP_SeekScan 124 /* synopsis: Scan-ahead up to P1 rows */ -#define OP_SeekHit 125 /* synopsis: set P2<=seekHit<=P3 */ -#define OP_Sequence 126 /* synopsis: r[P2]=cursor[P1].ctr++ */ -#define OP_NewRowid 127 /* synopsis: r[P2]=rowid */ -#define OP_Insert 128 /* synopsis: intkey=r[P3] data=r[P2] */ -#define OP_RowCell 129 -#define OP_Delete 130 -#define OP_ResetCount 131 -#define OP_SorterCompare 132 /* synopsis: if key(P1)!=trim(r[P3],P4) goto P2 */ -#define OP_SorterData 133 /* synopsis: r[P2]=data */ -#define OP_RowData 134 /* synopsis: r[P2]=data */ -#define OP_Rowid 135 /* synopsis: r[P2]=PX rowid of P1 */ -#define OP_NullRow 136 -#define OP_SeekEnd 137 -#define OP_IdxInsert 138 /* synopsis: key=r[P2] */ -#define OP_SorterInsert 139 /* synopsis: key=r[P2] */ -#define OP_IdxDelete 140 /* synopsis: key=r[P2@P3] */ -#define OP_DeferredSeek 141 /* synopsis: Move P3 to P1.rowid if needed */ -#define OP_IdxRowid 142 /* synopsis: r[P2]=rowid */ -#define OP_FinishSeek 143 -#define OP_Destroy 144 -#define OP_Clear 145 -#define OP_ResetSorter 146 -#define OP_CreateBtree 147 /* synopsis: r[P2]=root iDb=P1 flags=P3 */ -#define OP_SqlExec 148 -#define OP_ParseSchema 149 -#define OP_LoadAnalysis 150 -#define OP_DropTable 151 -#define OP_DropIndex 152 -#define OP_Real 153 /* same as TK_FLOAT, synopsis: r[P2]=P4 */ -#define OP_DropTrigger 154 -#define OP_IntegrityCk 155 -#define OP_RowSetAdd 156 /* synopsis: rowset(P1)=r[P2] */ -#define OP_Param 157 -#define OP_FkCounter 158 /* synopsis: fkctr[P1]+=P2 */ -#define OP_MemMax 159 /* synopsis: r[P1]=max(r[P1],r[P2]) */ -#define OP_OffsetLimit 160 /* synopsis: if r[P1]>0 then r[P2]=r[P1]+max(0,r[P3]) else r[P2]=(-1) */ -#define OP_AggInverse 161 /* synopsis: accum=r[P3] inverse(r[P2@P5]) */ -#define OP_AggStep 162 /* synopsis: accum=r[P3] step(r[P2@P5]) */ -#define OP_AggStep1 163 /* synopsis: accum=r[P3] step(r[P2@P5]) */ -#define OP_AggValue 164 /* synopsis: r[P3]=value N=P2 */ -#define OP_AggFinal 165 /* synopsis: accum=r[P1] N=P2 */ -#define OP_Expire 166 -#define OP_CursorLock 167 -#define OP_CursorUnlock 168 -#define OP_TableLock 169 /* synopsis: iDb=P1 root=P2 write=P3 */ -#define OP_VBegin 170 -#define OP_VCreate 171 -#define OP_VDestroy 172 -#define OP_VOpen 173 -#define OP_VCheck 174 -#define OP_VInitIn 175 /* synopsis: r[P2]=ValueList(P1,P3) */ -#define OP_VColumn 176 /* synopsis: r[P3]=vcolumn(P2) */ -#define OP_VRename 177 -#define OP_Pagecount 178 -#define OP_MaxPgcnt 179 -#define OP_ClrSubtype 180 /* synopsis: r[P1].subtype = 0 */ -#define OP_GetSubtype 181 /* synopsis: r[P2] = r[P1].subtype */ -#define OP_SetSubtype 182 /* synopsis: r[P2].subtype = r[P1] */ -#define OP_FilterAdd 183 /* synopsis: filter(P1) += key(P3@P4) */ -#define OP_Trace 184 -#define OP_CursorHint 185 -#define OP_ReleaseReg 186 /* synopsis: release r[P1@P2] mask P3 */ -#define OP_Noop 187 -#define OP_Explain 188 -#define OP_Abortable 189 +#define OP_IdxLT 45 /* jump, synopsis: key=r[P3@P4] */ +#define OP_IdxGE 46 /* jump, synopsis: key=r[P3@P4] */ +#define OP_RowSetRead 47 /* jump, synopsis: r[P3]=rowset(P1) */ +#define OP_RowSetTest 48 /* jump, synopsis: if r[P3] in rowset(P1) goto P2 */ +#define OP_Program 49 /* jump0 */ +#define OP_FkIfZero 50 /* jump, synopsis: if fkctr[P1]==0 goto P2 */ +#define OP_IsNull 51 /* jump, same as TK_ISNULL, synopsis: if r[P1]==NULL goto P2 */ +#define OP_NotNull 52 /* jump, same as TK_NOTNULL, synopsis: if r[P1]!=NULL goto P2 */ +#define OP_Ne 53 /* jump, same as TK_NE, synopsis: IF r[P3]!=r[P1] */ +#define OP_Eq 54 /* jump, same as TK_EQ, synopsis: IF r[P3]==r[P1] */ +#define OP_Gt 55 /* jump, same as TK_GT, synopsis: IF r[P3]>r[P1] */ +#define OP_Le 56 /* jump, same as TK_LE, synopsis: IF r[P3]<=r[P1] */ +#define OP_Lt 57 /* jump, same as TK_LT, synopsis: IF r[P3]=r[P1] */ +#define OP_ElseEq 59 /* jump, same as TK_ESCAPE */ +#define OP_IfPos 60 /* jump, synopsis: if r[P1]>0 then r[P1]-=P3, goto P2 */ +#define OP_IfNotZero 61 /* jump, synopsis: if r[P1]!=0 then r[P1]--, goto P2 */ +#define OP_DecrJumpZero 62 /* jump, synopsis: if (--r[P1])==0 goto P2 */ +#define OP_IncrVacuum 63 /* jump */ +#define OP_VNext 64 /* jump */ +#define OP_Filter 65 /* jump, synopsis: if key(P3@P4) not in filter(P1) goto P2 */ +#define OP_PureFunc 66 /* synopsis: r[P3]=func(r[P2@NP]) */ +#define OP_Function 67 /* synopsis: r[P3]=func(r[P2@NP]) */ +#define OP_Return 68 +#define OP_EndCoroutine 69 +#define OP_HaltIfNull 70 /* synopsis: if r[P3]=null halt */ +#define OP_Halt 71 +#define OP_Integer 72 /* synopsis: r[P2]=P1 */ +#define OP_Int64 73 /* synopsis: r[P2]=P4 */ +#define OP_String 74 /* synopsis: r[P2]='P4' (len=P1) */ +#define OP_BeginSubrtn 75 /* synopsis: r[P2]=NULL */ +#define OP_Null 76 /* synopsis: r[P2..P3]=NULL */ +#define OP_SoftNull 77 /* synopsis: r[P1]=NULL */ +#define OP_Blob 78 /* synopsis: r[P2]=P4 (len=P1) */ +#define OP_Variable 79 /* synopsis: r[P2]=parameter(P1) */ +#define OP_Move 80 /* synopsis: r[P2@P3]=r[P1@P3] */ +#define OP_Copy 81 /* synopsis: r[P2@P3+1]=r[P1@P3+1] */ +#define OP_SCopy 82 /* synopsis: r[P2]=r[P1] */ +#define OP_IntCopy 83 /* synopsis: r[P2]=r[P1] */ +#define OP_FkCheck 84 +#define OP_ResultRow 85 /* synopsis: output=r[P1@P2] */ +#define OP_CollSeq 86 +#define OP_AddImm 87 /* synopsis: r[P1]=r[P1]+P2 */ +#define OP_RealAffinity 88 +#define OP_Cast 89 /* synopsis: affinity(r[P1]) */ +#define OP_Permutation 90 +#define OP_Compare 91 /* synopsis: r[P1@P3] <-> r[P2@P3] */ +#define OP_IsTrue 92 /* synopsis: r[P2] = coalesce(r[P1]==TRUE,P3) ^ P4 */ +#define OP_ZeroOrNull 93 /* synopsis: r[P2] = 0 OR NULL */ +#define OP_Offset 94 /* synopsis: r[P3] = sqlite_offset(P1) */ +#define OP_Column 95 /* synopsis: r[P3]=PX cursor P1 column P2 */ +#define OP_TypeCheck 96 /* synopsis: typecheck(r[P1@P2]) */ +#define OP_Affinity 97 /* synopsis: affinity(r[P1@P2]) */ +#define OP_MakeRecord 98 /* synopsis: r[P3]=mkrec(r[P1@P2]) */ +#define OP_Count 99 /* synopsis: r[P2]=count() */ +#define OP_ReadCookie 100 +#define OP_SetCookie 101 +#define OP_ReopenIdx 102 /* synopsis: root=P2 iDb=P3 */ +#define OP_BitAnd 103 /* same as TK_BITAND, synopsis: r[P3]=r[P1]&r[P2] */ +#define OP_BitOr 104 /* same as TK_BITOR, synopsis: r[P3]=r[P1]|r[P2] */ +#define OP_ShiftLeft 105 /* same as TK_LSHIFT, synopsis: r[P3]=r[P2]<>r[P1] */ +#define OP_Add 107 /* same as TK_PLUS, synopsis: r[P3]=r[P1]+r[P2] */ +#define OP_Subtract 108 /* same as TK_MINUS, synopsis: r[P3]=r[P2]-r[P1] */ +#define OP_Multiply 109 /* same as TK_STAR, synopsis: r[P3]=r[P1]*r[P2] */ +#define OP_Divide 110 /* same as TK_SLASH, synopsis: r[P3]=r[P2]/r[P1] */ +#define OP_Remainder 111 /* same as TK_REM, synopsis: r[P3]=r[P2]%r[P1] */ +#define OP_Concat 112 /* same as TK_CONCAT, synopsis: r[P3]=r[P2]+r[P1] */ +#define OP_OpenRead 113 /* synopsis: root=P2 iDb=P3 */ +#define OP_OpenWrite 114 /* synopsis: root=P2 iDb=P3 */ +#define OP_BitNot 115 /* same as TK_BITNOT, synopsis: r[P2]= ~r[P1] */ +#define OP_OpenDup 116 +#define OP_OpenAutoindex 117 /* synopsis: nColumn=P2 */ +#define OP_String8 118 /* same as TK_STRING, synopsis: r[P2]='P4' */ +#define OP_OpenEphemeral 119 /* synopsis: nColumn=P2 */ +#define OP_SorterOpen 120 +#define OP_SequenceTest 121 /* synopsis: if( cursor[P1].ctr++ ) pc = P2 */ +#define OP_OpenPseudo 122 /* synopsis: P3 columns in r[P2] */ +#define OP_Close 123 +#define OP_ColumnsUsed 124 +#define OP_SeekScan 125 /* synopsis: Scan-ahead up to P1 rows */ +#define OP_SeekHit 126 /* synopsis: set P2<=seekHit<=P3 */ +#define OP_Sequence 127 /* synopsis: r[P2]=cursor[P1].ctr++ */ +#define OP_NewRowid 128 /* synopsis: r[P2]=rowid */ +#define OP_Insert 129 /* synopsis: intkey=r[P3] data=r[P2] */ +#define OP_RowCell 130 +#define OP_Delete 131 +#define OP_ResetCount 132 +#define OP_SorterCompare 133 /* synopsis: if key(P1)!=trim(r[P3],P4) goto P2 */ +#define OP_SorterData 134 /* synopsis: r[P2]=data */ +#define OP_RowData 135 /* synopsis: r[P2]=data */ +#define OP_Rowid 136 /* synopsis: r[P2]=PX rowid of P1 */ +#define OP_NullRow 137 +#define OP_SeekEnd 138 +#define OP_IdxInsert 139 /* synopsis: key=r[P2] */ +#define OP_SorterInsert 140 /* synopsis: key=r[P2] */ +#define OP_IdxDelete 141 /* synopsis: key=r[P2@P3] */ +#define OP_DeferredSeek 142 /* synopsis: Move P3 to P1.rowid if needed */ +#define OP_IdxRowid 143 /* synopsis: r[P2]=rowid */ +#define OP_FinishSeek 144 +#define OP_Destroy 145 +#define OP_Clear 146 +#define OP_ResetSorter 147 +#define OP_CreateBtree 148 /* synopsis: r[P2]=root iDb=P1 flags=P3 */ +#define OP_SqlExec 149 +#define OP_ParseSchema 150 +#define OP_LoadAnalysis 151 +#define OP_DropTable 152 +#define OP_DropIndex 153 +#define OP_Real 154 /* same as TK_FLOAT, synopsis: r[P2]=P4 */ +#define OP_DropTrigger 155 +#define OP_IntegrityCk 156 +#define OP_RowSetAdd 157 /* synopsis: rowset(P1)=r[P2] */ +#define OP_Param 158 +#define OP_FkCounter 159 /* synopsis: fkctr[P1]+=P2 */ +#define OP_MemMax 160 /* synopsis: r[P1]=max(r[P1],r[P2]) */ +#define OP_OffsetLimit 161 /* synopsis: if r[P1]>0 then r[P2]=r[P1]+max(0,r[P3]) else r[P2]=(-1) */ +#define OP_AggInverse 162 /* synopsis: accum=r[P3] inverse(r[P2@P5]) */ +#define OP_AggStep 163 /* synopsis: accum=r[P3] step(r[P2@P5]) */ +#define OP_AggStep1 164 /* synopsis: accum=r[P3] step(r[P2@P5]) */ +#define OP_AggValue 165 /* synopsis: r[P3]=value N=P2 */ +#define OP_AggFinal 166 /* synopsis: accum=r[P1] N=P2 */ +#define OP_Expire 167 +#define OP_CursorLock 168 +#define OP_CursorUnlock 169 +#define OP_TableLock 170 /* synopsis: iDb=P1 root=P2 write=P3 */ +#define OP_VBegin 171 +#define OP_VCreate 172 +#define OP_VDestroy 173 +#define OP_VOpen 174 +#define OP_VCheck 175 +#define OP_VInitIn 176 /* synopsis: r[P2]=ValueList(P1,P3) */ +#define OP_VColumn 177 /* synopsis: r[P3]=vcolumn(P2) */ +#define OP_VRename 178 +#define OP_Pagecount 179 +#define OP_MaxPgcnt 180 +#define OP_ClrSubtype 181 /* synopsis: r[P1].subtype = 0 */ +#define OP_GetSubtype 182 /* synopsis: r[P2] = r[P1].subtype */ +#define OP_SetSubtype 183 /* synopsis: r[P2].subtype = r[P1] */ +#define OP_FilterAdd 184 /* synopsis: filter(P1) += key(P3@P4) */ +#define OP_Trace 185 +#define OP_CursorHint 186 +#define OP_ReleaseReg 187 /* synopsis: release r[P1@P2] mask P3 */ +#define OP_Noop 188 +#define OP_Explain 189 +#define OP_Abortable 190 /* Properties such as "out2" or "jump" that are specified in ** comments following the "case" for each opcode in the vdbe.c @@ -16743,31 +17448,32 @@ typedef struct VdbeOpList VdbeOpList; #define OPFLG_OUT2 0x10 /* out2: P2 is an output */ #define OPFLG_OUT3 0x20 /* out3: P3 is an output */ #define OPFLG_NCYCLE 0x40 /* ncycle:Cycles count against P1 */ +#define OPFLG_JUMP0 0x80 /* jump0: P2 might be zero */ #define OPFLG_INITIALIZER {\ /* 0 */ 0x00, 0x00, 0x00, 0x00, 0x10, 0x00, 0x41, 0x00,\ -/* 8 */ 0x01, 0x01, 0x01, 0x01, 0x03, 0x03, 0x01, 0x01,\ -/* 16 */ 0x03, 0x03, 0x01, 0x12, 0x01, 0x49, 0x49, 0x49,\ -/* 24 */ 0x49, 0x01, 0x49, 0x49, 0x49, 0x49, 0x49, 0x49,\ -/* 32 */ 0x41, 0x01, 0x41, 0x41, 0x41, 0x01, 0x41, 0x41,\ -/* 40 */ 0x41, 0x41, 0x41, 0x26, 0x26, 0x41, 0x23, 0x0b,\ -/* 48 */ 0x01, 0x01, 0x03, 0x03, 0x0b, 0x0b, 0x0b, 0x0b,\ -/* 56 */ 0x0b, 0x0b, 0x01, 0x03, 0x03, 0x03, 0x01, 0x41,\ -/* 64 */ 0x01, 0x00, 0x00, 0x02, 0x02, 0x08, 0x00, 0x10,\ -/* 72 */ 0x10, 0x10, 0x00, 0x10, 0x00, 0x10, 0x10, 0x00,\ -/* 80 */ 0x00, 0x10, 0x10, 0x00, 0x00, 0x00, 0x02, 0x02,\ -/* 88 */ 0x02, 0x00, 0x00, 0x12, 0x1e, 0x20, 0x40, 0x00,\ -/* 96 */ 0x00, 0x00, 0x10, 0x10, 0x00, 0x40, 0x26, 0x26,\ +/* 8 */ 0x81, 0x01, 0x01, 0x81, 0x83, 0x83, 0x01, 0x01,\ +/* 16 */ 0x03, 0x03, 0x01, 0x12, 0x01, 0xc9, 0xc9, 0xc9,\ +/* 24 */ 0xc9, 0x01, 0x49, 0x49, 0x49, 0x49, 0xc9, 0x49,\ +/* 32 */ 0xc1, 0x01, 0x41, 0x41, 0xc1, 0x01, 0x01, 0x41,\ +/* 40 */ 0x41, 0x41, 0x41, 0x26, 0x26, 0x41, 0x41, 0x23,\ +/* 48 */ 0x0b, 0x81, 0x01, 0x03, 0x03, 0x0b, 0x0b, 0x0b,\ +/* 56 */ 0x0b, 0x0b, 0x0b, 0x01, 0x03, 0x03, 0x03, 0x01,\ +/* 64 */ 0x41, 0x01, 0x00, 0x00, 0x02, 0x02, 0x08, 0x00,\ +/* 72 */ 0x10, 0x10, 0x10, 0x00, 0x10, 0x00, 0x10, 0x10,\ +/* 80 */ 0x00, 0x00, 0x10, 0x10, 0x00, 0x00, 0x00, 0x02,\ +/* 88 */ 0x02, 0x02, 0x00, 0x00, 0x12, 0x1e, 0x20, 0x40,\ +/* 96 */ 0x00, 0x00, 0x00, 0x10, 0x10, 0x00, 0x40, 0x26,\ /* 104 */ 0x26, 0x26, 0x26, 0x26, 0x26, 0x26, 0x26, 0x26,\ -/* 112 */ 0x40, 0x00, 0x12, 0x40, 0x40, 0x10, 0x40, 0x00,\ -/* 120 */ 0x00, 0x00, 0x40, 0x00, 0x40, 0x40, 0x10, 0x10,\ -/* 128 */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x00, 0x50,\ -/* 136 */ 0x00, 0x40, 0x04, 0x04, 0x00, 0x40, 0x50, 0x40,\ -/* 144 */ 0x10, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x00,\ -/* 152 */ 0x00, 0x10, 0x00, 0x00, 0x06, 0x10, 0x00, 0x04,\ -/* 160 */ 0x1a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,\ -/* 168 */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x10, 0x50,\ -/* 176 */ 0x40, 0x00, 0x10, 0x10, 0x02, 0x12, 0x12, 0x00,\ -/* 184 */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,} +/* 112 */ 0x26, 0x40, 0x00, 0x12, 0x40, 0x40, 0x10, 0x40,\ +/* 120 */ 0x00, 0x00, 0x00, 0x40, 0x00, 0x40, 0x40, 0x10,\ +/* 128 */ 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x00,\ +/* 136 */ 0x50, 0x00, 0x40, 0x04, 0x04, 0x00, 0x40, 0x50,\ +/* 144 */ 0x40, 0x10, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00,\ +/* 152 */ 0x00, 0x00, 0x10, 0x00, 0x00, 0x06, 0x10, 0x00,\ +/* 160 */ 0x04, 0x1a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,\ +/* 168 */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x10,\ +/* 176 */ 0x50, 0x40, 0x00, 0x10, 0x10, 0x02, 0x12, 0x12,\ +/* 184 */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,} /* The resolve3P2Values() routine is able to run faster if it knows ** the value of the largest JUMP opcode. The smaller the maximum @@ -16775,7 +17481,7 @@ typedef struct VdbeOpList VdbeOpList; ** generated this include file strives to group all JUMP opcodes ** together near the beginning of the list. */ -#define SQLITE_MX_JUMP_OPCODE 64 /* Maximum JUMP opcode */ +#define SQLITE_MX_JUMP_OPCODE 65 /* Maximum JUMP opcode */ /************** End of opcodes.h *********************************************/ /************** Continuing where we left off in vdbe.h ***********************/ @@ -16784,7 +17490,7 @@ typedef struct VdbeOpList VdbeOpList; ** Additional non-public SQLITE_PREPARE_* flags */ #define SQLITE_PREPARE_SAVESQL 0x80 /* Preserve SQL text */ -#define SQLITE_PREPARE_MASK 0x0f /* Mask of public flags */ +#define SQLITE_PREPARE_MASK 0x1f /* Mask of public flags */ /* ** Prototypes for the VDBE interface. See comments on the implementation @@ -16898,8 +17604,11 @@ SQLITE_PRIVATE char *sqlite3VdbeExpandSql(Vdbe*, const char*); #endif SQLITE_PRIVATE int sqlite3MemCompare(const Mem*, const Mem*, const CollSeq*); SQLITE_PRIVATE int sqlite3BlobCompare(const Mem*, const Mem*); +#ifdef SQLITE_ENABLE_PERCENTILE +SQLITE_PRIVATE const char *sqlite3VdbeFuncName(const sqlite3_context*); +#endif -SQLITE_PRIVATE void sqlite3VdbeRecordUnpack(KeyInfo*,int,const void*,UnpackedRecord*); +SQLITE_PRIVATE void sqlite3VdbeRecordUnpack(int,const void*,UnpackedRecord*); SQLITE_PRIVATE int sqlite3VdbeRecordCompare(int,const void*,UnpackedRecord*); SQLITE_PRIVATE int sqlite3VdbeRecordCompareWithSkip(int, const void *, UnpackedRecord *, int); SQLITE_PRIVATE UnpackedRecord *sqlite3VdbeAllocUnpackedRecord(KeyInfo*); @@ -16910,13 +17619,17 @@ SQLITE_PRIVATE RecordCompare sqlite3VdbeFindCompare(UnpackedRecord*); SQLITE_PRIVATE void sqlite3VdbeLinkSubProgram(Vdbe *, SubProgram *); SQLITE_PRIVATE int sqlite3VdbeHasSubProgram(Vdbe*); +SQLITE_PRIVATE void sqlite3MemSetArrayInt64(sqlite3_value *aMem, int iIdx, i64 val); + +#ifndef SQLITE_OMIT_DATETIME_FUNCS SQLITE_PRIVATE int sqlite3NotPureFunc(sqlite3_context*); +#endif #ifdef SQLITE_ENABLE_BYTECODE_VTAB SQLITE_PRIVATE int sqlite3VdbeBytecodeVtabInit(sqlite3*); #endif -/* Use SQLITE_ENABLE_COMMENTS to enable generation of extra comments on -** each VDBE opcode. +/* Use SQLITE_ENABLE_EXPLAIN_COMMENTS to enable generation of extra +** comments on each VDBE opcode. ** ** Use the SQLITE_ENABLE_MODULE_COMMENTS macro to see some extra no-op ** comments in VDBE programs that show key decision points in the code @@ -17497,43 +18210,11 @@ struct FuncDefHash { }; #define SQLITE_FUNC_HASH(C,L) (((C)+(L))%SQLITE_FUNC_HASH_SZ) -#ifdef SQLITE_USER_AUTHENTICATION -/* -** Information held in the "sqlite3" database connection object and used -** to manage user authentication. -*/ -typedef struct sqlite3_userauth sqlite3_userauth; -struct sqlite3_userauth { - u8 authLevel; /* Current authentication level */ - int nAuthPW; /* Size of the zAuthPW in bytes */ - char *zAuthPW; /* Password used to authenticate */ - char *zAuthUser; /* User name used to authenticate */ -}; - -/* Allowed values for sqlite3_userauth.authLevel */ -#define UAUTH_Unknown 0 /* Authentication not yet checked */ -#define UAUTH_Fail 1 /* User authentication failed */ -#define UAUTH_User 2 /* Authenticated as a normal user */ -#define UAUTH_Admin 3 /* Authenticated as an administrator */ - -/* Functions used only by user authorization logic */ -SQLITE_PRIVATE int sqlite3UserAuthTable(const char*); -SQLITE_PRIVATE int sqlite3UserAuthCheckLogin(sqlite3*,const char*,u8*); -SQLITE_PRIVATE void sqlite3UserAuthInit(sqlite3*); -SQLITE_PRIVATE void sqlite3CryptFunc(sqlite3_context*,int,sqlite3_value**); - -#endif /* SQLITE_USER_AUTHENTICATION */ - /* ** typedef for the authorization callback function. */ -#ifdef SQLITE_USER_AUTHENTICATION - typedef int (*sqlite3_xauth)(void*,int,const char*,const char*,const char*, - const char*, const char*); -#else - typedef int (*sqlite3_xauth)(void*,int,const char*,const char*,const char*, - const char*); -#endif +typedef int (*sqlite3_xauth)(void*,int,const char*,const char*,const char*, + const char*); #ifndef SQLITE_OMIT_DEPRECATED /* This is an extra SQLITE_TRACE macro that indicates "legacy" tracing @@ -17598,7 +18279,7 @@ struct sqlite3 { u8 iDb; /* Which db file is being initialized */ u8 busy; /* TRUE if currently initializing */ unsigned orphanTrigger : 1; /* Last statement is orphaned TEMP trigger */ - unsigned imposterTable : 1; /* Building an imposter table */ + unsigned imposterTable : 2; /* Building an imposter table */ unsigned reopenMemdb : 1; /* ATTACH is really a reopen using MemDB */ const char **azInit; /* "type", "name", and "tbl_name" columns */ } init; @@ -17671,12 +18352,17 @@ struct sqlite3 { Savepoint *pSavepoint; /* List of active savepoints */ int nAnalysisLimit; /* Number of index rows to ANALYZE */ int busyTimeout; /* Busy handler timeout, in msec */ +#ifdef SQLITE_ENABLE_SETLK_TIMEOUT + int setlkTimeout; /* Blocking lock timeout, in msec. -1 -> inf. */ + int setlkFlags; /* Flags passed to setlk_timeout() */ +#endif int nSavepoint; /* Number of non-transaction savepoints */ int nStatement; /* Number of nested statement-transactions */ i64 nDeferredCons; /* Net deferred constraints this transaction. */ i64 nDeferredImmCons; /* Net deferred immediate constraints */ int *pnBytesFreed; /* If not NULL, increment this in DbFree() */ DbClientData *pDbData; /* sqlite3_set_clientdata() content */ + u64 nSpill; /* TEMP content spilled to disk */ #ifdef SQLITE_ENABLE_UNLOCK_NOTIFY /* The following variables are all protected by the STATIC_MAIN ** mutex, not by sqlite3.mutex. They are used by code in notify.c. @@ -17694,9 +18380,6 @@ struct sqlite3 { void (*xUnlockNotify)(void **, int); /* Unlock notify callback */ sqlite3 *pNextBlocked; /* Next in list of all blocked connections */ #endif -#ifdef SQLITE_USER_AUTHENTICATION - sqlite3_userauth auth; /* User authentication information */ -#endif }; /* @@ -17760,6 +18443,9 @@ struct sqlite3 { #define SQLITE_CorruptRdOnly HI(0x00002) /* Prohibit writes due to error */ #define SQLITE_ReadUncommit HI(0x00004) /* READ UNCOMMITTED in shared-cache */ #define SQLITE_FkNoAction HI(0x00008) /* Treat all FK as NO ACTION */ +#define SQLITE_AttachCreate HI(0x00010) /* ATTACH allowed to create new dbs */ +#define SQLITE_AttachWrite HI(0x00020) /* ATTACH allowed to open for write */ +#define SQLITE_Comments HI(0x00040) /* Enable SQL comments */ /* Flags used only if debugging */ #ifdef SQLITE_DEBUG @@ -17800,7 +18486,7 @@ struct sqlite3 { #define SQLITE_CursorHints 0x00000400 /* Add OP_CursorHint opcodes */ #define SQLITE_Stat4 0x00000800 /* Use STAT4 data */ /* TH3 expects this value ^^^^^^^^^^ to be 0x0000800. Don't change it */ -#define SQLITE_PushDown 0x00001000 /* The push-down optimization */ +#define SQLITE_PushDown 0x00001000 /* WHERE-clause push-down opt */ #define SQLITE_SimplifyJoin 0x00002000 /* Convert LEFT JOIN to JOIN */ #define SQLITE_SkipScan 0x00004000 /* Skip-scans */ #define SQLITE_PropagateConst 0x00008000 /* The constant propagation opt */ @@ -17818,6 +18504,9 @@ struct sqlite3 { #define SQLITE_Coroutines 0x02000000 /* Co-routines for subqueries */ #define SQLITE_NullUnusedCols 0x04000000 /* NULL unused columns in subqueries */ #define SQLITE_OnePass 0x08000000 /* Single-pass DELETE and UPDATE */ +#define SQLITE_OrderBySubq 0x10000000 /* ORDER BY in subquery helps outer */ +#define SQLITE_StarQuery 0x20000000 /* Heurists for star queries */ +#define SQLITE_ExistsToJoin 0x40000000 /* The EXISTS-to-JOIN optimization */ #define SQLITE_AllOpts 0xffffffff /* All optimizations */ /* @@ -17854,7 +18543,7 @@ struct sqlite3 { ** field is used by per-connection app-def functions. */ struct FuncDef { - i8 nArg; /* Number of arguments. -1 means unlimited */ + i16 nArg; /* Number of arguments. -1 means unlimited */ u32 funcFlags; /* Some combination of SQLITE_FUNC_* */ void *pUserData; /* User data parameter */ FuncDef *pNext; /* Next function with same name */ @@ -18056,7 +18745,7 @@ struct FuncDestructor { #define STR_FUNCTION(zName, nArg, pArg, bNC, xFunc) \ {nArg, SQLITE_FUNC_BUILTIN|\ SQLITE_FUNC_SLOCHNG|SQLITE_UTF8|(bNC*SQLITE_FUNC_NEEDCOLL), \ - pArg, 0, xFunc, 0, 0, 0, #zName, } + pArg, 0, xFunc, 0, 0, 0, #zName, {0} } #define LIKEFUNC(zName, nArg, arg, flags) \ {nArg, SQLITE_FUNC_BUILTIN|SQLITE_FUNC_CONSTANT|SQLITE_UTF8|flags, \ (void *)arg, 0, likeFunc, 0, 0, 0, #zName, {0} } @@ -18223,6 +18912,7 @@ struct CollSeq { #define SQLITE_AFF_INTEGER 0x44 /* 'D' */ #define SQLITE_AFF_REAL 0x45 /* 'E' */ #define SQLITE_AFF_FLEXNUM 0x46 /* 'F' */ +#define SQLITE_AFF_DEFER 0x58 /* 'X' - defer computation until later */ #define sqlite3IsNumericAffinity(X) ((X)>=SQLITE_AFF_NUMERIC) @@ -18347,6 +19037,7 @@ struct Table { } u; Trigger *pTrigger; /* List of triggers on this object */ Schema *pSchema; /* Schema that contains this table */ + u8 aHx[16]; /* Column aHt[K%sizeof(aHt)] might have hash K */ }; /* @@ -18373,8 +19064,7 @@ struct Table { #define TF_HasStored 0x00000040 /* Has one or more STORED columns */ #define TF_HasGenerated 0x00000060 /* Combo: HasVirtual + HasStored */ #define TF_WithoutRowid 0x00000080 /* No rowid. PRIMARY KEY is the key */ -#define TF_StatsUsed 0x00000100 /* Query planner decisions affected by - ** Index.aiRowLogEst[] values */ +#define TF_MaybeReanalyze 0x00000100 /* Maybe run ANALYZE on this table */ #define TF_NoVisibleRowid 0x00000200 /* No user-visible "rowid" column */ #define TF_OOOHidden 0x00000400 /* Out-of-Order hidden columns */ #define TF_HasNotNull 0x00000800 /* Contains NOT NULL constraints */ @@ -18383,6 +19073,7 @@ struct Table { #define TF_Ephemeral 0x00004000 /* An ephemeral table */ #define TF_Eponymous 0x00008000 /* An eponymous virtual table */ #define TF_Strict 0x00010000 /* STRICT mode */ +#define TF_Imposter 0x00020000 /* An imposter table */ /* ** Allowed values for Table.eTabType @@ -18430,6 +19121,15 @@ struct Table { #define HasRowid(X) (((X)->tabFlags & TF_WithoutRowid)==0) #define VisibleRowid(X) (((X)->tabFlags & TF_NoVisibleRowid)==0) +/* Macro is true if the SQLITE_ALLOW_ROWID_IN_VIEW (mis-)feature is +** available. By default, this macro is false +*/ +#ifndef SQLITE_ALLOW_ROWID_IN_VIEW +# define ViewCanHaveRowid 0 +#else +# define ViewCanHaveRowid (sqlite3Config.mNoVisibleRowid==0) +#endif + /* ** Each foreign key constraint is an instance of the following structure. ** @@ -18472,9 +19172,13 @@ struct FKey { struct sColMap { /* Mapping of columns in pFrom to columns in zTo */ int iFrom; /* Index of column in pFrom */ char *zCol; /* Name of column in zTo. If NULL use PRIMARY KEY */ - } aCol[1]; /* One entry for each of nCol columns */ + } aCol[FLEXARRAY]; /* One entry for each of nCol columns */ }; +/* The size (in bytes) of an FKey object holding N columns. The answer +** does NOT include space to hold the zTo name. */ +#define SZ_FKEY(N) (offsetof(FKey,aCol)+(N)*sizeof(struct sColMap)) + /* ** SQLite supports many different ways to resolve a constraint ** error. ROLLBACK processing means that a constraint violation @@ -18525,9 +19229,15 @@ struct FKey { ** argument to sqlite3VdbeKeyCompare and is used to control the ** comparison of the two index keys. ** -** Note that aSortOrder[] and aColl[] have nField+1 slots. There -** are nField slots for the columns of an index then one extra slot -** for the rowid at the end. +** The aSortOrder[] and aColl[] arrays have nAllField slots each. There +** are nKeyField slots for the columns of an index then extra slots +** for the rowid or key at the end. The aSortOrder array is located after +** the aColl[] array. +** +** If SQLITE_ENABLE_PREUPDATE_HOOK is defined, then aSortFlags might be NULL +** to indicate that this object is for use by a preupdate hook. When aSortFlags +** is NULL, then nAllField is uninitialized and no space is allocated for +** aColl[], so those fields may not be used. */ struct KeyInfo { u32 nRef; /* Number of references to this KeyInfo object */ @@ -18536,9 +19246,21 @@ struct KeyInfo { u16 nAllField; /* Total columns, including key plus others */ sqlite3 *db; /* The database connection */ u8 *aSortFlags; /* Sort order for each column. */ - CollSeq *aColl[1]; /* Collating sequence for each term of the key */ + CollSeq *aColl[FLEXARRAY]; /* Collating sequence for each term of the key */ }; +/* The size (in bytes) of a KeyInfo object with up to N fields. This includes +** the main body of the KeyInfo object and the aColl[] array of N elements, +** but does not count the memory used to hold aSortFlags[]. */ +#define SZ_KEYINFO(N) (offsetof(KeyInfo,aColl) + (N)*sizeof(CollSeq*)) + +/* The size of a bare KeyInfo with no aColl[] entries */ +#if FLEXARRAY+1 > 1 +# define SZ_KEYINFO_0 offsetof(KeyInfo,aColl) +#else +# define SZ_KEYINFO_0 sizeof(KeyInfo) +#endif + /* ** Allowed bit values for entries in the KeyInfo.aSortFlags[] array. */ @@ -18557,9 +19279,8 @@ struct KeyInfo { ** ** An instance of this object serves as a "key" for doing a search on ** an index b+tree. The goal of the search is to find the entry that -** is closed to the key described by this object. This object might hold -** just a prefix of the key. The number of fields is given by -** pKeyInfo->nField. +** is closest to the key described by this object. This object might hold +** just a prefix of the key. The number of fields is given by nField. ** ** The r1 and r2 fields are the values to return if this key is less than ** or greater than a key in the btree, respectively. These are normally @@ -18569,7 +19290,7 @@ struct KeyInfo { ** The key comparison functions actually return default_rc when they find ** an equals comparison. default_rc can be -1, 0, or +1. If there are ** multiple entries in the b-tree with the same key (when only looking -** at the first pKeyInfo->nFields,) then default_rc can be set to -1 to +** at the first nField elements) then default_rc can be set to -1 to ** cause the search to find the last match, or +1 to cause the search to ** find the first match. ** @@ -18581,8 +19302,8 @@ struct KeyInfo { ** b-tree. */ struct UnpackedRecord { - KeyInfo *pKeyInfo; /* Collation and sort-order information */ - Mem *aMem; /* Values */ + KeyInfo *pKeyInfo; /* Comparison info for the index that is unpacked */ + Mem *aMem; /* Values for columns of the index */ union { char *z; /* Cache of aMem[0].z for vdbeRecordCompareString() */ i64 i; /* Cache of aMem[0].u.i for vdbeRecordCompareInt() */ @@ -18658,7 +19379,7 @@ struct Index { Pgno tnum; /* DB Page containing root of this index */ LogEst szIdxRow; /* Estimated average row size in bytes */ u16 nKeyCol; /* Number of columns forming the key */ - u16 nColumn; /* Number of columns stored in the index */ + u16 nColumn; /* Nr columns in btree. Can be 2*Table.nCol */ u8 onError; /* OE_Abort, OE_Ignore, OE_Replace, or OE_None */ unsigned idxType:2; /* 0:Normal 1:UNIQUE, 2:PRIMARY KEY, 3:IPK */ unsigned bUnordered:1; /* Use this index for == or IN queries only */ @@ -18667,7 +19388,6 @@ struct Index { unsigned isCovering:1; /* True if this is a covering index */ unsigned noSkipScan:1; /* Do not try to use skip-scan if true */ unsigned hasStat1:1; /* aiRowLogEst values come from sqlite_stat1 */ - unsigned bLowQual:1; /* sqlite_stat1 says this is a low-quality index */ unsigned bNoQuery:1; /* Do not use this index to optimize queries */ unsigned bAscKeyBug:1; /* True if the bba7b69f9849b5bf bug applies */ unsigned bHasVCol:1; /* Index references one or more VIRTUAL columns */ @@ -18757,7 +19477,7 @@ struct AggInfo { ** from source tables rather than from accumulators */ u8 useSortingIdx; /* In direct mode, reference the sorting index rather ** than the source table */ - u16 nSortingColumn; /* Number of columns in the sorting index */ + u32 nSortingColumn; /* Number of columns in the sorting index */ int sortingIdx; /* Cursor number of the sorting index */ int sortingIdxPTab; /* Cursor number of pseudo-table */ int iFirstReg; /* First register in range for aCol[] and aFunc[] */ @@ -18766,8 +19486,8 @@ struct AggInfo { Table *pTab; /* Source table */ Expr *pCExpr; /* The original expression */ int iTable; /* Cursor number of the source table */ - i16 iColumn; /* Column number within the source table */ - i16 iSorterColumn; /* Column number in the sorting index */ + int iColumn; /* Column number within the source table */ + int iSorterColumn; /* Column number in the sorting index */ } *aCol; int nColumn; /* Number of used entries in aCol[] */ int nAccumulator; /* Number of columns that show through to the output. @@ -18797,9 +19517,15 @@ struct AggInfo { ** assignAggregateRegisters() that computes the value of pAggInfo->iFirstReg. ** The assert()s that are part of this macro verify that constraint. */ +#ifndef NDEBUG #define AggInfoColumnReg(A,I) (assert((A)->iFirstReg),(A)->iFirstReg+(I)) #define AggInfoFuncReg(A,I) \ (assert((A)->iFirstReg),(A)->iFirstReg+(A)->nColumn+(I)) +#else +#define AggInfoColumnReg(A,I) ((A)->iFirstReg+(I)) +#define AggInfoFuncReg(A,I) \ + ((A)->iFirstReg+(A)->nColumn+(I)) +#endif /* ** The datatype ynVar is a signed integer, either 16-bit or 32-bit. @@ -18936,6 +19662,7 @@ struct Expr { Table *pTab; /* TK_COLUMN: Table containing column. Can be NULL ** for a column of an index on an expression */ Window *pWin; /* EP_WinFunc: Window/Filter defn for a function */ + int nReg; /* TK_NULLS: Number of registers to NULL out */ struct { /* TK_IN, TK_SELECT, and TK_EXISTS */ int iAddr; /* Subroutine entry address */ int regReturn; /* Register used to hold return address */ @@ -18980,7 +19707,7 @@ struct Expr { #define EP_IsTrue 0x10000000 /* Always has boolean value of TRUE */ #define EP_IsFalse 0x20000000 /* Always has boolean value of FALSE */ #define EP_FromDDL 0x40000000 /* Originates from sqlite_schema */ - /* 0x80000000 // Available */ +#define EP_SubtArg 0x80000000 /* Is argument to SQLITE_SUBTYPE function */ /* The EP_Propagate mask is a set of properties that automatically propagate ** upwards into parent nodes. @@ -18990,10 +19717,10 @@ struct Expr { /* Macros can be used to test, set, or clear bits in the ** Expr.flags field. */ -#define ExprHasProperty(E,P) (((E)->flags&(P))!=0) -#define ExprHasAllProperty(E,P) (((E)->flags&(P))==(P)) -#define ExprSetProperty(E,P) (E)->flags|=(P) -#define ExprClearProperty(E,P) (E)->flags&=~(P) +#define ExprHasProperty(E,P) (((E)->flags&(u32)(P))!=0) +#define ExprHasAllProperty(E,P) (((E)->flags&(u32)(P))==(u32)(P)) +#define ExprSetProperty(E,P) (E)->flags|=(u32)(P) +#define ExprClearProperty(E,P) (E)->flags&=~(u32)(P) #define ExprAlwaysTrue(E) (((E)->flags&(EP_OuterON|EP_IsTrue))==EP_IsTrue) #define ExprAlwaysFalse(E) (((E)->flags&(EP_OuterON|EP_IsFalse))==EP_IsFalse) #define ExprIsFullSize(E) (((E)->flags&(EP_Reduced|EP_TokenOnly))==0) @@ -19105,9 +19832,14 @@ struct ExprList { int iConstExprReg; /* Register in which Expr value is cached. Used only ** by Parse.pConstExpr */ } u; - } a[1]; /* One slot for each expression in the list */ + } a[FLEXARRAY]; /* One slot for each expression in the list */ }; +/* The size (in bytes) of an ExprList object that is big enough to hold +** as many as N expressions. */ +#define SZ_EXPRLIST(N) \ + (offsetof(ExprList,a) + (N)*sizeof(struct ExprList_item)) + /* ** Allowed values for Expr.a.eEName */ @@ -19133,16 +19865,14 @@ struct ExprList { */ struct IdList { int nId; /* Number of identifiers on the list */ - u8 eU4; /* Which element of a.u4 is valid */ struct IdList_item { char *zName; /* Name of the identifier */ - union { - int idx; /* Index in some Table.aCol[] of a column named zName */ - Expr *pExpr; /* Expr to implement a USING variable -- NOT USED */ - } u4; - } a[1]; + } a[FLEXARRAY]; }; +/* The size (in bytes) of an IdList object that can hold up to N IDs. */ +#define SZ_IDLIST(N) (offsetof(IdList,a)+(N)*sizeof(struct IdList_item)) + /* ** Allowed values for IdList.eType, which determines which value of the a.u4 ** is valid. @@ -19151,6 +19881,16 @@ struct IdList { #define EU4_IDX 1 /* Uses IdList.a.u4.idx */ #define EU4_EXPR 2 /* Uses IdList.a.u4.pExpr -- NOT CURRENTLY USED */ +/* +** Details of the implementation of a subquery. +*/ +struct Subquery { + Select *pSelect; /* A SELECT statement used in place of a table name */ + int addrFillSub; /* Address of subroutine to initialize a subquery */ + int regReturn; /* Register holding return address of addrFillSub */ + int regResult; /* Registers holding results of a co-routine */ +}; + /* ** The SrcItem object represents a single term in the FROM clause of a query. ** The SrcList object is mostly an array of SrcItems. @@ -19163,27 +19903,40 @@ struct IdList { ** In the colUsed field, the high-order bit (bit 63) is set if the table ** contains more than 63 columns and the 64-th or later column is used. ** -** Union member validity: +** Aggressive use of "union" helps keep the size of the object small. This +** has been shown to boost performance, in addition to saving memory. +** Access to union elements is gated by the following rules which should +** always be checked, either by an if-statement or by an assert(). +** +** Field Only access if this is true +** --------------- ----------------------------------- +** u1.zIndexedBy fg.isIndexedBy +** u1.pFuncArg fg.isTabFunc +** u1.nRow !fg.isTabFunc && !fg.isIndexedBy +** +** u2.pIBIndex fg.isIndexedBy +** u2.pCteUse fg.isCte ** -** u1.zIndexedBy fg.isIndexedBy && !fg.isTabFunc -** u1.pFuncArg fg.isTabFunc && !fg.isIndexedBy -** u2.pIBIndex fg.isIndexedBy && !fg.isCte -** u2.pCteUse fg.isCte && !fg.isIndexedBy +** u3.pOn !fg.isUsing +** u3.pUsing fg.isUsing +** +** u4.zDatabase !fg.fixedSchema && !fg.isSubquery +** u4.pSchema fg.fixedSchema +** u4.pSubq fg.isSubquery +** +** See also the sqlite3SrcListDelete() routine for assert() statements that +** check invariants on the fields of this object, especially the flags +** inside the fg struct. */ struct SrcItem { - Schema *pSchema; /* Schema to which this item is fixed */ - char *zDatabase; /* Name of database holding this table */ char *zName; /* Name of the table */ char *zAlias; /* The "B" part of a "A AS B" phrase. zName is the "A" */ - Table *pTab; /* An SQL table corresponding to zName */ - Select *pSelect; /* A SELECT statement used in place of a table name */ - int addrFillSub; /* Address of subroutine to manifest a subquery */ - int regReturn; /* Register holding return address of addrFillSub */ - int regResult; /* Registers holding results of a co-routine */ + Table *pSTab; /* Table object for zName. Mnemonic: Srcitem-TABle */ struct { u8 jointype; /* Type of join between this table and the previous */ unsigned notIndexed :1; /* True if there is a NOT INDEXED clause */ unsigned isIndexedBy :1; /* True if there is an INDEXED BY clause */ + unsigned isSubquery :1; /* True if this term is a subquery */ unsigned isTabFunc :1; /* True if table-valued-function syntax */ unsigned isCorrelated :1; /* True if sub-query is correlated */ unsigned isMaterialized:1; /* This is a materialized view */ @@ -19196,21 +19949,31 @@ struct SrcItem { unsigned isOn :1; /* u3.pOn was once valid and non-NULL */ unsigned isSynthUsing :1; /* u3.pUsing is synthesized from NATURAL */ unsigned isNestedFrom :1; /* pSelect is a SF_NestedFrom subquery */ + unsigned rowidUsed :1; /* The ROWID of this table is referenced */ + unsigned fixedSchema :1; /* Uses u4.pSchema, not u4.zDatabase */ + unsigned hadSchema :1; /* Had u4.zDatabase before u4.pSchema */ + unsigned fromExists :1; /* Comes from WHERE EXISTS(...) */ } fg; int iCursor; /* The VDBE cursor number used to access this table */ - union { - Expr *pOn; /* fg.isUsing==0 => The ON clause of a join */ - IdList *pUsing; /* fg.isUsing==1 => The USING clause of a join */ - } u3; Bitmask colUsed; /* Bit N set if column N used. Details above for N>62 */ union { char *zIndexedBy; /* Identifier from "INDEXED BY " clause */ ExprList *pFuncArg; /* Arguments to table-valued-function */ + u32 nRow; /* Number of rows in a VALUES clause */ } u1; union { Index *pIBIndex; /* Index structure corresponding to u1.zIndexedBy */ CteUse *pCteUse; /* CTE Usage info when fg.isCte is true */ } u2; + union { + Expr *pOn; /* fg.isUsing==0 => The ON clause of a join */ + IdList *pUsing; /* fg.isUsing==1 => The USING clause of a join */ + } u3; + union { + Schema *pSchema; /* Schema to which this item is fixed */ + char *zDatabase; /* Name of database holding this table */ + Subquery *pSubq; /* Description of a subquery */ + } u4; }; /* @@ -19230,11 +19993,19 @@ struct OnOrUsing { ** */ struct SrcList { - int nSrc; /* Number of tables or subqueries in the FROM clause */ - u32 nAlloc; /* Number of entries allocated in a[] below */ - SrcItem a[1]; /* One entry for each identifier on the list */ + int nSrc; /* Number of tables or subqueries in the FROM clause */ + u32 nAlloc; /* Number of entries allocated in a[] below */ + SrcItem a[FLEXARRAY]; /* One entry for each identifier on the list */ }; +/* Size (in bytes) of a SrcList object that can hold as many as N +** SrcItem objects. */ +#define SZ_SRCLIST(N) (offsetof(SrcList,a)+(N)*sizeof(SrcItem)) + +/* Size (in bytes( of a SrcList object that holds 1 SrcItem. This is a +** special case of SZ_SRCITEM(1) that comes up often. */ +#define SZ_SRCLIST_1 (offsetof(SrcList,a)+sizeof(SrcItem)) + /* ** Permitted values of the SrcList.a.jointype field */ @@ -19270,7 +20041,7 @@ struct SrcList { #define WHERE_AGG_DISTINCT 0x0400 /* Query is "SELECT agg(DISTINCT ...)" */ #define WHERE_ORDERBY_LIMIT 0x0800 /* ORDERBY+LIMIT on the inner loop */ #define WHERE_RIGHT_JOIN 0x1000 /* Processing a RIGHT JOIN */ - /* 0x2000 not currently used */ +#define WHERE_KEEP_ALL_JOINS 0x2000 /* Do not do the omit-noop-join opt */ #define WHERE_USE_LIMIT 0x4000 /* Use the LIMIT in cost estimates */ /* 0x8000 not currently used */ @@ -19342,7 +20113,7 @@ struct NameContext { #define NC_UUpsert 0x000200 /* True if uNC.pUpsert is used */ #define NC_UBaseReg 0x000400 /* True if uNC.iBaseReg is used */ #define NC_MinMaxAgg 0x001000 /* min/max aggregates seen. See note above */ -#define NC_Complex 0x002000 /* True if a function or subquery seen */ +/* 0x002000 // available for reuse */ #define NC_AllowWin 0x004000 /* Window functions are allowed here */ #define NC_HasWin 0x008000 /* One or more window functions seen */ #define NC_IsDDL 0x010000 /* Resolving names in a CREATE statement */ @@ -19463,14 +20234,18 @@ struct Select { #define SF_View 0x0200000 /* SELECT statement is a view */ #define SF_NoopOrderBy 0x0400000 /* ORDER BY is ignored for this query */ #define SF_UFSrcCheck 0x0800000 /* Check pSrc as required by UPDATE...FROM */ -#define SF_PushDown 0x1000000 /* SELECT has be modified by push-down opt */ +#define SF_PushDown 0x1000000 /* Modified by WHERE-clause push-down opt */ #define SF_MultiPart 0x2000000 /* Has multiple incompatible PARTITIONs */ #define SF_CopyCte 0x4000000 /* SELECT statement is a copy of a CTE */ #define SF_OrderByReqd 0x8000000 /* The ORDER BY clause may not be omitted */ #define SF_UpdateFrom 0x10000000 /* Query originates with UPDATE FROM */ +#define SF_Correlated 0x20000000 /* True if references the outer context */ +#define SF_OnToWhere 0x40000000 /* One or more ON clauses moved to WHERE */ -/* True if S exists and has SF_NestedFrom */ -#define IsNestedFrom(S) ((S)!=0 && ((S)->selFlags&SF_NestedFrom)!=0) +/* True if SrcItem X is a subquery that has SF_NestedFrom */ +#define IsNestedFrom(X) \ + ((X)->fg.isSubquery && \ + ((X)->u4.pSubq->pSelect->selFlags&SF_NestedFrom)!=0) /* ** The results of a SELECT can be distributed in several ways, as defined @@ -19500,7 +20275,11 @@ struct Select { ** SRT_Set The result must be a single column. Store each ** row of result as the key in table pDest->iSDParm. ** Apply the affinity pDest->affSdst before storing -** results. Used to implement "IN (SELECT ...)". +** results. if pDest->iSDParm2 is positive, then it is +** a register holding a Bloom filter for the IN operator +** that should be populated in addition to the +** pDest->iSDParm table. This SRT is used to +** implement "IN (SELECT ...)". ** ** SRT_EphemTab Create an temporary table pDest->iSDParm and store ** the result there. The cursor is left open after @@ -19696,23 +20475,33 @@ struct Parse { char *zErrMsg; /* An error message */ Vdbe *pVdbe; /* An engine for executing database bytecode */ int rc; /* Return code from execution */ - u8 colNamesSet; /* TRUE after OP_ColumnName has been issued to pVdbe */ - u8 checkSchema; /* Causes schema cookie check after an error */ + LogEst nQueryLoop; /* Est number of iterations of a query (10*log2(N)) */ u8 nested; /* Number of nested calls to the parser/code generator */ u8 nTempReg; /* Number of temporary registers in aTempReg[] */ u8 isMultiWrite; /* True if statement may modify/insert multiple rows */ u8 mayAbort; /* True if statement may throw an ABORT exception */ u8 hasCompound; /* Need to invoke convertCompoundSelectToSubquery() */ - u8 okConstFactor; /* OK to factor out constants */ u8 disableLookaside; /* Number of times lookaside has been disabled */ u8 prepFlags; /* SQLITE_PREPARE_* flags */ u8 withinRJSubrtn; /* Nesting level for RIGHT JOIN body subroutines */ + u8 bHasExists; /* Has a correlated "EXISTS (SELECT ....)" expression */ + u8 mSubrtnSig; /* mini Bloom filter on available SubrtnSig.selId */ + u8 eTriggerOp; /* TK_UPDATE, TK_INSERT or TK_DELETE */ + u8 bReturning; /* Coding a RETURNING trigger */ + u8 eOrconf; /* Default ON CONFLICT policy for trigger steps */ + u8 disableTriggers; /* True to disable triggers */ #if defined(SQLITE_DEBUG) || defined(SQLITE_COVERAGE_TEST) u8 earlyCleanup; /* OOM inside sqlite3ParserAddCleanup() */ #endif #ifdef SQLITE_DEBUG u8 ifNotExists; /* Might be true if IF NOT EXISTS. Assert()s only */ + u8 isCreate; /* CREATE TABLE, INDEX, or VIEW (but not TRIGGER) + ** and ALTER TABLE ADD COLUMN. */ #endif + bft colNamesSet :1; /* TRUE after OP_ColumnName has been issued to pVdbe */ + bft bHasWith :1; /* True if statement contains WITH */ + bft okConstFactor :1; /* OK to factor out constants */ + bft checkSchema :1; /* Causes schema cookie check after an error */ int nRangeReg; /* Size of the temporary register block */ int iRangeReg; /* First register in temporary register block */ int nErr; /* Number of errors seen */ @@ -19727,12 +20516,9 @@ struct Parse { ExprList *pConstExpr;/* Constant expressions */ IndexedExpr *pIdxEpr;/* List of expressions used by active indexes */ IndexedExpr *pIdxPartExpr; /* Exprs constrained by index WHERE clauses */ - Token constraintName;/* Name of the constraint currently being parsed */ yDbMask writeMask; /* Start a write transaction on these databases */ yDbMask cookieMask; /* Bitmask of schema verified databases */ - int regRowid; /* Register holding rowid of CREATE TABLE entry */ - int regRoot; /* Register holding root page number for new objects */ - int nMaxArg; /* Max args passed to user function by sub-program */ + int nMaxArg; /* Max args to xUpdate and xFilter vtab methods */ int nSelect; /* Number of SELECT stmts. Counter for Select.selId */ #ifndef SQLITE_OMIT_PROGRESS_CALLBACK u32 nProgressSteps; /* xProgress steps taken during sqlite3_prepare() */ @@ -19746,17 +20532,6 @@ struct Parse { Table *pTriggerTab; /* Table triggers are being coded for */ TriggerPrg *pTriggerPrg; /* Linked list of coded triggers */ ParseCleanup *pCleanup; /* List of cleanup operations to run after parse */ - union { - int addrCrTab; /* Address of OP_CreateBtree on CREATE TABLE */ - Returning *pReturning; /* The RETURNING clause */ - } u1; - u32 oldmask; /* Mask of old.* columns referenced */ - u32 newmask; /* Mask of new.* columns referenced */ - LogEst nQueryLoop; /* Est number of iterations of a query (10*log2(N)) */ - u8 eTriggerOp; /* TK_UPDATE, TK_INSERT or TK_DELETE */ - u8 bReturning; /* Coding a RETURNING trigger */ - u8 eOrconf; /* Default ON CONFLICT policy for trigger steps */ - u8 disableTriggers; /* True to disable triggers */ /************************************************************************** ** Fields above must be initialized to zero. The fields that follow, @@ -19768,6 +20543,19 @@ struct Parse { int aTempReg[8]; /* Holding area for temporary registers */ Parse *pOuterParse; /* Outer Parse object when nested */ Token sNameToken; /* Token with unqualified schema object name */ + u32 oldmask; /* Mask of old.* columns referenced */ + u32 newmask; /* Mask of new.* columns referenced */ + union { + struct { /* These fields available when isCreate is true */ + int addrCrTab; /* Address of OP_CreateBtree on CREATE TABLE */ + int regRowid; /* Register holding rowid of CREATE TABLE entry */ + int regRoot; /* Register holding root page for new objects */ + Token constraintName; /* Name of the constraint currently being parsed */ + } cr; + struct { /* These fields available to all other statements */ + Returning *pReturning; /* The RETURNING clause */ + } d; + } u1; /************************************************************************ ** Above is constant between recursions. Below is reset before and after @@ -19785,9 +20573,7 @@ struct Parse { int nVtabLock; /* Number of virtual tables to lock */ #endif int nHeight; /* Expression tree height of current sub-select */ -#ifndef SQLITE_OMIT_EXPLAIN int addrExplain; /* Address of current OP_Explain opcode */ -#endif VList *pVList; /* Mapping between variable names and numbers */ Vdbe *pReprepare; /* VM being reprepared (sqlite3Reprepare()) */ const char *zTail; /* All SQL text past the last semicolon parsed */ @@ -20002,7 +20788,7 @@ struct Returning { }; /* -** An objected used to accumulate the text of a string where we +** An object used to accumulate the text of a string where we ** do not necessarily know how big the string will be in the end. */ struct sqlite3_str { @@ -20016,7 +20802,7 @@ struct sqlite3_str { }; #define SQLITE_PRINTF_INTERNAL 0x01 /* Internal-use-only converters allowed */ #define SQLITE_PRINTF_SQLFUNC 0x02 /* SQL function arguments to VXPrintf */ -#define SQLITE_PRINTF_MALLOCED 0x04 /* True if xText is allocated space */ +#define SQLITE_PRINTF_MALLOCED 0x04 /* True if zText is allocated space */ #define isMalloced(X) (((X)->printfFlags & SQLITE_PRINTF_MALLOCED)!=0) @@ -20094,7 +20880,6 @@ struct Sqlite3Config { u8 bUseCis; /* Use covering indices for full-scans */ u8 bSmallMalloc; /* Avoid large memory allocations if true */ u8 bExtraSchemaChecks; /* Verify type,name,tbl_name in schema */ - u8 bUseLongDouble; /* Make use of long double */ #ifdef SQLITE_DEBUG u8 bJsonSelfcheck; /* Double-check JSON parsing */ #endif @@ -20144,6 +20929,11 @@ struct Sqlite3Config { #endif #ifndef SQLITE_UNTESTABLE int (*xTestCallback)(int); /* Invoked by sqlite3FaultSim() */ +#endif +#ifdef SQLITE_ALLOW_ROWID_IN_VIEW + u32 mNoVisibleRowid; /* TF_NoVisibleRowid if the ROWID_IN_VIEW + ** feature is disabled. 0 if rowids can + ** occur in views. */ #endif int bLocaltimeFault; /* True to fail localtime() calls */ int (*xAltLocaltime)(const void*,void*); /* Alternative localtime() routine */ @@ -20204,6 +20994,7 @@ struct Walker { SrcItem *pSrcItem; /* A single FROM clause item */ DbFixer *pFix; /* See sqlite3FixSelect() */ Mem *aMem; /* See sqlite3BtreeCursorHint() */ + struct CheckOnCtx *pCheckOnCtx; /* See selectCheckOnClauses() */ } u; }; @@ -20281,9 +21072,13 @@ struct With { int nCte; /* Number of CTEs in the WITH clause */ int bView; /* Belongs to the outermost Select of a view */ With *pOuter; /* Containing WITH clause, or NULL */ - Cte a[1]; /* For each CTE in the WITH clause.... */ + Cte a[FLEXARRAY]; /* For each CTE in the WITH clause.... */ }; +/* The size (in bytes) of a With object that can hold as many +** as N different CTEs. */ +#define SZ_WITH(N) (offsetof(With,a) + (N)*sizeof(Cte)) + /* ** The Cte object is not guaranteed to persist for the entire duration ** of code generation. (The query flattener or other parser tree @@ -20312,9 +21107,13 @@ struct DbClientData { DbClientData *pNext; /* Next in a linked list */ void *pData; /* The data */ void (*xDestructor)(void*); /* Destructor. Might be NULL */ - char zName[1]; /* Name of this client data. MUST BE LAST */ + char zName[FLEXARRAY]; /* Name of this client data. MUST BE LAST */ }; +/* The size (in bytes) of a DbClientData object that can has a name +** that is N bytes long, including the zero-terminator. */ +#define SZ_DBCLIENTDATA(N) (offsetof(DbClientData,zName)+(N)) + #ifdef SQLITE_DEBUG /* ** An instance of the TreeView object is used for printing the content of @@ -20381,6 +21180,9 @@ struct Window { ** due to the SQLITE_SUBTYPE flag */ }; +SQLITE_PRIVATE Select *sqlite3MultiValues(Parse *pParse, Select *pLeft, ExprList *pRow); +SQLITE_PRIVATE void sqlite3MultiValuesEnd(Parse *pParse, Select *pVal); + #ifndef SQLITE_OMIT_WINDOWFUNC SQLITE_PRIVATE void sqlite3WindowDelete(sqlite3*, Window*); SQLITE_PRIVATE void sqlite3WindowUnlinkFromSelect(Window*); @@ -20461,15 +21263,6 @@ SQLITE_PRIVATE int sqlite3CorruptPgnoError(int,Pgno); # define SQLITE_ENABLE_FTS3 1 #endif -/* -** The ctype.h header is needed for non-ASCII systems. It is also -** needed by FTS3 when FTS3 is included in the amalgamation. -*/ -#if !defined(SQLITE_ASCII) || \ - (defined(SQLITE_ENABLE_FTS3) && defined(SQLITE_AMALGAMATION)) -# include -#endif - /* ** The following macros mimic the standard library functions toupper(), ** isspace(), isalnum(), isdigit() and isxdigit(), respectively. The @@ -20600,10 +21393,13 @@ SQLITE_PRIVATE void sqlite3MutexWarnOnContention(sqlite3_mutex*); # define EXP754 (((u64)0x7ff)<<52) # define MAN754 ((((u64)1)<<52)-1) # define IsNaN(X) (((X)&EXP754)==EXP754 && ((X)&MAN754)!=0) +# define IsOvfl(X) (((X)&EXP754)==EXP754) SQLITE_PRIVATE int sqlite3IsNaN(double); +SQLITE_PRIVATE int sqlite3IsOverflow(double); #else -# define IsNaN(X) 0 -# define sqlite3IsNaN(X) 0 +# define IsNaN(X) 0 +# define sqlite3IsNaN(X) 0 +# define sqlite3IsOVerflow(X) 0 #endif /* @@ -20686,6 +21482,7 @@ SQLITE_PRIVATE void sqlite3ShowTriggerList(const Trigger*); SQLITE_PRIVATE void sqlite3ShowWindow(const Window*); SQLITE_PRIVATE void sqlite3ShowWinFunc(const Window*); #endif +SQLITE_PRIVATE void sqlite3ShowBitvec(Bitvec*); #endif SQLITE_PRIVATE void sqlite3SetString(char **, sqlite3*, const char*); @@ -20695,6 +21492,7 @@ SQLITE_PRIVATE int sqlite3ErrorToParser(sqlite3*,int); SQLITE_PRIVATE void sqlite3Dequote(char*); SQLITE_PRIVATE void sqlite3DequoteExpr(Expr*); SQLITE_PRIVATE void sqlite3DequoteToken(Token*); +SQLITE_PRIVATE void sqlite3DequoteNumber(Parse*, Expr*); SQLITE_PRIVATE void sqlite3TokenInit(Token*,char*); SQLITE_PRIVATE int sqlite3KeywordCode(const unsigned char*, int); SQLITE_PRIVATE int sqlite3RunParser(Parse*, const char*); @@ -20725,7 +21523,7 @@ SQLITE_PRIVATE void sqlite3ExprFunctionUsable(Parse*,const Expr*,const FuncDef*) SQLITE_PRIVATE void sqlite3ExprAssignVarNumber(Parse*, Expr*, u32); SQLITE_PRIVATE void sqlite3ExprDelete(sqlite3*, Expr*); SQLITE_PRIVATE void sqlite3ExprDeleteGeneric(sqlite3*,void*); -SQLITE_PRIVATE void sqlite3ExprDeferredDelete(Parse*, Expr*); +SQLITE_PRIVATE int sqlite3ExprDeferredDelete(Parse*, Expr*); SQLITE_PRIVATE void sqlite3ExprUnmapAndDelete(Parse*, Expr*); SQLITE_PRIVATE ExprList *sqlite3ExprListAppend(Parse*,ExprList*,Expr*); SQLITE_PRIVATE ExprList *sqlite3ExprListAppendVector(Parse*,ExprList*,IdList*,Expr*); @@ -20759,7 +21557,7 @@ SQLITE_PRIVATE void sqlite3SubqueryColumnTypes(Parse*,Table*,Select*,char); SQLITE_PRIVATE Table *sqlite3ResultSetOfSelect(Parse*,Select*,char); SQLITE_PRIVATE void sqlite3OpenSchemaTable(Parse *, int); SQLITE_PRIVATE Index *sqlite3PrimaryKeyIndex(Table*); -SQLITE_PRIVATE i16 sqlite3TableColumnToIndex(Index*, i16); +SQLITE_PRIVATE int sqlite3TableColumnToIndex(Index*, int); #ifdef SQLITE_OMIT_GENERATED_COLUMNS # define sqlite3TableColumnToStorage(T,X) (X) /* No-op pass-through */ # define sqlite3StorageColumnToTable(T,X) (X) /* No-op pass-through */ @@ -20844,6 +21642,9 @@ SQLITE_PRIVATE int sqlite3IdListIndex(IdList*,const char*); SQLITE_PRIVATE SrcList *sqlite3SrcListEnlarge(Parse*, SrcList*, int, int); SQLITE_PRIVATE SrcList *sqlite3SrcListAppendList(Parse *pParse, SrcList *p1, SrcList *p2); SQLITE_PRIVATE SrcList *sqlite3SrcListAppend(Parse*, SrcList*, Token*, Token*); +SQLITE_PRIVATE void sqlite3SubqueryDelete(sqlite3*,Subquery*); +SQLITE_PRIVATE Select *sqlite3SubqueryDetach(sqlite3*,SrcItem*); +SQLITE_PRIVATE int sqlite3SrcItemAttachSubquery(Parse*, SrcItem*, Select*, int); SQLITE_PRIVATE SrcList *sqlite3SrcListAppendFromTerm(Parse*, SrcList*, Token*, Token*, Token*, Select*, OnOrUsing*); SQLITE_PRIVATE void sqlite3SrcListIndexedBy(Parse *, SrcList *, Token *); @@ -20854,7 +21655,7 @@ SQLITE_PRIVATE void sqlite3SrcListAssignCursors(Parse*, SrcList*); SQLITE_PRIVATE void sqlite3IdListDelete(sqlite3*, IdList*); SQLITE_PRIVATE void sqlite3ClearOnOrUsing(sqlite3*, OnOrUsing*); SQLITE_PRIVATE void sqlite3SrcListDelete(sqlite3*, SrcList*); -SQLITE_PRIVATE Index *sqlite3AllocateIndexObject(sqlite3*,i16,int,char**); +SQLITE_PRIVATE Index *sqlite3AllocateIndexObject(sqlite3*,int,int,char**); SQLITE_PRIVATE void sqlite3CreateIndex(Parse*,Token*,Token*,SrcList*,ExprList*,int,Token*, Expr*, int, int, u8); SQLITE_PRIVATE void sqlite3DropIndex(Parse*, SrcList*, int); @@ -20893,6 +21694,7 @@ SQLITE_PRIVATE void sqlite3ExprCodeLoadIndexColumn(Parse*, Index*, int, int, int SQLITE_PRIVATE int sqlite3ExprCodeGetColumn(Parse*, Table*, int, int, int, u8); SQLITE_PRIVATE void sqlite3ExprCodeGetColumnOfTable(Vdbe*, Table*, int, int, int); SQLITE_PRIVATE void sqlite3ExprCodeMove(Parse*, int, int, int); +SQLITE_PRIVATE void sqlite3ExprToRegister(Expr *pExpr, int iReg); SQLITE_PRIVATE void sqlite3ExprCode(Parse*, Expr*, int); #ifndef SQLITE_OMIT_GENERATED_COLUMNS SQLITE_PRIVATE void sqlite3ExprCodeGeneratedColumn(Parse*, Table*, Column*, int); @@ -20900,6 +21702,7 @@ SQLITE_PRIVATE void sqlite3ExprCodeGeneratedColumn(Parse*, Table*, Column*, int) SQLITE_PRIVATE void sqlite3ExprCodeCopy(Parse*, Expr*, int); SQLITE_PRIVATE void sqlite3ExprCodeFactorable(Parse*, Expr*, int); SQLITE_PRIVATE int sqlite3ExprCodeRunJustOnce(Parse*, Expr*, int); +SQLITE_PRIVATE void sqlite3ExprNullRegisterRange(Parse*, int, int); SQLITE_PRIVATE int sqlite3ExprCodeTemp(Parse*, Expr*, int*); SQLITE_PRIVATE int sqlite3ExprCodeTarget(Parse*, Expr*, int); SQLITE_PRIVATE int sqlite3ExprCodeExprList(Parse*, ExprList*, int, int, u8); @@ -20948,16 +21751,14 @@ SQLITE_PRIVATE void sqlite3LeaveMutexAndCloseZombie(sqlite3*); SQLITE_PRIVATE u32 sqlite3IsTrueOrFalse(const char*); SQLITE_PRIVATE int sqlite3ExprIdToTrueFalse(Expr*); SQLITE_PRIVATE int sqlite3ExprTruthValue(const Expr*); -SQLITE_PRIVATE int sqlite3ExprIsConstant(Expr*); -SQLITE_PRIVATE int sqlite3ExprIsConstantNotJoin(Expr*); +SQLITE_PRIVATE int sqlite3ExprIsConstant(Parse*,Expr*); SQLITE_PRIVATE int sqlite3ExprIsConstantOrFunction(Expr*, u8); SQLITE_PRIVATE int sqlite3ExprIsConstantOrGroupBy(Parse*, Expr*, ExprList*); -SQLITE_PRIVATE int sqlite3ExprIsTableConstant(Expr*,int); -SQLITE_PRIVATE int sqlite3ExprIsSingleTableConstraint(Expr*,const SrcList*,int); +SQLITE_PRIVATE int sqlite3ExprIsSingleTableConstraint(Expr*,const SrcList*,int,int); #ifdef SQLITE_ENABLE_CURSOR_HINTS SQLITE_PRIVATE int sqlite3ExprContainsSubquery(Expr*); #endif -SQLITE_PRIVATE int sqlite3ExprIsInteger(const Expr*, int*); +SQLITE_PRIVATE int sqlite3ExprIsInteger(const Expr*, int*, Parse*); SQLITE_PRIVATE int sqlite3ExprCanBeNull(const Expr*); SQLITE_PRIVATE int sqlite3ExprNeedsNoAffinityChange(const Expr*, char); SQLITE_PRIVATE int sqlite3IsRowid(const char*); @@ -20991,19 +21792,24 @@ SQLITE_PRIVATE Select *sqlite3SelectDup(sqlite3*,const Select*,int); SQLITE_PRIVATE FuncDef *sqlite3FunctionSearch(int,const char*); SQLITE_PRIVATE void sqlite3InsertBuiltinFuncs(FuncDef*,int); SQLITE_PRIVATE FuncDef *sqlite3FindFunction(sqlite3*,const char*,int,u8,u8); -SQLITE_PRIVATE void sqlite3QuoteValue(StrAccum*,sqlite3_value*); +SQLITE_PRIVATE void sqlite3QuoteValue(StrAccum*,sqlite3_value*,int); +SQLITE_PRIVATE int sqlite3AppendOneUtf8Character(char*, u32); SQLITE_PRIVATE void sqlite3RegisterBuiltinFunctions(void); SQLITE_PRIVATE void sqlite3RegisterDateTimeFunctions(void); SQLITE_PRIVATE void sqlite3RegisterJsonFunctions(void); SQLITE_PRIVATE void sqlite3RegisterPerConnectionBuiltinFunctions(sqlite3*); #if !defined(SQLITE_OMIT_VIRTUALTABLE) && !defined(SQLITE_OMIT_JSON) -SQLITE_PRIVATE int sqlite3JsonTableFunctions(sqlite3*); +SQLITE_PRIVATE Module *sqlite3JsonVtabRegister(sqlite3*,const char*); #endif SQLITE_PRIVATE int sqlite3SafetyCheckOk(sqlite3*); SQLITE_PRIVATE int sqlite3SafetyCheckSickOrOk(sqlite3*); SQLITE_PRIVATE void sqlite3ChangeCookie(Parse*, int); SQLITE_PRIVATE With *sqlite3WithDup(sqlite3 *db, With *p); +#if !defined(SQLITE_OMIT_VIRTUALTABLE) && defined(SQLITE_ENABLE_CARRAY) +SQLITE_PRIVATE Module *sqlite3CarrayRegister(sqlite3*); +#endif + #if !defined(SQLITE_OMIT_VIEW) && !defined(SQLITE_OMIT_TRIGGER) SQLITE_PRIVATE void sqlite3MaterializeView(Parse*, Table*, Expr*, ExprList*,Expr*,int); #endif @@ -21085,7 +21891,7 @@ SQLITE_PRIVATE int sqlite3GetInt32(const char *, int*); SQLITE_PRIVATE int sqlite3GetUInt32(const char*, u32*); SQLITE_PRIVATE int sqlite3Atoi(const char*); #ifndef SQLITE_OMIT_UTF16 -SQLITE_PRIVATE int sqlite3Utf16ByteLen(const void *pData, int nChar); +SQLITE_PRIVATE int sqlite3Utf16ByteLen(const void *pData, int nByte, int nChar); #endif SQLITE_PRIVATE int sqlite3Utf8CharLen(const char *pData, int nByte); SQLITE_PRIVATE u32 sqlite3Utf8Read(const u8**); @@ -21138,7 +21944,9 @@ SQLITE_PRIVATE void sqlite3ErrorWithMsg(sqlite3*, int, const char*,...); SQLITE_PRIVATE void sqlite3Error(sqlite3*,int); SQLITE_PRIVATE void sqlite3ErrorClear(sqlite3*); SQLITE_PRIVATE void sqlite3SystemError(sqlite3*,int); +#if !defined(SQLITE_OMIT_BLOB_LITERAL) SQLITE_PRIVATE void *sqlite3HexToBlob(sqlite3*, const char *z, int n); +#endif SQLITE_PRIVATE u8 sqlite3HexToInt(int h); SQLITE_PRIVATE int sqlite3TwoPartName(Parse *, Token *, Token *, Token **); @@ -21222,7 +22030,7 @@ SQLITE_PRIVATE void sqlite3Reindex(Parse*, Token*, Token*); SQLITE_PRIVATE void sqlite3AlterFunctions(void); SQLITE_PRIVATE void sqlite3AlterRenameTable(Parse*, SrcList*, Token*); SQLITE_PRIVATE void sqlite3AlterRenameColumn(Parse*, SrcList*, Token*, Token*); -SQLITE_PRIVATE int sqlite3GetToken(const unsigned char *, int *); +SQLITE_PRIVATE i64 sqlite3GetToken(const unsigned char *, int *); SQLITE_PRIVATE void sqlite3NestedParse(Parse*, const char*, ...); SQLITE_PRIVATE void sqlite3ExpirePreparedStatements(sqlite3*, int); SQLITE_PRIVATE void sqlite3CodeRhsOfIN(Parse*, Expr*, int); @@ -21839,6 +22647,9 @@ static const char * const sqlite3azCompileOpt[] = { "ALLOW_COVERING_INDEX_SCAN=" CTIMEOPT_VAL(SQLITE_ALLOW_COVERING_INDEX_SCAN), # endif #endif +#ifdef SQLITE_ALLOW_ROWID_IN_VIEW + "ALLOW_ROWID_IN_VIEW", +#endif #ifdef SQLITE_ALLOW_URI_AUTHORITY "ALLOW_URI_AUTHORITY", #endif @@ -21851,6 +22662,9 @@ static const char * const sqlite3azCompileOpt[] = { #ifdef SQLITE_BUG_COMPATIBLE_20160819 "BUG_COMPATIBLE_20160819", #endif +#ifdef SQLITE_BUG_COMPATIBLE_20250510 + "BUG_COMPATIBLE_20250510", +#endif #ifdef SQLITE_CASE_SENSITIVE_LIKE "CASE_SENSITIVE_LIKE", #endif @@ -21982,6 +22796,9 @@ static const char * const sqlite3azCompileOpt[] = { #ifdef SQLITE_ENABLE_BYTECODE_VTAB "ENABLE_BYTECODE_VTAB", #endif +#ifdef SQLITE_ENABLE_CARRAY + "ENABLE_CARRAY", +#endif #ifdef SQLITE_ENABLE_CEROD "ENABLE_CEROD=" CTIMEOPT_VAL(SQLITE_ENABLE_CEROD), #endif @@ -22066,9 +22883,15 @@ static const char * const sqlite3azCompileOpt[] = { #ifdef SQLITE_ENABLE_OFFSET_SQL_FUNC "ENABLE_OFFSET_SQL_FUNC", #endif +#ifdef SQLITE_ENABLE_ORDERED_SET_AGGREGATES + "ENABLE_ORDERED_SET_AGGREGATES", +#endif #ifdef SQLITE_ENABLE_OVERSIZE_CELL_CHECK "ENABLE_OVERSIZE_CELL_CHECK", #endif +#ifdef SQLITE_ENABLE_PERCENTILE + "ENABLE_PERCENTILE", +#endif #ifdef SQLITE_ENABLE_PREUPDATE_HOOK "ENABLE_PREUPDATE_HOOK", #endif @@ -22084,6 +22907,9 @@ static const char * const sqlite3azCompileOpt[] = { #ifdef SQLITE_ENABLE_SESSION "ENABLE_SESSION", #endif +#ifdef SQLITE_ENABLE_SETLK_TIMEOUT + "ENABLE_SETLK_TIMEOUT", +#endif #ifdef SQLITE_ENABLE_SNAPSHOT "ENABLE_SNAPSHOT", #endif @@ -22138,6 +22964,9 @@ static const char * const sqlite3azCompileOpt[] = { #ifdef SQLITE_EXTRA_INIT "EXTRA_INIT=" CTIMEOPT_VAL(SQLITE_EXTRA_INIT), #endif +#ifdef SQLITE_EXTRA_INIT_MUTEXED + "EXTRA_INIT_MUTEXED=" CTIMEOPT_VAL(SQLITE_EXTRA_INIT_MUTEXED), +#endif #ifdef SQLITE_EXTRA_SHUTDOWN "EXTRA_SHUTDOWN=" CTIMEOPT_VAL(SQLITE_EXTRA_SHUTDOWN), #endif @@ -22535,9 +23364,6 @@ static const char * const sqlite3azCompileOpt[] = { #ifdef SQLITE_UNTESTABLE "UNTESTABLE", #endif -#ifdef SQLITE_USER_AUTHENTICATION - "USER_AUTHENTICATION", -#endif #ifdef SQLITE_USE_ALLOCA "USE_ALLOCA", #endif @@ -22813,7 +23639,6 @@ SQLITE_PRIVATE SQLITE_WSD struct Sqlite3Config sqlite3Config = { SQLITE_ALLOW_COVERING_INDEX_SCAN, /* bUseCis */ 0, /* bSmallMalloc */ 1, /* bExtraSchemaChecks */ - sizeof(LONGDOUBLE_TYPE)>8, /* bUseLongDouble */ #ifdef SQLITE_DEBUG 0, /* bJsonSelfcheck */ #endif @@ -22858,6 +23683,9 @@ SQLITE_PRIVATE SQLITE_WSD struct Sqlite3Config sqlite3Config = { #endif #ifndef SQLITE_UNTESTABLE 0, /* xTestCallback */ +#endif +#ifdef SQLITE_ALLOW_ROWID_IN_VIEW + 0, /* mNoVisibleRowid. 0 == allow rowid-in-view */ #endif 0, /* bLocaltimeFault */ 0, /* xAltLocaltime */ @@ -23123,12 +23951,19 @@ struct VdbeCursor { #endif VdbeTxtBlbCache *pCache; /* Cache of large TEXT or BLOB values */ - /* 2*nField extra array elements allocated for aType[], beyond the one - ** static element declared in the structure. nField total array slots for - ** aType[] and nField+1 array slots for aOffset[] */ - u32 aType[1]; /* Type values record decode. MUST BE LAST */ + /* Space is allocated for aType to hold at least 2*nField+1 entries: + ** nField slots for aType[] and nField+1 array slots for aOffset[] */ + u32 aType[FLEXARRAY]; /* Type values record decode. MUST BE LAST */ }; +/* +** The size (in bytes) of a VdbeCursor object that has an nField value of N +** or less. The value of SZ_VDBECURSOR(n) is guaranteed to be a multiple +** of 8. +*/ +#define SZ_VDBECURSOR(N) \ + (ROUND8(offsetof(VdbeCursor,aType)) + ((N)+1)*sizeof(u64)) + /* Return true if P is a null-only cursor */ #define IsNullCursor(P) \ @@ -23234,6 +24069,7 @@ struct sqlite3_value { #ifdef SQLITE_DEBUG Mem *pScopyFrom; /* This Mem is a shallow copy of pScopyFrom */ u16 mScopyFlags; /* flags value immediately after the shallow copy */ + u8 bScopy; /* The pScopyFrom of some other Mem *might* point here */ #endif }; @@ -23270,7 +24106,7 @@ struct sqlite3_value { ** MEM_Int, MEM_Real, and MEM_IntReal. ** ** * MEM_Blob|MEM_Zero A blob in Mem.z of length Mem.n plus -** MEM.u.i extra 0x00 bytes at the end. +** Mem.u.nZero extra 0x00 bytes at the end. ** ** * MEM_Int Integer stored in Mem.u.i. ** @@ -23383,14 +24219,17 @@ struct sqlite3_context { int isError; /* Error code returned by the function. */ u8 enc; /* Encoding to use for results */ u8 skipFlag; /* Skip accumulator loading if true */ - u8 argc; /* Number of arguments */ - sqlite3_value *argv[1]; /* Argument set */ + u16 argc; /* Number of arguments */ + sqlite3_value *argv[FLEXARRAY]; /* Argument set */ }; -/* A bitfield type for use inside of structures. Always follow with :N where -** N is the number of bits. +/* +** The size (in bytes) of an sqlite3_context object that holds N +** argv[] arguments. */ -typedef unsigned bft; /* Bit Field Type */ +#define SZ_CONTEXT(N) \ + (offsetof(sqlite3_context,argv)+(N)*sizeof(sqlite3_value*)) + /* The ScanStatus object holds a single value for the ** sqlite3_stmt_scanstatus() interface. @@ -23451,7 +24290,7 @@ struct Vdbe { i64 nStmtDefCons; /* Number of def. constraints when stmt started */ i64 nStmtDefImmCons; /* Number of def. imm constraints when stmt started */ Mem *aMem; /* The memory locations */ - Mem **apArg; /* Arguments to currently executing user function */ + Mem **apArg; /* Arguments xUpdate and xFilter vtab methods */ VdbeCursor **apCsr; /* One element of this array for each open cursor */ Mem *aVar; /* Values for the OP_Variable opcode. */ @@ -23471,6 +24310,7 @@ struct Vdbe { #ifdef SQLITE_DEBUG int rcApp; /* errcode set by sqlite3_result_error_code() */ u32 nWrite; /* Number of write operations that have occurred */ + int napArg; /* Size of the apArg[] array */ #endif u16 nResColumn; /* Number of columns in one row of the result set */ u16 nResAlloc; /* Column slots allocated to aColName[] */ @@ -23523,16 +24363,21 @@ struct PreUpdate { VdbeCursor *pCsr; /* Cursor to read old values from */ int op; /* One of SQLITE_INSERT, UPDATE, DELETE */ u8 *aRecord; /* old.* database record */ - KeyInfo keyinfo; + KeyInfo *pKeyinfo; /* Key information */ UnpackedRecord *pUnpacked; /* Unpacked version of aRecord[] */ UnpackedRecord *pNewUnpacked; /* Unpacked version of new.* record */ int iNewReg; /* Register for new.* values */ int iBlobWrite; /* Value returned by preupdate_blobwrite() */ i64 iKey1; /* First key value passed to hook */ i64 iKey2; /* Second key value passed to hook */ + Mem oldipk; /* Memory cell holding "old" IPK value */ Mem *aNew; /* Array of new.* values */ Table *pTab; /* Schema object being updated */ Index *pPk; /* PK index if pTab is WITHOUT ROWID */ + sqlite3_value **apDflt; /* Array of default values, if required */ + struct { + u8 keyinfoSpace[SZ_KEYINFO_0]; /* Space to hold pKeyinfo[0] content */ + } uKey; }; /* @@ -23696,9 +24541,11 @@ SQLITE_PRIVATE int sqlite3VdbeCheckMemInvariants(Mem*); #endif #ifndef SQLITE_OMIT_FOREIGN_KEY -SQLITE_PRIVATE int sqlite3VdbeCheckFk(Vdbe *, int); +SQLITE_PRIVATE int sqlite3VdbeCheckFkImmediate(Vdbe*); +SQLITE_PRIVATE int sqlite3VdbeCheckFkDeferred(Vdbe*); #else -# define sqlite3VdbeCheckFk(p,i) 0 +# define sqlite3VdbeCheckFkImmediate(p) 0 +# define sqlite3VdbeCheckFkDeferred(p) 0 #endif #ifdef SQLITE_DEBUG @@ -23899,30 +24746,33 @@ SQLITE_PRIVATE int sqlite3LookasideUsed(sqlite3 *db, int *pHighwater){ nInit += countLookasideSlots(db->lookaside.pSmallInit); nFree += countLookasideSlots(db->lookaside.pSmallFree); #endif /* SQLITE_OMIT_TWOSIZE_LOOKASIDE */ - if( pHighwater ) *pHighwater = db->lookaside.nSlot - nInit; - return db->lookaside.nSlot - (nInit+nFree); + assert( db->lookaside.nSlot >= nInit+nFree ); + if( pHighwater ) *pHighwater = (int)(db->lookaside.nSlot - nInit); + return (int)(db->lookaside.nSlot - (nInit+nFree)); } /* ** Query status information for a single database connection */ -SQLITE_API int sqlite3_db_status( - sqlite3 *db, /* The database connection whose status is desired */ - int op, /* Status verb */ - int *pCurrent, /* Write current value here */ - int *pHighwater, /* Write high-water mark here */ - int resetFlag /* Reset high-water mark if true */ +SQLITE_API int sqlite3_db_status64( + sqlite3 *db, /* The database connection whose status is desired */ + int op, /* Status verb */ + sqlite3_int64 *pCurrent, /* Write current value here */ + sqlite3_int64 *pHighwtr, /* Write high-water mark here */ + int resetFlag /* Reset high-water mark if true */ ){ int rc = SQLITE_OK; /* Return code */ #ifdef SQLITE_ENABLE_API_ARMOR - if( !sqlite3SafetyCheckOk(db) || pCurrent==0|| pHighwater==0 ){ + if( !sqlite3SafetyCheckOk(db) || pCurrent==0|| pHighwtr==0 ){ return SQLITE_MISUSE_BKPT; } #endif sqlite3_mutex_enter(db->mutex); switch( op ){ case SQLITE_DBSTATUS_LOOKASIDE_USED: { - *pCurrent = sqlite3LookasideUsed(db, pHighwater); + int H = 0; + *pCurrent = sqlite3LookasideUsed(db, &H); + *pHighwtr = H; if( resetFlag ){ LookasideSlot *p = db->lookaside.pFree; if( p ){ @@ -23953,7 +24803,7 @@ SQLITE_API int sqlite3_db_status( assert( (op-SQLITE_DBSTATUS_LOOKASIDE_HIT)>=0 ); assert( (op-SQLITE_DBSTATUS_LOOKASIDE_HIT)<3 ); *pCurrent = 0; - *pHighwater = db->lookaside.anStat[op - SQLITE_DBSTATUS_LOOKASIDE_HIT]; + *pHighwtr = db->lookaside.anStat[op-SQLITE_DBSTATUS_LOOKASIDE_HIT]; if( resetFlag ){ db->lookaside.anStat[op - SQLITE_DBSTATUS_LOOKASIDE_HIT] = 0; } @@ -23967,7 +24817,7 @@ SQLITE_API int sqlite3_db_status( */ case SQLITE_DBSTATUS_CACHE_USED_SHARED: case SQLITE_DBSTATUS_CACHE_USED: { - int totalUsed = 0; + sqlite3_int64 totalUsed = 0; int i; sqlite3BtreeEnterAll(db); for(i=0; inDb; i++){ @@ -23983,18 +24833,18 @@ SQLITE_API int sqlite3_db_status( } sqlite3BtreeLeaveAll(db); *pCurrent = totalUsed; - *pHighwater = 0; + *pHighwtr = 0; break; } /* ** *pCurrent gets an accurate estimate of the amount of memory used ** to store the schema for all databases (main, temp, and any ATTACHed - ** databases. *pHighwater is set to zero. + ** databases. *pHighwtr is set to zero. */ case SQLITE_DBSTATUS_SCHEMA_USED: { - int i; /* Used to iterate through schemas */ - int nByte = 0; /* Used to accumulate return value */ + int i; /* Used to iterate through schemas */ + int nByte = 0; /* Used to accumulate return value */ sqlite3BtreeEnterAll(db); db->pnBytesFreed = &nByte; @@ -24028,7 +24878,7 @@ SQLITE_API int sqlite3_db_status( db->lookaside.pEnd = db->lookaside.pTrueEnd; sqlite3BtreeLeaveAll(db); - *pHighwater = 0; + *pHighwtr = 0; *pCurrent = nByte; break; } @@ -24036,7 +24886,7 @@ SQLITE_API int sqlite3_db_status( /* ** *pCurrent gets an accurate estimate of the amount of memory used ** to store all prepared statements. - ** *pHighwater is set to zero. + ** *pHighwtr is set to zero. */ case SQLITE_DBSTATUS_STMT_USED: { struct Vdbe *pVdbe; /* Used to iterate through VMs */ @@ -24051,7 +24901,7 @@ SQLITE_API int sqlite3_db_status( db->lookaside.pEnd = db->lookaside.pTrueEnd; db->pnBytesFreed = 0; - *pHighwater = 0; /* IMP: R-64479-57858 */ + *pHighwtr = 0; /* IMP: R-64479-57858 */ *pCurrent = nByte; break; @@ -24059,7 +24909,7 @@ SQLITE_API int sqlite3_db_status( /* ** Set *pCurrent to the total cache hits or misses encountered by all - ** pagers the database handle is connected to. *pHighwater is always set + ** pagers the database handle is connected to. *pHighwtr is always set ** to zero. */ case SQLITE_DBSTATUS_CACHE_SPILL: @@ -24079,19 +24929,39 @@ SQLITE_API int sqlite3_db_status( sqlite3PagerCacheStat(pPager, op, resetFlag, &nRet); } } - *pHighwater = 0; /* IMP: R-42420-56072 */ + *pHighwtr = 0; /* IMP: R-42420-56072 */ /* IMP: R-54100-20147 */ /* IMP: R-29431-39229 */ - *pCurrent = (int)nRet & 0x7fffffff; + *pCurrent = nRet; + break; + } + + /* Set *pCurrent to the number of bytes that the db database connection + ** has spilled to the filesystem in temporary files that could have been + ** stored in memory, had sufficient memory been available. + ** The *pHighwater is always set to zero. + */ + case SQLITE_DBSTATUS_TEMPBUF_SPILL: { + u64 nRet = 0; + if( db->aDb[1].pBt ){ + Pager *pPager = sqlite3BtreePager(db->aDb[1].pBt); + sqlite3PagerCacheStat(pPager, SQLITE_DBSTATUS_CACHE_WRITE, + resetFlag, &nRet); + nRet *= sqlite3BtreeGetPageSize(db->aDb[1].pBt); + } + nRet += db->nSpill; + if( resetFlag ) db->nSpill = 0; + *pHighwtr = 0; + *pCurrent = nRet; break; } /* Set *pCurrent to non-zero if there are unresolved deferred foreign ** key constraints. Set *pCurrent to zero if all foreign key constraints - ** have been satisfied. The *pHighwater is always set to zero. + ** have been satisfied. The *pHighwtr is always set to zero. */ case SQLITE_DBSTATUS_DEFERRED_FKS: { - *pHighwater = 0; /* IMP: R-11967-56545 */ + *pHighwtr = 0; /* IMP: R-11967-56545 */ *pCurrent = db->nDeferredImmCons>0 || db->nDeferredCons>0; break; } @@ -24104,6 +24974,31 @@ SQLITE_API int sqlite3_db_status( return rc; } +/* +** 32-bit variant of sqlite3_db_status64() +*/ +SQLITE_API int sqlite3_db_status( + sqlite3 *db, /* The database connection whose status is desired */ + int op, /* Status verb */ + int *pCurrent, /* Write current value here */ + int *pHighwtr, /* Write high-water mark here */ + int resetFlag /* Reset high-water mark if true */ +){ + sqlite3_int64 C = 0, H = 0; + int rc; +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) || pCurrent==0|| pHighwtr==0 ){ + return SQLITE_MISUSE_BKPT; + } +#endif + rc = sqlite3_db_status64(db, op, &C, &H, resetFlag); + if( rc==0 ){ + *pCurrent = C & 0x7fffffff; + *pHighwtr = H & 0x7fffffff; + } + return rc; +} + /************** End of status.c **********************************************/ /************** Begin file date.c ********************************************/ /* @@ -24179,13 +25074,14 @@ struct DateTime { int tz; /* Timezone offset in minutes */ double s; /* Seconds */ char validJD; /* True (1) if iJD is valid */ - char rawS; /* Raw numeric value stored in s */ char validYMD; /* True (1) if Y,M,D are valid */ char validHMS; /* True (1) if h,m,s are valid */ - char validTZ; /* True (1) if tz is valid */ - char tzSet; /* Timezone was set explicitly */ - char isError; /* An overflow has occurred */ - char useSubsec; /* Display subsecond precision */ + char nFloor; /* Days to implement "floor" */ + unsigned rawS : 1; /* Raw numeric value stored in s */ + unsigned isError : 1; /* An overflow has occurred */ + unsigned useSubsec : 1; /* Display subsecond precision */ + unsigned isUtc : 1; /* Time is known to be UTC */ + unsigned isLocal : 1; /* Time is known to be localtime */ }; @@ -24283,6 +25179,8 @@ static int parseTimezone(const char *zDate, DateTime *p){ sgn = +1; }else if( c=='Z' || c=='z' ){ zDate++; + p->isLocal = 0; + p->isUtc = 1; goto zulu_time; }else{ return c!=0; @@ -24293,9 +25191,12 @@ static int parseTimezone(const char *zDate, DateTime *p){ } zDate += 5; p->tz = sgn*(nMn + nHr*60); + if( p->tz==0 ){ /* Forum post 2025-09-17T10:12:14z */ + p->isLocal = 0; + p->isUtc = 1; + } zulu_time: while( sqlite3Isspace(*zDate) ){ zDate++; } - p->tzSet = 1; return *zDate!=0; } @@ -24328,6 +25229,9 @@ static int parseHhMmSs(const char *zDate, DateTime *p){ zDate++; } ms /= rScale; + /* Truncate to avoid problems with sub-milliseconds + ** rounding. https://sqlite.org/forum/forumpost/766a2c9231 */ + if( ms>0.999 ) ms = 0.999; } }else{ s = 0; @@ -24339,7 +25243,6 @@ static int parseHhMmSs(const char *zDate, DateTime *p){ p->m = m; p->s = s + ms; if( parseTimezone(zDate, p) ) return 1; - p->validTZ = (p->tz!=0)?1:0; return 0; } @@ -24378,23 +25281,48 @@ static void computeJD(DateTime *p){ Y--; M += 12; } - A = Y/100; - B = 2 - A + (A/4); + A = (Y+4800)/100; + B = 38 - A + (A/4); X1 = 36525*(Y+4716)/100; X2 = 306001*(M+1)/10000; p->iJD = (sqlite3_int64)((X1 + X2 + D + B - 1524.5 ) * 86400000); p->validJD = 1; if( p->validHMS ){ p->iJD += p->h*3600000 + p->m*60000 + (sqlite3_int64)(p->s*1000 + 0.5); - if( p->validTZ ){ + if( p->tz ){ p->iJD -= p->tz*60000; p->validYMD = 0; p->validHMS = 0; - p->validTZ = 0; + p->tz = 0; + p->isUtc = 1; + p->isLocal = 0; } } } +/* +** Given the YYYY-MM-DD information current in p, determine if there +** is day-of-month overflow and set nFloor to the number of days that +** would need to be subtracted from the date in order to bring the +** date back to the end of the month. +*/ +static void computeFloor(DateTime *p){ + assert( p->validYMD || p->isError ); + assert( p->D>=0 && p->D<=31 ); + assert( p->M>=0 && p->M<=12 ); + if( p->D<=28 ){ + p->nFloor = 0; + }else if( (1<M) & 0x15aa ){ + p->nFloor = 0; + }else if( p->M!=2 ){ + p->nFloor = (p->D==31); + }else if( p->Y%4!=0 || (p->Y%100==0 && p->Y%400!=0) ){ + p->nFloor = p->D - 28; + }else{ + p->nFloor = p->D - 29; + } +} + /* ** Parse dates of the form ** @@ -24433,12 +25361,16 @@ static int parseYyyyMmDd(const char *zDate, DateTime *p){ p->Y = neg ? -Y : Y; p->M = M; p->D = D; - if( p->validTZ ){ + computeFloor(p); + if( p->tz ){ computeJD(p); } return 0; } + +static void clearYMD_HMS_TZ(DateTime *p); /* Forward declaration */ + /* ** Set the time to the current time reported by the VFS. ** @@ -24448,6 +25380,9 @@ static int setDateTimeToCurrent(sqlite3_context *context, DateTime *p){ p->iJD = sqlite3StmtCurrentTime(context); if( p->iJD>0 ){ p->validJD = 1; + p->isUtc = 1; + p->isLocal = 0; + clearYMD_HMS_TZ(p); return 0; }else{ return 1; @@ -24531,7 +25466,7 @@ static int validJulianDay(sqlite3_int64 iJD){ ** Compute the Year, Month, and Day from the julian day number. */ static void computeYMD(DateTime *p){ - int Z, A, B, C, D, E, X1; + int Z, alpha, A, B, C, D, E, X1; if( p->validYMD ) return; if( !p->validJD ){ p->Y = 2000; @@ -24542,8 +25477,8 @@ static void computeYMD(DateTime *p){ return; }else{ Z = (int)((p->iJD + 43200000)/86400000); - A = (int)((Z - 1867216.25)/36524.25); - A = Z + 1 + A - (A/4); + alpha = (int)((Z + 32044.75)/36524.25) - 52; + A = Z + 1 + alpha - ((alpha+100)/4) + 25; B = A + 1524; C = (int)((B - 122.1)/365.25); D = (36525*(C&32767))/100; @@ -24586,7 +25521,7 @@ static void computeYMD_HMS(DateTime *p){ static void clearYMD_HMS_TZ(DateTime *p){ p->validYMD = 0; p->validHMS = 0; - p->validTZ = 0; + p->tz = 0; } #ifndef SQLITE_OMIT_LOCALTIME @@ -24718,7 +25653,7 @@ static int toLocaltime( p->validHMS = 1; p->validJD = 0; p->rawS = 0; - p->validTZ = 0; + p->tz = 0; p->isError = 0; return SQLITE_OK; } @@ -24738,12 +25673,12 @@ static const struct { float rLimit; /* Maximum NNN value for this transform */ float rXform; /* Constant used for this transform */ } aXformType[] = { - { 6, "second", 4.6427e+14, 1.0 }, - { 6, "minute", 7.7379e+12, 60.0 }, - { 4, "hour", 1.2897e+11, 3600.0 }, - { 3, "day", 5373485.0, 86400.0 }, - { 5, "month", 176546.0, 2592000.0 }, - { 4, "year", 14713.0, 31536000.0 }, + /* 0 */ { 6, "second", 4.6427e+14, 1.0 }, + /* 1 */ { 6, "minute", 7.7379e+12, 60.0 }, + /* 2 */ { 4, "hour", 1.2897e+11, 3600.0 }, + /* 3 */ { 3, "day", 5373485.0, 86400.0 }, + /* 4 */ { 5, "month", 176546.0, 2592000.0 }, + /* 5 */ { 4, "year", 14713.0, 31536000.0 }, }; /* @@ -24775,14 +25710,20 @@ static void autoAdjustDate(DateTime *p){ ** NNN.NNNN seconds ** NNN months ** NNN years +** +/-YYYY-MM-DD HH:MM:SS.SSS +** ceiling +** floor ** start of month ** start of year ** start of week ** start of day ** weekday N ** unixepoch +** auto ** localtime ** utc +** subsec +** subsecond ** ** Return 0 on success and 1 if there is any kind of error. If the error ** is in a system call (i.e. localtime()), then an error message is written @@ -24813,6 +25754,37 @@ static int parseModifier( } break; } + case 'c': { + /* + ** ceiling + ** + ** Resolve day-of-month overflow by rolling forward into the next + ** month. As this is the default action, this modifier is really + ** a no-op that is only included for symmetry. See "floor". + */ + if( sqlite3_stricmp(z, "ceiling")==0 ){ + computeJD(p); + clearYMD_HMS_TZ(p); + rc = 0; + p->nFloor = 0; + } + break; + } + case 'f': { + /* + ** floor + ** + ** Resolve day-of-month overflow by rolling back to the end of the + ** previous month. + */ + if( sqlite3_stricmp(z, "floor")==0 ){ + computeJD(p); + p->iJD -= p->nFloor*86400000; + clearYMD_HMS_TZ(p); + rc = 0; + } + break; + } case 'j': { /* ** julianday @@ -24839,7 +25811,9 @@ static int parseModifier( ** show local time. */ if( sqlite3_stricmp(z, "localtime")==0 && sqlite3NotPureFunc(pCtx) ){ - rc = toLocaltime(p, pCtx); + rc = p->isLocal ? SQLITE_OK : toLocaltime(p, pCtx); + p->isUtc = 0; + p->isLocal = 1; } break; } @@ -24864,7 +25838,7 @@ static int parseModifier( } #ifndef SQLITE_OMIT_LOCALTIME else if( sqlite3_stricmp(z, "utc")==0 && sqlite3NotPureFunc(pCtx) ){ - if( p->tzSet==0 ){ + if( p->isUtc==0 ){ i64 iOrigJD; /* Original localtime */ i64 iGuess; /* Guess at the corresponding utc time */ int cnt = 0; /* Safety to prevent infinite loop */ @@ -24887,7 +25861,8 @@ static int parseModifier( memset(p, 0, sizeof(*p)); p->iJD = iGuess; p->validJD = 1; - p->tzSet = 1; + p->isUtc = 1; + p->isLocal = 0; } rc = SQLITE_OK; } @@ -24907,7 +25882,7 @@ static int parseModifier( && r>=0.0 && r<7.0 && (n=(int)r)==r ){ sqlite3_int64 Z; computeYMD_HMS(p); - p->validTZ = 0; + p->tz = 0; p->validJD = 0; computeJD(p); Z = ((p->iJD + 129600000)/86400000) % 7; @@ -24947,7 +25922,7 @@ static int parseModifier( p->h = p->m = 0; p->s = 0.0; p->rawS = 0; - p->validTZ = 0; + p->tz = 0; p->validJD = 0; if( sqlite3_stricmp(z,"month")==0 ){ p->D = 1; @@ -25018,6 +25993,7 @@ static int parseModifier( x = p->M>0 ? (p->M-1)/12 : (p->M-12)/12; p->Y += x; p->M -= x*12; + computeFloor(p); computeJD(p); p->validHMS = 0; p->validYMD = 0; @@ -25064,11 +26040,12 @@ static int parseModifier( z += n; while( sqlite3Isspace(*z) ) z++; n = sqlite3Strlen30(z); - if( n>10 || n<3 ) break; + if( n<3 || n>10 ) break; if( sqlite3UpperToLower[(u8)z[n-1]]=='s' ) n--; computeJD(p); assert( rc==1 ); rRounder = r<0 ? -0.5 : +0.5; + p->nFloor = 0; for(i=0; iM += (int)r; x = p->M>0 ? (p->M-1)/12 : (p->M-12)/12; p->Y += x; p->M -= x*12; + computeFloor(p); p->validJD = 0; r -= (int)r; break; } case 5: { /* Special processing to add years */ int y = (int)r; - assert( strcmp(aXformType[i].zName,"year")==0 ); + assert( strcmp(aXformType[5].zName,"year")==0 ); computeYMD_HMS(p); + assert( p->M>=0 && p->M<=12 ); p->Y += y; + computeFloor(p); p->validJD = 0; r -= (int)r; break; @@ -25344,22 +26324,83 @@ static void dateFunc( } } +/* +** Compute the number of days after the most recent January 1. +** +** In other words, compute the zero-based day number for the +** current year: +** +** Jan01 = 0, Jan02 = 1, ..., Jan31 = 30, Feb01 = 31, ... +** Dec31 = 364 or 365. +*/ +static int daysAfterJan01(DateTime *pDate){ + DateTime jan01 = *pDate; + assert( jan01.validYMD ); + assert( jan01.validHMS ); + assert( pDate->validJD ); + jan01.validJD = 0; + jan01.M = 1; + jan01.D = 1; + computeJD(&jan01); + return (int)((pDate->iJD-jan01.iJD+43200000)/86400000); +} + +/* +** Return the number of days after the most recent Monday. +** +** In other words, return the day of the week according +** to this code: +** +** 0=Monday, 1=Tuesday, 2=Wednesday, ..., 6=Sunday. +*/ +static int daysAfterMonday(DateTime *pDate){ + assert( pDate->validJD ); + return (int)((pDate->iJD+43200000)/86400000) % 7; +} + +/* +** Return the number of days after the most recent Sunday. +** +** In other words, return the day of the week according +** to this code: +** +** 0=Sunday, 1=Monday, 2=Tuesday, ..., 6=Saturday +*/ +static int daysAfterSunday(DateTime *pDate){ + assert( pDate->validJD ); + return (int)((pDate->iJD+129600000)/86400000) % 7; +} + /* ** strftime( FORMAT, TIMESTRING, MOD, MOD, ...) ** ** Return a string described by FORMAT. Conversions as follows: ** -** %d day of month +** %d day of month 01-31 +** %e day of month 1-31 ** %f ** fractional seconds SS.SSS +** %F ISO date. YYYY-MM-DD +** %G ISO year corresponding to %V 0000-9999. +** %g 2-digit ISO year corresponding to %V 00-99 ** %H hour 00-24 -** %j day of year 000-366 +** %k hour 0-24 (leading zero converted to space) +** %I hour 01-12 +** %j day of year 001-366 ** %J ** julian day number +** %l hour 1-12 (leading zero converted to space) ** %m month 01-12 ** %M minute 00-59 +** %p "AM" or "PM" +** %P "am" or "pm" +** %R time as HH:MM ** %s seconds since 1970-01-01 ** %S seconds 00-59 -** %w day of week 0-6 Sunday==0 -** %W week of year 00-53 +** %T time as HH:MM:SS +** %u day of week 1-7 Monday==1, Sunday==7 +** %w day of week 0-6 Sunday==0, Monday==1 +** %U week of year 00-53 (First Sunday is start of week 01) +** %V week of year 01-53 (First week containing Thursday is week 01) +** %W week of year 00-53 (First Monday is start of week 01) ** %Y year 0000-9999 ** %% % */ @@ -25396,9 +26437,9 @@ static void strftimeFunc( sqlite3_str_appendf(&sRes, cf=='d' ? "%02d" : "%2d", x.D); break; } - case 'f': { + case 'f': { /* Fractional seconds. (Non-standard) */ double s = x.s; - if( s>59.999 ) s = 59.999; + if( NEVER(s>59.999) ) s = 59.999; sqlite3_str_appendf(&sRes, "%06.3f", s); break; } @@ -25406,6 +26447,21 @@ static void strftimeFunc( sqlite3_str_appendf(&sRes, "%04d-%02d-%02d", x.Y, x.M, x.D); break; } + case 'G': /* Fall thru */ + case 'g': { + DateTime y = x; + assert( y.validJD ); + /* Move y so that it is the Thursday in the same week as x */ + y.iJD += (3 - daysAfterMonday(&x))*86400000; + y.validYMD = 0; + computeYMD(&y); + if( cf=='g' ){ + sqlite3_str_appendf(&sRes, "%02d", y.Y%100); + }else{ + sqlite3_str_appendf(&sRes, "%04d", y.Y); + } + break; + } case 'H': case 'k': { sqlite3_str_appendf(&sRes, cf=='H' ? "%02d" : "%2d", x.h); @@ -25419,25 +26475,11 @@ static void strftimeFunc( sqlite3_str_appendf(&sRes, cf=='I' ? "%02d" : "%2d", h); break; } - case 'W': /* Fall thru */ - case 'j': { - int nDay; /* Number of days since 1st day of year */ - DateTime y = x; - y.validJD = 0; - y.M = 1; - y.D = 1; - computeJD(&y); - nDay = (int)((x.iJD-y.iJD+43200000)/86400000); - if( cf=='W' ){ - int wd; /* 0=Monday, 1=Tuesday, ... 6=Sunday */ - wd = (int)(((x.iJD+43200000)/86400000)%7); - sqlite3_str_appendf(&sRes,"%02d",(nDay+7-wd)/7); - }else{ - sqlite3_str_appendf(&sRes,"%03d",nDay+1); - } + case 'j': { /* Day of year. Jan01==1, Jan02==2, and so forth */ + sqlite3_str_appendf(&sRes,"%03d",daysAfterJan01(&x)+1); break; } - case 'J': { + case 'J': { /* Julian day number. (Non-standard) */ sqlite3_str_appendf(&sRes,"%.16g",x.iJD/86400000.0); break; } @@ -25480,13 +26522,33 @@ static void strftimeFunc( sqlite3_str_appendf(&sRes,"%02d:%02d:%02d", x.h, x.m, (int)x.s); break; } - case 'u': /* Fall thru */ - case 'w': { - char c = (char)(((x.iJD+129600000)/86400000) % 7) + '0'; + case 'u': /* Day of week. 1 to 7. Monday==1, Sunday==7 */ + case 'w': { /* Day of week. 0 to 6. Sunday==0, Monday==1 */ + char c = (char)daysAfterSunday(&x) + '0'; if( c=='0' && cf=='u' ) c = '7'; sqlite3_str_appendchar(&sRes, 1, c); break; } + case 'U': { /* Week num. 00-53. First Sun of the year is week 01 */ + sqlite3_str_appendf(&sRes,"%02d", + (daysAfterJan01(&x)-daysAfterSunday(&x)+7)/7); + break; + } + case 'V': { /* Week num. 01-53. First week with a Thur is week 01 */ + DateTime y = x; + /* Adjust y so that is the Thursday in the same week as x */ + assert( y.validJD ); + y.iJD += (3 - daysAfterMonday(&x))*86400000; + y.validYMD = 0; + computeYMD(&y); + sqlite3_str_appendf(&sRes,"%02d", daysAfterJan01(&y)/7+1); + break; + } + case 'W': { /* Week num. 00-53. First Mon of the year is week 01 */ + sqlite3_str_appendf(&sRes,"%02d", + (daysAfterJan01(&x)-daysAfterMonday(&x)+7)/7); + break; + } case 'Y': { sqlite3_str_appendf(&sRes,"%04d",x.Y); break; @@ -25633,9 +26695,7 @@ static void timediffFunc( d1.iJD = d2.iJD - d1.iJD; d1.iJD += (u64)1486995408 * (u64)100000; } - d1.validYMD = 0; - d1.validHMS = 0; - d1.validTZ = 0; + clearYMD_HMS_TZ(&d1); computeYMD_HMS(&d1); sqlite3StrAccumInit(&sRes, 0, 0, 0, 100); sqlite3_str_appendf(&sRes, "%c%04d-%02d-%02d %02d:%02d:%06.3f", @@ -25704,6 +26764,36 @@ static void currentTimeFunc( } #endif +#if !defined(SQLITE_OMIT_DATETIME_FUNCS) && defined(SQLITE_DEBUG) +/* +** datedebug(...) +** +** This routine returns JSON that describes the internal DateTime object. +** Used for debugging and testing only. Subject to change. +*/ +static void datedebugFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + DateTime x; + if( isDate(context, argc, argv, &x)==0 ){ + char *zJson; + zJson = sqlite3_mprintf( + "{iJD:%lld,Y:%d,M:%d,D:%d,h:%d,m:%d,tz:%d," + "s:%.3f,validJD:%d,validYMS:%d,validHMS:%d," + "nFloor:%d,rawS:%d,isError:%d,useSubsec:%d," + "isUtc:%d,isLocal:%d}", + x.iJD, x.Y, x.M, x.D, x.h, x.m, x.tz, + x.s, x.validJD, x.validYMD, x.validHMS, + x.nFloor, x.rawS, x.isError, x.useSubsec, + x.isUtc, x.isLocal); + sqlite3_result_text(context, zJson, -1, sqlite3_free); + } +} +#endif /* !SQLITE_OMIT_DATETIME_FUNCS && SQLITE_DEBUG */ + + /* ** This function registered all of the above C functions as SQL ** functions. This should be the only routine in this file with @@ -25719,6 +26809,9 @@ SQLITE_PRIVATE void sqlite3RegisterDateTimeFunctions(void){ PURE_DATE(datetime, -1, 0, 0, datetimeFunc ), PURE_DATE(strftime, -1, 0, 0, strftimeFunc ), PURE_DATE(timediff, 2, 0, 0, timediffFunc ), +#ifdef SQLITE_DEBUG + PURE_DATE(datedebug, -1, 0, 0, datedebugFunc ), +#endif DFUNCTION(current_time, 0, 0, 0, ctimeFunc ), DFUNCTION(current_timestamp, 0, 0, 0, ctimestampFunc), DFUNCTION(current_date, 0, 0, 0, cdateFunc ), @@ -28787,16 +29880,29 @@ SQLITE_API void sqlite3_mutex_leave(sqlite3_mutex *p){ /* ** The sqlite3_mutex_held() and sqlite3_mutex_notheld() routine are ** intended for use inside assert() statements. +** +** Because these routines raise false-positive alerts in TSAN, disable +** them (make them always return 1) when compiling with TSAN. */ SQLITE_API int sqlite3_mutex_held(sqlite3_mutex *p){ +# if defined(__has_feature) +# if __has_feature(thread_sanitizer) + p = 0; +# endif +# endif assert( p==0 || sqlite3GlobalConfig.mutex.xMutexHeld ); return p==0 || sqlite3GlobalConfig.mutex.xMutexHeld(p); } SQLITE_API int sqlite3_mutex_notheld(sqlite3_mutex *p){ +# if defined(__has_feature) +# if __has_feature(thread_sanitizer) + p = 0; +# endif +# endif assert( p==0 || sqlite3GlobalConfig.mutex.xMutexNotheld ); return p==0 || sqlite3GlobalConfig.mutex.xMutexNotheld(p); } -#endif +#endif /* NDEBUG */ #endif /* !defined(SQLITE_MUTEX_OMIT) */ @@ -29467,6 +30573,8 @@ SQLITE_PRIVATE sqlite3_mutex_methods const *sqlite3DefaultMutex(void){ #ifdef __CYGWIN__ # include +# include /* amalgamator: dontcache */ +# include /* amalgamator: dontcache */ # include /* amalgamator: dontcache */ #endif @@ -30134,6 +31242,24 @@ static void sqlite3MallocAlarm(int nByte){ sqlite3_mutex_enter(mem0.mutex); } +#ifdef SQLITE_DEBUG +/* +** This routine is called whenever an out-of-memory condition is seen, +** It's only purpose to to serve as a breakpoint for gdb or similar +** code debuggers when working on out-of-memory conditions, for example +** caused by PRAGMA hard_heap_limit=N. +*/ +static SQLITE_NOINLINE void test_oom_breakpoint(u64 n){ + static u64 nOomFault = 0; + nOomFault += n; + /* The assert() is never reached in a human lifetime. It is here mostly + ** to prevent code optimizers from optimizing out this function. */ + assert( (nOomFault>>32) < 0xffffffff ); +} +#else +# define test_oom_breakpoint(X) /* No-op for production builds */ +#endif + /* ** Do a memory allocation with statistics and alarms. Assume the ** lock is already held. @@ -30160,6 +31286,7 @@ static void mallocWithAlarm(int n, void **pp){ if( mem0.hardLimit ){ nUsed = sqlite3StatusValue(SQLITE_STATUS_MEMORY_USED); if( nUsed >= mem0.hardLimit - nFull ){ + test_oom_breakpoint(1); *pp = 0; return; } @@ -30448,6 +31575,7 @@ SQLITE_PRIVATE void *sqlite3Realloc(void *pOld, u64 nBytes){ sqlite3MallocAlarm(nDiff); if( mem0.hardLimit>0 && nUsed >= mem0.hardLimit - nDiff ){ sqlite3_mutex_leave(mem0.mutex); + test_oom_breakpoint(1); return 0; } } @@ -30841,17 +31969,17 @@ SQLITE_PRIVATE int sqlite3ApiExit(sqlite3* db, int rc){ #define etPERCENT 7 /* Percent symbol. %% */ #define etCHARX 8 /* Characters. %c */ /* The rest are extensions, not normally found in printf() */ -#define etSQLESCAPE 9 /* Strings with '\'' doubled. %q */ -#define etSQLESCAPE2 10 /* Strings with '\'' doubled and enclosed in '', - NULL pointers replaced by SQL NULL. %Q */ -#define etTOKEN 11 /* a pointer to a Token structure */ -#define etSRCITEM 12 /* a pointer to a SrcItem */ -#define etPOINTER 13 /* The %p conversion */ -#define etSQLESCAPE3 14 /* %w -> Strings with '\"' doubled */ -#define etORDINAL 15 /* %r -> 1st, 2nd, 3rd, 4th, etc. English only */ -#define etDECIMAL 16 /* %d or %u, but not %x, %o */ +#define etESCAPE_q 9 /* Strings with '\'' doubled. %q */ +#define etESCAPE_Q 10 /* Strings with '\'' doubled and enclosed in '', + NULL pointers replaced by SQL NULL. %Q */ +#define etTOKEN 11 /* a pointer to a Token structure */ +#define etSRCITEM 12 /* a pointer to a SrcItem */ +#define etPOINTER 13 /* The %p conversion */ +#define etESCAPE_w 14 /* %w -> Strings with '\"' doubled */ +#define etORDINAL 15 /* %r -> 1st, 2nd, 3rd, 4th, etc. English only */ +#define etDECIMAL 16 /* %d or %u, but not %x, %o */ -#define etINVALID 17 /* Any unrecognized conversion type */ +#define etINVALID 17 /* Any unrecognized conversion type */ /* @@ -30870,6 +31998,7 @@ typedef struct et_info { /* Information about each format field */ etByte type; /* Conversion paradigm */ etByte charset; /* Offset into aDigits[] of the digits string */ etByte prefix; /* Offset into aPrefix[] of the prefix string */ + char iNxt; /* Next with same hash, or 0 for end of chain */ } et_info; /* @@ -30878,44 +32007,61 @@ typedef struct et_info { /* Information about each format field */ #define FLAG_SIGNED 1 /* True if the value to convert is signed */ #define FLAG_STRING 4 /* Allow infinite precision */ - /* -** The following table is searched linearly, so it is good to put the -** most frequently used conversion types first. +** The table is searched by hash. In the case of %C where C is the character +** and that character has ASCII value j, then the hash is j%23. +** +** The order of the entries in fmtinfo[] and the hash chain was entered +** manually, but based on the output of the following TCL script: */ +#if 0 /***** Beginning of script ******/ +foreach c {d s g z q Q w c o u x X f e E G i n % p T S r} { + scan $c %c x + set n($c) $x +} +set mx [llength [array names n]] +puts "count: $mx" + +set mx 27 +puts "*********** mx=$mx ************" +for {set r 0} {$r<$mx} {incr r} { + puts -nonewline [format %2d: $r] + foreach c [array names n] { + if {($n($c))%$mx==$r} {puts -nonewline " $c"} + } + puts "" +} +#endif /***** End of script ********/ + static const char aDigits[] = "0123456789ABCDEF0123456789abcdef"; static const char aPrefix[] = "-x0\000X0"; -static const et_info fmtinfo[] = { - { 'd', 10, 1, etDECIMAL, 0, 0 }, - { 's', 0, 4, etSTRING, 0, 0 }, - { 'g', 0, 1, etGENERIC, 30, 0 }, - { 'z', 0, 4, etDYNSTRING, 0, 0 }, - { 'q', 0, 4, etSQLESCAPE, 0, 0 }, - { 'Q', 0, 4, etSQLESCAPE2, 0, 0 }, - { 'w', 0, 4, etSQLESCAPE3, 0, 0 }, - { 'c', 0, 0, etCHARX, 0, 0 }, - { 'o', 8, 0, etRADIX, 0, 2 }, - { 'u', 10, 0, etDECIMAL, 0, 0 }, - { 'x', 16, 0, etRADIX, 16, 1 }, - { 'X', 16, 0, etRADIX, 0, 4 }, -#ifndef SQLITE_OMIT_FLOATING_POINT - { 'f', 0, 1, etFLOAT, 0, 0 }, - { 'e', 0, 1, etEXP, 30, 0 }, - { 'E', 0, 1, etEXP, 14, 0 }, - { 'G', 0, 1, etGENERIC, 14, 0 }, -#endif - { 'i', 10, 1, etDECIMAL, 0, 0 }, - { 'n', 0, 0, etSIZE, 0, 0 }, - { '%', 0, 0, etPERCENT, 0, 0 }, - { 'p', 16, 0, etPOINTER, 0, 1 }, - - /* All the rest are undocumented and are for internal use only */ - { 'T', 0, 0, etTOKEN, 0, 0 }, - { 'S', 0, 0, etSRCITEM, 0, 0 }, - { 'r', 10, 1, etORDINAL, 0, 0 }, +static const et_info fmtinfo[23] = { + /* 0 */ { 's', 0, 4, etSTRING, 0, 0, 1 }, + /* 1 */ { 'E', 0, 1, etEXP, 14, 0, 0 }, /* Hash: 0 */ + /* 2 */ { 'u', 10, 0, etDECIMAL, 0, 0, 3 }, + /* 3 */ { 'G', 0, 1, etGENERIC, 14, 0, 0 }, /* Hash: 2 */ + /* 4 */ { 'w', 0, 4, etESCAPE_w, 0, 0, 0 }, + /* 5 */ { 'x', 16, 0, etRADIX, 16, 1, 0 }, + /* 6 */ { 'c', 0, 0, etCHARX, 0, 0, 0 }, /* Hash: 7 */ + /* 7 */ { 'z', 0, 4, etDYNSTRING, 0, 0, 6 }, + /* 8 */ { 'd', 10, 1, etDECIMAL, 0, 0, 0 }, + /* 9 */ { 'e', 0, 1, etEXP, 30, 0, 0 }, + /* 10 */ { 'f', 0, 1, etFLOAT, 0, 0, 0 }, + /* 11 */ { 'g', 0, 1, etGENERIC, 30, 0, 0 }, + /* 12 */ { 'Q', 0, 4, etESCAPE_Q, 0, 0, 0 }, + /* 13 */ { 'i', 10, 1, etDECIMAL, 0, 0, 0 }, + /* 14 */ { '%', 0, 0, etPERCENT, 0, 0, 16 }, + /* 15 */ { 'T', 0, 0, etTOKEN, 0, 0, 0 }, + /* 16 */ { 'S', 0, 0, etSRCITEM, 0, 0, 0 }, /* Hash: 14 */ + /* 17 */ { 'X', 16, 0, etRADIX, 0, 4, 0 }, /* Hash: 19 */ + /* 18 */ { 'n', 0, 0, etSIZE, 0, 0, 0 }, + /* 19 */ { 'o', 8, 0, etRADIX, 0, 2, 17 }, + /* 20 */ { 'p', 16, 0, etPOINTER, 0, 1, 0 }, + /* 21 */ { 'q', 0, 4, etESCAPE_q, 0, 0, 0 }, + /* 22 */ { 'r', 10, 1, etORDINAL, 0, 0, 0 } }; -/* Notes: +/* Additional Notes: ** ** %S Takes a pointer to SrcItem. Shows name or database.name ** %!S Like %S but prefer the zName over the zAlias @@ -31042,7 +32188,10 @@ SQLITE_API void sqlite3_str_vappendf( #if HAVE_STRCHRNUL fmt = strchrnul(fmt, '%'); #else - do{ fmt++; }while( *fmt && *fmt != '%' ); + fmt = strchr(fmt, '%'); + if( fmt==0 ){ + fmt = bufpt + strlen(bufpt); + } #endif sqlite3_str_append(pAccum, bufpt, (int)(fmt - bufpt)); if( *fmt==0 ) break; @@ -31156,6 +32305,9 @@ SQLITE_API void sqlite3_str_vappendf( }while( !done && (c=(*++fmt))!=0 ); /* Fetch the info entry for the field */ +#ifdef SQLITE_EBCDIC + /* The hash table only works for ASCII. For EBCDIC, we need to do + ** a linear search of the table */ infop = &fmtinfo[0]; xtype = etINVALID; for(idx=0; idxtype; + }else{ + infop = &fmtinfo[0]; + xtype = etINVALID; + } +#endif /* ** At this point, variables are initialized as follows: @@ -31232,6 +32398,14 @@ SQLITE_API void sqlite3_str_vappendf( } prefix = 0; } + +#if WHERETRACE_ENABLED + if( xtype==etPOINTER && sqlite3WhereTrace & 0x100000 ) longvalue = 0; +#endif +#if TREETRACE_ENABLED + if( xtype==etPOINTER && sqlite3TreeTrace & 0x100000 ) longvalue = 0; +#endif + if( longvalue==0 ) flag_alternateform = 0; if( flag_zeropad && precision0 ) precision--; /* ** If the field type is etGENERIC, then convert to either etEXP ** or etFLOAT, as appropriate. */ if( xtype==etGENERIC ){ + assert( precision>0 ); + precision--; flag_rtz = !flag_alternateform; if( exp<-4 || exp>precision ){ xtype = etEXP; @@ -31488,25 +32677,7 @@ SQLITE_API void sqlite3_str_vappendf( } }else{ unsigned int ch = va_arg(ap,unsigned int); - if( ch<0x00080 ){ - buf[0] = ch & 0xff; - length = 1; - }else if( ch<0x00800 ){ - buf[0] = 0xc0 + (u8)((ch>>6)&0x1f); - buf[1] = 0x80 + (u8)(ch & 0x3f); - length = 2; - }else if( ch<0x10000 ){ - buf[0] = 0xe0 + (u8)((ch>>12)&0x0f); - buf[1] = 0x80 + (u8)((ch>>6) & 0x3f); - buf[2] = 0x80 + (u8)(ch & 0x3f); - length = 3; - }else{ - buf[0] = 0xf0 + (u8)((ch>>18) & 0x07); - buf[1] = 0x80 + (u8)((ch>>12) & 0x3f); - buf[2] = 0x80 + (u8)((ch>>6) & 0x3f); - buf[3] = 0x80 + (u8)(ch & 0x3f); - length = 4; - } + length = sqlite3AppendOneUtf8Character(buf, ch); } if( precision>1 ){ i64 nPrior = 1; @@ -31586,22 +32757,31 @@ SQLITE_API void sqlite3_str_vappendf( while( ii>=0 ) if( (bufpt[ii--] & 0xc0)==0x80 ) width++; } break; - case etSQLESCAPE: /* %q: Escape ' characters */ - case etSQLESCAPE2: /* %Q: Escape ' and enclose in '...' */ - case etSQLESCAPE3: { /* %w: Escape " characters */ + case etESCAPE_q: /* %q: Escape ' characters */ + case etESCAPE_Q: /* %Q: Escape ' and enclose in '...' */ + case etESCAPE_w: { /* %w: Escape " characters */ i64 i, j, k, n; - int needQuote, isnull; + int needQuote = 0; char ch; - char q = ((xtype==etSQLESCAPE3)?'"':'\''); /* Quote character */ char *escarg; + char q; if( bArgList ){ escarg = getTextArg(pArgList); }else{ escarg = va_arg(ap,char*); } - isnull = escarg==0; - if( isnull ) escarg = (xtype==etSQLESCAPE2 ? "NULL" : "(NULL)"); + if( escarg==0 ){ + escarg = (xtype==etESCAPE_Q ? "NULL" : "(NULL)"); + }else if( xtype==etESCAPE_Q ){ + needQuote = 1; + } + if( xtype==etESCAPE_w ){ + q = '"'; + flag_alternateform = 0; + }else{ + q = '\''; + } /* For %q, %Q, and %w, the precision is the number of bytes (or ** characters if the ! flags is present) to use from the input. ** Because of the extra quoting characters inserted, the number @@ -31614,7 +32794,30 @@ SQLITE_API void sqlite3_str_vappendf( while( (escarg[i+1]&0xc0)==0x80 ){ i++; } } } - needQuote = !isnull && xtype==etSQLESCAPE2; + if( flag_alternateform ){ + /* For %#q, do unistr()-style backslash escapes for + ** all control characters, and for backslash itself. + ** For %#Q, do the same but only if there is at least + ** one control character. */ + u32 nBack = 0; + u32 nCtrl = 0; + for(k=0; ketBUFSIZE ){ bufpt = zExtra = printfTempBuf(pAccum, n); @@ -31623,13 +32826,41 @@ SQLITE_API void sqlite3_str_vappendf( bufpt = buf; } j = 0; - if( needQuote ) bufpt[j++] = q; + if( needQuote ){ + if( needQuote==2 ){ + memcpy(&bufpt[j], "unistr('", 8); + j += 8; + }else{ + bufpt[j++] = '\''; + } + } k = i; - for(i=0; i=0x10 ? '1' : '0'; + bufpt[j++] = "0123456789abcdef"[ch&0xf]; + } + } + }else{ + for(i=0; izAlias && !flag_altform2 ){ sqlite3_str_appendall(pAccum, pItem->zAlias); }else if( pItem->zName ){ - if( pItem->zDatabase ){ - sqlite3_str_appendall(pAccum, pItem->zDatabase); + if( pItem->fg.fixedSchema==0 + && pItem->fg.isSubquery==0 + && pItem->u4.zDatabase!=0 + ){ + sqlite3_str_appendall(pAccum, pItem->u4.zDatabase); sqlite3_str_append(pAccum, ".", 1); } sqlite3_str_appendall(pAccum, pItem->zName); }else if( pItem->zAlias ){ sqlite3_str_appendall(pAccum, pItem->zAlias); - }else{ - Select *pSel = pItem->pSelect; + }else if( ALWAYS(pItem->fg.isSubquery) ){/* Because of tag-20240424-1 */ + Select *pSel = pItem->u4.pSubq->pSelect; assert( pSel!=0 ); if( pSel->selFlags & SF_NestedFrom ){ sqlite3_str_appendf(pAccum, "(join-%u)", pSel->selId); + }else if( pSel->selFlags & SF_MultiValue ){ + assert( !pItem->fg.isTabFunc && !pItem->fg.isIndexedBy ); + sqlite3_str_appendf(pAccum, "%u-ROW VALUES CLAUSE", + pItem->u1.nRow); }else{ sqlite3_str_appendf(pAccum, "(subquery-%u)", pSel->selId); } @@ -31746,6 +32984,7 @@ SQLITE_PRIVATE void sqlite3RecordErrorOffsetOfExpr(sqlite3 *db, const Expr *pExp pExpr = pExpr->pLeft; } if( pExpr==0 ) return; + if( ExprHasProperty(pExpr, EP_FromDDL) ) return; db->errByteOffset = pExpr->w.iOfst; } @@ -31864,7 +33103,7 @@ SQLITE_API void sqlite3_str_appendall(sqlite3_str *p, const char *z){ static SQLITE_NOINLINE char *strAccumFinishRealloc(StrAccum *p){ char *zText; assert( p->mxAlloc>0 && !isMalloced(p) ); - zText = sqlite3DbMallocRaw(p->db, p->nChar+1 ); + zText = sqlite3DbMallocRaw(p->db, 1+(u64)p->nChar ); if( zText ){ memcpy(zText, p->zText, p->nChar+1); p->printfFlags |= SQLITE_PRINTF_MALLOCED; @@ -32109,6 +33348,15 @@ SQLITE_API char *sqlite3_snprintf(int n, char *zBuf, const char *zFormat, ...){ return zBuf; } +/* Maximum size of an sqlite3_log() message. */ +#if defined(SQLITE_MAX_LOG_MESSAGE) + /* Leave the definition as supplied */ +#elif SQLITE_PRINT_BUF_SIZE*10>10000 +# define SQLITE_MAX_LOG_MESSAGE 10000 +#else +# define SQLITE_MAX_LOG_MESSAGE (SQLITE_PRINT_BUF_SIZE*10) +#endif + /* ** This is the routine that actually formats the sqlite3_log() message. ** We house it in a separate routine from sqlite3_log() to avoid using @@ -32125,7 +33373,7 @@ SQLITE_API char *sqlite3_snprintf(int n, char *zBuf, const char *zFormat, ...){ */ static void renderLogMsg(int iErrCode, const char *zFormat, va_list ap){ StrAccum acc; /* String accumulator */ - char zMsg[SQLITE_PRINT_BUF_SIZE*3]; /* Complete log message */ + char zMsg[SQLITE_MAX_LOG_MESSAGE]; /* Complete log message */ sqlite3StrAccumInit(&acc, 0, zMsg, sizeof(zMsg), 0); sqlite3_str_vappendf(&acc, zFormat, ap); @@ -32450,9 +33698,11 @@ SQLITE_PRIVATE void sqlite3TreeViewSrcList(TreeView *pView, const SrcList *pSrc) sqlite3StrAccumInit(&x, 0, zLine, sizeof(zLine), 0); x.printfFlags |= SQLITE_PRINTF_INTERNAL; sqlite3_str_appendf(&x, "{%d:*} %!S", pItem->iCursor, pItem); - if( pItem->pTab ){ - sqlite3_str_appendf(&x, " tab=%Q nCol=%d ptr=%p used=%llx", - pItem->pTab->zName, pItem->pTab->nCol, pItem->pTab, pItem->colUsed); + if( pItem->pSTab ){ + sqlite3_str_appendf(&x, " tab=%Q nCol=%d ptr=%p used=%llx%s", + pItem->pSTab->zName, pItem->pSTab->nCol, pItem->pSTab, + pItem->colUsed, + pItem->fg.rowidUsed ? "+rowid" : ""); } if( (pItem->fg.jointype & (JT_LEFT|JT_RIGHT))==(JT_LEFT|JT_RIGHT) ){ sqlite3_str_appendf(&x, " FULL-OUTER-JOIN"); @@ -32470,10 +33720,13 @@ SQLITE_PRIVATE void sqlite3TreeViewSrcList(TreeView *pView, const SrcList *pSrc) sqlite3_str_appendf(&x, " DDL"); } if( pItem->fg.isCte ){ - sqlite3_str_appendf(&x, " CteUse=0x%p", pItem->u2.pCteUse); + static const char *aMat[] = {",MAT", "", ",NO-MAT"}; + sqlite3_str_appendf(&x, " CteUse=%d%s", + pItem->u2.pCteUse->nUse, + aMat[pItem->u2.pCteUse->eM10d]); } if( pItem->fg.isOn || (pItem->fg.isUsing==0 && pItem->u3.pOn!=0) ){ - sqlite3_str_appendf(&x, " ON"); + sqlite3_str_appendf(&x, " isOn"); } if( pItem->fg.isTabFunc ) sqlite3_str_appendf(&x, " isTabFunc"); if( pItem->fg.isCorrelated ) sqlite3_str_appendf(&x, " isCorrelated"); @@ -32481,23 +33734,31 @@ SQLITE_PRIVATE void sqlite3TreeViewSrcList(TreeView *pView, const SrcList *pSrc) if( pItem->fg.viaCoroutine ) sqlite3_str_appendf(&x, " viaCoroutine"); if( pItem->fg.notCte ) sqlite3_str_appendf(&x, " notCte"); if( pItem->fg.isNestedFrom ) sqlite3_str_appendf(&x, " isNestedFrom"); + if( pItem->fg.fixedSchema ) sqlite3_str_appendf(&x, " fixedSchema"); + if( pItem->fg.hadSchema ) sqlite3_str_appendf(&x, " hadSchema"); + if( pItem->fg.isSubquery ) sqlite3_str_appendf(&x, " isSubquery"); sqlite3StrAccumFinish(&x); sqlite3TreeViewItem(pView, zLine, inSrc-1); n = 0; - if( pItem->pSelect ) n++; + if( pItem->fg.isSubquery ) n++; if( pItem->fg.isTabFunc ) n++; - if( pItem->fg.isUsing ) n++; + if( pItem->fg.isUsing || pItem->u3.pOn!=0 ) n++; if( pItem->fg.isUsing ){ sqlite3TreeViewIdList(pView, pItem->u3.pUsing, (--n)>0, "USING"); + }else if( pItem->u3.pOn!=0 ){ + sqlite3TreeViewItem(pView, "ON", (--n)>0); + sqlite3TreeViewExpr(pView, pItem->u3.pOn, 0); + sqlite3TreeViewPop(&pView); } - if( pItem->pSelect ){ - if( pItem->pTab ){ - Table *pTab = pItem->pTab; + if( pItem->fg.isSubquery ){ + assert( n==1 ); + if( pItem->pSTab ){ + Table *pTab = pItem->pSTab; sqlite3TreeViewColumnList(pView, pTab->aCol, pTab->nCol, 1); } - assert( (int)pItem->fg.isNestedFrom == IsNestedFrom(pItem->pSelect) ); - sqlite3TreeViewSelect(pView, pItem->pSelect, (--n)>0); + assert( (int)pItem->fg.isNestedFrom == IsNestedFrom(pItem) ); + sqlite3TreeViewSelect(pView, pItem->u4.pSubq->pSelect, 0); } if( pItem->fg.isTabFunc ){ sqlite3TreeViewExprList(pView, pItem->u1.pFuncArg, 0, "func-args:"); @@ -32539,7 +33800,7 @@ SQLITE_PRIVATE void sqlite3TreeViewSelect(TreeView *pView, const Select *p, u8 m n = 1000; }else{ n = 0; - if( p->pSrc && p->pSrc->nSrc ) n++; + if( p->pSrc && p->pSrc->nSrc && p->pSrc->nAlloc ) n++; if( p->pWhere ) n++; if( p->pGroupBy ) n++; if( p->pHaving ) n++; @@ -32565,7 +33826,7 @@ SQLITE_PRIVATE void sqlite3TreeViewSelect(TreeView *pView, const Select *p, u8 m sqlite3TreeViewPop(&pView); } #endif - if( p->pSrc && p->pSrc->nSrc ){ + if( p->pSrc && p->pSrc->nSrc && p->pSrc->nAlloc ){ sqlite3TreeViewPush(&pView, (n--)>0); sqlite3TreeViewLine(pView, "FROM"); sqlite3TreeViewSrcList(pView, p->pSrc); @@ -32601,7 +33862,7 @@ SQLITE_PRIVATE void sqlite3TreeViewSelect(TreeView *pView, const Select *p, u8 m sqlite3TreeViewItem(pView, "LIMIT", (n--)>0); sqlite3TreeViewExpr(pView, p->pLimit->pLeft, p->pLimit->pRight!=0); if( p->pLimit->pRight ){ - sqlite3TreeViewItem(pView, "OFFSET", (n--)>0); + sqlite3TreeViewItem(pView, "OFFSET", 0); sqlite3TreeViewExpr(pView, p->pLimit->pRight, 0); sqlite3TreeViewPop(&pView); } @@ -33073,7 +34334,8 @@ SQLITE_PRIVATE void sqlite3TreeViewExpr(TreeView *pView, const Expr *pExpr, u8 m case OE_Ignore: zType = "ignore"; break; } assert( !ExprHasProperty(pExpr, EP_IntValue) ); - sqlite3TreeViewLine(pView, "RAISE %s(%Q)", zType, pExpr->u.zToken); + sqlite3TreeViewLine(pView, "RAISE %s", zType); + sqlite3TreeViewExpr(pView, pExpr->pLeft, 0); break; } #endif @@ -33153,9 +34415,10 @@ SQLITE_PRIVATE void sqlite3TreeViewBareExprList( sqlite3TreeViewLine(pView, "%s", zLabel); for(i=0; inExpr; i++){ int j = pList->a[i].u.x.iOrderByCol; + u8 sortFlags = pList->a[i].fg.sortFlags; char *zName = pList->a[i].zEName; int moreToFollow = inExpr - 1; - if( j || zName ){ + if( j || zName || sortFlags ){ sqlite3TreeViewPush(&pView, moreToFollow); moreToFollow = 0; sqlite3TreeViewLine(pView, 0); @@ -33176,13 +34439,18 @@ SQLITE_PRIVATE void sqlite3TreeViewBareExprList( } } if( j ){ - fprintf(stdout, "iOrderByCol=%d", j); + fprintf(stdout, "iOrderByCol=%d ", j); + } + if( sortFlags & KEYINFO_ORDER_DESC ){ + fprintf(stdout, "DESC "); + }else if( sortFlags & KEYINFO_ORDER_BIGNULL ){ + fprintf(stdout, "NULLS-LAST"); } fprintf(stdout, "\n"); fflush(stdout); } sqlite3TreeViewExpr(pView, pList->a[i].pExpr, moreToFollow); - if( j || zName ){ + if( j || zName || sortFlags ){ sqlite3TreeViewPop(&pView); } } @@ -33219,21 +34487,7 @@ SQLITE_PRIVATE void sqlite3TreeViewBareIdList( if( zName==0 ) zName = "(null)"; sqlite3TreeViewPush(&pView, moreToFollow); sqlite3TreeViewLine(pView, 0); - if( pList->eU4==EU4_NONE ){ - fprintf(stdout, "%s\n", zName); - }else if( pList->eU4==EU4_IDX ){ - fprintf(stdout, "%s (%d)\n", zName, pList->a[i].u4.idx); - }else{ - assert( pList->eU4==EU4_EXPR ); - if( pList->a[i].u4.pExpr==0 ){ - fprintf(stdout, "%s (pExpr=NULL)\n", zName); - }else{ - fprintf(stdout, "%s\n", zName); - sqlite3TreeViewPush(&pView, inId-1); - sqlite3TreeViewExpr(pView, pList->a[i].u4.pExpr, 0); - sqlite3TreeViewPop(&pView); - } - } + fprintf(stdout, "%s\n", zName); sqlite3TreeViewPop(&pView); } } @@ -33543,6 +34797,10 @@ SQLITE_PRIVATE void sqlite3TreeViewTrigger( ** accessible to the debugging, and to avoid warnings about unused ** functions. But these routines only exist in debugging builds, so they ** do not contaminate the interface. +** +** See Also: +** +** sqlite3ShowWhereTerm() in where.c */ SQLITE_PRIVATE void sqlite3ShowExpr(const Expr *p){ sqlite3TreeViewExpr(0,p,0); } SQLITE_PRIVATE void sqlite3ShowExprList(const ExprList *p){ sqlite3TreeViewExprList(0,p,0,0);} @@ -34114,6 +35372,35 @@ static const unsigned char sqlite3Utf8Trans1[] = { } \ } +/* +** Write a single UTF8 character whose value is v into the +** buffer starting at zOut. zOut must be sized to hold at +** least four bytes. Return the number of bytes needed +** to encode the new character. +*/ +SQLITE_PRIVATE int sqlite3AppendOneUtf8Character(char *zOut, u32 v){ + if( v<0x00080 ){ + zOut[0] = (u8)(v & 0xff); + return 1; + } + if( v<0x00800 ){ + zOut[0] = 0xc0 + (u8)((v>>6) & 0x1f); + zOut[1] = 0x80 + (u8)(v & 0x3f); + return 2; + } + if( v<0x10000 ){ + zOut[0] = 0xe0 + (u8)((v>>12) & 0x0f); + zOut[1] = 0x80 + (u8)((v>>6) & 0x3f); + zOut[2] = 0x80 + (u8)(v & 0x3f); + return 3; + } + zOut[0] = 0xf0 + (u8)((v>>18) & 0x07); + zOut[1] = 0x80 + (u8)((v>>12) & 0x3f); + zOut[2] = 0x80 + (u8)((v>>6) & 0x3f); + zOut[3] = 0x80 + (u8)(v & 0x3f); + return 4; +} + /* ** Translate a single UTF-8 character. Return the unicode value. ** @@ -34145,7 +35432,7 @@ static const unsigned char sqlite3Utf8Trans1[] = { c = *(zIn++); \ if( c>=0xc0 ){ \ c = sqlite3Utf8Trans1[c-0xc0]; \ - while( zIn!=zTerm && (*zIn & 0xc0)==0x80 ){ \ + while( zIn=0xd8 && c<0xdc && z[0]>=0xdc && z[0]<0xe0 ) z += 2; + if( c>=0xd8 && c<0xdc && z<=zEnd && z[0]>=0xdc && z[0]<0xe0 ) z += 2; n++; } return (int)(z-(unsigned char const *)zIn) @@ -34646,6 +35935,19 @@ SQLITE_PRIVATE int sqlite3IsNaN(double x){ } #endif /* SQLITE_OMIT_FLOATING_POINT */ +#ifndef SQLITE_OMIT_FLOATING_POINT +/* +** Return true if the floating point value is NaN or +Inf or -Inf. +*/ +SQLITE_PRIVATE int sqlite3IsOverflow(double x){ + int rc; /* The value return */ + u64 y; + memcpy(&y,&x,sizeof(y)); + rc = IsOvfl(y); + return rc; +} +#endif /* SQLITE_OMIT_FLOATING_POINT */ + /* ** Compute a string length that is limited to what can be stored in ** lower 30 bits of a 32-bit signed integer. @@ -34889,6 +36191,44 @@ SQLITE_PRIVATE void sqlite3DequoteExpr(Expr *p){ sqlite3Dequote(p->u.zToken); } +/* +** Expression p is a QNUMBER (quoted number). Dequote the value in p->u.zToken +** and set the type to INTEGER or FLOAT. "Quoted" integers or floats are those +** that contain '_' characters that must be removed before further processing. +*/ +SQLITE_PRIVATE void sqlite3DequoteNumber(Parse *pParse, Expr *p){ + assert( p!=0 || pParse->db->mallocFailed ); + if( p ){ + const char *pIn = p->u.zToken; + char *pOut = p->u.zToken; + int bHex = (pIn[0]=='0' && (pIn[1]=='x' || pIn[1]=='X')); + int iValue; + assert( p->op==TK_QNUMBER ); + p->op = TK_INTEGER; + do { + if( *pIn!=SQLITE_DIGIT_SEPARATOR ){ + *pOut++ = *pIn; + if( *pIn=='e' || *pIn=='E' || *pIn=='.' ) p->op = TK_FLOAT; + }else{ + if( (bHex==0 && (!sqlite3Isdigit(pIn[-1]) || !sqlite3Isdigit(pIn[1]))) + || (bHex==1 && (!sqlite3Isxdigit(pIn[-1]) || !sqlite3Isxdigit(pIn[1]))) + ){ + sqlite3ErrorMsg(pParse, "unrecognized token: \"%s\"", p->u.zToken); + } + } + }while( *pIn++ ); + if( bHex ) p->op = TK_INTEGER; + + /* tag-20240227-a: If after dequoting, the number is an integer that + ** fits in 32 bits, then it must be converted into EP_IntValue. Other + ** parts of the code expect this. See also tag-20240227-b. */ + if( p->op==TK_INTEGER && sqlite3GetInt32(p->u.zToken, &iValue) ){ + p->u.iValue = iValue; + p->flags |= EP_IntValue; + } + } +} + /* ** If the input token p is quoted, try to adjust the token to remove ** the quotes. This is not always possible: @@ -35066,6 +36406,8 @@ SQLITE_PRIVATE int sqlite3AtoF(const char *z, double *pResult, int length, u8 en int eValid = 1; /* True exponent is either not used or is well-formed */ int nDigit = 0; /* Number of digits processed */ int eType = 1; /* 1: pure integer, 2+: fractional -1 or less: bad UTF16 */ + u64 s2; /* round-tripped significand */ + double rr[2]; assert( enc==SQLITE_UTF8 || enc==SQLITE_UTF16LE || enc==SQLITE_UTF16BE ); *pResult = 0.0; /* Default return value, in case of an error */ @@ -35168,7 +36510,7 @@ SQLITE_PRIVATE int sqlite3AtoF(const char *z, double *pResult, int length, u8 en e = (e*esign) + d; /* Try to adjust the exponent to make it smaller */ - while( e>0 && s<(LARGEST_UINT64/10) ){ + while( e>0 && s<((LARGEST_UINT64-0x7ff)/10) ){ s *= 10; e--; } @@ -35177,68 +36519,52 @@ SQLITE_PRIVATE int sqlite3AtoF(const char *z, double *pResult, int length, u8 en e++; } - if( e==0 ){ - *pResult = s; - }else if( sqlite3Config.bUseLongDouble ){ - LONGDOUBLE_TYPE r = (LONGDOUBLE_TYPE)s; - if( e>0 ){ - while( e>=100 ){ e-=100; r *= 1.0e+100L; } - while( e>=10 ){ e-=10; r *= 1.0e+10L; } - while( e>=1 ){ e-=1; r *= 1.0e+01L; } - }else{ - while( e<=-100 ){ e+=100; r *= 1.0e-100L; } - while( e<=-10 ){ e+=10; r *= 1.0e-10L; } - while( e<=-1 ){ e+=1; r *= 1.0e-01L; } - } - assert( r>=0.0 ); - if( r>+1.7976931348623157081452742373e+308L ){ -#ifdef INFINITY - *pResult = +INFINITY; -#else - *pResult = 1.0e308*10.0; + rr[0] = (double)s; + assert( sizeof(s2)==sizeof(rr[0]) ); +#ifdef SQLITE_DEBUG + rr[1] = 18446744073709549568.0; + memcpy(&s2, &rr[1], sizeof(s2)); + assert( s2==0x43efffffffffffffLL ); #endif - }else{ - *pResult = (double)r; - } - }else{ - double rr[2]; - u64 s2; - rr[0] = (double)s; + /* Largest double that can be safely converted to u64 + ** vvvvvvvvvvvvvvvvvvvvvv */ + if( rr[0]<=18446744073709549568.0 ){ s2 = (u64)rr[0]; -#if defined(_MSC_VER) && _MSC_VER<1700 - if( s2==0x8000000000000000LL ){ s2 = 2*(u64)(0.5*rr[0]); } -#endif rr[1] = s>=s2 ? (double)(s - s2) : -(double)(s2 - s); - if( e>0 ){ - while( e>=100 ){ - e -= 100; - dekkerMul2(rr, 1.0e+100, -1.5902891109759918046e+83); - } - while( e>=10 ){ - e -= 10; - dekkerMul2(rr, 1.0e+10, 0.0); - } - while( e>=1 ){ - e -= 1; - dekkerMul2(rr, 1.0e+01, 0.0); - } - }else{ - while( e<=-100 ){ - e += 100; - dekkerMul2(rr, 1.0e-100, -1.99918998026028836196e-117); - } - while( e<=-10 ){ - e += 10; - dekkerMul2(rr, 1.0e-10, -3.6432197315497741579e-27); - } - while( e<=-1 ){ - e += 1; - dekkerMul2(rr, 1.0e-01, -5.5511151231257827021e-18); - } + }else{ + rr[1] = 0.0; + } + assert( rr[1]<=1.0e-10*rr[0] ); /* Equal only when rr[0]==0.0 */ + + if( e>0 ){ + while( e>=100 ){ + e -= 100; + dekkerMul2(rr, 1.0e+100, -1.5902891109759918046e+83); + } + while( e>=10 ){ + e -= 10; + dekkerMul2(rr, 1.0e+10, 0.0); + } + while( e>=1 ){ + e -= 1; + dekkerMul2(rr, 1.0e+01, 0.0); + } + }else{ + while( e<=-100 ){ + e += 100; + dekkerMul2(rr, 1.0e-100, -1.99918998026028836196e-117); + } + while( e<=-10 ){ + e += 10; + dekkerMul2(rr, 1.0e-10, -3.6432197315497741579e-27); + } + while( e<=-1 ){ + e += 1; + dekkerMul2(rr, 1.0e-01, -5.5511151231257827021e-18); } - *pResult = rr[0]+rr[1]; - if( sqlite3IsNaN(*pResult) ) *pResult = 1e300*1e300; } + *pResult = rr[0]+rr[1]; + if( sqlite3IsNaN(*pResult) ) *pResult = 1e300*1e300; if( sign<0 ) *pResult = -*pResult; assert( !sqlite3IsNaN(*pResult) ); @@ -35542,10 +36868,13 @@ SQLITE_PRIVATE int sqlite3Atoi(const char *z){ ** Decode a floating-point value into an approximate decimal ** representation. ** -** Round the decimal representation to n significant digits if -** n is positive. Or round to -n signficant digits after the -** decimal point if n is negative. No rounding is performed if -** n is zero. +** If iRound<=0 then round to -iRound significant digits to the +** the left of the decimal point, or to a maximum of mxRound total +** significant digits. +** +** If iRound>0 round to min(iRound,mxRound) significant digits total. +** +** mxRound must be positive. ** ** The significant digits of the decimal representation are ** stored in p->z[] which is a often (but not always) a pointer @@ -35556,8 +36885,11 @@ SQLITE_PRIVATE void sqlite3FpDecode(FpDecode *p, double r, int iRound, int mxRou int i; u64 v; int e, exp = 0; + double rr[2]; + p->isSpecial = 0; p->z = p->zBuf; + assert( mxRound>0 ); /* Convert negative numbers to positive. Deal with Infinity, 0.0, and ** NaN. */ @@ -35584,62 +36916,45 @@ SQLITE_PRIVATE void sqlite3FpDecode(FpDecode *p, double r, int iRound, int mxRou /* Multiply r by powers of ten until it lands somewhere in between ** 1.0e+19 and 1.0e+17. + ** + ** Use Dekker-style double-double computation to increase the + ** precision. + ** + ** The error terms on constants like 1.0e+100 computed using the + ** decimal extension, for example as follows: + ** + ** SELECT decimal_exp(decimal_sub('1.0e+100',decimal(1.0e+100))); */ - if( sqlite3Config.bUseLongDouble ){ - LONGDOUBLE_TYPE rr = r; - if( rr>=1.0e+19 ){ - while( rr>=1.0e+119L ){ exp+=100; rr *= 1.0e-100L; } - while( rr>=1.0e+29L ){ exp+=10; rr *= 1.0e-10L; } - while( rr>=1.0e+19L ){ exp++; rr *= 1.0e-1L; } - }else{ - while( rr<1.0e-97L ){ exp-=100; rr *= 1.0e+100L; } - while( rr<1.0e+07L ){ exp-=10; rr *= 1.0e+10L; } - while( rr<1.0e+17L ){ exp--; rr *= 1.0e+1L; } + rr[0] = r; + rr[1] = 0.0; + if( rr[0]>9.223372036854774784e+18 ){ + while( rr[0]>9.223372036854774784e+118 ){ + exp += 100; + dekkerMul2(rr, 1.0e-100, -1.99918998026028836196e-117); + } + while( rr[0]>9.223372036854774784e+28 ){ + exp += 10; + dekkerMul2(rr, 1.0e-10, -3.6432197315497741579e-27); + } + while( rr[0]>9.223372036854774784e+18 ){ + exp += 1; + dekkerMul2(rr, 1.0e-01, -5.5511151231257827021e-18); } - v = (u64)rr; }else{ - /* If high-precision floating point is not available using "long double", - ** then use Dekker-style double-double computation to increase the - ** precision. - ** - ** The error terms on constants like 1.0e+100 computed using the - ** decimal extension, for example as follows: - ** - ** SELECT decimal_exp(decimal_sub('1.0e+100',decimal(1.0e+100))); - */ - double rr[2]; - rr[0] = r; - rr[1] = 0.0; - if( rr[0]>9.223372036854774784e+18 ){ - while( rr[0]>9.223372036854774784e+118 ){ - exp += 100; - dekkerMul2(rr, 1.0e-100, -1.99918998026028836196e-117); - } - while( rr[0]>9.223372036854774784e+28 ){ - exp += 10; - dekkerMul2(rr, 1.0e-10, -3.6432197315497741579e-27); - } - while( rr[0]>9.223372036854774784e+18 ){ - exp += 1; - dekkerMul2(rr, 1.0e-01, -5.5511151231257827021e-18); - } - }else{ - while( rr[0]<9.223372036854774784e-83 ){ - exp -= 100; - dekkerMul2(rr, 1.0e+100, -1.5902891109759918046e+83); - } - while( rr[0]<9.223372036854774784e+07 ){ - exp -= 10; - dekkerMul2(rr, 1.0e+10, 0.0); - } - while( rr[0]<9.22337203685477478e+17 ){ - exp -= 1; - dekkerMul2(rr, 1.0e+01, 0.0); - } + while( rr[0]<9.223372036854774784e-83 ){ + exp -= 100; + dekkerMul2(rr, 1.0e+100, -1.5902891109759918046e+83); + } + while( rr[0]<9.223372036854774784e+07 ){ + exp -= 10; + dekkerMul2(rr, 1.0e+10, 0.0); + } + while( rr[0]<9.22337203685477478e+17 ){ + exp -= 1; + dekkerMul2(rr, 1.0e+01, 0.0); } - v = rr[1]<0.0 ? (u64)rr[0]-(u64)(-rr[1]) : (u64)rr[0]+(u64)rr[1]; } - + v = rr[1]<0.0 ? (u64)rr[0]-(u64)(-rr[1]) : (u64)rr[0]+(u64)rr[1]; /* Extract significant digits. */ i = sizeof(p->zBuf)-1; @@ -35682,7 +36997,11 @@ SQLITE_PRIVATE void sqlite3FpDecode(FpDecode *p, double r, int iRound, int mxRou } p->z = &p->zBuf[i+1]; assert( i+p->n < sizeof(p->zBuf) ); - while( ALWAYS(p->n>0) && p->z[p->n-1]=='0' ){ p->n--; } + assert( p->n>0 ); + while( p->z[p->n-1]=='0' ){ + p->n--; + assert( p->n>0 ); + } } /* @@ -36187,7 +37506,7 @@ SQLITE_PRIVATE int sqlite3MulInt64(i64 *pA, i64 iB){ } /* -** Compute the absolute value of a 32-bit signed integer, of possible. Or +** Compute the absolute value of a 32-bit signed integer, if possible. Or ** if the integer has a value of -2147483648, return +2147483647 */ SQLITE_PRIVATE int sqlite3AbsInt32(int x){ @@ -36410,104 +37729,6 @@ SQLITE_PRIVATE int sqlite3VListNameToNum(VList *pIn, const char *zName, int nNam return 0; } -/* -** High-resolution hardware timer used for debugging and testing only. -*/ -#if defined(VDBE_PROFILE) \ - || defined(SQLITE_PERFORMANCE_TRACE) \ - || defined(SQLITE_ENABLE_STMT_SCANSTATUS) -/************** Include hwtime.h in the middle of util.c *********************/ -/************** Begin file hwtime.h ******************************************/ -/* -** 2008 May 27 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -****************************************************************************** -** -** This file contains inline asm code for retrieving "high-performance" -** counters for x86 and x86_64 class CPUs. -*/ -#ifndef SQLITE_HWTIME_H -#define SQLITE_HWTIME_H - -/* -** The following routine only works on Pentium-class (or newer) processors. -** It uses the RDTSC opcode to read the cycle count value out of the -** processor and returns that value. This can be used for high-res -** profiling. -*/ -#if !defined(__STRICT_ANSI__) && \ - (defined(__GNUC__) || defined(_MSC_VER)) && \ - (defined(i386) || defined(__i386__) || defined(_M_IX86)) - - #if defined(__GNUC__) - - __inline__ sqlite_uint64 sqlite3Hwtime(void){ - unsigned int lo, hi; - __asm__ __volatile__ ("rdtsc" : "=a" (lo), "=d" (hi)); - return (sqlite_uint64)hi << 32 | lo; - } - - #elif defined(_MSC_VER) - - __declspec(naked) __inline sqlite_uint64 __cdecl sqlite3Hwtime(void){ - __asm { - rdtsc - ret ; return value at EDX:EAX - } - } - - #endif - -#elif !defined(__STRICT_ANSI__) && (defined(__GNUC__) && defined(__x86_64__)) - - __inline__ sqlite_uint64 sqlite3Hwtime(void){ - unsigned int lo, hi; - __asm__ __volatile__ ("rdtsc" : "=a" (lo), "=d" (hi)); - return (sqlite_uint64)hi << 32 | lo; - } - -#elif !defined(__STRICT_ANSI__) && (defined(__GNUC__) && defined(__ppc__)) - - __inline__ sqlite_uint64 sqlite3Hwtime(void){ - unsigned long long retval; - unsigned long junk; - __asm__ __volatile__ ("\n\ - 1: mftbu %1\n\ - mftb %L0\n\ - mftbu %0\n\ - cmpw %0,%1\n\ - bne 1b" - : "=r" (retval), "=r" (junk)); - return retval; - } - -#else - - /* - ** asm() is needed for hardware timing support. Without asm(), - ** disable the sqlite3Hwtime() routine. - ** - ** sqlite3Hwtime() is only used for some obscure debugging - ** and analysis configurations, not in any deliverable, so this - ** should not be a great loss. - */ -SQLITE_PRIVATE sqlite_uint64 sqlite3Hwtime(void){ return ((sqlite_uint64)0); } - -#endif - -#endif /* !defined(SQLITE_HWTIME_H) */ - -/************** End of hwtime.h **********************************************/ -/************** Continuing where we left off in util.c ***********************/ -#endif - /************** End of util.c ************************************************/ /************** Begin file hash.c ********************************************/ /* @@ -36566,12 +37787,19 @@ SQLITE_PRIVATE void sqlite3HashClear(Hash *pH){ */ static unsigned int strHash(const char *z){ unsigned int h = 0; - unsigned char c; - while( (c = (unsigned char)*z++)!=0 ){ /*OPTIMIZATION-IF-TRUE*/ + while( z[0] ){ /*OPTIMIZATION-IF-TRUE*/ /* Knuth multiplicative hashing. (Sorting & Searching, p. 510). ** 0x9e3779b1 is 2654435761 which is the closest prime number to - ** (2**32)*golden_ratio, where golden_ratio = (sqrt(5) - 1)/2. */ - h += sqlite3UpperToLower[c]; + ** (2**32)*golden_ratio, where golden_ratio = (sqrt(5) - 1)/2. + ** + ** Only bits 0xdf for ASCII and bits 0xbf for EBCDIC each octet are + ** hashed since the omitted bits determine the upper/lower case difference. + */ +#ifdef SQLITE_EBCDIC + h += 0xbf & (unsigned char)*(z++); +#else + h += 0xdf & (unsigned char)*(z++); +#endif h *= 0x9e3779b1; } return h; @@ -36644,9 +37872,8 @@ static int rehash(Hash *pH, unsigned int new_size){ pH->htsize = new_size = sqlite3MallocSize(new_ht)/sizeof(struct _ht); memset(new_ht, 0, new_size*sizeof(struct _ht)); for(elem=pH->first, pH->first=0; elem; elem = next_elem){ - unsigned int h = strHash(elem->pKey) % new_size; next_elem = elem->next; - insertElement(pH, &new_ht[h], elem); + insertElement(pH, &new_ht[elem->h % new_size], elem); } return 1; } @@ -36664,23 +37891,22 @@ static HashElem *findElementWithHash( HashElem *elem; /* Used to loop thru the element list */ unsigned int count; /* Number of elements left to test */ unsigned int h; /* The computed hash */ - static HashElem nullElement = { 0, 0, 0, 0 }; + static HashElem nullElement = { 0, 0, 0, 0, 0 }; + h = strHash(pKey); if( pH->ht ){ /*OPTIMIZATION-IF-TRUE*/ struct _ht *pEntry; - h = strHash(pKey) % pH->htsize; - pEntry = &pH->ht[h]; + pEntry = &pH->ht[h % pH->htsize]; elem = pEntry->chain; count = pEntry->count; }else{ - h = 0; elem = pH->first; count = pH->count; } if( pHash ) *pHash = h; while( count ){ assert( elem!=0 ); - if( sqlite3StrICmp(elem->pKey,pKey)==0 ){ + if( h==elem->h && sqlite3StrICmp(elem->pKey,pKey)==0 ){ return elem; } elem = elem->next; @@ -36692,10 +37918,9 @@ static HashElem *findElementWithHash( /* Remove a single entry from the hash table given a pointer to that ** element and a hash on the element's key. */ -static void removeElementGivenHash( +static void removeElement( Hash *pH, /* The pH containing "elem" */ - HashElem* elem, /* The element to be removed from the pH */ - unsigned int h /* Hash value for the element */ + HashElem *elem /* The element to be removed from the pH */ ){ struct _ht *pEntry; if( elem->prev ){ @@ -36707,7 +37932,7 @@ static void removeElementGivenHash( elem->next->prev = elem->prev; } if( pH->ht ){ - pEntry = &pH->ht[h]; + pEntry = &pH->ht[elem->h % pH->htsize]; if( pEntry->chain==elem ){ pEntry->chain = elem->next; } @@ -36758,7 +37983,7 @@ SQLITE_PRIVATE void *sqlite3HashInsert(Hash *pH, const char *pKey, void *data){ if( elem->data ){ void *old_data = elem->data; if( data==0 ){ - removeElementGivenHash(pH,elem,h); + removeElement(pH,elem); }else{ elem->data = data; elem->pKey = pKey; @@ -36769,18 +37994,17 @@ SQLITE_PRIVATE void *sqlite3HashInsert(Hash *pH, const char *pKey, void *data){ new_elem = (HashElem*)sqlite3Malloc( sizeof(HashElem) ); if( new_elem==0 ) return data; new_elem->pKey = pKey; + new_elem->h = h; new_elem->data = data; pH->count++; - if( pH->count>=10 && pH->count > 2*pH->htsize ){ - if( rehash(pH, pH->count*2) ){ - assert( pH->htsize>0 ); - h = strHash(pKey) % pH->htsize; - } + if( pH->count>=5 && pH->count > 2*pH->htsize ){ + rehash(pH, pH->count*3); } - insertElement(pH, pH->ht ? &pH->ht[h] : 0, new_elem); + insertElement(pH, pH->ht ? &pH->ht[new_elem->h % pH->htsize] : 0, new_elem); return 0; } + /************** End of hash.c ************************************************/ /************** Begin file opcodes.c *****************************************/ /* Automatically generated. Do not edit */ @@ -36828,163 +38052,164 @@ SQLITE_PRIVATE const char *sqlite3OpcodeName(int i){ /* 30 */ "SeekRowid" OpHelp("intkey=r[P3]"), /* 31 */ "NotExists" OpHelp("intkey=r[P3]"), /* 32 */ "Last" OpHelp(""), - /* 33 */ "IfSmaller" OpHelp(""), + /* 33 */ "IfSizeBetween" OpHelp(""), /* 34 */ "SorterSort" OpHelp(""), /* 35 */ "Sort" OpHelp(""), /* 36 */ "Rewind" OpHelp(""), - /* 37 */ "SorterNext" OpHelp(""), - /* 38 */ "Prev" OpHelp(""), - /* 39 */ "Next" OpHelp(""), - /* 40 */ "IdxLE" OpHelp("key=r[P3@P4]"), - /* 41 */ "IdxGT" OpHelp("key=r[P3@P4]"), - /* 42 */ "IdxLT" OpHelp("key=r[P3@P4]"), + /* 37 */ "IfEmpty" OpHelp("if( empty(P1) ) goto P2"), + /* 38 */ "SorterNext" OpHelp(""), + /* 39 */ "Prev" OpHelp(""), + /* 40 */ "Next" OpHelp(""), + /* 41 */ "IdxLE" OpHelp("key=r[P3@P4]"), + /* 42 */ "IdxGT" OpHelp("key=r[P3@P4]"), /* 43 */ "Or" OpHelp("r[P3]=(r[P1] || r[P2])"), /* 44 */ "And" OpHelp("r[P3]=(r[P1] && r[P2])"), - /* 45 */ "IdxGE" OpHelp("key=r[P3@P4]"), - /* 46 */ "RowSetRead" OpHelp("r[P3]=rowset(P1)"), - /* 47 */ "RowSetTest" OpHelp("if r[P3] in rowset(P1) goto P2"), - /* 48 */ "Program" OpHelp(""), - /* 49 */ "FkIfZero" OpHelp("if fkctr[P1]==0 goto P2"), - /* 50 */ "IsNull" OpHelp("if r[P1]==NULL goto P2"), - /* 51 */ "NotNull" OpHelp("if r[P1]!=NULL goto P2"), - /* 52 */ "Ne" OpHelp("IF r[P3]!=r[P1]"), - /* 53 */ "Eq" OpHelp("IF r[P3]==r[P1]"), - /* 54 */ "Gt" OpHelp("IF r[P3]>r[P1]"), - /* 55 */ "Le" OpHelp("IF r[P3]<=r[P1]"), - /* 56 */ "Lt" OpHelp("IF r[P3]=r[P1]"), - /* 58 */ "ElseEq" OpHelp(""), - /* 59 */ "IfPos" OpHelp("if r[P1]>0 then r[P1]-=P3, goto P2"), - /* 60 */ "IfNotZero" OpHelp("if r[P1]!=0 then r[P1]--, goto P2"), - /* 61 */ "DecrJumpZero" OpHelp("if (--r[P1])==0 goto P2"), - /* 62 */ "IncrVacuum" OpHelp(""), - /* 63 */ "VNext" OpHelp(""), - /* 64 */ "Filter" OpHelp("if key(P3@P4) not in filter(P1) goto P2"), - /* 65 */ "PureFunc" OpHelp("r[P3]=func(r[P2@NP])"), - /* 66 */ "Function" OpHelp("r[P3]=func(r[P2@NP])"), - /* 67 */ "Return" OpHelp(""), - /* 68 */ "EndCoroutine" OpHelp(""), - /* 69 */ "HaltIfNull" OpHelp("if r[P3]=null halt"), - /* 70 */ "Halt" OpHelp(""), - /* 71 */ "Integer" OpHelp("r[P2]=P1"), - /* 72 */ "Int64" OpHelp("r[P2]=P4"), - /* 73 */ "String" OpHelp("r[P2]='P4' (len=P1)"), - /* 74 */ "BeginSubrtn" OpHelp("r[P2]=NULL"), - /* 75 */ "Null" OpHelp("r[P2..P3]=NULL"), - /* 76 */ "SoftNull" OpHelp("r[P1]=NULL"), - /* 77 */ "Blob" OpHelp("r[P2]=P4 (len=P1)"), - /* 78 */ "Variable" OpHelp("r[P2]=parameter(P1,P4)"), - /* 79 */ "Move" OpHelp("r[P2@P3]=r[P1@P3]"), - /* 80 */ "Copy" OpHelp("r[P2@P3+1]=r[P1@P3+1]"), - /* 81 */ "SCopy" OpHelp("r[P2]=r[P1]"), - /* 82 */ "IntCopy" OpHelp("r[P2]=r[P1]"), - /* 83 */ "FkCheck" OpHelp(""), - /* 84 */ "ResultRow" OpHelp("output=r[P1@P2]"), - /* 85 */ "CollSeq" OpHelp(""), - /* 86 */ "AddImm" OpHelp("r[P1]=r[P1]+P2"), - /* 87 */ "RealAffinity" OpHelp(""), - /* 88 */ "Cast" OpHelp("affinity(r[P1])"), - /* 89 */ "Permutation" OpHelp(""), - /* 90 */ "Compare" OpHelp("r[P1@P3] <-> r[P2@P3]"), - /* 91 */ "IsTrue" OpHelp("r[P2] = coalesce(r[P1]==TRUE,P3) ^ P4"), - /* 92 */ "ZeroOrNull" OpHelp("r[P2] = 0 OR NULL"), - /* 93 */ "Offset" OpHelp("r[P3] = sqlite_offset(P1)"), - /* 94 */ "Column" OpHelp("r[P3]=PX cursor P1 column P2"), - /* 95 */ "TypeCheck" OpHelp("typecheck(r[P1@P2])"), - /* 96 */ "Affinity" OpHelp("affinity(r[P1@P2])"), - /* 97 */ "MakeRecord" OpHelp("r[P3]=mkrec(r[P1@P2])"), - /* 98 */ "Count" OpHelp("r[P2]=count()"), - /* 99 */ "ReadCookie" OpHelp(""), - /* 100 */ "SetCookie" OpHelp(""), - /* 101 */ "ReopenIdx" OpHelp("root=P2 iDb=P3"), - /* 102 */ "BitAnd" OpHelp("r[P3]=r[P1]&r[P2]"), - /* 103 */ "BitOr" OpHelp("r[P3]=r[P1]|r[P2]"), - /* 104 */ "ShiftLeft" OpHelp("r[P3]=r[P2]<>r[P1]"), - /* 106 */ "Add" OpHelp("r[P3]=r[P1]+r[P2]"), - /* 107 */ "Subtract" OpHelp("r[P3]=r[P2]-r[P1]"), - /* 108 */ "Multiply" OpHelp("r[P3]=r[P1]*r[P2]"), - /* 109 */ "Divide" OpHelp("r[P3]=r[P2]/r[P1]"), - /* 110 */ "Remainder" OpHelp("r[P3]=r[P2]%r[P1]"), - /* 111 */ "Concat" OpHelp("r[P3]=r[P2]+r[P1]"), - /* 112 */ "OpenRead" OpHelp("root=P2 iDb=P3"), - /* 113 */ "OpenWrite" OpHelp("root=P2 iDb=P3"), - /* 114 */ "BitNot" OpHelp("r[P2]= ~r[P1]"), - /* 115 */ "OpenDup" OpHelp(""), - /* 116 */ "OpenAutoindex" OpHelp("nColumn=P2"), - /* 117 */ "String8" OpHelp("r[P2]='P4'"), - /* 118 */ "OpenEphemeral" OpHelp("nColumn=P2"), - /* 119 */ "SorterOpen" OpHelp(""), - /* 120 */ "SequenceTest" OpHelp("if( cursor[P1].ctr++ ) pc = P2"), - /* 121 */ "OpenPseudo" OpHelp("P3 columns in r[P2]"), - /* 122 */ "Close" OpHelp(""), - /* 123 */ "ColumnsUsed" OpHelp(""), - /* 124 */ "SeekScan" OpHelp("Scan-ahead up to P1 rows"), - /* 125 */ "SeekHit" OpHelp("set P2<=seekHit<=P3"), - /* 126 */ "Sequence" OpHelp("r[P2]=cursor[P1].ctr++"), - /* 127 */ "NewRowid" OpHelp("r[P2]=rowid"), - /* 128 */ "Insert" OpHelp("intkey=r[P3] data=r[P2]"), - /* 129 */ "RowCell" OpHelp(""), - /* 130 */ "Delete" OpHelp(""), - /* 131 */ "ResetCount" OpHelp(""), - /* 132 */ "SorterCompare" OpHelp("if key(P1)!=trim(r[P3],P4) goto P2"), - /* 133 */ "SorterData" OpHelp("r[P2]=data"), - /* 134 */ "RowData" OpHelp("r[P2]=data"), - /* 135 */ "Rowid" OpHelp("r[P2]=PX rowid of P1"), - /* 136 */ "NullRow" OpHelp(""), - /* 137 */ "SeekEnd" OpHelp(""), - /* 138 */ "IdxInsert" OpHelp("key=r[P2]"), - /* 139 */ "SorterInsert" OpHelp("key=r[P2]"), - /* 140 */ "IdxDelete" OpHelp("key=r[P2@P3]"), - /* 141 */ "DeferredSeek" OpHelp("Move P3 to P1.rowid if needed"), - /* 142 */ "IdxRowid" OpHelp("r[P2]=rowid"), - /* 143 */ "FinishSeek" OpHelp(""), - /* 144 */ "Destroy" OpHelp(""), - /* 145 */ "Clear" OpHelp(""), - /* 146 */ "ResetSorter" OpHelp(""), - /* 147 */ "CreateBtree" OpHelp("r[P2]=root iDb=P1 flags=P3"), - /* 148 */ "SqlExec" OpHelp(""), - /* 149 */ "ParseSchema" OpHelp(""), - /* 150 */ "LoadAnalysis" OpHelp(""), - /* 151 */ "DropTable" OpHelp(""), - /* 152 */ "DropIndex" OpHelp(""), - /* 153 */ "Real" OpHelp("r[P2]=P4"), - /* 154 */ "DropTrigger" OpHelp(""), - /* 155 */ "IntegrityCk" OpHelp(""), - /* 156 */ "RowSetAdd" OpHelp("rowset(P1)=r[P2]"), - /* 157 */ "Param" OpHelp(""), - /* 158 */ "FkCounter" OpHelp("fkctr[P1]+=P2"), - /* 159 */ "MemMax" OpHelp("r[P1]=max(r[P1],r[P2])"), - /* 160 */ "OffsetLimit" OpHelp("if r[P1]>0 then r[P2]=r[P1]+max(0,r[P3]) else r[P2]=(-1)"), - /* 161 */ "AggInverse" OpHelp("accum=r[P3] inverse(r[P2@P5])"), - /* 162 */ "AggStep" OpHelp("accum=r[P3] step(r[P2@P5])"), - /* 163 */ "AggStep1" OpHelp("accum=r[P3] step(r[P2@P5])"), - /* 164 */ "AggValue" OpHelp("r[P3]=value N=P2"), - /* 165 */ "AggFinal" OpHelp("accum=r[P1] N=P2"), - /* 166 */ "Expire" OpHelp(""), - /* 167 */ "CursorLock" OpHelp(""), - /* 168 */ "CursorUnlock" OpHelp(""), - /* 169 */ "TableLock" OpHelp("iDb=P1 root=P2 write=P3"), - /* 170 */ "VBegin" OpHelp(""), - /* 171 */ "VCreate" OpHelp(""), - /* 172 */ "VDestroy" OpHelp(""), - /* 173 */ "VOpen" OpHelp(""), - /* 174 */ "VCheck" OpHelp(""), - /* 175 */ "VInitIn" OpHelp("r[P2]=ValueList(P1,P3)"), - /* 176 */ "VColumn" OpHelp("r[P3]=vcolumn(P2)"), - /* 177 */ "VRename" OpHelp(""), - /* 178 */ "Pagecount" OpHelp(""), - /* 179 */ "MaxPgcnt" OpHelp(""), - /* 180 */ "ClrSubtype" OpHelp("r[P1].subtype = 0"), - /* 181 */ "GetSubtype" OpHelp("r[P2] = r[P1].subtype"), - /* 182 */ "SetSubtype" OpHelp("r[P2].subtype = r[P1]"), - /* 183 */ "FilterAdd" OpHelp("filter(P1) += key(P3@P4)"), - /* 184 */ "Trace" OpHelp(""), - /* 185 */ "CursorHint" OpHelp(""), - /* 186 */ "ReleaseReg" OpHelp("release r[P1@P2] mask P3"), - /* 187 */ "Noop" OpHelp(""), - /* 188 */ "Explain" OpHelp(""), - /* 189 */ "Abortable" OpHelp(""), + /* 45 */ "IdxLT" OpHelp("key=r[P3@P4]"), + /* 46 */ "IdxGE" OpHelp("key=r[P3@P4]"), + /* 47 */ "RowSetRead" OpHelp("r[P3]=rowset(P1)"), + /* 48 */ "RowSetTest" OpHelp("if r[P3] in rowset(P1) goto P2"), + /* 49 */ "Program" OpHelp(""), + /* 50 */ "FkIfZero" OpHelp("if fkctr[P1]==0 goto P2"), + /* 51 */ "IsNull" OpHelp("if r[P1]==NULL goto P2"), + /* 52 */ "NotNull" OpHelp("if r[P1]!=NULL goto P2"), + /* 53 */ "Ne" OpHelp("IF r[P3]!=r[P1]"), + /* 54 */ "Eq" OpHelp("IF r[P3]==r[P1]"), + /* 55 */ "Gt" OpHelp("IF r[P3]>r[P1]"), + /* 56 */ "Le" OpHelp("IF r[P3]<=r[P1]"), + /* 57 */ "Lt" OpHelp("IF r[P3]=r[P1]"), + /* 59 */ "ElseEq" OpHelp(""), + /* 60 */ "IfPos" OpHelp("if r[P1]>0 then r[P1]-=P3, goto P2"), + /* 61 */ "IfNotZero" OpHelp("if r[P1]!=0 then r[P1]--, goto P2"), + /* 62 */ "DecrJumpZero" OpHelp("if (--r[P1])==0 goto P2"), + /* 63 */ "IncrVacuum" OpHelp(""), + /* 64 */ "VNext" OpHelp(""), + /* 65 */ "Filter" OpHelp("if key(P3@P4) not in filter(P1) goto P2"), + /* 66 */ "PureFunc" OpHelp("r[P3]=func(r[P2@NP])"), + /* 67 */ "Function" OpHelp("r[P3]=func(r[P2@NP])"), + /* 68 */ "Return" OpHelp(""), + /* 69 */ "EndCoroutine" OpHelp(""), + /* 70 */ "HaltIfNull" OpHelp("if r[P3]=null halt"), + /* 71 */ "Halt" OpHelp(""), + /* 72 */ "Integer" OpHelp("r[P2]=P1"), + /* 73 */ "Int64" OpHelp("r[P2]=P4"), + /* 74 */ "String" OpHelp("r[P2]='P4' (len=P1)"), + /* 75 */ "BeginSubrtn" OpHelp("r[P2]=NULL"), + /* 76 */ "Null" OpHelp("r[P2..P3]=NULL"), + /* 77 */ "SoftNull" OpHelp("r[P1]=NULL"), + /* 78 */ "Blob" OpHelp("r[P2]=P4 (len=P1)"), + /* 79 */ "Variable" OpHelp("r[P2]=parameter(P1)"), + /* 80 */ "Move" OpHelp("r[P2@P3]=r[P1@P3]"), + /* 81 */ "Copy" OpHelp("r[P2@P3+1]=r[P1@P3+1]"), + /* 82 */ "SCopy" OpHelp("r[P2]=r[P1]"), + /* 83 */ "IntCopy" OpHelp("r[P2]=r[P1]"), + /* 84 */ "FkCheck" OpHelp(""), + /* 85 */ "ResultRow" OpHelp("output=r[P1@P2]"), + /* 86 */ "CollSeq" OpHelp(""), + /* 87 */ "AddImm" OpHelp("r[P1]=r[P1]+P2"), + /* 88 */ "RealAffinity" OpHelp(""), + /* 89 */ "Cast" OpHelp("affinity(r[P1])"), + /* 90 */ "Permutation" OpHelp(""), + /* 91 */ "Compare" OpHelp("r[P1@P3] <-> r[P2@P3]"), + /* 92 */ "IsTrue" OpHelp("r[P2] = coalesce(r[P1]==TRUE,P3) ^ P4"), + /* 93 */ "ZeroOrNull" OpHelp("r[P2] = 0 OR NULL"), + /* 94 */ "Offset" OpHelp("r[P3] = sqlite_offset(P1)"), + /* 95 */ "Column" OpHelp("r[P3]=PX cursor P1 column P2"), + /* 96 */ "TypeCheck" OpHelp("typecheck(r[P1@P2])"), + /* 97 */ "Affinity" OpHelp("affinity(r[P1@P2])"), + /* 98 */ "MakeRecord" OpHelp("r[P3]=mkrec(r[P1@P2])"), + /* 99 */ "Count" OpHelp("r[P2]=count()"), + /* 100 */ "ReadCookie" OpHelp(""), + /* 101 */ "SetCookie" OpHelp(""), + /* 102 */ "ReopenIdx" OpHelp("root=P2 iDb=P3"), + /* 103 */ "BitAnd" OpHelp("r[P3]=r[P1]&r[P2]"), + /* 104 */ "BitOr" OpHelp("r[P3]=r[P1]|r[P2]"), + /* 105 */ "ShiftLeft" OpHelp("r[P3]=r[P2]<>r[P1]"), + /* 107 */ "Add" OpHelp("r[P3]=r[P1]+r[P2]"), + /* 108 */ "Subtract" OpHelp("r[P3]=r[P2]-r[P1]"), + /* 109 */ "Multiply" OpHelp("r[P3]=r[P1]*r[P2]"), + /* 110 */ "Divide" OpHelp("r[P3]=r[P2]/r[P1]"), + /* 111 */ "Remainder" OpHelp("r[P3]=r[P2]%r[P1]"), + /* 112 */ "Concat" OpHelp("r[P3]=r[P2]+r[P1]"), + /* 113 */ "OpenRead" OpHelp("root=P2 iDb=P3"), + /* 114 */ "OpenWrite" OpHelp("root=P2 iDb=P3"), + /* 115 */ "BitNot" OpHelp("r[P2]= ~r[P1]"), + /* 116 */ "OpenDup" OpHelp(""), + /* 117 */ "OpenAutoindex" OpHelp("nColumn=P2"), + /* 118 */ "String8" OpHelp("r[P2]='P4'"), + /* 119 */ "OpenEphemeral" OpHelp("nColumn=P2"), + /* 120 */ "SorterOpen" OpHelp(""), + /* 121 */ "SequenceTest" OpHelp("if( cursor[P1].ctr++ ) pc = P2"), + /* 122 */ "OpenPseudo" OpHelp("P3 columns in r[P2]"), + /* 123 */ "Close" OpHelp(""), + /* 124 */ "ColumnsUsed" OpHelp(""), + /* 125 */ "SeekScan" OpHelp("Scan-ahead up to P1 rows"), + /* 126 */ "SeekHit" OpHelp("set P2<=seekHit<=P3"), + /* 127 */ "Sequence" OpHelp("r[P2]=cursor[P1].ctr++"), + /* 128 */ "NewRowid" OpHelp("r[P2]=rowid"), + /* 129 */ "Insert" OpHelp("intkey=r[P3] data=r[P2]"), + /* 130 */ "RowCell" OpHelp(""), + /* 131 */ "Delete" OpHelp(""), + /* 132 */ "ResetCount" OpHelp(""), + /* 133 */ "SorterCompare" OpHelp("if key(P1)!=trim(r[P3],P4) goto P2"), + /* 134 */ "SorterData" OpHelp("r[P2]=data"), + /* 135 */ "RowData" OpHelp("r[P2]=data"), + /* 136 */ "Rowid" OpHelp("r[P2]=PX rowid of P1"), + /* 137 */ "NullRow" OpHelp(""), + /* 138 */ "SeekEnd" OpHelp(""), + /* 139 */ "IdxInsert" OpHelp("key=r[P2]"), + /* 140 */ "SorterInsert" OpHelp("key=r[P2]"), + /* 141 */ "IdxDelete" OpHelp("key=r[P2@P3]"), + /* 142 */ "DeferredSeek" OpHelp("Move P3 to P1.rowid if needed"), + /* 143 */ "IdxRowid" OpHelp("r[P2]=rowid"), + /* 144 */ "FinishSeek" OpHelp(""), + /* 145 */ "Destroy" OpHelp(""), + /* 146 */ "Clear" OpHelp(""), + /* 147 */ "ResetSorter" OpHelp(""), + /* 148 */ "CreateBtree" OpHelp("r[P2]=root iDb=P1 flags=P3"), + /* 149 */ "SqlExec" OpHelp(""), + /* 150 */ "ParseSchema" OpHelp(""), + /* 151 */ "LoadAnalysis" OpHelp(""), + /* 152 */ "DropTable" OpHelp(""), + /* 153 */ "DropIndex" OpHelp(""), + /* 154 */ "Real" OpHelp("r[P2]=P4"), + /* 155 */ "DropTrigger" OpHelp(""), + /* 156 */ "IntegrityCk" OpHelp(""), + /* 157 */ "RowSetAdd" OpHelp("rowset(P1)=r[P2]"), + /* 158 */ "Param" OpHelp(""), + /* 159 */ "FkCounter" OpHelp("fkctr[P1]+=P2"), + /* 160 */ "MemMax" OpHelp("r[P1]=max(r[P1],r[P2])"), + /* 161 */ "OffsetLimit" OpHelp("if r[P1]>0 then r[P2]=r[P1]+max(0,r[P3]) else r[P2]=(-1)"), + /* 162 */ "AggInverse" OpHelp("accum=r[P3] inverse(r[P2@P5])"), + /* 163 */ "AggStep" OpHelp("accum=r[P3] step(r[P2@P5])"), + /* 164 */ "AggStep1" OpHelp("accum=r[P3] step(r[P2@P5])"), + /* 165 */ "AggValue" OpHelp("r[P3]=value N=P2"), + /* 166 */ "AggFinal" OpHelp("accum=r[P1] N=P2"), + /* 167 */ "Expire" OpHelp(""), + /* 168 */ "CursorLock" OpHelp(""), + /* 169 */ "CursorUnlock" OpHelp(""), + /* 170 */ "TableLock" OpHelp("iDb=P1 root=P2 write=P3"), + /* 171 */ "VBegin" OpHelp(""), + /* 172 */ "VCreate" OpHelp(""), + /* 173 */ "VDestroy" OpHelp(""), + /* 174 */ "VOpen" OpHelp(""), + /* 175 */ "VCheck" OpHelp(""), + /* 176 */ "VInitIn" OpHelp("r[P2]=ValueList(P1,P3)"), + /* 177 */ "VColumn" OpHelp("r[P3]=vcolumn(P2)"), + /* 178 */ "VRename" OpHelp(""), + /* 179 */ "Pagecount" OpHelp(""), + /* 180 */ "MaxPgcnt" OpHelp(""), + /* 181 */ "ClrSubtype" OpHelp("r[P1].subtype = 0"), + /* 182 */ "GetSubtype" OpHelp("r[P2] = r[P1].subtype"), + /* 183 */ "SetSubtype" OpHelp("r[P2].subtype = r[P1]"), + /* 184 */ "FilterAdd" OpHelp("filter(P1) += key(P3@P4)"), + /* 185 */ "Trace" OpHelp(""), + /* 186 */ "CursorHint" OpHelp(""), + /* 187 */ "ReleaseReg" OpHelp("release r[P1@P2] mask P3"), + /* 188 */ "Noop" OpHelp(""), + /* 189 */ "Explain" OpHelp(""), + /* 190 */ "Abortable" OpHelp(""), }; return azName[i]; } @@ -37168,7 +38393,7 @@ static int kvstorageRead(const char*, const char *zKey, char *zBuf, int nBuf); #define KVSTORAGE_KEY_SZ 32 /* Expand the key name with an appropriate prefix and put the result -** zKeyOut[]. The zKeyOut[] buffer is assumed to hold at least +** in zKeyOut[]. The zKeyOut[] buffer is assumed to hold at least ** KVSTORAGE_KEY_SZ bytes. */ static void kvstorageMakeKey( @@ -37227,10 +38452,12 @@ static int kvstorageDelete(const char *zClass, const char *zKey){ ** ** Return the total number of bytes in the data, without truncation, and ** not counting the final zero terminator. Return -1 if the key does -** not exist. +** not exist or its key cannot be read. ** -** If nBuf<=0 then this routine simply returns the size of the data without -** actually reading it. +** If nBuf<=0 then this routine simply returns the size of the data +** without actually reading it. Similarly, if nBuf==1 then it +** zero-terminates zBuf at zBuf[0] and returns the size of the data +** without reading it. */ static int kvstorageRead( const char *zClass, @@ -37279,11 +38506,9 @@ static int kvstorageRead( ** kvvfs i/o methods with JavaScript implementations in WASM builds. ** Maintenance reminder: if this struct changes in any way, the JSON ** rendering of its structure must be updated in -** sqlite3_wasm_enum_json(). There are no binary compatibility -** concerns, so it does not need an iVersion member. This file is -** necessarily always compiled together with sqlite3_wasm_enum_json(), -** and JS code dynamically creates the mapping of members based on -** that JSON description. +** sqlite3-wasm.c:sqlite3__wasm_enum_json(). There are no binary +** compatibility concerns, so it does not need an iVersion +** member. */ typedef struct sqlite3_kvvfs_methods sqlite3_kvvfs_methods; struct sqlite3_kvvfs_methods { @@ -37300,8 +38525,8 @@ struct sqlite3_kvvfs_methods { ** the compiler can hopefully optimize this level of indirection out. ** That said, kvvfs is intended primarily for use in WASM builds. ** -** Note that this is not explicitly flagged as static because the -** amalgamation build will tag it with SQLITE_PRIVATE. +** This is not explicitly flagged as static because the amalgamation +** build will tag it with SQLITE_PRIVATE. */ #ifndef SQLITE_WASM const @@ -38189,7 +39414,7 @@ SQLITE_PRIVATE int sqlite3KvvfsInit(void){ # endif #else /* !SQLITE_WASI */ # ifndef HAVE_FCHMOD -# define HAVE_FCHMOD +# define HAVE_FCHMOD 1 # endif #endif /* SQLITE_WASI */ @@ -38260,6 +39485,7 @@ struct unixFile { #endif #ifdef SQLITE_ENABLE_SETLK_TIMEOUT unsigned iBusyTimeout; /* Wait this many millisec on locks */ + int bBlockOnConnect; /* True to block for SHARED locks */ #endif #if OS_VXWORKS struct vxworksFileId *pId; /* Unique file ID */ @@ -38298,7 +39524,7 @@ static pid_t randomnessPid = 0; #define UNIXFILE_EXCL 0x01 /* Connections from one process only */ #define UNIXFILE_RDONLY 0x02 /* Connection is read only */ #define UNIXFILE_PERSIST_WAL 0x04 /* Persistent WAL mode */ -#ifndef SQLITE_DISABLE_DIRSYNC +#if !defined(SQLITE_DISABLE_DIRSYNC) && !defined(_AIX) # define UNIXFILE_DIRSYNC 0x08 /* Directory sync needed */ #else # define UNIXFILE_DIRSYNC 0x00 @@ -38473,10 +39699,11 @@ static struct unix_syscall { #if defined(HAVE_FCHMOD) { "fchmod", (sqlite3_syscall_ptr)fchmod, 0 }, +#define osFchmod ((int(*)(int,mode_t))aSyscall[14].pCurrent) #else { "fchmod", (sqlite3_syscall_ptr)0, 0 }, +#define osFchmod(FID,MODE) 0 #endif -#define osFchmod ((int(*)(int,mode_t))aSyscall[14].pCurrent) #if defined(HAVE_POSIX_FALLOCATE) && HAVE_POSIX_FALLOCATE { "fallocate", (sqlite3_syscall_ptr)posix_fallocate, 0 }, @@ -38570,6 +39797,119 @@ static struct unix_syscall { }; /* End of the overrideable system calls */ +#if defined(SQLITE_DEBUG) || defined(SQLITE_ENABLE_FILESTAT) +/* +** Extract Posix Advisory Locking information about file description fd +** from the /proc/PID/fdinfo/FD pseudo-file. Fill the string buffer a[16] +** with characters to indicate which SQLite-relevant locks are held. +** a[16] will be a 15-character zero-terminated string with the following +** schema: +** +** AAA/B.DDD.DDDDD +** +** Each of character A-D will be "w" or "r" or "-" to indicate either a +** write-lock, a read-lock, or no-lock, respectively. The "." and "/" +** characters are delimiters intended to make the string more easily +** readable by humans. Here are the meaning of the specific letters: +** +** AAA -> The main database locks. PENDING_BYTE, RESERVED_BYTE, +** and SHARED_FIRST, respectively. +** +** B -> The deadman switch lock. Offset 128 of the -shm file. +** +** CCC -> WAL locks: WRITE, CKPT, RECOVER +** +** DDDDD -> WAL read-locks 0 through 5 +** +** Note that elements before the "/" apply to the main database file and +** elements after the "/" apply to the -shm file in WAL mode. +** +** Here is another way of thinking about the meaning of the result string: +** +** AAA/B.CCC.DDDDD +** ||| | ||| \___/ +** PENDING--'|| | ||| `----- READ 0-5 +** RESERVED--'| | ||`---- RECOVER +** SHARED ----' | |`----- CKPT +** DMS ------' `------ WRITE +** +** Return SQLITE_OK on success and SQLITE_ERROR_UNABLE if the /proc +** pseudo-filesystem is unavailable. +*/ +static int unixPosixAdvisoryLocks( + int fd, /* The file descriptor to analyze */ + char a[16] /* Write a text description of PALs here */ +){ + int in; + ssize_t n; + char *p, *pNext, *x; + char z[2000]; + + /* 1 */ + /* 012 4 678 01234 */ + memcpy(a, "---/-.---.-----", 16); + sqlite3_snprintf(sizeof(z), z, "/proc/%d/fdinfo/%d", getpid(), fd); + in = osOpen(z, O_RDONLY, 0); + if( in<0 ){ + return SQLITE_ERROR_UNABLE; + } + n = osRead(in, z, sizeof(z)-1); + osClose(in); + if( n<=0 ) return SQLITE_ERROR_UNABLE; + z[n] = 0; + + /* We are looking for lines that begin with "lock:\t". Examples: + ** + ** lock: 1: POSIX ADVISORY READ 494716 08:02:5277597 1073741826 1073742335 + ** lock: 1: POSIX ADVISORY WRITE 494716 08:02:5282282 120 120 + ** lock: 2: POSIX ADVISORY READ 494716 08:02:5282282 123 123 + ** lock: 3: POSIX ADVISORY READ 494716 08:02:5282282 128 128 + */ + pNext = strstr(z, "lock:\t"); + while( pNext ){ + char cType = 0; + sqlite3_int64 iFirst, iLast; + p = pNext+6; + pNext = strstr(p, "lock:\t"); + if( pNext ) pNext[-1] = 0; + if( (x = strstr(p, " READ "))!=0 ){ + cType = 'r'; + x += 6; + }else if( (x = strstr(p, " WRITE "))!=0 ){ + cType = 'w'; + x += 7; + }else{ + continue; + } + x = strrchr(x, ' '); + if( x==0 ) continue; + iLast = strtoll(x+1, 0, 10); + *x = 0; + x = strrchr(p, ' '); + if( x==0 ) continue; + iFirst = strtoll(x+1, 0, 10); + if( iLast>=PENDING_BYTE ){ + if( iFirst<=PENDING_BYTE && iLast>=PENDING_BYTE ) a[0] = cType; + if( iFirst<=PENDING_BYTE+1 && iLast>=PENDING_BYTE+1 ) a[1] = cType; + if( iFirst<=PENDING_BYTE+2 && iLast>=PENDING_BYTE+510 ) a[2] = cType; + }else if( iLast<=128 ){ + if( iFirst<=128 && iLast>=128 ) a[4] = cType; + if( iFirst<=120 && iLast>=120 ) a[6] = cType; + if( iFirst<=121 && iLast>=121 ) a[7] = cType; + if( iFirst<=122 && iLast>=122 ) a[8] = cType; + if( iFirst<=123 && iLast>=123 ) a[10] = cType; + if( iFirst<=124 && iLast>=124 ) a[11] = cType; + if( iFirst<=125 && iLast>=125 ) a[12] = cType; + if( iFirst<=126 && iLast>=126 ) a[13] = cType; + if( iFirst<=127 && iLast>=127 ) a[14] = cType; + } + } + return SQLITE_OK; +} +#else +# define unixPosixAdvisoryLocks(A,B) SQLITE_ERROR_UNABLE +#endif /* SQLITE_DEBUG || SQLITE_ENABLE_FILESTAT */ + /* ** On some systems, calls to fchown() will trigger a message in a security ** log if they come from non-root processes. So avoid calling fchown() if @@ -38734,9 +40074,8 @@ static int robust_open(const char *z, int f, mode_t m){ /* ** Helper functions to obtain and relinquish the global mutex. The -** global mutex is used to protect the unixInodeInfo and -** vxworksFileId objects used by this file, all of which may be -** shared by multiple threads. +** global mutex is used to protect the unixInodeInfo objects used by +** this file, all of which may be shared by multiple threads. ** ** Function unixMutexHeld() is used to assert() that the global mutex ** is held when required. This function is only used as part of assert() @@ -38938,6 +40277,7 @@ struct vxworksFileId { ** variable: */ static struct vxworksFileId *vxworksFileList = 0; +static sqlite3_mutex *vxworksMutex = 0; /* ** Simplify a filename into its canonical form @@ -39003,14 +40343,14 @@ static struct vxworksFileId *vxworksFindFileId(const char *zAbsoluteName){ ** If found, increment the reference count and return a pointer to ** the existing file ID. */ - unixEnterMutex(); + sqlite3_mutex_enter(vxworksMutex); for(pCandidate=vxworksFileList; pCandidate; pCandidate=pCandidate->pNext){ if( pCandidate->nName==n && memcmp(pCandidate->zCanonicalName, pNew->zCanonicalName, n)==0 ){ sqlite3_free(pNew); pCandidate->nRef++; - unixLeaveMutex(); + sqlite3_mutex_leave(vxworksMutex); return pCandidate; } } @@ -39020,7 +40360,7 @@ static struct vxworksFileId *vxworksFindFileId(const char *zAbsoluteName){ pNew->nName = n; pNew->pNext = vxworksFileList; vxworksFileList = pNew; - unixLeaveMutex(); + sqlite3_mutex_leave(vxworksMutex); return pNew; } @@ -39029,7 +40369,7 @@ static struct vxworksFileId *vxworksFindFileId(const char *zAbsoluteName){ ** the object when the reference count reaches zero. */ static void vxworksReleaseFileId(struct vxworksFileId *pId){ - unixEnterMutex(); + sqlite3_mutex_enter(vxworksMutex); assert( pId->nRef>0 ); pId->nRef--; if( pId->nRef==0 ){ @@ -39039,7 +40379,7 @@ static void vxworksReleaseFileId(struct vxworksFileId *pId){ *pp = pId->pNext; sqlite3_free(pId); } - unixLeaveMutex(); + sqlite3_mutex_leave(vxworksMutex); } #endif /* OS_VXWORKS */ /*************** End of Unique File ID Utility Used By VxWorks **************** @@ -39271,8 +40611,12 @@ static int unixLogErrorAtLine( ** available, the error message will often be an empty string. Not a ** huge problem. Incorrectly concluding that the GNU version is available ** could lead to a segfault though. + ** + ** Forum post 3f13857fa4062301 reports that the Android SDK may use + ** int-type return, depending on its version. */ -#if defined(STRERROR_R_CHAR_P) || defined(__USE_GNU) +#if (defined(STRERROR_R_CHAR_P) || defined(__USE_GNU)) \ + && !defined(ANDROID) && !defined(__ANDROID__) zErr = # endif strerror_r(iErrno, aErr, sizeof(aErr)-1); @@ -39423,6 +40767,10 @@ static int findInodeInfo( storeLastErrno(pFile, errno); return SQLITE_IOERR; } + if( fsync(fd) ){ + storeLastErrno(pFile, errno); + return SQLITE_IOERR_FSYNC; + } rc = osFstat(fd, &statbuf); if( rc!=0 ){ storeLastErrno(pFile, errno); @@ -39592,18 +40940,42 @@ static int osSetPosixAdvisoryLock( struct flock *pLock, /* The description of the lock */ unixFile *pFile /* Structure holding timeout value */ ){ - int tm = pFile->iBusyTimeout; - int rc = osFcntl(h,F_SETLK,pLock); - while( rc<0 && tm>0 ){ - /* On systems that support some kind of blocking file lock with a timeout, - ** make appropriate changes here to invoke that blocking file lock. On - ** generic posix, however, there is no such API. So we simply try the - ** lock once every millisecond until either the timeout expires, or until - ** the lock is obtained. */ - unixSleep(0,1000); + int rc = 0; + + if( pFile->iBusyTimeout==0 ){ + /* unixFile->iBusyTimeout is set to 0. In this case, attempt a + ** non-blocking lock. */ + rc = osFcntl(h,F_SETLK,pLock); + }else{ + /* unixFile->iBusyTimeout is set to greater than zero. In this case, + ** attempt a blocking-lock with a unixFile->iBusyTimeout ms timeout. + ** + ** On systems that support some kind of blocking file lock operation, + ** this block should be replaced by code to attempt a blocking lock + ** with a timeout of unixFile->iBusyTimeout ms. The code below is + ** placeholder code. If SQLITE_TEST is defined, the placeholder code + ** retries the lock once every 1ms until it succeeds or the timeout + ** is reached. Or, if SQLITE_TEST is not defined, the placeholder + ** code attempts a non-blocking lock and sets unixFile->iBusyTimeout + ** to 0. This causes the caller to return SQLITE_BUSY, instead of + ** SQLITE_BUSY_TIMEOUT to SQLite - as required by a VFS that does not + ** support blocking locks. + */ +#ifdef SQLITE_TEST + int tm = pFile->iBusyTimeout; + while( tm>0 ){ + rc = osFcntl(h,F_SETLK,pLock); + if( rc==0 ) break; + unixSleep(0,1000); + tm--; + } +#else rc = osFcntl(h,F_SETLK,pLock); - tm--; + pFile->iBusyTimeout = 0; +#endif + /* End of code to replace with real blocking-locks code. */ } + return rc; } #endif /* SQLITE_ENABLE_SETLK_TIMEOUT */ @@ -39636,7 +41008,7 @@ static int unixFileLock(unixFile *pFile, struct flock *pLock){ if( (pFile->ctrlFlags & (UNIXFILE_EXCL|UNIXFILE_RDONLY))==UNIXFILE_EXCL ){ if( pInode->bProcessLock==0 ){ struct flock lock; - assert( pInode->nLock==0 ); + /* assert( pInode->nLock==0 ); <-- Not true if unix-excl READONLY used */ lock.l_whence = SEEK_SET; lock.l_start = SHARED_FIRST; lock.l_len = SHARED_SIZE; @@ -39649,11 +41021,25 @@ static int unixFileLock(unixFile *pFile, struct flock *pLock){ rc = 0; } }else{ +#ifdef SQLITE_ENABLE_SETLK_TIMEOUT + if( pFile->bBlockOnConnect && pLock->l_type==F_RDLCK + && pLock->l_start==SHARED_FIRST && pLock->l_len==SHARED_SIZE + ){ + rc = osFcntl(pFile->h, F_SETLKW, pLock); + }else +#endif rc = osSetPosixAdvisoryLock(pFile->h, pLock, pFile); } return rc; } +#if !defined(SQLITE_WASI) && !defined(SQLITE_OMIT_WAL) +/* Forward reference */ +static int unixIsSharingShmNode(unixFile*); +#else +#define unixIsSharingShmNode(pFile) (0) +#endif + /* ** Lock the file with the lock specified by parameter eFileLock - one ** of the following: @@ -39842,7 +41228,9 @@ static int unixLock(sqlite3_file *id, int eFileLock){ pInode->nLock++; pInode->nShared = 1; } - }else if( eFileLock==EXCLUSIVE_LOCK && pInode->nShared>1 ){ + }else if( (eFileLock==EXCLUSIVE_LOCK && pInode->nShared>1) + || unixIsSharingShmNode(pFile) + ){ /* We are trying for an exclusive lock but another thread in this ** same process is still holding a shared lock. */ rc = SQLITE_BUSY; @@ -40251,26 +41639,22 @@ static int nolockClose(sqlite3_file *id) { /* ** This routine checks if there is a RESERVED lock held on the specified -** file by this or any other process. If such a lock is held, set *pResOut -** to a non-zero value otherwise *pResOut is set to zero. The return value -** is set to SQLITE_OK unless an I/O error occurs during lock checking. -** -** In dotfile locking, either a lock exists or it does not. So in this -** variation of CheckReservedLock(), *pResOut is set to true if any lock -** is held on the file and false if the file is unlocked. +** file by this or any other process. If the caller holds a SHARED +** or greater lock when it is called, then it is assumed that no other +** client may hold RESERVED. Or, if the caller holds no lock, then it +** is assumed another client holds RESERVED if the lock-file exists. */ static int dotlockCheckReservedLock(sqlite3_file *id, int *pResOut) { - int rc = SQLITE_OK; - int reserved = 0; unixFile *pFile = (unixFile*)id; - SimulateIOError( return SQLITE_IOERR_CHECKRESERVEDLOCK; ); - assert( pFile ); - reserved = osAccess((const char*)pFile->lockingContext, 0)==0; - OSTRACE(("TEST WR-LOCK %d %d %d (dotlock)\n", pFile->h, rc, reserved)); - *pResOut = reserved; - return rc; + if( pFile->eFileLock>=SHARED_LOCK ){ + *pResOut = 0; + }else{ + *pResOut = osAccess((const char*)pFile->lockingContext, 0)==0; + } + OSTRACE(("TEST WR-LOCK %d %d %d (dotlock)\n", pFile->h, 0, *pResOut)); + return SQLITE_OK; } /* @@ -40440,54 +41824,33 @@ static int robust_flock(int fd, int op){ ** is set to SQLITE_OK unless an I/O error occurs during lock checking. */ static int flockCheckReservedLock(sqlite3_file *id, int *pResOut){ - int rc = SQLITE_OK; - int reserved = 0; +#ifdef SQLITE_DEBUG unixFile *pFile = (unixFile*)id; +#else + UNUSED_PARAMETER(id); +#endif SimulateIOError( return SQLITE_IOERR_CHECKRESERVEDLOCK; ); assert( pFile ); + assert( pFile->eFileLock<=SHARED_LOCK ); - /* Check if a thread in this process holds such a lock */ - if( pFile->eFileLock>SHARED_LOCK ){ - reserved = 1; - } - - /* Otherwise see if some other process holds it. */ - if( !reserved ){ - /* attempt to get the lock */ - int lrc = robust_flock(pFile->h, LOCK_EX | LOCK_NB); - if( !lrc ){ - /* got the lock, unlock it */ - lrc = robust_flock(pFile->h, LOCK_UN); - if ( lrc ) { - int tErrno = errno; - /* unlock failed with an error */ - lrc = SQLITE_IOERR_UNLOCK; - storeLastErrno(pFile, tErrno); - rc = lrc; - } - } else { - int tErrno = errno; - reserved = 1; - /* someone else might have it reserved */ - lrc = sqliteErrorFromPosixError(tErrno, SQLITE_IOERR_LOCK); - if( IS_LOCK_ERROR(lrc) ){ - storeLastErrno(pFile, tErrno); - rc = lrc; - } - } - } - OSTRACE(("TEST WR-LOCK %d %d %d (flock)\n", pFile->h, rc, reserved)); + /* The flock VFS only ever takes exclusive locks (see function flockLock). + ** Therefore, if this connection is holding any lock at all, no other + ** connection may be holding a RESERVED lock. So set *pResOut to 0 + ** in this case. + ** + ** Or, this connection may be holding no lock. In that case, set *pResOut to + ** 0 as well. The caller will then attempt to take an EXCLUSIVE lock on the + ** db in order to roll the hot journal back. If there is another connection + ** holding a lock, that attempt will fail and an SQLITE_BUSY returned to + ** the user. With other VFS, we try to avoid this, in order to allow a reader + ** to proceed while a writer is preparing its transaction. But that won't + ** work with the flock VFS - as it always takes EXCLUSIVE locks - so it is + ** not a problem in this case. */ + *pResOut = 0; -#ifdef SQLITE_IGNORE_FLOCK_LOCK_ERRORS - if( (rc & 0xff) == SQLITE_IOERR ){ - rc = SQLITE_OK; - reserved=1; - } -#endif /* SQLITE_IGNORE_FLOCK_LOCK_ERRORS */ - *pResOut = reserved; - return rc; + return SQLITE_OK; } /* @@ -41959,9 +43322,13 @@ static void unixModeBit(unixFile *pFile, unsigned char mask, int *pArg){ /* Forward declaration */ static int unixGetTempname(int nBuf, char *zBuf); -#ifndef SQLITE_OMIT_WAL +#if !defined(SQLITE_WASI) && !defined(SQLITE_OMIT_WAL) static int unixFcntlExternalReader(unixFile*, int*); #endif +#if defined(SQLITE_DEBUG) || defined(SQLITE_ENABLE_FILESTAT) + static void unixDescribeShm(sqlite3_str*,unixShm*); +#endif + /* ** Information and control of an open file handle. @@ -41984,6 +43351,11 @@ static int unixFileControl(sqlite3_file *id, int op, void *pArg){ } #endif /* __linux__ && SQLITE_ENABLE_BATCH_ATOMIC_WRITE */ + case SQLITE_FCNTL_NULL_IO: { + osClose(pFile->h); + pFile->h = -1; + return SQLITE_OK; + } case SQLITE_FCNTL_LOCKSTATE: { *(int*)pArg = pFile->eFileLock; return SQLITE_OK; @@ -42030,8 +43402,9 @@ static int unixFileControl(sqlite3_file *id, int op, void *pArg){ #ifdef SQLITE_ENABLE_SETLK_TIMEOUT case SQLITE_FCNTL_LOCK_TIMEOUT: { int iOld = pFile->iBusyTimeout; + int iNew = *(int*)pArg; #if SQLITE_ENABLE_SETLK_TIMEOUT==1 - pFile->iBusyTimeout = *(int*)pArg; + pFile->iBusyTimeout = iNew<0 ? 0x7FFFFFFF : (unsigned)iNew; #elif SQLITE_ENABLE_SETLK_TIMEOUT==2 pFile->iBusyTimeout = !!(*(int*)pArg); #else @@ -42040,7 +43413,12 @@ static int unixFileControl(sqlite3_file *id, int op, void *pArg){ *(int*)pArg = iOld; return SQLITE_OK; } -#endif + case SQLITE_FCNTL_BLOCK_ON_CONNECT: { + int iNew = *(int*)pArg; + pFile->bBlockOnConnect = iNew; + return SQLITE_OK; + } +#endif /* SQLITE_ENABLE_SETLK_TIMEOUT */ #if SQLITE_MAX_MMAP_SIZE>0 case SQLITE_FCNTL_MMAP_SIZE: { i64 newLimit = *(i64*)pArg; @@ -42086,13 +43464,73 @@ static int unixFileControl(sqlite3_file *id, int op, void *pArg){ #endif /* SQLITE_ENABLE_LOCKING_STYLE && defined(__APPLE__) */ case SQLITE_FCNTL_EXTERNAL_READER: { -#ifndef SQLITE_OMIT_WAL +#if !defined(SQLITE_WASI) && !defined(SQLITE_OMIT_WAL) return unixFcntlExternalReader((unixFile*)id, (int*)pArg); #else *(int*)pArg = 0; return SQLITE_OK; #endif } + +#if defined(SQLITE_DEBUG) || defined(SQLITE_ENABLE_FILESTAT) + case SQLITE_FCNTL_FILESTAT: { + sqlite3_str *pStr = (sqlite3_str*)pArg; + char aLck[16]; + unixInodeInfo *pInode; + static const char *azLock[] = { "SHARED", "RESERVED", + "PENDING", "EXCLUSIVE" }; + sqlite3_str_appendf(pStr, "{\"h\":%d", pFile->h); + sqlite3_str_appendf(pStr, ",\"vfs\":\"%s\"", pFile->pVfs->zName); + if( pFile->eFileLock ){ + sqlite3_str_appendf(pStr, ",\"eFileLock\":\"%s\"", + azLock[pFile->eFileLock-1]); + if( unixPosixAdvisoryLocks(pFile->h, aLck)==SQLITE_OK ){ + sqlite3_str_appendf(pStr, ",\"pal\":\"%s\"", aLck); + } + } + unixEnterMutex(); + if( pFile->pShm ){ + sqlite3_str_appendall(pStr, ",\"shm\":"); + unixDescribeShm(pStr, pFile->pShm); + } +#if SQLITE_MAX_MMAP_SIZE>0 + if( pFile->mmapSize ){ + sqlite3_str_appendf(pStr, ",\"mmapSize\":%lld", pFile->mmapSize); + sqlite3_str_appendf(pStr, ",\"nFetchOut\":%d", pFile->nFetchOut); + } +#endif + if( (pInode = pFile->pInode)!=0 ){ + sqlite3_str_appendf(pStr, ",\"inode\":{\"nRef\":%d",pInode->nRef); + sqlite3_mutex_enter(pInode->pLockMutex); + sqlite3_str_appendf(pStr, ",\"nShared\":%d", pInode->nShared); + if( pInode->eFileLock ){ + sqlite3_str_appendf(pStr, ",\"eFileLock\":\"%s\"", + azLock[pInode->eFileLock-1]); + } + if( pInode->pUnused ){ + char cSep = '['; + UnixUnusedFd *pUFd = pFile->pInode->pUnused; + sqlite3_str_appendall(pStr, ",\"unusedFd\":"); + while( pUFd ){ + sqlite3_str_appendf(pStr, "%c{\"fd\":%d,\"flags\":%d", + cSep, pUFd->fd, pUFd->flags); + cSep = ','; + if( unixPosixAdvisoryLocks(pUFd->fd, aLck)==SQLITE_OK ){ + sqlite3_str_appendf(pStr, ",\"pal\":\"%s\"", aLck); + } + sqlite3_str_append(pStr, "}", 1); + pUFd = pUFd->pNext; + } + sqlite3_str_append(pStr, "]", 1); + } + sqlite3_mutex_leave(pInode->pLockMutex); + sqlite3_str_append(pStr, "}", 1); + } + unixLeaveMutex(); + sqlite3_str_append(pStr, "}", 1); + return SQLITE_OK; + } +#endif /* SQLITE_DEBUG || SQLITE_ENABLE_FILESTAT */ } return SQLITE_NOTFOUND; } @@ -42125,6 +43563,7 @@ static void setDeviceCharacteristics(unixFile *pFd){ if( pFd->ctrlFlags & UNIXFILE_PSOW ){ pFd->deviceCharacteristics |= SQLITE_IOCAP_POWERSAFE_OVERWRITE; } + pFd->deviceCharacteristics |= SQLITE_IOCAP_SUBPAGE_READ; pFd->sectorSize = SQLITE_DEFAULT_SECTOR_SIZE; } @@ -42175,7 +43614,7 @@ static void setDeviceCharacteristics(unixFile *pFile){ pFile->sectorSize = fsInfo.f_bsize; pFile->deviceCharacteristics = /* full bitset of atomics from max sector size and smaller */ - ((pFile->sectorSize / 512 * SQLITE_IOCAP_ATOMIC512) << 1) - 2 | + (((pFile->sectorSize / 512 * SQLITE_IOCAP_ATOMIC512) << 1) - 2) | SQLITE_IOCAP_SEQUENTIAL | /* The ram filesystem has no write behind ** so it is ordered */ 0; @@ -42183,7 +43622,7 @@ static void setDeviceCharacteristics(unixFile *pFile){ pFile->sectorSize = fsInfo.f_bsize; pFile->deviceCharacteristics = /* full bitset of atomics from max sector size and smaller */ - ((pFile->sectorSize / 512 * SQLITE_IOCAP_ATOMIC512) << 1) - 2 | + (((pFile->sectorSize / 512 * SQLITE_IOCAP_ATOMIC512) << 1) - 2) | SQLITE_IOCAP_SEQUENTIAL | /* The ram filesystem has no write behind ** so it is ordered */ 0; @@ -42259,7 +43698,7 @@ static int unixGetpagesize(void){ #endif /* !defined(SQLITE_OMIT_WAL) || SQLITE_MAX_MMAP_SIZE>0 */ -#ifndef SQLITE_OMIT_WAL +#if !defined(SQLITE_WASI) && !defined(SQLITE_OMIT_WAL) /* ** Object used to represent an shared memory buffer. @@ -42358,6 +43797,26 @@ struct unixShm { #define UNIX_SHM_BASE ((22+SQLITE_SHM_NLOCK)*4) /* first lock byte */ #define UNIX_SHM_DMS (UNIX_SHM_BASE+SQLITE_SHM_NLOCK) /* deadman switch */ +#if defined(SQLITE_DEBUG) || defined(SQLITE_ENABLE_FILESTAT) +/* +** Describe the pShm object using JSON. Used for diagnostics only. +*/ +static void unixDescribeShm(sqlite3_str *pStr, unixShm *pShm){ + unixShmNode *pNode = pShm->pShmNode; + char aLck[16]; + sqlite3_str_appendf(pStr, "{\"h\":%d", pNode->hShm); + assert( unixMutexHeld() ); + sqlite3_str_appendf(pStr, ",\"nRef\":%d", pNode->nRef); + sqlite3_str_appendf(pStr, ",\"id\":%d", pShm->id); + sqlite3_str_appendf(pStr, ",\"sharedMask\":%d", pShm->sharedMask); + sqlite3_str_appendf(pStr, ",\"exclMask\":%d", pShm->exclMask); + if( unixPosixAdvisoryLocks(pNode->hShm, aLck)==SQLITE_OK ){ + sqlite3_str_appendf(pStr, ",\"pal\":\"%s\"", aLck); + } + sqlite3_str_append(pStr, "}", 1); +} +#endif /* SQLITE_DEBUG || SQLITE_ENABLE_FILESTAT */ + /* ** Use F_GETLK to check whether or not there are any readers with open ** wal-mode transactions in other processes on database file pFile. If @@ -42391,6 +43850,49 @@ static int unixFcntlExternalReader(unixFile *pFile, int *piOut){ return rc; } +/* +** If pFile has a -shm file open and it is sharing that file with some +** other connection, either in the same process or in a separate process, +** then return true. Return false if either pFile does not have a -shm +** file open or if it is the only connection to that -shm file across the +** entire system. +** +** This routine is not required for correct operation. It can always return +** false and SQLite will continue to operate according to spec. However, +** when this routine does its job, it adds extra robustness in cases +** where database file locks have been erroneously deleted in a WAL-mode +** database by doing close(open(DATABASE_PATHNAME)) or similar. +** +** With false negatives, SQLite still operates to spec, though with less +** robustness. With false positives, the last database connection on a +** WAL-mode database will fail to unlink the -wal and -shm files, which +** is annoying but harmless. False positives will also prevent a database +** connection from running "PRAGMA journal_mode=DELETE" in order to take +** the database out of WAL mode, which is perhaps more serious, but is +** still not a disaster. +*/ +static int unixIsSharingShmNode(unixFile *pFile){ + int rc; + unixShmNode *pShmNode; + if( pFile->pShm==0 ) return 0; + if( pFile->ctrlFlags & UNIXFILE_EXCL ) return 0; + pShmNode = pFile->pShm->pShmNode; + rc = 1; + unixEnterMutex(); + if( ALWAYS(pShmNode->nRef==1) ){ + struct flock lock; + lock.l_whence = SEEK_SET; + lock.l_start = UNIX_SHM_DMS; + lock.l_len = 1; + lock.l_type = F_WRLCK; + osFcntl(pShmNode->hShm, F_GETLK, &lock); + if( lock.l_type==F_UNLCK ){ + rc = 0; + } + } + unixLeaveMutex(); + return rc; +} /* ** Apply posix advisory locks for all bytes from ofst through ofst+n-1. @@ -42436,7 +43938,8 @@ static int unixShmSystemLock( /* Locks are within range */ assert( n>=1 && n<=SQLITE_SHM_NLOCK ); - assert( ofst>=UNIX_SHM_BASE && ofst<=(UNIX_SHM_DMS+SQLITE_SHM_NLOCK) ); + assert( ofst>=UNIX_SHM_BASE && ofst<=UNIX_SHM_DMS ); + assert( ofst+n-1<=UNIX_SHM_DMS ); if( pShmNode->hShm>=0 ){ int res; @@ -42968,7 +44471,7 @@ static int assertLockingArrayOk(unixShmNode *pShmNode){ return (memcmp(pShmNode->aLock, aLock, sizeof(aLock))==0); #endif } -#endif +#endif /* !defined(SQLITE_WASI) && !defined(SQLITE_OMIT_WAL) */ /* ** Change the lock state for a shared-memory segment. @@ -43012,21 +44515,20 @@ static int unixShmLock( /* Check that, if this to be a blocking lock, no locks that occur later ** in the following list than the lock being obtained are already held: ** - ** 1. Checkpointer lock (ofst==1). - ** 2. Write lock (ofst==0). - ** 3. Read locks (ofst>=3 && ofst=3 && ofstexclMask|p->sharedMask); assert( (flags & SQLITE_SHM_UNLOCK) || pDbFd->iBusyTimeout==0 || ( - (ofst!=2) /* not RECOVER */ + (ofst!=2 || lockMask==0) && (ofst!=1 || lockMask==0 || lockMask==2) && (ofst!=0 || lockMask<3) && (ofst<3 || lockMask<(1<=0 ) robust_close(pNew, h, __LINE__); - h = -1; - osUnlink(zFilename); - pNew->ctrlFlags |= UNIXFILE_DELETE; + if( h>=0 ){ + robust_close(pNew, h, __LINE__); + h = -1; + } + if( pNew->ctrlFlags & UNIXFILE_DELETE ){ + osUnlink(zFilename); + } + if( pNew->pId ){ + vxworksReleaseFileId(pNew->pId); + pNew->pId = 0; + } } #endif if( rc!=SQLITE_OK ){ @@ -43978,6 +45487,9 @@ static const char *unixTempFileDir(void){ while(1){ if( zDir!=0 +#if OS_VXWORKS + && zDir[0]=='/' +#endif && osStat(zDir, &buf)==0 && S_ISDIR(buf.st_mode) && osAccess(zDir, 03)==0 @@ -44292,6 +45804,12 @@ static int unixOpen( || eType==SQLITE_OPEN_TRANSIENT_DB || eType==SQLITE_OPEN_WAL ); +#if OS_VXWORKS + /* The file-ID mechanism used in Vxworks requires that all pathnames + ** provided to unixOpen must be absolute pathnames. */ + if( zPath!=0 && zPath[0]!='/' ){ return SQLITE_CANTOPEN; } +#endif + /* Detect a pid change and reset the PRNG. There is a race condition ** here such that two or more threads all trying to open databases at ** the same instant might all reset the PRNG. But multiple resets @@ -44370,12 +45888,19 @@ static int unixOpen( rc = SQLITE_READONLY_DIRECTORY; }else if( errno!=EISDIR && isReadWrite ){ /* Failed to open the file for read/write access. Try read-only. */ + UnixUnusedFd *pReadonly = 0; flags &= ~(SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE); openFlags &= ~(O_RDWR|O_CREAT); flags |= SQLITE_OPEN_READONLY; openFlags |= O_RDONLY; isReadonly = 1; - fd = robust_open(zName, openFlags, openMode); + pReadonly = findReusableFd(zName, flags); + if( pReadonly ){ + fd = pReadonly->fd; + sqlite3_free(pReadonly); + }else{ + fd = robust_open(zName, openFlags, openMode); + } } } if( fd<0 ){ @@ -44485,8 +46010,11 @@ static int unixOpen( } #endif - assert( zPath==0 || zPath[0]=='/' - || eType==SQLITE_OPEN_SUPER_JOURNAL || eType==SQLITE_OPEN_MAIN_JOURNAL + assert( zPath==0 + || zPath[0]=='/' + || eType==SQLITE_OPEN_SUPER_JOURNAL + || eType==SQLITE_OPEN_MAIN_JOURNAL + || eType==SQLITE_OPEN_TEMP_JOURNAL ); rc = fillInUnixFile(pVfs, fd, pFile, zPath, ctrlFlags); @@ -44824,7 +46352,7 @@ static int unixSleep(sqlite3_vfs *NotUsed, int microseconds){ /* Almost all modern unix systems support nanosleep(). But if you are ** compiling for one of the rare exceptions, you can use - ** -DHAVE_NANOSLEEP=0 (perhaps in conjuction with -DHAVE_USLEEP if + ** -DHAVE_NANOSLEEP=0 (perhaps in conjunction with -DHAVE_USLEEP if ** usleep() is available) in order to bypass the use of nanosleep() */ nanosleep(&sp, NULL); @@ -46215,6 +47743,9 @@ SQLITE_API int sqlite3_os_init(void){ sqlite3KvvfsInit(); #endif unixBigLock = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_VFS1); +#if OS_VXWORKS + vxworksMutex = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_VFS2); +#endif #ifndef SQLITE_OMIT_WAL /* Validate lock assumptions */ @@ -46249,6 +47780,9 @@ SQLITE_API int sqlite3_os_init(void){ */ SQLITE_API int sqlite3_os_end(void){ unixBigLock = 0; +#if OS_VXWORKS + vxworksMutex = 0; +#endif return SQLITE_OK; } @@ -46545,8 +48079,18 @@ struct winFile { sqlite3_int64 mmapSize; /* Size of mapped region */ sqlite3_int64 mmapSizeMax; /* Configured FCNTL_MMAP_SIZE value */ #endif +#ifdef SQLITE_ENABLE_SETLK_TIMEOUT + DWORD iBusyTimeout; /* Wait this many millisec on locks */ + int bBlockOnConnect; +#endif }; +#ifdef SQLITE_ENABLE_SETLK_TIMEOUT +# define winFileBusyTimeout(pDbFd) pDbFd->iBusyTimeout +#else +# define winFileBusyTimeout(pDbFd) 0 +#endif + /* ** The winVfsAppData structure is used for the pAppData member for all of the ** Win32 VFS variants. @@ -46865,7 +48409,7 @@ static struct win_syscall { { "FileTimeToLocalFileTime", (SYSCALL)0, 0 }, #endif -#define osFileTimeToLocalFileTime ((BOOL(WINAPI*)(CONST FILETIME*, \ +#define osFileTimeToLocalFileTime ((BOOL(WINAPI*)(const FILETIME*, \ LPFILETIME))aSyscall[11].pCurrent) #if SQLITE_OS_WINCE @@ -46874,7 +48418,7 @@ static struct win_syscall { { "FileTimeToSystemTime", (SYSCALL)0, 0 }, #endif -#define osFileTimeToSystemTime ((BOOL(WINAPI*)(CONST FILETIME*, \ +#define osFileTimeToSystemTime ((BOOL(WINAPI*)(const FILETIME*, \ LPSYSTEMTIME))aSyscall[12].pCurrent) { "FlushFileBuffers", (SYSCALL)FlushFileBuffers, 0 }, @@ -46980,6 +48524,12 @@ static struct win_syscall { #define osGetFullPathNameW ((DWORD(WINAPI*)(LPCWSTR,DWORD,LPWSTR, \ LPWSTR*))aSyscall[25].pCurrent) +/* +** For GetLastError(), MSDN says: +** +** Minimum supported client: Windows XP [desktop apps | UWP apps] +** Minimum supported server: Windows Server 2003 [desktop apps | UWP apps] +*/ { "GetLastError", (SYSCALL)GetLastError, 0 }, #define osGetLastError ((DWORD(WINAPI*)(VOID))aSyscall[26].pCurrent) @@ -47148,7 +48698,7 @@ static struct win_syscall { { "LockFile", (SYSCALL)0, 0 }, #endif -#ifndef osLockFile +#if !defined(osLockFile) && defined(SQLITE_WIN32_HAS_ANSI) #define osLockFile ((BOOL(WINAPI*)(HANDLE,DWORD,DWORD,DWORD, \ DWORD))aSyscall[47].pCurrent) #endif @@ -47212,7 +48762,7 @@ static struct win_syscall { { "SystemTimeToFileTime", (SYSCALL)SystemTimeToFileTime, 0 }, -#define osSystemTimeToFileTime ((BOOL(WINAPI*)(CONST SYSTEMTIME*, \ +#define osSystemTimeToFileTime ((BOOL(WINAPI*)(const SYSTEMTIME*, \ LPFILETIME))aSyscall[56].pCurrent) #if !SQLITE_OS_WINCE && !SQLITE_OS_WINRT @@ -47221,7 +48771,7 @@ static struct win_syscall { { "UnlockFile", (SYSCALL)0, 0 }, #endif -#ifndef osUnlockFile +#if !defined(osUnlockFile) && defined(SQLITE_WIN32_HAS_ANSI) #define osUnlockFile ((BOOL(WINAPI*)(HANDLE,DWORD,DWORD,DWORD, \ DWORD))aSyscall[57].pCurrent) #endif @@ -47262,11 +48812,13 @@ static struct win_syscall { #define osCreateEventExW ((HANDLE(WINAPI*)(LPSECURITY_ATTRIBUTES,LPCWSTR, \ DWORD,DWORD))aSyscall[62].pCurrent) -#if !SQLITE_OS_WINRT +/* +** For WaitForSingleObject(), MSDN says: +** +** Minimum supported client: Windows XP [desktop apps | UWP apps] +** Minimum supported server: Windows Server 2003 [desktop apps | UWP apps] +*/ { "WaitForSingleObject", (SYSCALL)WaitForSingleObject, 0 }, -#else - { "WaitForSingleObject", (SYSCALL)0, 0 }, -#endif #define osWaitForSingleObject ((DWORD(WINAPI*)(HANDLE, \ DWORD))aSyscall[63].pCurrent) @@ -47413,6 +48965,97 @@ static struct win_syscall { #define osFlushViewOfFile \ ((BOOL(WINAPI*)(LPCVOID,SIZE_T))aSyscall[79].pCurrent) +/* +** If SQLITE_ENABLE_SETLK_TIMEOUT is defined, we require CreateEvent() +** to implement blocking locks with timeouts. MSDN says: +** +** Minimum supported client: Windows XP [desktop apps | UWP apps] +** Minimum supported server: Windows Server 2003 [desktop apps | UWP apps] +*/ +#ifdef SQLITE_ENABLE_SETLK_TIMEOUT + { "CreateEvent", (SYSCALL)CreateEvent, 0 }, +#else + { "CreateEvent", (SYSCALL)0, 0 }, +#endif + +#define osCreateEvent ( \ + (HANDLE(WINAPI*) (LPSECURITY_ATTRIBUTES,BOOL,BOOL,LPCSTR)) \ + aSyscall[80].pCurrent \ +) + +/* +** If SQLITE_ENABLE_SETLK_TIMEOUT is defined, we require CancelIo() +** for the case where a timeout expires and a lock request must be +** cancelled. +** +** Minimum supported client: Windows XP [desktop apps | UWP apps] +** Minimum supported server: Windows Server 2003 [desktop apps | UWP apps] +*/ +#ifdef SQLITE_ENABLE_SETLK_TIMEOUT + { "CancelIo", (SYSCALL)CancelIo, 0 }, +#else + { "CancelIo", (SYSCALL)0, 0 }, +#endif + +#define osCancelIo ((BOOL(WINAPI*)(HANDLE))aSyscall[81].pCurrent) + +#if defined(SQLITE_WIN32_HAS_WIDE) && defined(_WIN32) + { "GetModuleHandleW", (SYSCALL)GetModuleHandleW, 0 }, +#else + { "GetModuleHandleW", (SYSCALL)0, 0 }, +#endif + +#define osGetModuleHandleW ((HMODULE(WINAPI*)(LPCWSTR))aSyscall[82].pCurrent) + +#ifndef _WIN32 + { "getenv", (SYSCALL)getenv, 0 }, +#else + { "getenv", (SYSCALL)0, 0 }, +#endif + +#define osGetenv ((const char *(*)(const char *))aSyscall[83].pCurrent) + +#ifndef _WIN32 + { "getcwd", (SYSCALL)getcwd, 0 }, +#else + { "getcwd", (SYSCALL)0, 0 }, +#endif + +#define osGetcwd ((char*(*)(char*,size_t))aSyscall[84].pCurrent) + +#ifndef _WIN32 + { "readlink", (SYSCALL)readlink, 0 }, +#else + { "readlink", (SYSCALL)0, 0 }, +#endif + +#define osReadlink ((ssize_t(*)(const char*,char*,size_t))aSyscall[85].pCurrent) + +#ifndef _WIN32 + { "lstat", (SYSCALL)lstat, 0 }, +#else + { "lstat", (SYSCALL)0, 0 }, +#endif + +#define osLstat ((int(*)(const char*,struct stat*))aSyscall[86].pCurrent) + +#ifndef _WIN32 + { "__errno", (SYSCALL)__errno, 0 }, +#else + { "__errno", (SYSCALL)0, 0 }, +#endif + +#define osErrno (*((int*(*)(void))aSyscall[87].pCurrent)()) + +#ifndef _WIN32 + { "cygwin_conv_path", (SYSCALL)cygwin_conv_path, 0 }, +#else + { "cygwin_conv_path", (SYSCALL)0, 0 }, +#endif + +#define osCygwin_conv_path ((size_t(*)(unsigned int, \ + const void *, void *, size_t))aSyscall[88].pCurrent) + }; /* End of the overrideable system calls */ /* @@ -47586,6 +49229,7 @@ SQLITE_API int sqlite3_win32_reset_heap(){ } #endif /* SQLITE_WIN32_MALLOC */ +#ifdef _WIN32 /* ** This function outputs the specified (ANSI) string to the Win32 debugger ** (if available). @@ -47628,6 +49272,7 @@ SQLITE_API void sqlite3_win32_write_debug(const char *zBuf, int nBuf){ } #endif } +#endif /* _WIN32 */ /* ** The following routine suspends the current thread for at least ms @@ -47711,7 +49356,9 @@ SQLITE_API int sqlite3_win32_is_nt(void){ } return osInterlockedCompareExchange(&sqlite3_os_type, 2, 2)==2; #elif SQLITE_TEST - return osInterlockedCompareExchange(&sqlite3_os_type, 2, 2)==2; + return osInterlockedCompareExchange(&sqlite3_os_type, 2, 2)==2 + || osInterlockedCompareExchange(&sqlite3_os_type, 0, 0)==0 + ; #else /* ** NOTE: All sub-platforms where the GetVersionEx[AW] functions are @@ -47926,6 +49573,7 @@ SQLITE_PRIVATE void sqlite3MemSetDefault(void){ } #endif /* SQLITE_WIN32_MALLOC */ +#ifdef _WIN32 /* ** Convert a UTF-8 string to Microsoft Unicode. ** @@ -47951,6 +49599,7 @@ static LPWSTR winUtf8ToUnicode(const char *zText){ } return zWideText; } +#endif /* _WIN32 */ /* ** Convert a Microsoft Unicode string to UTF-8. @@ -47985,28 +49634,29 @@ static char *winUnicodeToUtf8(LPCWSTR zWideText){ ** Space to hold the returned string is obtained from sqlite3_malloc(). */ static LPWSTR winMbcsToUnicode(const char *zText, int useAnsi){ - int nByte; + int nWideChar; LPWSTR zMbcsText; int codepage = useAnsi ? CP_ACP : CP_OEMCP; - nByte = osMultiByteToWideChar(codepage, 0, zText, -1, NULL, - 0)*sizeof(WCHAR); - if( nByte==0 ){ + nWideChar = osMultiByteToWideChar(codepage, 0, zText, -1, NULL, + 0); + if( nWideChar==0 ){ return 0; } - zMbcsText = sqlite3MallocZero( nByte*sizeof(WCHAR) ); + zMbcsText = sqlite3MallocZero( nWideChar*sizeof(WCHAR) ); if( zMbcsText==0 ){ return 0; } - nByte = osMultiByteToWideChar(codepage, 0, zText, -1, zMbcsText, - nByte); - if( nByte==0 ){ + nWideChar = osMultiByteToWideChar(codepage, 0, zText, -1, zMbcsText, + nWideChar); + if( nWideChar==0 ){ sqlite3_free(zMbcsText); zMbcsText = 0; } return zMbcsText; } +#ifdef _WIN32 /* ** Convert a Microsoft Unicode string to a multi-byte character string, ** using the ANSI or OEM code page. @@ -48034,6 +49684,7 @@ static char *winUnicodeToMbcs(LPCWSTR zWideText, int useAnsi){ } return zText; } +#endif /* _WIN32 */ /* ** Convert a multi-byte character string to UTF-8. @@ -48053,6 +49704,7 @@ static char *winMbcsToUtf8(const char *zText, int useAnsi){ return zTextUtf8; } +#ifdef _WIN32 /* ** Convert a UTF-8 string to a multi-byte character string. ** @@ -48102,6 +49754,7 @@ SQLITE_API char *sqlite3_win32_unicode_to_utf8(LPCWSTR zWideText){ #endif return winUnicodeToUtf8(zWideText); } +#endif /* _WIN32 */ /* ** This is a public wrapper for the winMbcsToUtf8() function. @@ -48119,6 +49772,7 @@ SQLITE_API char *sqlite3_win32_mbcs_to_utf8(const char *zText){ return winMbcsToUtf8(zText, osAreFileApisANSI()); } +#ifdef _WIN32 /* ** This is a public wrapper for the winMbcsToUtf8() function. */ @@ -48243,6 +49897,7 @@ SQLITE_API int sqlite3_win32_set_directory( ){ return sqlite3_win32_set_directory16(type, zValue); } +#endif /* _WIN32 */ /* ** The return value of winGetLastErrorMsg @@ -48791,13 +50446,100 @@ static BOOL winLockFile( ovlp.Offset = offsetLow; ovlp.OffsetHigh = offsetHigh; return osLockFileEx(*phFile, flags, 0, numBytesLow, numBytesHigh, &ovlp); +#ifdef SQLITE_WIN32_HAS_ANSI }else{ return osLockFile(*phFile, offsetLow, offsetHigh, numBytesLow, numBytesHigh); +#endif } #endif } +#ifndef SQLITE_OMIT_WAL +/* +** Lock a region of nByte bytes starting at offset offset of file hFile. +** Take an EXCLUSIVE lock if parameter bExclusive is true, or a SHARED lock +** otherwise. If nMs is greater than zero and the lock cannot be obtained +** immediately, block for that many ms before giving up. +** +** This function returns SQLITE_OK if the lock is obtained successfully. If +** some other process holds the lock, SQLITE_BUSY is returned if nMs==0, or +** SQLITE_BUSY_TIMEOUT otherwise. Or, if an error occurs, SQLITE_IOERR. +*/ +static int winHandleLockTimeout( + HANDLE hFile, + DWORD offset, + DWORD nByte, + int bExcl, + DWORD nMs +){ + DWORD flags = LOCKFILE_FAIL_IMMEDIATELY | (bExcl?LOCKFILE_EXCLUSIVE_LOCK:0); + int rc = SQLITE_OK; + BOOL ret; + + if( !osIsNT() ){ + ret = winLockFile(&hFile, flags, offset, 0, nByte, 0); + }else{ + OVERLAPPED ovlp; + memset(&ovlp, 0, sizeof(OVERLAPPED)); + ovlp.Offset = offset; + +#ifdef SQLITE_ENABLE_SETLK_TIMEOUT + if( nMs!=0 ){ + flags &= ~LOCKFILE_FAIL_IMMEDIATELY; + } + ovlp.hEvent = osCreateEvent(NULL, TRUE, FALSE, NULL); + if( ovlp.hEvent==NULL ){ + return SQLITE_IOERR_LOCK; + } +#endif + + ret = osLockFileEx(hFile, flags, 0, nByte, 0, &ovlp); + +#ifdef SQLITE_ENABLE_SETLK_TIMEOUT + /* If SQLITE_ENABLE_SETLK_TIMEOUT is defined, then the file-handle was + ** opened with FILE_FLAG_OVERHEAD specified. In this case, the call to + ** LockFileEx() may fail because the request is still pending. This can + ** happen even if LOCKFILE_FAIL_IMMEDIATELY was specified. + ** + ** If nMs is 0, then LOCKFILE_FAIL_IMMEDIATELY was set in the flags + ** passed to LockFileEx(). In this case, if the operation is pending, + ** block indefinitely until it is finished. + ** + ** Otherwise, wait for up to nMs ms for the operation to finish. nMs + ** may be set to INFINITE. + */ + if( !ret && GetLastError()==ERROR_IO_PENDING ){ + DWORD nDelay = (nMs==0 ? INFINITE : nMs); + DWORD res = osWaitForSingleObject(ovlp.hEvent, nDelay); + if( res==WAIT_OBJECT_0 ){ + ret = TRUE; + }else if( res==WAIT_TIMEOUT ){ +#if SQLITE_ENABLE_SETLK_TIMEOUT==1 + rc = SQLITE_BUSY_TIMEOUT; +#else + rc = SQLITE_BUSY; +#endif + }else{ + /* Some other error has occurred */ + rc = SQLITE_IOERR_LOCK; + } + + /* If it is still pending, cancel the LockFileEx() call. */ + osCancelIo(hFile); + } + + osCloseHandle(ovlp.hEvent); +#endif + } + + if( rc==SQLITE_OK && !ret ){ + rc = SQLITE_BUSY; + } + return rc; +} +#endif /* #ifndef SQLITE_OMIT_WAL */ + /* ** Unlock a file region. */ @@ -48822,13 +50564,25 @@ static BOOL winUnlockFile( ovlp.Offset = offsetLow; ovlp.OffsetHigh = offsetHigh; return osUnlockFileEx(*phFile, 0, numBytesLow, numBytesHigh, &ovlp); +#ifdef SQLITE_WIN32_HAS_ANSI }else{ return osUnlockFile(*phFile, offsetLow, offsetHigh, numBytesLow, numBytesHigh); +#endif } #endif } +#ifndef SQLITE_OMIT_WAL +/* +** Remove an nByte lock starting at offset iOff from HANDLE h. +*/ +static int winHandleUnlock(HANDLE h, int iOff, int nByte){ + BOOL ret = winUnlockFile(&h, iOff, 0, nByte, 0); + return (ret ? SQLITE_OK : SQLITE_IOERR_UNLOCK); +} +#endif + /***************************************************************************** ** The next group of routines implement the I/O methods specified ** by the sqlite3_io_methods object. @@ -48842,66 +50596,70 @@ static BOOL winUnlockFile( #endif /* -** Move the current position of the file handle passed as the first -** argument to offset iOffset within the file. If successful, return 0. -** Otherwise, set pFile->lastErrno and return non-zero. +** Seek the file handle h to offset nByte of the file. +** +** If successful, return SQLITE_OK. Or, if an error occurs, return an SQLite +** error code. */ -static int winSeekFile(winFile *pFile, sqlite3_int64 iOffset){ +static int winHandleSeek(HANDLE h, sqlite3_int64 iOffset){ + int rc = SQLITE_OK; /* Return value */ + #if !SQLITE_OS_WINRT LONG upperBits; /* Most sig. 32 bits of new offset */ LONG lowerBits; /* Least sig. 32 bits of new offset */ DWORD dwRet; /* Value returned by SetFilePointer() */ - DWORD lastErrno; /* Value returned by GetLastError() */ - - OSTRACE(("SEEK file=%p, offset=%lld\n", pFile->h, iOffset)); upperBits = (LONG)((iOffset>>32) & 0x7fffffff); lowerBits = (LONG)(iOffset & 0xffffffff); + dwRet = osSetFilePointer(h, lowerBits, &upperBits, FILE_BEGIN); + /* API oddity: If successful, SetFilePointer() returns a dword ** containing the lower 32-bits of the new file-offset. Or, if it fails, ** it returns INVALID_SET_FILE_POINTER. However according to MSDN, ** INVALID_SET_FILE_POINTER may also be a valid new offset. So to determine ** whether an error has actually occurred, it is also necessary to call - ** GetLastError(). - */ - dwRet = osSetFilePointer(pFile->h, lowerBits, &upperBits, FILE_BEGIN); - - if( (dwRet==INVALID_SET_FILE_POINTER - && ((lastErrno = osGetLastError())!=NO_ERROR)) ){ - pFile->lastErrno = lastErrno; - winLogError(SQLITE_IOERR_SEEK, pFile->lastErrno, - "winSeekFile", pFile->zPath); - OSTRACE(("SEEK file=%p, rc=SQLITE_IOERR_SEEK\n", pFile->h)); - return 1; + ** GetLastError(). */ + if( dwRet==INVALID_SET_FILE_POINTER ){ + DWORD lastErrno = osGetLastError(); + if( lastErrno!=NO_ERROR ){ + rc = SQLITE_IOERR_SEEK; + } } - - OSTRACE(("SEEK file=%p, rc=SQLITE_OK\n", pFile->h)); - return 0; #else - /* - ** Same as above, except that this implementation works for WinRT. - */ - + /* This implementation works for WinRT. */ LARGE_INTEGER x; /* The new offset */ BOOL bRet; /* Value returned by SetFilePointerEx() */ x.QuadPart = iOffset; - bRet = osSetFilePointerEx(pFile->h, x, 0, FILE_BEGIN); + bRet = osSetFilePointerEx(h, x, 0, FILE_BEGIN); if(!bRet){ - pFile->lastErrno = osGetLastError(); - winLogError(SQLITE_IOERR_SEEK, pFile->lastErrno, - "winSeekFile", pFile->zPath); - OSTRACE(("SEEK file=%p, rc=SQLITE_IOERR_SEEK\n", pFile->h)); - return 1; + rc = SQLITE_IOERR_SEEK; } - - OSTRACE(("SEEK file=%p, rc=SQLITE_OK\n", pFile->h)); - return 0; #endif + + OSTRACE(("SEEK file=%p, offset=%lld rc=%s\n", h, iOffset, sqlite3ErrName(rc))); + return rc; } +/* +** Move the current position of the file handle passed as the first +** argument to offset iOffset within the file. If successful, return 0. +** Otherwise, set pFile->lastErrno and return non-zero. +*/ +static int winSeekFile(winFile *pFile, sqlite3_int64 iOffset){ + int rc; + + rc = winHandleSeek(pFile->h, iOffset); + if( rc!=SQLITE_OK ){ + pFile->lastErrno = osGetLastError(); + winLogError(rc, pFile->lastErrno, "winSeekFile", pFile->zPath); + } + return rc; +} + + #if SQLITE_MAX_MMAP_SIZE>0 /* Forward references to VFS helper methods used for memory mapped files */ static int winMapfile(winFile*, sqlite3_int64); @@ -49161,6 +50919,62 @@ static int winWrite( return SQLITE_OK; } +#ifndef SQLITE_OMIT_WAL +/* +** Truncate the file opened by handle h to nByte bytes in size. +*/ +static int winHandleTruncate(HANDLE h, sqlite3_int64 nByte){ + int rc = SQLITE_OK; /* Return code */ + rc = winHandleSeek(h, nByte); + if( rc==SQLITE_OK ){ + if( 0==osSetEndOfFile(h) ){ + rc = SQLITE_IOERR_TRUNCATE; + } + } + return rc; +} + +/* +** Determine the size in bytes of the file opened by the handle passed as +** the first argument. +*/ +static int winHandleSize(HANDLE h, sqlite3_int64 *pnByte){ + int rc = SQLITE_OK; + +#if SQLITE_OS_WINRT + FILE_STANDARD_INFO info; + BOOL b; + b = osGetFileInformationByHandleEx(h, FileStandardInfo, &info, sizeof(info)); + if( b ){ + *pnByte = info.EndOfFile.QuadPart; + }else{ + rc = SQLITE_IOERR_FSTAT; + } +#else + DWORD upperBits = 0; + DWORD lowerBits = 0; + + assert( pnByte ); + lowerBits = osGetFileSize(h, &upperBits); + *pnByte = (((sqlite3_int64)upperBits)<<32) + lowerBits; + if( lowerBits==INVALID_FILE_SIZE && osGetLastError()!=NO_ERROR ){ + rc = SQLITE_IOERR_FSTAT; + } +#endif + + return rc; +} + +/* +** Close the handle passed as the only argument. +*/ +static void winHandleClose(HANDLE h){ + if( h!=INVALID_HANDLE_VALUE ){ + osCloseHandle(h); + } +} +#endif /* #ifndef SQLITE_OMIT_WAL */ + /* ** Truncate an open file to a specified size */ @@ -49416,8 +51230,9 @@ static int winFileSize(sqlite3_file *id, sqlite3_int64 *pSize){ ** Different API routines are called depending on whether or not this ** is Win9x or WinNT. */ -static int winGetReadLock(winFile *pFile){ +static int winGetReadLock(winFile *pFile, int bBlock){ int res; + DWORD mask = ~(bBlock ? LOCKFILE_FAIL_IMMEDIATELY : 0); OSTRACE(("READ-LOCK file=%p, lock=%d\n", pFile->h, pFile->locktype)); if( osIsNT() ){ #if SQLITE_OS_WINCE @@ -49427,7 +51242,7 @@ static int winGetReadLock(winFile *pFile){ */ res = winceLockFile(&pFile->h, SHARED_FIRST, 0, 1, 0); #else - res = winLockFile(&pFile->h, SQLITE_LOCKFILEEX_FLAGS, SHARED_FIRST, 0, + res = winLockFile(&pFile->h, SQLITE_LOCKFILEEX_FLAGS&mask, SHARED_FIRST, 0, SHARED_SIZE, 0); #endif } @@ -49436,7 +51251,7 @@ static int winGetReadLock(winFile *pFile){ int lk; sqlite3_randomness(sizeof(lk), &lk); pFile->sharedLockByte = (short)((lk & 0x7fffffff)%(SHARED_SIZE - 1)); - res = winLockFile(&pFile->h, SQLITE_LOCKFILE_FLAGS, + res = winLockFile(&pFile->h, SQLITE_LOCKFILE_FLAGS&mask, SHARED_FIRST+pFile->sharedLockByte, 0, 1, 0); } #endif @@ -49531,46 +51346,62 @@ static int winLock(sqlite3_file *id, int locktype){ assert( locktype!=PENDING_LOCK ); assert( locktype!=RESERVED_LOCK || pFile->locktype==SHARED_LOCK ); - /* Lock the PENDING_LOCK byte if we need to acquire a PENDING lock or + /* Lock the PENDING_LOCK byte if we need to acquire an EXCLUSIVE lock or ** a SHARED lock. If we are acquiring a SHARED lock, the acquisition of ** the PENDING_LOCK byte is temporary. */ newLocktype = pFile->locktype; - if( pFile->locktype==NO_LOCK - || (locktype==EXCLUSIVE_LOCK && pFile->locktype<=RESERVED_LOCK) + if( locktype==SHARED_LOCK + || (locktype==EXCLUSIVE_LOCK && pFile->locktype==RESERVED_LOCK) ){ int cnt = 3; - while( cnt-->0 && (res = winLockFile(&pFile->h, SQLITE_LOCKFILE_FLAGS, - PENDING_BYTE, 0, 1, 0))==0 ){ + + /* Flags for the LockFileEx() call. This should be an exclusive lock if + ** this call is to obtain EXCLUSIVE, or a shared lock if this call is to + ** obtain SHARED. */ + int flags = LOCKFILE_FAIL_IMMEDIATELY; + if( locktype==EXCLUSIVE_LOCK ){ + flags |= LOCKFILE_EXCLUSIVE_LOCK; + } + while( cnt>0 ){ /* Try 3 times to get the pending lock. This is needed to work ** around problems caused by indexing and/or anti-virus software on ** Windows systems. + ** ** If you are using this code as a model for alternative VFSes, do not - ** copy this retry logic. It is a hack intended for Windows only. - */ + ** copy this retry logic. It is a hack intended for Windows only. */ + res = winLockFile(&pFile->h, flags, PENDING_BYTE, 0, 1, 0); + if( res ) break; + lastErrno = osGetLastError(); OSTRACE(("LOCK-PENDING-FAIL file=%p, count=%d, result=%d\n", - pFile->h, cnt, res)); + pFile->h, cnt, res + )); + if( lastErrno==ERROR_INVALID_HANDLE ){ pFile->lastErrno = lastErrno; rc = SQLITE_IOERR_LOCK; OSTRACE(("LOCK-FAIL file=%p, count=%d, rc=%s\n", - pFile->h, cnt, sqlite3ErrName(rc))); + pFile->h, cnt, sqlite3ErrName(rc) + )); return rc; } - if( cnt ) sqlite3_win32_sleep(1); + + cnt--; + if( cnt>0 ) sqlite3_win32_sleep(1); } gotPendingLock = res; - if( !res ){ - lastErrno = osGetLastError(); - } } /* Acquire a shared lock */ if( locktype==SHARED_LOCK && res ){ assert( pFile->locktype==NO_LOCK ); - res = winGetReadLock(pFile); +#ifdef SQLITE_ENABLE_SETLK_TIMEOUT + res = winGetReadLock(pFile, pFile->bBlockOnConnect); +#else + res = winGetReadLock(pFile, 0); +#endif if( res ){ newLocktype = SHARED_LOCK; }else{ @@ -49608,7 +51439,7 @@ static int winLock(sqlite3_file *id, int locktype){ newLocktype = EXCLUSIVE_LOCK; }else{ lastErrno = osGetLastError(); - winGetReadLock(pFile); + winGetReadLock(pFile, 0); } } @@ -49688,7 +51519,7 @@ static int winUnlock(sqlite3_file *id, int locktype){ type = pFile->locktype; if( type>=EXCLUSIVE_LOCK ){ winUnlockFile(&pFile->h, SHARED_FIRST, 0, SHARED_SIZE, 0); - if( locktype==SHARED_LOCK && !winGetReadLock(pFile) ){ + if( locktype==SHARED_LOCK && !winGetReadLock(pFile, 0) ){ /* This should never happen. We should always be able to ** reacquire the read lock */ rc = winLogError(SQLITE_IOERR_UNLOCK, osGetLastError(), @@ -49857,6 +51688,11 @@ static int winFileControl(sqlite3_file *id, int op, void *pArg){ return SQLITE_OK; } #endif + case SQLITE_FCNTL_NULL_IO: { + (void)osCloseHandle(pFile->h); + pFile->h = NULL; + return SQLITE_OK; + } case SQLITE_FCNTL_TEMPFILENAME: { char *zTFile = 0; int rc = winGetTempname(pFile->pVfs, &zTFile); @@ -49893,6 +51729,50 @@ static int winFileControl(sqlite3_file *id, int op, void *pArg){ return rc; } #endif + +#ifdef SQLITE_ENABLE_SETLK_TIMEOUT + case SQLITE_FCNTL_LOCK_TIMEOUT: { + int iOld = pFile->iBusyTimeout; + int iNew = *(int*)pArg; +#if SQLITE_ENABLE_SETLK_TIMEOUT==1 + pFile->iBusyTimeout = (iNew < 0) ? INFINITE : (DWORD)iNew; +#elif SQLITE_ENABLE_SETLK_TIMEOUT==2 + pFile->iBusyTimeout = (DWORD)(!!iNew); +#else +# error "SQLITE_ENABLE_SETLK_TIMEOUT must be set to 1 or 2" +#endif + *(int*)pArg = iOld; + return SQLITE_OK; + } + case SQLITE_FCNTL_BLOCK_ON_CONNECT: { + int iNew = *(int*)pArg; + pFile->bBlockOnConnect = iNew; + return SQLITE_OK; + } +#endif /* SQLITE_ENABLE_SETLK_TIMEOUT */ + +#if defined(SQLITE_DEBUG) || defined(SQLITE_ENABLE_FILESTAT) + case SQLITE_FCNTL_FILESTAT: { + sqlite3_str *pStr = (sqlite3_str*)pArg; + sqlite3_str_appendf(pStr, "{\"h\":%llu", (sqlite3_uint64)pFile->h); + sqlite3_str_appendf(pStr, ",\"vfs\":\"%s\"", pFile->pVfs->zName); + if( pFile->locktype ){ + static const char *azLock[] = { "SHARED", "RESERVED", + "PENDING", "EXCLUSIVE" }; + sqlite3_str_appendf(pStr, ",\"locktype\":\"%s\"", + azLock[pFile->locktype-1]); + } +#if SQLITE_MAX_MMAP_SIZE>0 + if( pFile->mmapSize ){ + sqlite3_str_appendf(pStr, ",\"mmapSize\":%lld", pFile->mmapSize); + sqlite3_str_appendf(pStr, ",\"nFetchOut\":%d", pFile->nFetchOut); + } +#endif + sqlite3_str_append(pStr, "}", 1); + return SQLITE_OK; + } +#endif /* SQLITE_DEBUG || SQLITE_ENABLE_FILESTAT */ + } OSTRACE(("FCNTL file=%p, rc=SQLITE_NOTFOUND\n", pFile->h)); return SQLITE_NOTFOUND; @@ -49918,7 +51798,7 @@ static int winSectorSize(sqlite3_file *id){ */ static int winDeviceCharacteristics(sqlite3_file *id){ winFile *p = (winFile*)id; - return SQLITE_IOCAP_UNDELETABLE_WHEN_OPEN | + return SQLITE_IOCAP_UNDELETABLE_WHEN_OPEN | SQLITE_IOCAP_SUBPAGE_READ | ((p->ctrlFlags & WINFILE_PSOW)?SQLITE_IOCAP_POWERSAFE_OVERWRITE:0); } @@ -49930,6 +51810,103 @@ static int winDeviceCharacteristics(sqlite3_file *id){ */ static SYSTEM_INFO winSysInfo; +/* +** Convert a UTF-8 filename into whatever form the underlying +** operating system wants filenames in. Space to hold the result +** is obtained from malloc and must be freed by the calling +** function +** +** On Cygwin, 3 possible input forms are accepted: +** - If the filename starts with ":/" or ":\", +** it is converted to UTF-16 as-is. +** - If the filename contains '/', it is assumed to be a +** Cygwin absolute path, it is converted to a win32 +** absolute path in UTF-16. +** - Otherwise it must be a filename only, the win32 filename +** is returned in UTF-16. +** Note: If the function cygwin_conv_path() fails, only +** UTF-8 -> UTF-16 conversion will be done. This can only +** happen when the file path >32k, in which case winUtf8ToUnicode() +** will fail too. +*/ +static void *winConvertFromUtf8Filename(const char *zFilename){ + void *zConverted = 0; + if( osIsNT() ){ +#ifdef __CYGWIN__ + int nChar; + LPWSTR zWideFilename; + + if( osCygwin_conv_path && !(winIsDriveLetterAndColon(zFilename) + && winIsDirSep(zFilename[2])) ){ + i64 nByte; + int convertflag = CCP_POSIX_TO_WIN_W; + if( !strchr(zFilename, '/') ) convertflag |= CCP_RELATIVE; + nByte = (i64)osCygwin_conv_path(convertflag, + zFilename, 0, 0); + if( nByte>0 ){ + zConverted = sqlite3MallocZero(12+(u64)nByte); + if ( zConverted==0 ){ + return zConverted; + } + zWideFilename = zConverted; + /* Filenames should be prefixed, except when converted + * full path already starts with "\\?\". */ + if( osCygwin_conv_path(convertflag, zFilename, + zWideFilename+4, nByte)==0 ){ + if( (convertflag&CCP_RELATIVE) ){ + memmove(zWideFilename, zWideFilename+4, nByte); + }else if( memcmp(zWideFilename+4, L"\\\\", 4) ){ + memcpy(zWideFilename, L"\\\\?\\", 8); + }else if( zWideFilename[6]!='?' ){ + memmove(zWideFilename+6, zWideFilename+4, nByte); + memcpy(zWideFilename, L"\\\\?\\UNC", 14); + }else{ + memmove(zWideFilename, zWideFilename+4, nByte); + } + return zConverted; + } + sqlite3_free(zConverted); + } + } + nChar = osMultiByteToWideChar(CP_UTF8, 0, zFilename, -1, NULL, 0); + if( nChar==0 ){ + return 0; + } + zWideFilename = sqlite3MallocZero( nChar*sizeof(WCHAR)+12 ); + if( zWideFilename==0 ){ + return 0; + } + nChar = osMultiByteToWideChar(CP_UTF8, 0, zFilename, -1, + zWideFilename, nChar); + if( nChar==0 ){ + sqlite3_free(zWideFilename); + zWideFilename = 0; + }else if( nChar>MAX_PATH + && winIsDriveLetterAndColon(zFilename) + && winIsDirSep(zFilename[2]) ){ + memmove(zWideFilename+4, zWideFilename, nChar*sizeof(WCHAR)); + zWideFilename[2] = '\\'; + memcpy(zWideFilename, L"\\\\?\\", 8); + }else if( nChar>MAX_PATH + && winIsDirSep(zFilename[0]) && winIsDirSep(zFilename[1]) + && zFilename[2] != '?' ){ + memmove(zWideFilename+6, zWideFilename, nChar*sizeof(WCHAR)); + memcpy(zWideFilename, L"\\\\?\\UNC", 14); + } + zConverted = zWideFilename; +#else + zConverted = winUtf8ToUnicode(zFilename); +#endif /* __CYGWIN__ */ + } +#if defined(SQLITE_WIN32_HAS_ANSI) && defined(_WIN32) + else{ + zConverted = winUtf8ToMbcs(zFilename, osAreFileApisANSI()); + } +#endif + /* caller will handle out of memory */ + return zConverted; +} + #ifndef SQLITE_OMIT_WAL /* @@ -49966,30 +51943,40 @@ static int winShmMutexHeld(void) { ** log-summary is opened only once per process. ** ** winShmMutexHeld() must be true when creating or destroying -** this object or while reading or writing the following fields: +** this object, or while editing the global linked list that starts +** at winShmNodeList. ** -** nRef -** pNext +** When reading or writing the linked list starting at winShmNode.pWinShmList, +** pShmNode->mutex must be held. ** -** The following fields are read-only after the object is created: +** The following fields are constant after the object is created: ** -** fid ** zFilename +** hSharedShm +** mutex +** bUseSharedLockHandle ** -** Either winShmNode.mutex must be held or winShmNode.nRef==0 and +** Either winShmNode.mutex must be held or winShmNode.pWinShmList==0 and ** winShmMutexHeld() is true when reading or writing any other field ** in this structure. ** +** File-handle hSharedShm is always used to (a) take the DMS lock, (b) +** truncate the *-shm file if the DMS-locking protocol demands it, and +** (c) map regions of the *-shm file into memory using MapViewOfFile() +** or similar. If bUseSharedLockHandle is true, then other locks are also +** taken on hSharedShm. Or, if bUseSharedLockHandle is false, then other +** locks are taken using each connection's winShm.hShm handles. */ struct winShmNode { sqlite3_mutex *mutex; /* Mutex to access this object */ char *zFilename; /* Name of the file */ - winFile hFile; /* File handle from winOpen */ + HANDLE hSharedShm; /* File handle open on zFilename */ + int bUseSharedLockHandle; /* True to use hSharedShm for everything */ + int isUnlocked; /* DMS lock has not yet been obtained */ + int isReadonly; /* True if read-only */ int szRegion; /* Size of shared-memory regions */ int nRegion; /* Size of array apRegion */ - u8 isReadonly; /* True if read-only */ - u8 isUnlocked; /* True if no DMS lock held */ struct ShmRegion { HANDLE hMap; /* File handle from CreateFileMapping */ @@ -49997,8 +51984,8 @@ struct winShmNode { } *aRegion; DWORD lastErrno; /* The Windows errno from the last I/O error */ - int nRef; /* Number of winShm objects pointing to this */ - winShm *pFirst; /* All winShm objects pointing to this */ + winShm *pWinShmList; /* List of winShm objects with ptrs to this */ + winShmNode *pNext; /* Next in list of all winShmNode objects */ #if defined(SQLITE_DEBUG) || defined(SQLITE_HAVE_OS_TRACE) u8 nextShmId; /* Next available winShm.id value */ @@ -50014,26 +52001,19 @@ static winShmNode *winShmNodeList = 0; /* ** Structure used internally by this VFS to record the state of an -** open shared memory connection. -** -** The following fields are initialized when this object is created and -** are read-only thereafter: -** -** winShm.pShmNode -** winShm.id -** -** All other fields are read/write. The winShm.pShmNode->mutex must be held -** while accessing any read/write fields. +** open shared memory connection. There is one such structure for each +** winFile open on a wal mode database. */ struct winShm { winShmNode *pShmNode; /* The underlying winShmNode object */ - winShm *pNext; /* Next winShm with the same winShmNode */ - u8 hasMutex; /* True if holding the winShmNode mutex */ u16 sharedMask; /* Mask of shared locks held */ u16 exclMask; /* Mask of exclusive locks held */ + HANDLE hShm; /* File-handle on *-shm file. For locking. */ + int bReadonly; /* True if hShm is opened read-only */ #if defined(SQLITE_DEBUG) || defined(SQLITE_HAVE_OS_TRACE) u8 id; /* Id of this connection with its winShmNode */ #endif + winShm *pWinShmNext; /* Next winShm object on same winShmNode */ }; /* @@ -50042,56 +52022,12 @@ struct winShm { #define WIN_SHM_BASE ((22+SQLITE_SHM_NLOCK)*4) /* first lock byte */ #define WIN_SHM_DMS (WIN_SHM_BASE+SQLITE_SHM_NLOCK) /* deadman switch */ -/* -** Apply advisory locks for all n bytes beginning at ofst. -*/ -#define WINSHM_UNLCK 1 -#define WINSHM_RDLCK 2 -#define WINSHM_WRLCK 3 -static int winShmSystemLock( - winShmNode *pFile, /* Apply locks to this open shared-memory segment */ - int lockType, /* WINSHM_UNLCK, WINSHM_RDLCK, or WINSHM_WRLCK */ - int ofst, /* Offset to first byte to be locked/unlocked */ - int nByte /* Number of bytes to lock or unlock */ -){ - int rc = 0; /* Result code form Lock/UnlockFileEx() */ - - /* Access to the winShmNode object is serialized by the caller */ - assert( pFile->nRef==0 || sqlite3_mutex_held(pFile->mutex) ); - - OSTRACE(("SHM-LOCK file=%p, lock=%d, offset=%d, size=%d\n", - pFile->hFile.h, lockType, ofst, nByte)); - - /* Release/Acquire the system-level lock */ - if( lockType==WINSHM_UNLCK ){ - rc = winUnlockFile(&pFile->hFile.h, ofst, 0, nByte, 0); - }else{ - /* Initialize the locking parameters */ - DWORD dwFlags = LOCKFILE_FAIL_IMMEDIATELY; - if( lockType == WINSHM_WRLCK ) dwFlags |= LOCKFILE_EXCLUSIVE_LOCK; - rc = winLockFile(&pFile->hFile.h, dwFlags, ofst, 0, nByte, 0); - } - - if( rc!= 0 ){ - rc = SQLITE_OK; - }else{ - pFile->lastErrno = osGetLastError(); - rc = SQLITE_BUSY; - } - - OSTRACE(("SHM-LOCK file=%p, func=%s, errno=%lu, rc=%s\n", - pFile->hFile.h, (lockType == WINSHM_UNLCK) ? "winUnlockFile" : - "winLockFile", pFile->lastErrno, sqlite3ErrName(rc))); - - return rc; -} - /* Forward references to VFS methods */ static int winOpen(sqlite3_vfs*,const char*,sqlite3_file*,int,int*); static int winDelete(sqlite3_vfs *,const char*,int); /* -** Purge the winShmNodeList list of all entries with winShmNode.nRef==0. +** Purge the winShmNodeList list of all entries with winShmNode.pWinShmList==0. ** ** This is not a VFS shared-memory method; it is a utility function called ** by VFS shared-memory methods. @@ -50104,7 +52040,7 @@ static void winShmPurge(sqlite3_vfs *pVfs, int deleteFlag){ osGetCurrentProcessId(), deleteFlag)); pp = &winShmNodeList; while( (p = *pp)!=0 ){ - if( p->nRef==0 ){ + if( p->pWinShmList==0 ){ int i; if( p->mutex ){ sqlite3_mutex_free(p->mutex); } for(i=0; inRegion; i++){ @@ -50117,11 +52053,7 @@ static void winShmPurge(sqlite3_vfs *pVfs, int deleteFlag){ osGetCurrentProcessId(), i, bRc ? "ok" : "failed")); UNUSED_VARIABLE_VALUE(bRc); } - if( p->hFile.h!=NULL && p->hFile.h!=INVALID_HANDLE_VALUE ){ - SimulateIOErrorBenign(1); - winClose((sqlite3_file *)&p->hFile); - SimulateIOErrorBenign(0); - } + winHandleClose(p->hSharedShm); if( deleteFlag ){ SimulateIOErrorBenign(1); sqlite3BeginBenignMalloc(); @@ -50139,42 +52071,196 @@ static void winShmPurge(sqlite3_vfs *pVfs, int deleteFlag){ } /* -** The DMS lock has not yet been taken on shm file pShmNode. Attempt to -** take it now. Return SQLITE_OK if successful, or an SQLite error -** code otherwise. -** -** If the DMS cannot be locked because this is a readonly_shm=1 -** connection and no other process already holds a lock, return -** SQLITE_READONLY_CANTINIT and set pShmNode->isUnlocked=1. +** The DMS lock has not yet been taken on the shm file associated with +** pShmNode. Take the lock. Truncate the *-shm file if required. +** Return SQLITE_OK if successful, or an SQLite error code otherwise. */ -static int winLockSharedMemory(winShmNode *pShmNode){ - int rc = winShmSystemLock(pShmNode, WINSHM_WRLCK, WIN_SHM_DMS, 1); +static int winLockSharedMemory(winShmNode *pShmNode, DWORD nMs){ + HANDLE h = pShmNode->hSharedShm; + int rc = SQLITE_OK; + assert( sqlite3_mutex_held(pShmNode->mutex) ); + rc = winHandleLockTimeout(h, WIN_SHM_DMS, 1, 1, 0); if( rc==SQLITE_OK ){ + /* We have an EXCLUSIVE lock on the DMS byte. This means that this + ** is the first process to open the file. Truncate it to zero bytes + ** in this case. */ if( pShmNode->isReadonly ){ - pShmNode->isUnlocked = 1; - winShmSystemLock(pShmNode, WINSHM_UNLCK, WIN_SHM_DMS, 1); - return SQLITE_READONLY_CANTINIT; - }else if( winTruncate((sqlite3_file*)&pShmNode->hFile, 0) ){ - winShmSystemLock(pShmNode, WINSHM_UNLCK, WIN_SHM_DMS, 1); - return winLogError(SQLITE_IOERR_SHMOPEN, osGetLastError(), - "winLockSharedMemory", pShmNode->zFilename); + rc = SQLITE_READONLY_CANTINIT; + }else{ + rc = winHandleTruncate(h, 0); } + + /* Release the EXCLUSIVE lock acquired above. */ + winUnlockFile(&h, WIN_SHM_DMS, 0, 1, 0); + }else if( (rc & 0xFF)==SQLITE_BUSY ){ + rc = SQLITE_OK; } if( rc==SQLITE_OK ){ - winShmSystemLock(pShmNode, WINSHM_UNLCK, WIN_SHM_DMS, 1); + /* Take a SHARED lock on the DMS byte. */ + rc = winHandleLockTimeout(h, WIN_SHM_DMS, 1, 0, nMs); + if( rc==SQLITE_OK ){ + pShmNode->isUnlocked = 0; + } } - return winShmSystemLock(pShmNode, WINSHM_RDLCK, WIN_SHM_DMS, 1); + return rc; } + /* -** Open the shared-memory area associated with database file pDbFd. +** This function is used to open a handle on a *-shm file. ** -** When opening a new shared-memory file, if no other instances of that -** file are currently open, in this process or in other processes, then -** the file must be truncated to zero length or have its header cleared. +** If SQLITE_ENABLE_SETLK_TIMEOUT is defined at build time, then the file +** is opened with FILE_FLAG_OVERLAPPED specified. If not, it is not. +*/ +static int winHandleOpen( + const char *zUtf8, /* File to open */ + int *pbReadonly, /* IN/OUT: True for readonly handle */ + HANDLE *ph /* OUT: New HANDLE for file */ +){ + int rc = SQLITE_OK; + void *zConverted = 0; + int bReadonly = *pbReadonly; + HANDLE h = INVALID_HANDLE_VALUE; + +#ifdef SQLITE_ENABLE_SETLK_TIMEOUT + const DWORD flag_overlapped = FILE_FLAG_OVERLAPPED; +#else + const DWORD flag_overlapped = 0; +#endif + + /* Convert the filename to the system encoding. */ + zConverted = winConvertFromUtf8Filename(zUtf8); + if( zConverted==0 ){ + OSTRACE(("OPEN name=%s, rc=SQLITE_IOERR_NOMEM", zUtf8)); + rc = SQLITE_IOERR_NOMEM_BKPT; + goto winopenfile_out; + } + + /* Ensure the file we are trying to open is not actually a directory. */ + if( winIsDir(zConverted) ){ + OSTRACE(("OPEN name=%s, rc=SQLITE_CANTOPEN_ISDIR", zUtf8)); + rc = SQLITE_CANTOPEN_ISDIR; + goto winopenfile_out; + } + + /* TODO: platforms. + ** TODO: retry-on-ioerr. + */ + if( osIsNT() ){ +#if SQLITE_OS_WINRT + CREATEFILE2_EXTENDED_PARAMETERS extendedParameters; + memset(&extendedParameters, 0, sizeof(extendedParameters)); + extendedParameters.dwSize = sizeof(extendedParameters); + extendedParameters.dwFileAttributes = FILE_ATTRIBUTE_NORMAL; + extendedParameters.dwFileFlags = flag_overlapped; + extendedParameters.dwSecurityQosFlags = SECURITY_ANONYMOUS; + h = osCreateFile2((LPCWSTR)zConverted, + (GENERIC_READ | (bReadonly ? 0 : GENERIC_WRITE)),/* dwDesiredAccess */ + FILE_SHARE_READ | FILE_SHARE_WRITE, /* dwShareMode */ + OPEN_ALWAYS, /* dwCreationDisposition */ + &extendedParameters + ); +#else + h = osCreateFileW((LPCWSTR)zConverted, /* lpFileName */ + (GENERIC_READ | (bReadonly ? 0 : GENERIC_WRITE)), /* dwDesiredAccess */ + FILE_SHARE_READ | FILE_SHARE_WRITE, /* dwShareMode */ + NULL, /* lpSecurityAttributes */ + OPEN_ALWAYS, /* dwCreationDisposition */ + FILE_ATTRIBUTE_NORMAL|flag_overlapped, + NULL + ); +#endif + }else{ + /* Due to pre-processor directives earlier in this file, + ** SQLITE_WIN32_HAS_ANSI is always defined if osIsNT() is false. */ +#ifdef SQLITE_WIN32_HAS_ANSI + h = osCreateFileA((LPCSTR)zConverted, + (GENERIC_READ | (bReadonly ? 0 : GENERIC_WRITE)), /* dwDesiredAccess */ + FILE_SHARE_READ | FILE_SHARE_WRITE, /* dwShareMode */ + NULL, /* lpSecurityAttributes */ + OPEN_ALWAYS, /* dwCreationDisposition */ + FILE_ATTRIBUTE_NORMAL|flag_overlapped, + NULL + ); +#endif + } + + if( h==INVALID_HANDLE_VALUE ){ + if( bReadonly==0 ){ + bReadonly = 1; + rc = winHandleOpen(zUtf8, &bReadonly, &h); + }else{ + rc = SQLITE_CANTOPEN_BKPT; + } + } + + winopenfile_out: + sqlite3_free(zConverted); + *pbReadonly = bReadonly; + *ph = h; + return rc; +} + +/* +** Close pDbFd's connection to shared-memory. Delete the underlying +** *-shm file if deleteFlag is true. +*/ +static int winCloseSharedMemory(winFile *pDbFd, int deleteFlag){ + winShm *p; /* The connection to be closed */ + winShm **pp; /* Iterator for pShmNode->pWinShmList */ + winShmNode *pShmNode; /* The underlying shared-memory file */ + + p = pDbFd->pShm; + if( p==0 ) return SQLITE_OK; + if( p->hShm!=INVALID_HANDLE_VALUE ){ + osCloseHandle(p->hShm); + } + + winShmEnterMutex(); + pShmNode = p->pShmNode; + + /* Remove this connection from the winShmNode.pWinShmList list */ + sqlite3_mutex_enter(pShmNode->mutex); + for(pp=&pShmNode->pWinShmList; *pp!=p; pp=&(*pp)->pWinShmNext){} + *pp = p->pWinShmNext; + sqlite3_mutex_leave(pShmNode->mutex); + + winShmPurge(pDbFd->pVfs, deleteFlag); + winShmLeaveMutex(); + + /* Free the connection p */ + sqlite3_free(p); + pDbFd->pShm = 0; + return SQLITE_OK; +} + +/* +** testfixture builds may set this global variable to true via a +** Tcl interface. This forces the VFS to use the locking normally +** only used for UNC paths for all files. +*/ +#ifdef SQLITE_TEST +SQLITE_API int sqlite3_win_test_unc_locking = 0; +#else +# define sqlite3_win_test_unc_locking 0 +#endif + +/* +** Return true if the string passed as the only argument is likely +** to be a UNC path. In other words, if it starts with "\\". +*/ +static int winIsUNCPath(const char *zFile){ + if( zFile[0]=='\\' && zFile[1]=='\\' ){ + return 1; + } + return sqlite3_win_test_unc_locking; +} + +/* +** Open the shared-memory area associated with database file pDbFd. */ static int winOpenSharedMemory(winFile *pDbFd){ struct winShm *p; /* The connection to be opened */ @@ -50186,98 +52272,93 @@ static int winOpenSharedMemory(winFile *pDbFd){ assert( pDbFd->pShm==0 ); /* Not previously opened */ /* Allocate space for the new sqlite3_shm object. Also speculatively - ** allocate space for a new winShmNode and filename. - */ + ** allocate space for a new winShmNode and filename. */ p = sqlite3MallocZero( sizeof(*p) ); if( p==0 ) return SQLITE_IOERR_NOMEM_BKPT; nName = sqlite3Strlen30(pDbFd->zPath); - pNew = sqlite3MallocZero( sizeof(*pShmNode) + nName + 17 ); + pNew = sqlite3MallocZero( sizeof(*pShmNode) + (i64)nName + 17 ); if( pNew==0 ){ sqlite3_free(p); return SQLITE_IOERR_NOMEM_BKPT; } pNew->zFilename = (char*)&pNew[1]; + pNew->hSharedShm = INVALID_HANDLE_VALUE; + pNew->isUnlocked = 1; + pNew->bUseSharedLockHandle = winIsUNCPath(pDbFd->zPath); sqlite3_snprintf(nName+15, pNew->zFilename, "%s-shm", pDbFd->zPath); sqlite3FileSuffix3(pDbFd->zPath, pNew->zFilename); /* Look to see if there is an existing winShmNode that can be used. - ** If no matching winShmNode currently exists, create a new one. - */ + ** If no matching winShmNode currently exists, then create a new one. */ winShmEnterMutex(); for(pShmNode = winShmNodeList; pShmNode; pShmNode=pShmNode->pNext){ /* TBD need to come up with better match here. Perhaps - ** use FILE_ID_BOTH_DIR_INFO Structure. - */ + ** use FILE_ID_BOTH_DIR_INFO Structure. */ if( sqlite3StrICmp(pShmNode->zFilename, pNew->zFilename)==0 ) break; } - if( pShmNode ){ - sqlite3_free(pNew); - }else{ - int inFlags = SQLITE_OPEN_WAL; - int outFlags = 0; - + if( pShmNode==0 ){ pShmNode = pNew; - pNew = 0; - ((winFile*)(&pShmNode->hFile))->h = INVALID_HANDLE_VALUE; - pShmNode->pNext = winShmNodeList; - winShmNodeList = pShmNode; + /* Allocate a mutex for this winShmNode object, if one is required. */ if( sqlite3GlobalConfig.bCoreMutex ){ pShmNode->mutex = sqlite3_mutex_alloc(SQLITE_MUTEX_FAST); - if( pShmNode->mutex==0 ){ - rc = SQLITE_IOERR_NOMEM_BKPT; - goto shm_open_err; - } + if( pShmNode->mutex==0 ) rc = SQLITE_IOERR_NOMEM_BKPT; } - if( 0==sqlite3_uri_boolean(pDbFd->zPath, "readonly_shm", 0) ){ - inFlags |= SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE; - }else{ - inFlags |= SQLITE_OPEN_READONLY; - } - rc = winOpen(pDbFd->pVfs, pShmNode->zFilename, - (sqlite3_file*)&pShmNode->hFile, - inFlags, &outFlags); - if( rc!=SQLITE_OK ){ - rc = winLogError(rc, osGetLastError(), "winOpenShm", - pShmNode->zFilename); - goto shm_open_err; + /* Open a file-handle to use for mappings, and for the DMS lock. */ + if( rc==SQLITE_OK ){ + HANDLE h = INVALID_HANDLE_VALUE; + pShmNode->isReadonly = sqlite3_uri_boolean(pDbFd->zPath,"readonly_shm",0); + rc = winHandleOpen(pNew->zFilename, &pShmNode->isReadonly, &h); + pShmNode->hSharedShm = h; } - if( outFlags==SQLITE_OPEN_READONLY ) pShmNode->isReadonly = 1; - rc = winLockSharedMemory(pShmNode); - if( rc!=SQLITE_OK && rc!=SQLITE_READONLY_CANTINIT ) goto shm_open_err; + /* If successful, link the new winShmNode into the global list. If an + ** error occurred, free the object. */ + if( rc==SQLITE_OK ){ + pShmNode->pNext = winShmNodeList; + winShmNodeList = pShmNode; + pNew = 0; + }else{ + sqlite3_mutex_free(pShmNode->mutex); + if( pShmNode->hSharedShm!=INVALID_HANDLE_VALUE ){ + osCloseHandle(pShmNode->hSharedShm); + } + } } - /* Make the new connection a child of the winShmNode */ - p->pShmNode = pShmNode; + /* If no error has occurred, link the winShm object to the winShmNode and + ** the winShm to pDbFd. */ + if( rc==SQLITE_OK ){ + sqlite3_mutex_enter(pShmNode->mutex); + p->pShmNode = pShmNode; + p->pWinShmNext = pShmNode->pWinShmList; + pShmNode->pWinShmList = p; #if defined(SQLITE_DEBUG) || defined(SQLITE_HAVE_OS_TRACE) - p->id = pShmNode->nextShmId++; + p->id = pShmNode->nextShmId++; #endif - pShmNode->nRef++; - pDbFd->pShm = p; + pDbFd->pShm = p; + sqlite3_mutex_leave(pShmNode->mutex); + }else if( p ){ + sqlite3_free(p); + } + + assert( rc!=SQLITE_OK || pShmNode->isUnlocked==0 || pShmNode->nRegion==0 ); winShmLeaveMutex(); + sqlite3_free(pNew); - /* The reference count on pShmNode has already been incremented under - ** the cover of the winShmEnterMutex() mutex and the pointer from the - ** new (struct winShm) object to the pShmNode has been set. All that is - ** left to do is to link the new object into the linked list starting - ** at pShmNode->pFirst. This must be done while holding the pShmNode->mutex - ** mutex. - */ - sqlite3_mutex_enter(pShmNode->mutex); - p->pNext = pShmNode->pFirst; - pShmNode->pFirst = p; - sqlite3_mutex_leave(pShmNode->mutex); - return rc; + /* Open a file-handle on the *-shm file for this connection. This file-handle + ** is only used for locking. The mapping of the *-shm file is created using + ** the shared file handle in winShmNode.hSharedShm. */ + if( rc==SQLITE_OK && pShmNode->bUseSharedLockHandle==0 ){ + p->bReadonly = sqlite3_uri_boolean(pDbFd->zPath, "readonly_shm", 0); + rc = winHandleOpen(pShmNode->zFilename, &p->bReadonly, &p->hShm); + if( rc!=SQLITE_OK ){ + assert( p->hShm==INVALID_HANDLE_VALUE ); + winCloseSharedMemory(pDbFd, 0); + } + } - /* Jump here on any error */ -shm_open_err: - winShmSystemLock(pShmNode, WINSHM_UNLCK, WIN_SHM_DMS, 1); - winShmPurge(pDbFd->pVfs, 0); /* This call frees pShmNode if required */ - sqlite3_free(p); - sqlite3_free(pNew); - winShmLeaveMutex(); return rc; } @@ -50289,38 +52370,7 @@ static int winShmUnmap( sqlite3_file *fd, /* Database holding shared memory */ int deleteFlag /* Delete after closing if true */ ){ - winFile *pDbFd; /* Database holding shared-memory */ - winShm *p; /* The connection to be closed */ - winShmNode *pShmNode; /* The underlying shared-memory file */ - winShm **pp; /* For looping over sibling connections */ - - pDbFd = (winFile*)fd; - p = pDbFd->pShm; - if( p==0 ) return SQLITE_OK; - pShmNode = p->pShmNode; - - /* Remove connection p from the set of connections associated - ** with pShmNode */ - sqlite3_mutex_enter(pShmNode->mutex); - for(pp=&pShmNode->pFirst; (*pp)!=p; pp = &(*pp)->pNext){} - *pp = p->pNext; - - /* Free the connection p */ - sqlite3_free(p); - pDbFd->pShm = 0; - sqlite3_mutex_leave(pShmNode->mutex); - - /* If pShmNode->nRef has reached 0, then close the underlying - ** shared-memory file, too */ - winShmEnterMutex(); - assert( pShmNode->nRef>0 ); - pShmNode->nRef--; - if( pShmNode->nRef==0 ){ - winShmPurge(pDbFd->pVfs, deleteFlag); - } - winShmLeaveMutex(); - - return SQLITE_OK; + return winCloseSharedMemory((winFile*)fd, deleteFlag); } /* @@ -50334,10 +52384,9 @@ static int winShmLock( ){ winFile *pDbFd = (winFile*)fd; /* Connection holding shared memory */ winShm *p = pDbFd->pShm; /* The shared memory being locked */ - winShm *pX; /* For looping over all siblings */ winShmNode *pShmNode; int rc = SQLITE_OK; /* Result code */ - u16 mask; /* Mask of locks to take or release */ + u16 mask = (u16)((1U<<(ofst+n)) - (1U<pShmNode; @@ -50351,85 +52400,127 @@ static int winShmLock( || flags==(SQLITE_SHM_UNLOCK | SQLITE_SHM_EXCLUSIVE) ); assert( n==1 || (flags & SQLITE_SHM_EXCLUSIVE)!=0 ); - mask = (u16)((1U<<(ofst+n)) - (1U<1 || mask==(1<mutex); - if( flags & SQLITE_SHM_UNLOCK ){ - u16 allMask = 0; /* Mask of locks held by siblings */ + /* Check that, if this to be a blocking lock, no locks that occur later + ** in the following list than the lock being obtained are already held: + ** + ** 1. Recovery lock (ofst==2). + ** 2. Checkpointer lock (ofst==1). + ** 3. Write lock (ofst==0). + ** 4. Read locks (ofst>=3 && ofstexclMask|p->sharedMask); + assert( (flags & SQLITE_SHM_UNLOCK) || pDbFd->iBusyTimeout==0 || ( + (ofst!=2 || lockMask==0) + && (ofst!=1 || lockMask==0 || lockMask==2) + && (ofst!=0 || lockMask<3) + && (ofst<3 || lockMask<(1<pFirst; pX; pX=pX->pNext){ - if( pX==p ) continue; - assert( (pX->exclMask & (p->exclMask|p->sharedMask))==0 ); - allMask |= pX->sharedMask; - } + /* Check if there is any work to do. There are three cases: + ** + ** a) An unlock operation where there are locks to unlock, + ** b) An shared lock where the requested lock is not already held + ** c) An exclusive lock where the requested lock is not already held + ** + ** The SQLite core never requests an exclusive lock that it already holds. + ** This is assert()ed immediately below. */ + assert( flags!=(SQLITE_SHM_EXCLUSIVE|SQLITE_SHM_LOCK) + || 0==(p->exclMask & mask) + ); + if( ((flags & SQLITE_SHM_UNLOCK) && ((p->exclMask|p->sharedMask) & mask)) + || (flags==(SQLITE_SHM_SHARED|SQLITE_SHM_LOCK) && 0==(p->sharedMask & mask)) + || (flags==(SQLITE_SHM_EXCLUSIVE|SQLITE_SHM_LOCK)) + ){ + HANDLE h = p->hShm; - /* Unlock the system-level locks */ - if( (mask & allMask)==0 ){ - rc = winShmSystemLock(pShmNode, WINSHM_UNLCK, ofst+WIN_SHM_BASE, n); - }else{ - rc = SQLITE_OK; - } + if( flags & SQLITE_SHM_UNLOCK ){ + /* Case (a) - unlock. */ - /* Undo the local locks */ - if( rc==SQLITE_OK ){ - p->exclMask &= ~mask; - p->sharedMask &= ~mask; - } - }else if( flags & SQLITE_SHM_SHARED ){ - u16 allShared = 0; /* Union of locks held by connections other than "p" */ + assert( (p->exclMask & p->sharedMask)==0 ); + assert( !(flags & SQLITE_SHM_EXCLUSIVE) || (p->exclMask & mask)==mask ); + assert( !(flags & SQLITE_SHM_SHARED) || (p->sharedMask & mask)==mask ); - /* Find out which shared locks are already held by sibling connections. - ** If any sibling already holds an exclusive lock, go ahead and return - ** SQLITE_BUSY. - */ - for(pX=pShmNode->pFirst; pX; pX=pX->pNext){ - if( (pX->exclMask & mask)!=0 ){ - rc = SQLITE_BUSY; - break; + assert( !(flags & SQLITE_SHM_SHARED) || n==1 ); + if( pShmNode->bUseSharedLockHandle ){ + h = pShmNode->hSharedShm; + if( flags & SQLITE_SHM_SHARED ){ + winShm *pShm; + sqlite3_mutex_enter(pShmNode->mutex); + for(pShm=pShmNode->pWinShmList; pShm; pShm=pShm->pWinShmNext){ + if( pShm!=p && (pShm->sharedMask & mask) ){ + /* Another connection within this process is also holding this + ** SHARED lock. So do not actually release the OS lock. */ + h = INVALID_HANDLE_VALUE; + break; + } + } + sqlite3_mutex_leave(pShmNode->mutex); + } } - allShared |= pX->sharedMask; - } - /* Get shared locks at the system level, if necessary */ - if( rc==SQLITE_OK ){ - if( (allShared & mask)==0 ){ - rc = winShmSystemLock(pShmNode, WINSHM_RDLCK, ofst+WIN_SHM_BASE, n); - }else{ - rc = SQLITE_OK; + if( h!=INVALID_HANDLE_VALUE ){ + rc = winHandleUnlock(h, ofst+WIN_SHM_BASE, n); } - } - /* Get the local shared locks */ - if( rc==SQLITE_OK ){ - p->sharedMask |= mask; - } - }else{ - /* Make sure no sibling connections hold locks that will block this - ** lock. If any do, return SQLITE_BUSY right away. - */ - for(pX=pShmNode->pFirst; pX; pX=pX->pNext){ - if( (pX->exclMask & mask)!=0 || (pX->sharedMask & mask)!=0 ){ - rc = SQLITE_BUSY; - break; + /* If successful, also clear the bits in sharedMask/exclMask */ + if( rc==SQLITE_OK ){ + p->exclMask = (p->exclMask & ~mask); + p->sharedMask = (p->sharedMask & ~mask); + } + }else{ + int bExcl = ((flags & SQLITE_SHM_EXCLUSIVE) ? 1 : 0); + DWORD nMs = winFileBusyTimeout(pDbFd); + + if( pShmNode->bUseSharedLockHandle ){ + winShm *pShm; + h = pShmNode->hSharedShm; + sqlite3_mutex_enter(pShmNode->mutex); + for(pShm=pShmNode->pWinShmList; pShm; pShm=pShm->pWinShmNext){ + if( bExcl ){ + if( (pShm->sharedMask|pShm->exclMask) & mask ){ + rc = SQLITE_BUSY; + h = INVALID_HANDLE_VALUE; + } + }else{ + if( pShm->sharedMask & mask ){ + h = INVALID_HANDLE_VALUE; + }else if( pShm->exclMask & mask ){ + rc = SQLITE_BUSY; + h = INVALID_HANDLE_VALUE; + } + } + } + sqlite3_mutex_leave(pShmNode->mutex); } - } - /* Get the exclusive locks at the system level. Then if successful - ** also mark the local connection as being locked. - */ - if( rc==SQLITE_OK ){ - rc = winShmSystemLock(pShmNode, WINSHM_WRLCK, ofst+WIN_SHM_BASE, n); + if( h!=INVALID_HANDLE_VALUE ){ + rc = winHandleLockTimeout(h, ofst+WIN_SHM_BASE, n, bExcl, nMs); + } if( rc==SQLITE_OK ){ - assert( (p->sharedMask & mask)==0 ); - p->exclMask |= mask; + if( bExcl ){ + p->exclMask = (p->exclMask | mask); + }else{ + p->sharedMask = (p->sharedMask | mask); + } } } } - sqlite3_mutex_leave(pShmNode->mutex); - OSTRACE(("SHM-LOCK pid=%lu, id=%d, sharedMask=%03x, exclMask=%03x, rc=%s\n", - osGetCurrentProcessId(), p->id, p->sharedMask, p->exclMask, - sqlite3ErrName(rc))); + + OSTRACE(( + "SHM-LOCK(%d,%d,%d) pid=%lu, id=%d, sharedMask=%03x, exclMask=%03x," + " rc=%s\n", + ofst, n, flags, + osGetCurrentProcessId(), p->id, p->sharedMask, p->exclMask, + sqlite3ErrName(rc)) + ); return rc; } @@ -50491,13 +52582,15 @@ static int winShmMap( sqlite3_mutex_enter(pShmNode->mutex); if( pShmNode->isUnlocked ){ - rc = winLockSharedMemory(pShmNode); + /* Take the DMS lock. */ + assert( pShmNode->nRegion==0 ); + rc = winLockSharedMemory(pShmNode, winFileBusyTimeout(pDbFd)); if( rc!=SQLITE_OK ) goto shmpage_out; - pShmNode->isUnlocked = 0; } - assert( szRegion==pShmNode->szRegion || pShmNode->nRegion==0 ); + assert( szRegion==pShmNode->szRegion || pShmNode->nRegion==0 ); if( pShmNode->nRegion<=iRegion ){ + HANDLE hShared = pShmNode->hSharedShm; struct ShmRegion *apNew; /* New aRegion[] array */ int nByte = (iRegion+1)*szRegion; /* Minimum required file size */ sqlite3_int64 sz; /* Current size of wal-index file */ @@ -50508,10 +52601,9 @@ static int winShmMap( ** Check to see if it has been allocated (i.e. if the wal-index file is ** large enough to contain the requested region). */ - rc = winFileSize((sqlite3_file *)&pShmNode->hFile, &sz); + rc = winHandleSize(hShared, &sz); if( rc!=SQLITE_OK ){ - rc = winLogError(SQLITE_IOERR_SHMSIZE, osGetLastError(), - "winShmMap1", pDbFd->zPath); + rc = winLogError(rc, osGetLastError(), "winShmMap1", pDbFd->zPath); goto shmpage_out; } @@ -50520,19 +52612,17 @@ static int winShmMap( ** zero, exit early. *pp will be set to NULL and SQLITE_OK returned. ** ** Alternatively, if isWrite is non-zero, use ftruncate() to allocate - ** the requested memory region. - */ + ** the requested memory region. */ if( !isWrite ) goto shmpage_out; - rc = winTruncate((sqlite3_file *)&pShmNode->hFile, nByte); + rc = winHandleTruncate(hShared, nByte); if( rc!=SQLITE_OK ){ - rc = winLogError(SQLITE_IOERR_SHMSIZE, osGetLastError(), - "winShmMap2", pDbFd->zPath); + rc = winLogError(rc, osGetLastError(), "winShmMap2", pDbFd->zPath); goto shmpage_out; } } /* Map the requested memory region into this processes address space. */ - apNew = (struct ShmRegion *)sqlite3_realloc64( + apNew = (struct ShmRegion*)sqlite3_realloc64( pShmNode->aRegion, (iRegion+1)*sizeof(apNew[0]) ); if( !apNew ){ @@ -50551,18 +52641,13 @@ static int winShmMap( void *pMap = 0; /* Mapped memory region */ #if SQLITE_OS_WINRT - hMap = osCreateFileMappingFromApp(pShmNode->hFile.h, - NULL, protect, nByte, NULL - ); + hMap = osCreateFileMappingFromApp(hShared, NULL, protect, nByte, NULL); #elif defined(SQLITE_WIN32_HAS_WIDE) - hMap = osCreateFileMappingW(pShmNode->hFile.h, - NULL, protect, 0, nByte, NULL - ); + hMap = osCreateFileMappingW(hShared, NULL, protect, 0, nByte, NULL); #elif defined(SQLITE_WIN32_HAS_ANSI) && SQLITE_WIN32_CREATEFILEMAPPINGA - hMap = osCreateFileMappingA(pShmNode->hFile.h, - NULL, protect, 0, nByte, NULL - ); + hMap = osCreateFileMappingA(hShared, NULL, protect, 0, nByte, NULL); #endif + OSTRACE(("SHM-MAP-CREATE pid=%lu, region=%d, size=%d, rc=%s\n", osGetCurrentProcessId(), pShmNode->nRegion, nByte, hMap ? "ok" : "failed")); @@ -50605,7 +52690,9 @@ static int winShmMap( }else{ *pp = 0; } - if( pShmNode->isReadonly && rc==SQLITE_OK ) rc = SQLITE_READONLY; + if( pShmNode->isReadonly && rc==SQLITE_OK ){ + rc = SQLITE_READONLY; + } sqlite3_mutex_leave(pShmNode->mutex); return rc; } @@ -50925,47 +53012,6 @@ static winVfsAppData winNolockAppData = { ** sqlite3_vfs object. */ -#if defined(__CYGWIN__) -/* -** Convert a filename from whatever the underlying operating system -** supports for filenames into UTF-8. Space to hold the result is -** obtained from malloc and must be freed by the calling function. -*/ -static char *winConvertToUtf8Filename(const void *zFilename){ - char *zConverted = 0; - if( osIsNT() ){ - zConverted = winUnicodeToUtf8(zFilename); - } -#ifdef SQLITE_WIN32_HAS_ANSI - else{ - zConverted = winMbcsToUtf8(zFilename, osAreFileApisANSI()); - } -#endif - /* caller will handle out of memory */ - return zConverted; -} -#endif - -/* -** Convert a UTF-8 filename into whatever form the underlying -** operating system wants filenames in. Space to hold the result -** is obtained from malloc and must be freed by the calling -** function. -*/ -static void *winConvertFromUtf8Filename(const char *zFilename){ - void *zConverted = 0; - if( osIsNT() ){ - zConverted = winUtf8ToUnicode(zFilename); - } -#ifdef SQLITE_WIN32_HAS_ANSI - else{ - zConverted = winUtf8ToMbcs(zFilename, osAreFileApisANSI()); - } -#endif - /* caller will handle out of memory */ - return zConverted; -} - /* ** This function returns non-zero if the specified UTF-8 string buffer ** ends with a directory separator character or one was successfully @@ -50978,7 +53024,14 @@ static int winMakeEndInDirSep(int nBuf, char *zBuf){ if( winIsDirSep(zBuf[nLen-1]) ){ return 1; }else if( nLen+1mxPathname; nBuf = nMax + 2; + nMax = pVfs->mxPathname; + nBuf = 2 + (i64)nMax; zBuf = sqlite3MallocZero( nBuf ); if( !zBuf ){ OSTRACE(("TEMP-FILENAME rc=SQLITE_IOERR_NOMEM\n")); @@ -51055,7 +53109,7 @@ static int winGetTempname(sqlite3_vfs *pVfs, char **pzBuf){ } #if defined(__CYGWIN__) - else{ + else if( osGetenv!=NULL ){ static const char *azDirs[] = { 0, /* getenv("SQLITE_TMPDIR") */ 0, /* getenv("TMPDIR") */ @@ -51071,11 +53125,11 @@ static int winGetTempname(sqlite3_vfs *pVfs, char **pzBuf){ unsigned int i; const char *zDir = 0; - if( !azDirs[0] ) azDirs[0] = getenv("SQLITE_TMPDIR"); - if( !azDirs[1] ) azDirs[1] = getenv("TMPDIR"); - if( !azDirs[2] ) azDirs[2] = getenv("TMP"); - if( !azDirs[3] ) azDirs[3] = getenv("TEMP"); - if( !azDirs[4] ) azDirs[4] = getenv("USERPROFILE"); + if( !azDirs[0] ) azDirs[0] = osGetenv("SQLITE_TMPDIR"); + if( !azDirs[1] ) azDirs[1] = osGetenv("TMPDIR"); + if( !azDirs[2] ) azDirs[2] = osGetenv("TMP"); + if( !azDirs[3] ) azDirs[3] = osGetenv("TEMP"); + if( !azDirs[4] ) azDirs[4] = osGetenv("USERPROFILE"); for(i=0; inOut ){ + /* SQLite assumes that xFullPathname() nul-terminates the output buffer + ** even if it returns an error. */ + zOut[iOff] = '\0'; + return SQLITE_CANTOPEN_BKPT; + } + sqlite3_snprintf(nOut-iOff, &zOut[iOff], "%s", zPath); + return SQLITE_OK; +} +#endif /* __CYGWIN__ */ /* ** Turn a relative pathname into a full pathname. Write the full @@ -51858,8 +53961,8 @@ static int winFullPathnameNoMutex( int nFull, /* Size of output buffer in bytes */ char *zFull /* Output buffer */ ){ -#if !SQLITE_OS_WINCE && !SQLITE_OS_WINRT && !defined(__CYGWIN__) - DWORD nByte; +#if !SQLITE_OS_WINCE && !SQLITE_OS_WINRT + int nByte; void *zConverted; char *zOut; #endif @@ -51872,64 +53975,82 @@ static int winFullPathnameNoMutex( zRelative++; } -#if defined(__CYGWIN__) SimulateIOError( return SQLITE_ERROR ); - UNUSED_PARAMETER(nFull); - assert( nFull>=pVfs->mxPathname ); - if ( sqlite3_data_directory && !winIsVerbatimPathname(zRelative) ){ - /* - ** NOTE: We are dealing with a relative path name and the data - ** directory has been set. Therefore, use it as the basis - ** for converting the relative path name to an absolute - ** one by prepending the data directory and a slash. - */ - char *zOut = sqlite3MallocZero( pVfs->mxPathname+1 ); - if( !zOut ){ - return SQLITE_IOERR_NOMEM_BKPT; - } - if( cygwin_conv_path( - (osIsNT() ? CCP_POSIX_TO_WIN_W : CCP_POSIX_TO_WIN_A) | - CCP_RELATIVE, zRelative, zOut, pVfs->mxPathname+1)<0 ){ - sqlite3_free(zOut); - return winLogError(SQLITE_CANTOPEN_CONVPATH, (DWORD)errno, - "winFullPathname1", zRelative); - }else{ - char *zUtf8 = winConvertToUtf8Filename(zOut); - if( !zUtf8 ){ - sqlite3_free(zOut); - return SQLITE_IOERR_NOMEM_BKPT; - } - sqlite3_snprintf(MIN(nFull, pVfs->mxPathname), zFull, "%s%c%s", - sqlite3_data_directory, winGetDirSep(), zUtf8); - sqlite3_free(zUtf8); - sqlite3_free(zOut); - } - }else{ - char *zOut = sqlite3MallocZero( pVfs->mxPathname+1 ); - if( !zOut ){ - return SQLITE_IOERR_NOMEM_BKPT; - } - if( cygwin_conv_path( - (osIsNT() ? CCP_POSIX_TO_WIN_W : CCP_POSIX_TO_WIN_A), - zRelative, zOut, pVfs->mxPathname+1)<0 ){ - sqlite3_free(zOut); - return winLogError(SQLITE_CANTOPEN_CONVPATH, (DWORD)errno, - "winFullPathname2", zRelative); - }else{ - char *zUtf8 = winConvertToUtf8Filename(zOut); - if( !zUtf8 ){ - sqlite3_free(zOut); - return SQLITE_IOERR_NOMEM_BKPT; - } - sqlite3_snprintf(MIN(nFull, pVfs->mxPathname), zFull, "%s", zUtf8); - sqlite3_free(zUtf8); - sqlite3_free(zOut); + +#ifdef __CYGWIN__ + if( osGetcwd ){ + zFull[nFull-1] = '\0'; + if( !winIsDriveLetterAndColon(zRelative) || !winIsDirSep(zRelative[2]) ){ + int rc = SQLITE_OK; + int nLink = 1; /* Number of symbolic links followed so far */ + const char *zIn = zRelative; /* Input path for each iteration of loop */ + char *zDel = 0; + struct stat buf; + + UNUSED_PARAMETER(pVfs); + + do { + /* Call lstat() on path zIn. Set bLink to true if the path is a symbolic + ** link, or false otherwise. */ + int bLink = 0; + if( osLstat && osReadlink ) { + if( osLstat(zIn, &buf)!=0 ){ + int myErrno = osErrno; + if( myErrno!=ENOENT ){ + rc = winLogError(SQLITE_CANTOPEN_BKPT, (DWORD)myErrno, "lstat", zIn); + } + }else{ + bLink = ((buf.st_mode & 0170000) == 0120000); + } + + if( bLink ){ + if( zDel==0 ){ + zDel = sqlite3MallocZero(nFull); + if( zDel==0 ) rc = SQLITE_NOMEM; + }else if( ++nLink>SQLITE_MAX_SYMLINKS ){ + rc = SQLITE_CANTOPEN_BKPT; + } + + if( rc==SQLITE_OK ){ + nByte = osReadlink(zIn, zDel, nFull-1); + if( nByte ==(DWORD)-1 ){ + rc = winLogError(SQLITE_CANTOPEN_BKPT, (DWORD)osErrno, "readlink", zIn); + }else{ + if( zDel[0]!='/' ){ + int n; + for(n = sqlite3Strlen30(zIn); n>0 && zIn[n-1]!='/'; n--); + if( nByte+n+1>nFull ){ + rc = SQLITE_CANTOPEN_BKPT; + }else{ + memmove(&zDel[n], zDel, nByte+1); + memcpy(zDel, zIn, n); + nByte += n; + } + } + zDel[nByte] = '\0'; + } + } + + zIn = zDel; + } + } + + assert( rc!=SQLITE_OK || zIn!=zFull || zIn[0]=='/' ); + if( rc==SQLITE_OK && zIn!=zFull ){ + rc = mkFullPathname(zIn, zFull, nFull); + } + if( bLink==0 ) break; + zIn = zFull; + }while( rc==SQLITE_OK ); + + sqlite3_free(zDel); + winSimplifyName(zFull); + return rc; } } - return SQLITE_OK; -#endif +#endif /* __CYGWIN__ */ -#if (SQLITE_OS_WINCE || SQLITE_OS_WINRT) && !defined(__CYGWIN__) +#if (SQLITE_OS_WINCE || SQLITE_OS_WINRT) && defined(_WIN32) SimulateIOError( return SQLITE_ERROR ); /* WinCE has no concept of a relative pathname, or so I am told. */ /* WinRT has no way to convert a relative path to an absolute one. */ @@ -51948,7 +54069,8 @@ static int winFullPathnameNoMutex( return SQLITE_OK; #endif -#if !SQLITE_OS_WINCE && !SQLITE_OS_WINRT && !defined(__CYGWIN__) +#if !SQLITE_OS_WINCE && !SQLITE_OS_WINRT +#if defined(_WIN32) /* It's odd to simulate an io-error here, but really this is just ** using the io-error infrastructure to test that SQLite handles this ** function failing. This function could fail if, for example, the @@ -51966,6 +54088,7 @@ static int winFullPathnameNoMutex( sqlite3_data_directory, winGetDirSep(), zRelative); return SQLITE_OK; } +#endif zConverted = winConvertFromUtf8Filename(zRelative); if( zConverted==0 ){ return SQLITE_IOERR_NOMEM_BKPT; @@ -52004,13 +54127,12 @@ static int winFullPathnameNoMutex( return winLogError(SQLITE_CANTOPEN_FULLPATH, osGetLastError(), "winFullPathname3", zRelative); } - nByte += 3; - zTemp = sqlite3MallocZero( nByte*sizeof(zTemp[0]) ); + zTemp = sqlite3MallocZero( nByte*sizeof(zTemp[0]) + 3*sizeof(zTemp[0]) ); if( zTemp==0 ){ sqlite3_free(zConverted); return SQLITE_IOERR_NOMEM_BKPT; } - nByte = osGetFullPathNameA((char*)zConverted, nByte, zTemp, 0); + nByte = osGetFullPathNameA((char*)zConverted, nByte+3, zTemp, 0); if( nByte==0 ){ sqlite3_free(zConverted); sqlite3_free(zTemp); @@ -52023,7 +54145,26 @@ static int winFullPathnameNoMutex( } #endif if( zOut ){ +#ifdef __CYGWIN__ + if( memcmp(zOut, "\\\\?\\", 4) ){ + sqlite3_snprintf(MIN(nFull, pVfs->mxPathname), zFull, "%s", zOut); + }else if( memcmp(zOut+4, "UNC\\", 4) ){ + sqlite3_snprintf(MIN(nFull, pVfs->mxPathname), zFull, "%s", zOut+4); + }else{ + char *p = zOut+6; + *p = '\\'; + if( osGetcwd ){ + /* On Cygwin, UNC paths use forward slashes */ + while( *p ){ + if( *p=='\\' ) *p = '/'; + ++p; + } + } + sqlite3_snprintf(MIN(nFull, pVfs->mxPathname), zFull, "%s", zOut+6); + } +#else sqlite3_snprintf(MIN(nFull, pVfs->mxPathname), zFull, "%s", zOut); +#endif /* __CYGWIN__ */ sqlite3_free(zOut); return SQLITE_OK; }else{ @@ -52053,25 +54194,8 @@ static int winFullPathname( */ static void *winDlOpen(sqlite3_vfs *pVfs, const char *zFilename){ HANDLE h; -#if defined(__CYGWIN__) - int nFull = pVfs->mxPathname+1; - char *zFull = sqlite3MallocZero( nFull ); - void *zConverted = 0; - if( zFull==0 ){ - OSTRACE(("DLOPEN name=%s, handle=%p\n", zFilename, (void*)0)); - return 0; - } - if( winFullPathname(pVfs, zFilename, nFull, zFull)!=SQLITE_OK ){ - sqlite3_free(zFull); - OSTRACE(("DLOPEN name=%s, handle=%p\n", zFilename, (void*)0)); - return 0; - } - zConverted = winConvertFromUtf8Filename(zFull); - sqlite3_free(zFull); -#else void *zConverted = winConvertFromUtf8Filename(zFilename); UNUSED_PARAMETER(pVfs); -#endif if( zConverted==0 ){ OSTRACE(("DLOPEN name=%s, handle=%p\n", zFilename, (void*)0)); return 0; @@ -52420,7 +54544,7 @@ SQLITE_API int sqlite3_os_init(void){ /* Double-check that the aSyscall[] array has been constructed ** correctly. See ticket [bb3a86e890c8e96ab] */ - assert( ArraySize(aSyscall)==80 ); + assert( ArraySize(aSyscall)==89 ); /* get memory map allocation granularity */ memset(&winSysInfo, 0, sizeof(SYSTEM_INFO)); @@ -53039,13 +55163,13 @@ static int memdbOpen( } if( p==0 ){ MemStore **apNew; - p = sqlite3Malloc( sizeof(*p) + szName + 3 ); + p = sqlite3Malloc( sizeof(*p) + (i64)szName + 3 ); if( p==0 ){ sqlite3_mutex_leave(pVfsMutex); return SQLITE_NOMEM; } apNew = sqlite3Realloc(memdb_g.apMemStore, - sizeof(apNew[0])*(memdb_g.nMemStore+1) ); + sizeof(apNew[0])*(1+(i64)memdb_g.nMemStore) ); if( apNew==0 ){ sqlite3_free(p); sqlite3_mutex_leave(pVfsMutex); @@ -53478,7 +55602,7 @@ SQLITE_PRIVATE int sqlite3MemdbInit(void){ ** no fewer collisions than the no-op *1. */ #define BITVEC_HASH(X) (((X)*1)%BITVEC_NINT) -#define BITVEC_NPTR (BITVEC_USIZE/sizeof(Bitvec *)) +#define BITVEC_NPTR ((u32)(BITVEC_USIZE/sizeof(Bitvec *))) /* @@ -53518,6 +55642,7 @@ struct Bitvec { } u; }; + /* ** Create a new bitmap object able to handle bits between 0 and iSize, ** inclusive. Return a pointer to the new object. Return NULL if @@ -53627,7 +55752,9 @@ SQLITE_PRIVATE int sqlite3BitvecSet(Bitvec *p, u32 i){ }else{ memcpy(aiValues, p->u.aHash, sizeof(p->u.aHash)); memset(p->u.apSub, 0, sizeof(p->u.apSub)); - p->iDivisor = (p->iSize + BITVEC_NPTR - 1)/BITVEC_NPTR; + p->iDivisor = p->iSize/BITVEC_NPTR; + if( (p->iSize%BITVEC_NPTR)!=0 ) p->iDivisor++; + if( p->iDivisoriDivisor = BITVEC_NBIT; rc = sqlite3BitvecSet(p, i); for(j=0; jiSize<=BITVEC_NBIT ){ - p->u.aBitmap[i/BITVEC_SZELEM] &= ~(1 << (i&(BITVEC_SZELEM-1))); + p->u.aBitmap[i/BITVEC_SZELEM] &= ~(BITVEC_TELEM)(1<<(i&(BITVEC_SZELEM-1))); }else{ unsigned int j; u32 *aiValues = pBuf; @@ -53704,6 +55831,52 @@ SQLITE_PRIVATE u32 sqlite3BitvecSize(Bitvec *p){ return p->iSize; } +#ifdef SQLITE_DEBUG +/* +** Show the content of a Bitvec option and its children. Indent +** everything by n spaces. Add x to each bitvec value. +** +** From a debugger such as gdb, one can type: +** +** call sqlite3ShowBitvec(p) +** +** For some Bitvec p and see a recursive view of the Bitvec's content. +*/ +static void showBitvec(Bitvec *p, int n, unsigned x){ + int i; + if( p==0 ){ + printf("NULL\n"); + return; + } + printf("Bitvec 0x%p iSize=%u", p, p->iSize); + if( p->iSize<=BITVEC_NBIT ){ + printf(" bitmap\n"); + printf("%*s bits:", n, ""); + for(i=1; i<=BITVEC_NBIT; i++){ + if( sqlite3BitvecTest(p,i) ) printf(" %u", x+(unsigned)i); + } + printf("\n"); + }else if( p->iDivisor==0 ){ + printf(" hash with %u entries\n", p->nSet); + printf("%*s bits:", n, ""); + for(i=0; iu.aHash[i] ) printf(" %u", x+(unsigned)p->u.aHash[i]); + } + printf("\n"); + }else{ + printf(" sub-bitvec with iDivisor=%u\n", p->iDivisor); + for(i=0; iu.apSub[i]==0 ) continue; + printf("%*s apSub[%d]=", n, "", i); + showBitvec(p->u.apSub[i], n+4, i*p->iDivisor); + } + } +} +SQLITE_PRIVATE void sqlite3ShowBitvec(Bitvec *p){ + showBitvec(p, 0, 0); +} +#endif + #ifndef SQLITE_UNTESTABLE /* ** Let V[] be an array of unsigned characters sufficient to hold @@ -53712,9 +55885,10 @@ SQLITE_PRIVATE u32 sqlite3BitvecSize(Bitvec *p){ ** individual bits within V. */ #define SETBIT(V,I) V[I>>3] |= (1<<(I&7)) -#define CLEARBIT(V,I) V[I>>3] &= ~(1<<(I&7)) +#define CLEARBIT(V,I) V[I>>3] &= ~(BITVEC_TELEM)(1<<(I&7)) #define TESTBIT(V,I) (V[I>>3]&(1<<(I&7)))!=0 + /* ** This routine runs an extensive test of the Bitvec code. ** @@ -53723,7 +55897,7 @@ SQLITE_PRIVATE u32 sqlite3BitvecSize(Bitvec *p){ ** by 0, 1, or 3 operands, depending on the opcode. Another ** opcode follows immediately after the last operand. ** -** There are 6 opcodes numbered from 0 through 5. 0 is the +** There are opcodes numbered starting with 0. 0 is the ** "halt" opcode and causes the test to end. ** ** 0 Halt and return the number of errors @@ -53732,18 +55906,25 @@ SQLITE_PRIVATE u32 sqlite3BitvecSize(Bitvec *p){ ** 3 N Set N randomly chosen bits ** 4 N Clear N randomly chosen bits ** 5 N S X Set N bits from S increment X in array only, not in bitvec +** 6 Invoice sqlite3ShowBitvec() on the Bitvec object so far +** 7 X Show compile-time parameters and the hash of X ** ** The opcodes 1 through 4 perform set and clear operations are performed ** on both a Bitvec object and on a linear array of bits obtained from malloc. ** Opcode 5 works on the linear array only, not on the Bitvec. ** Opcode 5 is used to deliberately induce a fault in order to -** confirm that error detection works. +** confirm that error detection works. Opcodes 6 and greater are +** state output opcodes. Opcodes 6 and greater are no-ops unless +** SQLite has been compiled with SQLITE_DEBUG. ** ** At the conclusion of the test the linear array is compared ** against the Bitvec object. If there are any differences, ** an error is returned. If they are the same, zero is returned. ** ** If a memory allocation error occurs, return -1. +** +** sz is the size of the Bitvec. Or if sz is negative, make the size +** 2*(unsigned)(-sz) and disabled the linear vector check. */ SQLITE_PRIVATE int sqlite3BitvecBuiltinTest(int sz, int *aOp){ Bitvec *pBitvec = 0; @@ -53754,10 +55935,15 @@ SQLITE_PRIVATE int sqlite3BitvecBuiltinTest(int sz, int *aOp){ /* Allocate the Bitvec to be tested and a linear array of ** bits to act as the reference */ - pBitvec = sqlite3BitvecCreate( sz ); - pV = sqlite3MallocZero( (sz+7)/8 + 1 ); + if( sz<=0 ){ + pBitvec = sqlite3BitvecCreate( 2*(unsigned)(-sz) ); + pV = 0; + }else{ + pBitvec = sqlite3BitvecCreate( sz ); + pV = sqlite3MallocZero( (7+(i64)sz)/8 + 1 ); + } pTmpSpace = sqlite3_malloc64(BITVEC_SZ); - if( pBitvec==0 || pV==0 || pTmpSpace==0 ) goto bitvec_end; + if( pBitvec==0 || pTmpSpace==0 || (pV==0 && sz>0) ) goto bitvec_end; /* NULL pBitvec tests */ sqlite3BitvecSet(0, 1); @@ -53766,6 +55952,24 @@ SQLITE_PRIVATE int sqlite3BitvecBuiltinTest(int sz, int *aOp){ /* Run the program */ pc = i = 0; while( (op = aOp[pc])!=0 ){ + if( op>=6 ){ +#ifdef SQLITE_DEBUG + if( op==6 ){ + sqlite3ShowBitvec(pBitvec); + }else if( op==7 ){ + printf("BITVEC_SZ = %d (%d by sizeof)\n", + BITVEC_SZ, (int)sizeof(Bitvec)); + printf("BITVEC_USIZE = %d\n", (int)BITVEC_USIZE); + printf("BITVEC_NELEM = %d\n", (int)BITVEC_NELEM); + printf("BITVEC_NBIT = %d\n", (int)BITVEC_NBIT); + printf("BITVEC_NINT = %d\n", (int)BITVEC_NINT); + printf("BITVEC_MXHASH = %d\n", (int)BITVEC_MXHASH); + printf("BITVEC_NPTR = %d\n", (int)BITVEC_NPTR); + } +#endif + pc++; + continue; + } switch( op ){ case 1: case 2: @@ -53787,12 +55991,12 @@ SQLITE_PRIVATE int sqlite3BitvecBuiltinTest(int sz, int *aOp){ pc += nx; i = (i & 0x7fffffff)%sz; if( (op & 1)!=0 ){ - SETBIT(pV, (i+1)); + if( pV ) SETBIT(pV, (i+1)); if( op!=5 ){ if( sqlite3BitvecSet(pBitvec, i+1) ) goto bitvec_end; } }else{ - CLEARBIT(pV, (i+1)); + if( pV ) CLEARBIT(pV, (i+1)); sqlite3BitvecClear(pBitvec, i+1, pTmpSpace); } } @@ -53802,14 +56006,18 @@ SQLITE_PRIVATE int sqlite3BitvecBuiltinTest(int sz, int *aOp){ ** match (rc==0). Change rc to non-zero if a discrepancy ** is found. */ - rc = sqlite3BitvecTest(0,0) + sqlite3BitvecTest(pBitvec, sz+1) - + sqlite3BitvecTest(pBitvec, 0) - + (sqlite3BitvecSize(pBitvec) - sz); - for(i=1; i<=sz; i++){ - if( (TESTBIT(pV,i))!=sqlite3BitvecTest(pBitvec,i) ){ - rc = i; - break; + if( pV ){ + rc = sqlite3BitvecTest(0,0) + sqlite3BitvecTest(pBitvec, sz+1) + + sqlite3BitvecTest(pBitvec, 0) + + (sqlite3BitvecSize(pBitvec) - sz); + for(i=1; i<=sz; i++){ + if( (TESTBIT(pV,i))!=sqlite3BitvecTest(pBitvec,i) ){ + rc = i; + break; + } } + }else{ + rc = 0; } /* Free allocated structure */ @@ -54337,6 +56545,7 @@ static SQLITE_NOINLINE PgHdr *pcacheFetchFinishWithInit( pPgHdr->pData = pPage->pBuf; pPgHdr->pExtra = (void *)&pPgHdr[1]; memset(pPgHdr->pExtra, 0, 8); + assert( EIGHT_BYTE_ALIGNMENT( pPgHdr->pExtra ) ); pPgHdr->pCache = pCache; pPgHdr->pgno = pgno; pPgHdr->flags = PGHDR_CLEAN; @@ -54995,10 +57204,6 @@ static SQLITE_WSD struct PCacheGlobal { sqlite3_mutex *mutex; /* Mutex for accessing the following: */ PgFreeslot *pFree; /* Free page blocks */ int nFreeSlot; /* Number of unused pcache slots */ - /* The following value requires a mutex to change. We skip the mutex on - ** reading because (1) most platforms read a 32-bit integer atomically and - ** (2) even if an incorrect value is read, no great harm is done since this - ** is really just an optimization. */ int bUnderPressure; /* True if low on PAGECACHE memory */ } pcache1_g; @@ -55046,7 +57251,7 @@ SQLITE_PRIVATE void sqlite3PCacheBufferSetup(void *pBuf, int sz, int n){ pcache1.nReserve = n>90 ? 10 : (n/10 + 1); pcache1.pStart = pBuf; pcache1.pFree = 0; - pcache1.bUnderPressure = 0; + AtomicStore(&pcache1.bUnderPressure,0); while( n-- ){ p = (PgFreeslot*)pBuf; p->pNext = pcache1.pFree; @@ -55083,7 +57288,8 @@ static int pcache1InitBulk(PCache1 *pCache){ do{ PgHdr1 *pX = (PgHdr1*)&zBulk[pCache->szPage]; pX->page.pBuf = zBulk; - pX->page.pExtra = &pX[1]; + pX->page.pExtra = (u8*)pX + ROUND8(sizeof(*pX)); + assert( EIGHT_BYTE_ALIGNMENT( pX->page.pExtra ) ); pX->isBulkLocal = 1; pX->isAnchor = 0; pX->pNext = pCache->pFree; @@ -55113,7 +57319,7 @@ static void *pcache1Alloc(int nByte){ if( p ){ pcache1.pFree = pcache1.pFree->pNext; pcache1.nFreeSlot--; - pcache1.bUnderPressure = pcache1.nFreeSlot=0 ); sqlite3StatusHighwater(SQLITE_STATUS_PAGECACHE_SIZE, nByte); sqlite3StatusUp(SQLITE_STATUS_PAGECACHE_USED, 1); @@ -55152,7 +57358,7 @@ static void pcache1Free(void *p){ pSlot->pNext = pcache1.pFree; pcache1.pFree = pSlot; pcache1.nFreeSlot++; - pcache1.bUnderPressure = pcache1.nFreeSlotszPage]; p->page.pBuf = pPg; - p->page.pExtra = &p[1]; + p->page.pExtra = (u8*)p + ROUND8(sizeof(*p)); + assert( EIGHT_BYTE_ALIGNMENT( p->page.pExtra ) ); p->isBulkLocal = 0; p->isAnchor = 0; p->pLruPrev = 0; /* Initializing this saves a valgrind error */ @@ -55282,7 +57489,7 @@ SQLITE_PRIVATE void sqlite3PageFree(void *p){ */ static int pcache1UnderMemoryPressure(PCache1 *pCache){ if( pcache1.nSlot && (pCache->szPage+pCache->szExtra)<=pcache1.szSlot ){ - return pcache1.bUnderPressure; + return AtomicLoad(&pcache1.bUnderPressure); }else{ return sqlite3HeapNearlyFull(); } @@ -55299,12 +57506,12 @@ static int pcache1UnderMemoryPressure(PCache1 *pCache){ */ static void pcache1ResizeHash(PCache1 *p){ PgHdr1 **apNew; - unsigned int nNew; - unsigned int i; + u64 nNew; + u32 i; assert( sqlite3_mutex_held(p->pGroup->mutex) ); - nNew = p->nHash*2; + nNew = 2*(u64)p->nHash; if( nNew<256 ){ nNew = 256; } @@ -55527,7 +57734,7 @@ static void pcache1Destroy(sqlite3_pcache *p); static sqlite3_pcache *pcache1Create(int szPage, int szExtra, int bPurgeable){ PCache1 *pCache; /* The newly created page cache */ PGroup *pGroup; /* The group the new page cache will belong to */ - int sz; /* Bytes of memory required to allocate the new cache */ + i64 sz; /* Bytes of memory required to allocate the new cache */ assert( (szPage & (szPage-1))==0 && szPage>=512 && szPage<=65536 ); assert( szExtra < 300 ); @@ -57415,6 +59622,9 @@ struct Pager { Wal *pWal; /* Write-ahead log used by "journal_mode=wal" */ char *zWal; /* File name for write-ahead log */ #endif +#ifdef SQLITE_ENABLE_SETLK_TIMEOUT + sqlite3 *dbWal; +#endif }; /* @@ -57504,39 +59714,33 @@ static const unsigned char aJournalMagic[] = { # define USEFETCH(x) 0 #endif -/* -** The argument to this macro is a file descriptor (type sqlite3_file*). -** Return 0 if it is not open, or non-zero (but not 1) if it is. -** -** This is so that expressions can be written as: -** -** if( isOpen(pPager->jfd) ){ ... -** -** instead of -** -** if( pPager->jfd->pMethods ){ ... -*/ -#define isOpen(pFd) ((pFd)->pMethods!=0) - #ifdef SQLITE_DIRECT_OVERFLOW_READ /* ** Return true if page pgno can be read directly from the database file ** by the b-tree layer. This is the case if: ** -** * the database file is open, -** * there are no dirty pages in the cache, and -** * the desired page is not currently in the wal file. +** (1) the database file is open +** (2) the VFS for the database is able to do unaligned sub-page reads +** (3) there are no dirty pages in the cache, and +** (4) the desired page is not currently in the wal file. */ SQLITE_PRIVATE int sqlite3PagerDirectReadOk(Pager *pPager, Pgno pgno){ - if( pPager->fd->pMethods==0 ) return 0; - if( sqlite3PCacheIsDirty(pPager->pPCache) ) return 0; + assert( pPager!=0 ); + assert( pPager->fd!=0 ); + if( pPager->fd->pMethods==0 ) return 0; /* Case (1) */ + if( sqlite3PCacheIsDirty(pPager->pPCache) ) return 0; /* Failed (3) */ #ifndef SQLITE_OMIT_WAL if( pPager->pWal ){ u32 iRead = 0; (void)sqlite3WalFindFrame(pPager->pWal, pgno, &iRead); - return iRead==0; + if( iRead ) return 0; /* Case (4) */ } #endif + assert( pPager->fd->pMethods->xDeviceCharacteristics!=0 ); + if( (pPager->fd->pMethods->xDeviceCharacteristics(pPager->fd) + & SQLITE_IOCAP_SUBPAGE_READ)==0 ){ + return 0; /* Case (2) */ + } return 1; } #endif @@ -58012,7 +60216,7 @@ static void checkPage(PgHdr *pPg){ ** If an error occurs while reading from the journal file, an SQLite ** error code is returned. */ -static int readSuperJournal(sqlite3_file *pJrnl, char *zSuper, u32 nSuper){ +static int readSuperJournal(sqlite3_file *pJrnl, char *zSuper, u64 nSuper){ int rc; /* Return code */ u32 len; /* Length in bytes of super-journal name */ i64 szJ; /* Total size in bytes of journal file pJrnl */ @@ -58567,6 +60771,15 @@ static void pager_unlock(Pager *pPager){ if( pagerUseWal(pPager) ){ assert( !isOpen(pPager->jfd) ); + if( pPager->eState==PAGER_ERROR ){ + /* If an IO error occurs in wal.c while attempting to wrap the wal file, + ** then the Wal object may be holding a write-lock but no read-lock. + ** This call ensures that the write-lock is dropped as well. We cannot + ** have sqlite3WalEndReadTransaction() drop the write-lock, as it once + ** did, because this would break "BEGIN EXCLUSIVE" handling for + ** SQLITE_ENABLE_SETLK_TIMEOUT builds. */ + (void)sqlite3WalEndWriteTransaction(pPager->pWal); + } sqlite3WalEndReadTransaction(pPager->pWal); pPager->eState = PAGER_OPEN; }else if( !pPager->exclusiveMode ){ @@ -58795,7 +61008,7 @@ static int pager_end_transaction(Pager *pPager, int hasSuper, int bCommit){ } pPager->journalOff = 0; }else if( pPager->journalMode==PAGER_JOURNALMODE_PERSIST - || (pPager->exclusiveMode && pPager->journalMode!=PAGER_JOURNALMODE_WAL) + || (pPager->exclusiveMode && pPager->journalModetempFile); pPager->journalOff = 0; @@ -59248,12 +61461,12 @@ static int pager_delsuper(Pager *pPager, const char *zSuper){ char *zJournal; /* Pointer to one journal within MJ file */ char *zSuperPtr; /* Space to hold super-journal filename */ char *zFree = 0; /* Free this buffer */ - int nSuperPtr; /* Amount of space allocated to zSuperPtr[] */ + i64 nSuperPtr; /* Amount of space allocated to zSuperPtr[] */ /* Allocate space for both the pJournal and pSuper file descriptors. ** If successful, open the super-journal file for reading. */ - pSuper = (sqlite3_file *)sqlite3MallocZero(pVfs->szOsFile * 2); + pSuper = (sqlite3_file *)sqlite3MallocZero(2 * (i64)pVfs->szOsFile); if( !pSuper ){ rc = SQLITE_NOMEM_BKPT; pJournal = 0; @@ -59271,11 +61484,14 @@ static int pager_delsuper(Pager *pPager, const char *zSuper){ */ rc = sqlite3OsFileSize(pSuper, &nSuperJournal); if( rc!=SQLITE_OK ) goto delsuper_out; - nSuperPtr = pVfs->mxPathname+1; + nSuperPtr = 1 + (i64)pVfs->mxPathname; + assert( nSuperJournal>=0 && nSuperPtr>0 ); zFree = sqlite3Malloc(4 + nSuperJournal + nSuperPtr + 2); if( !zFree ){ rc = SQLITE_NOMEM_BKPT; goto delsuper_out; + }else{ + assert( nSuperJournal<=0x7fffffff ); } zFree[0] = zFree[1] = zFree[2] = zFree[3] = 0; zSuperJournal = &zFree[4]; @@ -59536,7 +61752,7 @@ static int pager_playback(Pager *pPager, int isHot){ ** for pageSize. */ zSuper = pPager->pTmpSpace; - rc = readSuperJournal(pPager->jfd, zSuper, pPager->pVfs->mxPathname+1); + rc = readSuperJournal(pPager->jfd, zSuper, 1+(i64)pPager->pVfs->mxPathname); if( rc==SQLITE_OK && zSuper[0] ){ rc = sqlite3OsAccess(pVfs, zSuper, SQLITE_ACCESS_EXISTS, &res); } @@ -59675,7 +61891,7 @@ static int pager_playback(Pager *pPager, int isHot){ ** which case it requires 4 0x00 bytes in memory immediately before ** the filename. */ zSuper = &pPager->pTmpSpace[4]; - rc = readSuperJournal(pPager->jfd, zSuper, pPager->pVfs->mxPathname+1); + rc = readSuperJournal(pPager->jfd, zSuper, 1+(i64)pPager->pVfs->mxPathname); testcase( rc!=SQLITE_OK ); } if( rc==SQLITE_OK @@ -60318,14 +62534,27 @@ SQLITE_PRIVATE void sqlite3PagerSetFlags( unsigned pgFlags /* Various flags */ ){ unsigned level = pgFlags & PAGER_SYNCHRONOUS_MASK; - if( pPager->tempFile ){ + if( pPager->tempFile || level==PAGER_SYNCHRONOUS_OFF ){ pPager->noSync = 1; pPager->fullSync = 0; pPager->extraSync = 0; }else{ - pPager->noSync = level==PAGER_SYNCHRONOUS_OFF ?1:0; + pPager->noSync = 0; pPager->fullSync = level>=PAGER_SYNCHRONOUS_FULL ?1:0; - pPager->extraSync = level==PAGER_SYNCHRONOUS_EXTRA ?1:0; + + /* Set Pager.extraSync if "PRAGMA synchronous=EXTRA" is requested, or + ** if the file-system supports F2FS style atomic writes. If this flag + ** is set, SQLite syncs the directory to disk immediately after deleting + ** a journal file in "PRAGMA journal_mode=DELETE" mode. */ + if( level==PAGER_SYNCHRONOUS_EXTRA +#ifdef SQLITE_ENABLE_BATCH_ATOMIC_WRITE + || (sqlite3OsDeviceCharacteristics(pPager->fd) & SQLITE_IOCAP_BATCH_ATOMIC) +#endif + ){ + pPager->extraSync = 1; + }else{ + pPager->extraSync = 0; + } } if( pPager->noSync ){ pPager->syncFlags = 0; @@ -60779,6 +63008,7 @@ static int pagerAcquireMapPage( return SQLITE_NOMEM_BKPT; } p->pExtra = (void *)&p[1]; + assert( EIGHT_BYTE_ALIGNMENT( p->pExtra ) ); p->flags = PGHDR_MMAP; p->nRef = 1; p->pPager = pPager; @@ -61445,6 +63675,7 @@ SQLITE_PRIVATE int sqlite3PagerOpen( const char *zUri = 0; /* URI args to copy */ int nUriByte = 1; /* Number of bytes of URI args at *zUri */ + /* Figure out how much space is required for each journal file-handle ** (there are two of them, the main journal and the sub-journal). */ journalFileSize = ROUND8(sqlite3JournalSize(pVfs)); @@ -61470,8 +63701,8 @@ SQLITE_PRIVATE int sqlite3PagerOpen( */ if( zFilename && zFilename[0] ){ const char *z; - nPathname = pVfs->mxPathname+1; - zPathname = sqlite3DbMallocRaw(0, nPathname*2); + nPathname = pVfs->mxPathname + 1; + zPathname = sqlite3DbMallocRaw(0, 2*(i64)nPathname); if( zPathname==0 ){ return SQLITE_NOMEM_BKPT; } @@ -61558,14 +63789,14 @@ SQLITE_PRIVATE int sqlite3PagerOpen( ROUND8(sizeof(*pPager)) + /* Pager structure */ ROUND8(pcacheSize) + /* PCache object */ ROUND8(pVfs->szOsFile) + /* The main db file */ - journalFileSize * 2 + /* The two journal files */ + (u64)journalFileSize * 2 + /* The two journal files */ SQLITE_PTRSIZE + /* Space to hold a pointer */ 4 + /* Database prefix */ - nPathname + 1 + /* database filename */ - nUriByte + /* query parameters */ - nPathname + 8 + 1 + /* Journal filename */ + (u64)nPathname + 1 + /* database filename */ + (u64)nUriByte + /* query parameters */ + (u64)nPathname + 8 + 1 + /* Journal filename */ #ifndef SQLITE_OMIT_WAL - nPathname + 4 + 1 + /* WAL filename */ + (u64)nPathname + 4 + 1 + /* WAL filename */ #endif 3 /* Terminator */ ); @@ -63802,7 +66033,7 @@ SQLITE_PRIVATE sqlite3_file *sqlite3PagerFile(Pager *pPager){ ** This will be either the rollback journal or the WAL file. */ SQLITE_PRIVATE sqlite3_file *sqlite3PagerJrnlFile(Pager *pPager){ -#if SQLITE_OMIT_WAL +#ifdef SQLITE_OMIT_WAL return pPager->jfd; #else return pPager->pWal ? sqlite3WalFile(pPager->pWal) : pPager->jfd; @@ -64216,7 +66447,7 @@ SQLITE_PRIVATE int sqlite3PagerCheckpoint( } if( pPager->pWal ){ rc = sqlite3WalCheckpoint(pPager->pWal, db, eMode, - (eMode==SQLITE_CHECKPOINT_PASSIVE ? 0 : pPager->xBusyHandler), + (eMode<=SQLITE_CHECKPOINT_PASSIVE ? 0 : pPager->xBusyHandler), pPager->pBusyHandlerArg, pPager->walSyncFlags, pPager->pageSize, (u8 *)pPager->pTmpSpace, pnLog, pnCkpt @@ -64288,6 +66519,11 @@ static int pagerOpenWal(Pager *pPager){ pPager->fd, pPager->zWal, pPager->exclusiveMode, pPager->journalSizeLimit, &pPager->pWal ); +#ifdef SQLITE_ENABLE_SETLK_TIMEOUT + if( rc==SQLITE_OK ){ + sqlite3WalDb(pPager->pWal, pPager->dbWal); + } +#endif } pagerFixMaplimit(pPager); @@ -64407,6 +66643,7 @@ SQLITE_PRIVATE int sqlite3PagerWalWriteLock(Pager *pPager, int bLock){ ** blocking locks are required. */ SQLITE_PRIVATE void sqlite3PagerWalDb(Pager *pPager, sqlite3 *db){ + pPager->dbWal = db; if( pagerUseWal(pPager) ){ sqlite3WalDb(pPager->pWal, db); } @@ -64562,7 +66799,7 @@ SQLITE_PRIVATE int sqlite3PagerWalSystemErrno(Pager *pPager){ ** 28: Checksum-2 (second part of checksum for first 24 bytes of header). ** ** Immediately following the wal-header are zero or more frames. Each -** frame consists of a 24-byte frame-header followed by a bytes +** frame consists of a 24-byte frame-header followed by bytes ** of page data. The frame-header is six big-endian 32-bit unsigned ** integer values, as follows: ** @@ -65020,6 +67257,11 @@ struct WalCkptInfo { /* ** An open write-ahead log file is represented by an instance of the ** following object. +** +** writeLock: +** This is usually set to 1 whenever the WRITER lock is held. However, +** if it is set to 2, then the WRITER lock is held but must be released +** by walHandleException() if a SEH exception is thrown. */ struct Wal { sqlite3_vfs *pVfs; /* The VFS used to create pDbFd */ @@ -65059,6 +67301,7 @@ struct Wal { #endif #ifdef SQLITE_ENABLE_SNAPSHOT WalIndexHdr *pSnapshot; /* Start transaction here if not NULL */ + int bGetSnapshot; /* Transaction opened for sqlite3_get_snapshot() */ #endif #ifdef SQLITE_ENABLE_SETLK_TIMEOUT sqlite3 *db; @@ -65109,9 +67352,13 @@ struct WalIterator { u32 *aPgno; /* Array of page numbers. */ int nEntry; /* Nr. of entries in aPgno[] and aIndex[] */ int iZero; /* Frame number associated with aPgno[0] */ - } aSegment[1]; /* One for every 32KB page in the wal-index */ + } aSegment[FLEXARRAY]; /* One for every 32KB page in the wal-index */ }; +/* Size (in bytes) of a WalIterator object suitable for N or fewer segments */ +#define SZ_WALITERATOR(N) \ + (offsetof(WalIterator,aSegment)+(N)*sizeof(struct WalSegment)) + /* ** Define the parameters of the hash tables in the wal-index file. There ** is a hash-table following every HASHTABLE_NPAGE page numbers in the @@ -65270,7 +67517,7 @@ static SQLITE_NOINLINE int walIndexPageRealloc( /* Enlarge the pWal->apWiData[] array if required */ if( pWal->nWiData<=iPage ){ - sqlite3_int64 nByte = sizeof(u32*)*(iPage+1); + sqlite3_int64 nByte = sizeof(u32*)*(1+(i64)iPage); volatile u32 **apNew; apNew = (volatile u32 **)sqlite3Realloc((void *)pWal->apWiData, nByte); if( !apNew ){ @@ -65379,10 +67626,8 @@ static void walChecksumBytes( s1 = s2 = 0; } - assert( nByte>=8 ); - assert( (nByte&0x00000007)==0 ); - assert( nByte<=65536 ); - assert( nByte%4==0 ); + /* nByte is a multiple of 8 between 8 and 65536 */ + assert( nByte>=8 && (nByte&7)==0 && nByte<=65536 ); if( !nativeCksum ){ do { @@ -66472,8 +68717,7 @@ static int walIteratorInit(Wal *pWal, u32 nBackfill, WalIterator **pp){ /* Allocate space for the WalIterator object. */ nSegment = walFramePage(iLast) + 1; - nByte = sizeof(WalIterator) - + (nSegment-1)*sizeof(struct WalSegment) + nByte = SZ_WALITERATOR(nSegment) + iLast*sizeof(ht_slot); p = (WalIterator *)sqlite3_malloc64(nByte + sizeof(ht_slot) * (iLast>HASHTABLE_NPAGE?HASHTABLE_NPAGE:iLast) @@ -66544,7 +68788,7 @@ static int walEnableBlockingMs(Wal *pWal, int nMs){ static int walEnableBlocking(Wal *pWal){ int res = 0; if( pWal->db ){ - int tmout = pWal->db->busyTimeout; + int tmout = pWal->db->setlkTimeout; if( tmout ){ res = walEnableBlockingMs(pWal, tmout); } @@ -66930,7 +69174,9 @@ static int walHandleException(Wal *pWal){ static const int S = 1; static const int E = (1<lockMask & ~( + u32 mUnlock; + if( pWal->writeLock==2 ) pWal->writeLock = 0; + mUnlock = pWal->lockMask & ~( (pWal->readLock<0 ? 0 : (S << WAL_READ_LOCK(pWal->readLock))) | (pWal->writeLock ? (E << WAL_WRITE_LOCK) : 0) | (pWal->ckptLock ? (E << WAL_CKPT_LOCK) : 0) @@ -66951,7 +69197,7 @@ static int walHandleException(Wal *pWal){ /* ** Assert that the Wal.lockMask mask, which indicates the locks held -** by the connenction, is consistent with the Wal.readLock, Wal.writeLock +** by the connection, is consistent with the Wal.readLock, Wal.writeLock ** and Wal.ckptLock variables. To be used as: ** ** assert( walAssertLockmask(pWal) ); @@ -67202,7 +69448,12 @@ static int walIndexReadHdr(Wal *pWal, int *pChanged){ if( bWriteLock || SQLITE_OK==(rc = walLockExclusive(pWal, WAL_WRITE_LOCK, 1)) ){ - pWal->writeLock = 1; + /* If the write-lock was just obtained, set writeLock to 2 instead of + ** the usual 1. This causes walIndexPage() to behave as if the + ** write-lock were held (so that it allocates new pages as required), + ** and walHandleException() to unlock the write-lock if a SEH exception + ** is thrown. */ + if( !bWriteLock ) pWal->writeLock = 2; if( SQLITE_OK==(rc = walIndexPage(pWal, 0, &page0)) ){ badHdr = walIndexTryHdr(pWal, pChanged); if( badHdr ){ @@ -67503,11 +69754,7 @@ static int walBeginShmUnreliable(Wal *pWal, int *pChanged){ */ static int walTryBeginRead(Wal *pWal, int *pChanged, int useWal, int *pCnt){ volatile WalCkptInfo *pInfo; /* Checkpoint information in wal-index */ - u32 mxReadMark; /* Largest aReadMark[] value */ - int mxI; /* Index of largest aReadMark[] value */ - int i; /* Loop counter */ int rc = SQLITE_OK; /* Return code */ - u32 mxFrame; /* Wal frame to lock to */ #ifdef SQLITE_ENABLE_SETLK_TIMEOUT int nBlockTmout = 0; #endif @@ -67570,7 +69817,6 @@ static int walTryBeginRead(Wal *pWal, int *pChanged, int useWal, int *pCnt){ rc = walIndexReadHdr(pWal, pChanged); } #ifdef SQLITE_ENABLE_SETLK_TIMEOUT - walDisableBlocking(pWal); if( rc==SQLITE_BUSY_TIMEOUT ){ rc = SQLITE_BUSY; *pCnt |= WAL_RETRY_BLOCKED_MASK; @@ -67585,6 +69831,7 @@ static int walTryBeginRead(Wal *pWal, int *pChanged, int useWal, int *pCnt){ ** WAL_RETRY this routine will be called again and will probably be ** right on the second iteration. */ + (void)walEnableBlocking(pWal); if( pWal->apWiData[0]==0 ){ /* This branch is taken when the xShmMap() method returns SQLITE_BUSY. ** We assume this is a transient condition, so return WAL_RETRY. The @@ -67601,6 +69848,7 @@ static int walTryBeginRead(Wal *pWal, int *pChanged, int useWal, int *pCnt){ rc = SQLITE_BUSY_RECOVERY; } } + walDisableBlocking(pWal); if( rc!=SQLITE_OK ){ return rc; } @@ -67613,141 +69861,147 @@ static int walTryBeginRead(Wal *pWal, int *pChanged, int useWal, int *pCnt){ assert( pWal->apWiData[0]!=0 ); pInfo = walCkptInfo(pWal); SEH_INJECT_FAULT; - if( !useWal && AtomicLoad(&pInfo->nBackfill)==pWal->hdr.mxFrame + { + u32 mxReadMark; /* Largest aReadMark[] value */ + int mxI; /* Index of largest aReadMark[] value */ + int i; /* Loop counter */ + u32 mxFrame; /* Wal frame to lock to */ + if( !useWal && AtomicLoad(&pInfo->nBackfill)==pWal->hdr.mxFrame #ifdef SQLITE_ENABLE_SNAPSHOT - && (pWal->pSnapshot==0 || pWal->hdr.mxFrame==0) + && ((pWal->bGetSnapshot==0 && pWal->pSnapshot==0) || pWal->hdr.mxFrame==0) #endif - ){ - /* The WAL has been completely backfilled (or it is empty). - ** and can be safely ignored. - */ - rc = walLockShared(pWal, WAL_READ_LOCK(0)); - walShmBarrier(pWal); - if( rc==SQLITE_OK ){ - if( memcmp((void *)walIndexHdr(pWal), &pWal->hdr, sizeof(WalIndexHdr)) ){ - /* It is not safe to allow the reader to continue here if frames - ** may have been appended to the log before READ_LOCK(0) was obtained. - ** When holding READ_LOCK(0), the reader ignores the entire log file, - ** which implies that the database file contains a trustworthy - ** snapshot. Since holding READ_LOCK(0) prevents a checkpoint from - ** happening, this is usually correct. - ** - ** However, if frames have been appended to the log (or if the log - ** is wrapped and written for that matter) before the READ_LOCK(0) - ** is obtained, that is not necessarily true. A checkpointer may - ** have started to backfill the appended frames but crashed before - ** it finished. Leaving a corrupt image in the database file. - */ - walUnlockShared(pWal, WAL_READ_LOCK(0)); - return WAL_RETRY; + ){ + /* The WAL has been completely backfilled (or it is empty). + ** and can be safely ignored. + */ + rc = walLockShared(pWal, WAL_READ_LOCK(0)); + walShmBarrier(pWal); + if( rc==SQLITE_OK ){ + if( memcmp((void *)walIndexHdr(pWal), &pWal->hdr,sizeof(WalIndexHdr)) ){ + /* It is not safe to allow the reader to continue here if frames + ** may have been appended to the log before READ_LOCK(0) was obtained. + ** When holding READ_LOCK(0), the reader ignores the entire log file, + ** which implies that the database file contains a trustworthy + ** snapshot. Since holding READ_LOCK(0) prevents a checkpoint from + ** happening, this is usually correct. + ** + ** However, if frames have been appended to the log (or if the log + ** is wrapped and written for that matter) before the READ_LOCK(0) + ** is obtained, that is not necessarily true. A checkpointer may + ** have started to backfill the appended frames but crashed before + ** it finished. Leaving a corrupt image in the database file. + */ + walUnlockShared(pWal, WAL_READ_LOCK(0)); + return WAL_RETRY; + } + pWal->readLock = 0; + return SQLITE_OK; + }else if( rc!=SQLITE_BUSY ){ + return rc; } - pWal->readLock = 0; - return SQLITE_OK; - }else if( rc!=SQLITE_BUSY ){ - return rc; } - } - /* If we get this far, it means that the reader will want to use - ** the WAL to get at content from recent commits. The job now is - ** to select one of the aReadMark[] entries that is closest to - ** but not exceeding pWal->hdr.mxFrame and lock that entry. - */ - mxReadMark = 0; - mxI = 0; - mxFrame = pWal->hdr.mxFrame; + /* If we get this far, it means that the reader will want to use + ** the WAL to get at content from recent commits. The job now is + ** to select one of the aReadMark[] entries that is closest to + ** but not exceeding pWal->hdr.mxFrame and lock that entry. + */ + mxReadMark = 0; + mxI = 0; + mxFrame = pWal->hdr.mxFrame; #ifdef SQLITE_ENABLE_SNAPSHOT - if( pWal->pSnapshot && pWal->pSnapshot->mxFramepSnapshot->mxFrame; - } -#endif - for(i=1; iaReadMark+i); SEH_INJECT_FAULT; - if( mxReadMark<=thisMark && thisMark<=mxFrame ){ - assert( thisMark!=READMARK_NOT_USED ); - mxReadMark = thisMark; - mxI = i; + if( pWal->pSnapshot && pWal->pSnapshot->mxFramepSnapshot->mxFrame; } - } - if( (pWal->readOnly & WAL_SHM_RDONLY)==0 - && (mxReadMarkaReadMark+i,mxFrame); - mxReadMark = mxFrame; + u32 thisMark = AtomicLoad(pInfo->aReadMark+i); SEH_INJECT_FAULT; + if( mxReadMark<=thisMark && thisMark<=mxFrame ){ + assert( thisMark!=READMARK_NOT_USED ); + mxReadMark = thisMark; mxI = i; - walUnlockExclusive(pWal, WAL_READ_LOCK(i), 1); - break; - }else if( rc!=SQLITE_BUSY ){ - return rc; } } - } - if( mxI==0 ){ - assert( rc==SQLITE_BUSY || (pWal->readOnly & WAL_SHM_RDONLY)!=0 ); - return rc==SQLITE_BUSY ? WAL_RETRY : SQLITE_READONLY_CANTINIT; - } + if( (pWal->readOnly & WAL_SHM_RDONLY)==0 + && (mxReadMarkaReadMark+i,mxFrame); + mxReadMark = mxFrame; + mxI = i; + walUnlockExclusive(pWal, WAL_READ_LOCK(i), 1); + break; + }else if( rc!=SQLITE_BUSY ){ + return rc; + } + } + } + if( mxI==0 ){ + assert( rc==SQLITE_BUSY || (pWal->readOnly & WAL_SHM_RDONLY)!=0 ); + return rc==SQLITE_BUSY ? WAL_RETRY : SQLITE_READONLY_CANTINIT; + } - (void)walEnableBlockingMs(pWal, nBlockTmout); - rc = walLockShared(pWal, WAL_READ_LOCK(mxI)); - walDisableBlocking(pWal); - if( rc ){ + (void)walEnableBlockingMs(pWal, nBlockTmout); + rc = walLockShared(pWal, WAL_READ_LOCK(mxI)); + walDisableBlocking(pWal); + if( rc ){ #ifdef SQLITE_ENABLE_SETLK_TIMEOUT - if( rc==SQLITE_BUSY_TIMEOUT ){ - *pCnt |= WAL_RETRY_BLOCKED_MASK; - } + if( rc==SQLITE_BUSY_TIMEOUT ){ + *pCnt |= WAL_RETRY_BLOCKED_MASK; + } #else - assert( rc!=SQLITE_BUSY_TIMEOUT ); + assert( rc!=SQLITE_BUSY_TIMEOUT ); #endif - assert( (rc&0xFF)!=SQLITE_BUSY||rc==SQLITE_BUSY||rc==SQLITE_BUSY_TIMEOUT ); - return (rc&0xFF)==SQLITE_BUSY ? WAL_RETRY : rc; - } - /* Now that the read-lock has been obtained, check that neither the - ** value in the aReadMark[] array or the contents of the wal-index - ** header have changed. - ** - ** It is necessary to check that the wal-index header did not change - ** between the time it was read and when the shared-lock was obtained - ** on WAL_READ_LOCK(mxI) was obtained to account for the possibility - ** that the log file may have been wrapped by a writer, or that frames - ** that occur later in the log than pWal->hdr.mxFrame may have been - ** copied into the database by a checkpointer. If either of these things - ** happened, then reading the database with the current value of - ** pWal->hdr.mxFrame risks reading a corrupted snapshot. So, retry - ** instead. - ** - ** Before checking that the live wal-index header has not changed - ** since it was read, set Wal.minFrame to the first frame in the wal - ** file that has not yet been checkpointed. This client will not need - ** to read any frames earlier than minFrame from the wal file - they - ** can be safely read directly from the database file. - ** - ** Because a ShmBarrier() call is made between taking the copy of - ** nBackfill and checking that the wal-header in shared-memory still - ** matches the one cached in pWal->hdr, it is guaranteed that the - ** checkpointer that set nBackfill was not working with a wal-index - ** header newer than that cached in pWal->hdr. If it were, that could - ** cause a problem. The checkpointer could omit to checkpoint - ** a version of page X that lies before pWal->minFrame (call that version - ** A) on the basis that there is a newer version (version B) of the same - ** page later in the wal file. But if version B happens to like past - ** frame pWal->hdr.mxFrame - then the client would incorrectly assume - ** that it can read version A from the database file. However, since - ** we can guarantee that the checkpointer that set nBackfill could not - ** see any pages past pWal->hdr.mxFrame, this problem does not come up. - */ - pWal->minFrame = AtomicLoad(&pInfo->nBackfill)+1; SEH_INJECT_FAULT; - walShmBarrier(pWal); - if( AtomicLoad(pInfo->aReadMark+mxI)!=mxReadMark - || memcmp((void *)walIndexHdr(pWal), &pWal->hdr, sizeof(WalIndexHdr)) - ){ - walUnlockShared(pWal, WAL_READ_LOCK(mxI)); - return WAL_RETRY; - }else{ - assert( mxReadMark<=pWal->hdr.mxFrame ); - pWal->readLock = (i16)mxI; + assert((rc&0xFF)!=SQLITE_BUSY||rc==SQLITE_BUSY||rc==SQLITE_BUSY_TIMEOUT); + return (rc&0xFF)==SQLITE_BUSY ? WAL_RETRY : rc; + } + /* Now that the read-lock has been obtained, check that neither the + ** value in the aReadMark[] array or the contents of the wal-index + ** header have changed. + ** + ** It is necessary to check that the wal-index header did not change + ** between the time it was read and when the shared-lock was obtained + ** on WAL_READ_LOCK(mxI) was obtained to account for the possibility + ** that the log file may have been wrapped by a writer, or that frames + ** that occur later in the log than pWal->hdr.mxFrame may have been + ** copied into the database by a checkpointer. If either of these things + ** happened, then reading the database with the current value of + ** pWal->hdr.mxFrame risks reading a corrupted snapshot. So, retry + ** instead. + ** + ** Before checking that the live wal-index header has not changed + ** since it was read, set Wal.minFrame to the first frame in the wal + ** file that has not yet been checkpointed. This client will not need + ** to read any frames earlier than minFrame from the wal file - they + ** can be safely read directly from the database file. + ** + ** Because a ShmBarrier() call is made between taking the copy of + ** nBackfill and checking that the wal-header in shared-memory still + ** matches the one cached in pWal->hdr, it is guaranteed that the + ** checkpointer that set nBackfill was not working with a wal-index + ** header newer than that cached in pWal->hdr. If it were, that could + ** cause a problem. The checkpointer could omit to checkpoint + ** a version of page X that lies before pWal->minFrame (call that version + ** A) on the basis that there is a newer version (version B) of the same + ** page later in the wal file. But if version B happens to like past + ** frame pWal->hdr.mxFrame - then the client would incorrectly assume + ** that it can read version A from the database file. However, since + ** we can guarantee that the checkpointer that set nBackfill could not + ** see any pages past pWal->hdr.mxFrame, this problem does not come up. + */ + pWal->minFrame = AtomicLoad(&pInfo->nBackfill)+1; SEH_INJECT_FAULT; + walShmBarrier(pWal); + if( AtomicLoad(pInfo->aReadMark+mxI)!=mxReadMark + || memcmp((void *)walIndexHdr(pWal), &pWal->hdr, sizeof(WalIndexHdr)) + ){ + walUnlockShared(pWal, WAL_READ_LOCK(mxI)); + return WAL_RETRY; + }else{ + assert( mxReadMark<=pWal->hdr.mxFrame ); + pWal->readLock = (i16)mxI; + } } return rc; } @@ -67985,8 +70239,11 @@ SQLITE_PRIVATE int sqlite3WalBeginReadTransaction(Wal *pWal, int *pChanged){ ** read-lock. */ SQLITE_PRIVATE void sqlite3WalEndReadTransaction(Wal *pWal){ - sqlite3WalEndWriteTransaction(pWal); +#ifndef SQLITE_ENABLE_SETLK_TIMEOUT + assert( pWal->writeLock==0 || pWal->readLock<0 ); +#endif if( pWal->readLock>=0 ){ + (void)sqlite3WalEndWriteTransaction(pWal); walUnlockShared(pWal, WAL_READ_LOCK(pWal->readLock)); pWal->readLock = -1; } @@ -68179,7 +70436,7 @@ SQLITE_PRIVATE int sqlite3WalBeginWriteTransaction(Wal *pWal){ ** read-transaction was even opened, making this call a no-op. ** Return early. */ if( pWal->writeLock ){ - assert( !memcmp(&pWal->hdr,(void *)walIndexHdr(pWal),sizeof(WalIndexHdr)) ); + assert( !memcmp(&pWal->hdr,(void*)pWal->apWiData[0],sizeof(WalIndexHdr)) ); return SQLITE_OK; } #endif @@ -68279,6 +70536,7 @@ SQLITE_PRIVATE int sqlite3WalUndo(Wal *pWal, int (*xUndo)(void *, Pgno), void *p if( iMax!=pWal->hdr.mxFrame ) walCleanupHash(pWal); } SEH_EXCEPT( rc = SQLITE_IOERR_IN_PAGE; ) + pWal->iReCksum = 0; } return rc; } @@ -68326,6 +70584,9 @@ SQLITE_PRIVATE int sqlite3WalSavepointUndo(Wal *pWal, u32 *aWalData){ walCleanupHash(pWal); } SEH_EXCEPT( rc = SQLITE_IOERR_IN_PAGE; ) + if( pWal->iReCksum>pWal->hdr.mxFrame ){ + pWal->iReCksum = 0; + } } return rc; @@ -68791,7 +71052,8 @@ SQLITE_PRIVATE int sqlite3WalCheckpoint( /* EVIDENCE-OF: R-62920-47450 The busy-handler callback is never invoked ** in the SQLITE_CHECKPOINT_PASSIVE mode. */ - assert( eMode!=SQLITE_CHECKPOINT_PASSIVE || xBusy==0 ); + assert( SQLITE_CHECKPOINT_NOOPSQLITE_CHECKPOINT_PASSIVE || xBusy==0 ); if( pWal->readOnly ) return SQLITE_READONLY; WALTRACE(("WAL%p: checkpoint begins\n", pWal)); @@ -68808,31 +71070,35 @@ SQLITE_PRIVATE int sqlite3WalCheckpoint( ** EVIDENCE-OF: R-53820-33897 Even if there is a busy-handler configured, ** it will not be invoked in this case. */ - rc = walLockExclusive(pWal, WAL_CKPT_LOCK, 1); - testcase( rc==SQLITE_BUSY ); - testcase( rc!=SQLITE_OK && xBusy2!=0 ); - if( rc==SQLITE_OK ){ - pWal->ckptLock = 1; + if( eMode!=SQLITE_CHECKPOINT_NOOP ){ + rc = walLockExclusive(pWal, WAL_CKPT_LOCK, 1); + testcase( rc==SQLITE_BUSY ); + testcase( rc!=SQLITE_OK && xBusy2!=0 ); + if( rc==SQLITE_OK ){ + pWal->ckptLock = 1; - /* IMPLEMENTATION-OF: R-59782-36818 The SQLITE_CHECKPOINT_FULL, RESTART and - ** TRUNCATE modes also obtain the exclusive "writer" lock on the database - ** file. - ** - ** EVIDENCE-OF: R-60642-04082 If the writer lock cannot be obtained - ** immediately, and a busy-handler is configured, it is invoked and the - ** writer lock retried until either the busy-handler returns 0 or the - ** lock is successfully obtained. - */ - if( eMode!=SQLITE_CHECKPOINT_PASSIVE ){ - rc = walBusyLock(pWal, xBusy2, pBusyArg, WAL_WRITE_LOCK, 1); - if( rc==SQLITE_OK ){ - pWal->writeLock = 1; - }else if( rc==SQLITE_BUSY ){ - eMode2 = SQLITE_CHECKPOINT_PASSIVE; - xBusy2 = 0; - rc = SQLITE_OK; + /* IMPLEMENTATION-OF: R-59782-36818 The SQLITE_CHECKPOINT_FULL, RESTART + ** and TRUNCATE modes also obtain the exclusive "writer" lock on the + ** database file. + ** + ** EVIDENCE-OF: R-60642-04082 If the writer lock cannot be obtained + ** immediately, and a busy-handler is configured, it is invoked and the + ** writer lock retried until either the busy-handler returns 0 or the + ** lock is successfully obtained. + */ + if( eMode!=SQLITE_CHECKPOINT_PASSIVE ){ + rc = walBusyLock(pWal, xBusy2, pBusyArg, WAL_WRITE_LOCK, 1); + if( rc==SQLITE_OK ){ + pWal->writeLock = 1; + }else if( rc==SQLITE_BUSY ){ + eMode2 = SQLITE_CHECKPOINT_PASSIVE; + xBusy2 = 0; + rc = SQLITE_OK; + } } } + }else{ + rc = SQLITE_OK; } @@ -68846,7 +71112,7 @@ SQLITE_PRIVATE int sqlite3WalCheckpoint( ** immediately and do a partial checkpoint if it cannot obtain it. */ walDisableBlocking(pWal); rc = walIndexReadHdr(pWal, &isChanged); - if( eMode2!=SQLITE_CHECKPOINT_PASSIVE ) (void)walEnableBlocking(pWal); + if( eMode2>SQLITE_CHECKPOINT_PASSIVE ) (void)walEnableBlocking(pWal); if( isChanged && pWal->pDbFd->pMethods->iVersion>=3 ){ sqlite3OsUnfetch(pWal->pDbFd, 0, 0); } @@ -68856,7 +71122,7 @@ SQLITE_PRIVATE int sqlite3WalCheckpoint( if( rc==SQLITE_OK ){ if( pWal->hdr.mxFrame && walPagesize(pWal)!=nBuf ){ rc = SQLITE_CORRUPT_BKPT; - }else{ + }else if( eMode2!=SQLITE_CHECKPOINT_NOOP ){ rc = walCheckpoint(pWal, db, eMode2, xBusy2, pBusyArg, sync_flags,zBuf); } @@ -68884,7 +71150,7 @@ SQLITE_PRIVATE int sqlite3WalCheckpoint( sqlite3WalDb(pWal, 0); /* Release the locks. */ - sqlite3WalEndWriteTransaction(pWal); + (void)sqlite3WalEndWriteTransaction(pWal); if( pWal->ckptLock ){ walUnlockExclusive(pWal, WAL_CKPT_LOCK, 1); pWal->ckptLock = 0; @@ -69015,7 +71281,20 @@ SQLITE_PRIVATE void sqlite3WalSnapshotOpen( Wal *pWal, sqlite3_snapshot *pSnapshot ){ - pWal->pSnapshot = (WalIndexHdr*)pSnapshot; + if( pSnapshot && ((WalIndexHdr*)pSnapshot)->iVersion==0 ){ + /* iVersion==0 means that this is a call to sqlite3_snapshot_get(). In + ** this case set the bGetSnapshot flag so that if the call to + ** sqlite3_snapshot_get() is about to read transaction on this wal + ** file, it does not take read-lock 0 if the wal file has been completely + ** checkpointed. Taking read-lock 0 would work, but then it would be + ** possible for a subsequent writer to destroy the snapshot even while + ** this connection is holding its read-transaction open. This is contrary + ** to user expectations, so we avoid it by not taking read-lock 0. */ + pWal->bGetSnapshot = 1; + }else{ + pWal->pSnapshot = (WalIndexHdr*)pSnapshot; + pWal->bGetSnapshot = 0; + } } /* @@ -69615,6 +71894,12 @@ struct CellInfo { */ #define BTCURSOR_MAX_DEPTH 20 +/* +** Maximum amount of storage local to a database page, regardless of +** page size. +*/ +#define BT_MAX_LOCAL 65501 /* 65536 - 35 */ + /* ** A cursor is a pointer to a particular entry within a particular ** b-tree within a database file. @@ -69826,6 +72111,7 @@ struct IntegrityCk { StrAccum errMsg; /* Accumulate the error message text here */ u32 *heap; /* Min-heap used for analyzing cell coverage */ sqlite3 *db; /* Database connection running the check */ + i64 nRow; /* Number of rows visited in current tree */ }; /* @@ -70022,7 +72308,7 @@ SQLITE_PRIVATE int sqlite3BtreeHoldsMutex(Btree *p){ */ static void SQLITE_NOINLINE btreeEnterAll(sqlite3 *db){ int i; - int skipOk = 1; + u8 skipOk = 1; Btree *p; assert( sqlite3_mutex_held(db->mutex) ); for(i=0; inDb; i++){ @@ -70300,8 +72586,47 @@ int corruptPageError(int lineno, MemPage *p){ # define SQLITE_CORRUPT_PAGE(pMemPage) SQLITE_CORRUPT_PGNO(pMemPage->pgno) #endif +/* Default value for SHARED_LOCK_TRACE macro if shared-cache is disabled +** or if the lock tracking is disabled. This is always the value for +** release builds. +*/ +#define SHARED_LOCK_TRACE(X,MSG,TAB,TYPE) /*no-op*/ + #ifndef SQLITE_OMIT_SHARED_CACHE +#if 0 +/* ^---- Change to 1 and recompile to enable shared-lock tracing +** for debugging purposes. +** +** Print all shared-cache locks on a BtShared. Debugging use only. +*/ +static void sharedLockTrace( + BtShared *pBt, + const char *zMsg, + int iRoot, + int eLockType +){ + BtLock *pLock; + if( iRoot>0 ){ + printf("%s-%p %u%s:", zMsg, pBt, iRoot, eLockType==READ_LOCK?"R":"W"); + }else{ + printf("%s-%p:", zMsg, pBt); + } + for(pLock=pBt->pLock; pLock; pLock=pLock->pNext){ + printf(" %p/%u%s", pLock->pBtree, pLock->iTable, + pLock->eLock==READ_LOCK ? "R" : "W"); + while( pLock->pNext && pLock->pBtree==pLock->pNext->pBtree ){ + pLock = pLock->pNext; + printf(",%u%s", pLock->iTable, pLock->eLock==READ_LOCK ? "R" : "W"); + } + } + printf("\n"); + fflush(stdout); +} +#undef SHARED_LOCK_TRACE +#define SHARED_LOCK_TRACE(X,MSG,TAB,TYPE) sharedLockTrace(X,MSG,TAB,TYPE) +#endif /* Shared-lock tracing */ + #ifdef SQLITE_DEBUG /* **** This function is only used as part of an assert() statement. *** @@ -70378,6 +72703,8 @@ static int hasSharedCacheTableLock( iTab = iRoot; } + SHARED_LOCK_TRACE(pBtree->pBt,"hasLock",iRoot,eLockType); + /* Search for the required lock. Either a write-lock on root-page iTab, a ** write-lock on the schema table, or (if the client is reading) a ** read-lock on iTab will suffice. Return 1 if any of these are found. */ @@ -70511,6 +72838,8 @@ static int setSharedCacheTableLock(Btree *p, Pgno iTable, u8 eLock){ BtLock *pLock = 0; BtLock *pIter; + SHARED_LOCK_TRACE(pBt,"setLock", iTable, eLock); + assert( sqlite3BtreeHoldsMutex(p) ); assert( eLock==READ_LOCK || eLock==WRITE_LOCK ); assert( p->db!=0 ); @@ -70578,6 +72907,8 @@ static void clearAllSharedCacheTableLocks(Btree *p){ assert( p->sharable || 0==*ppIter ); assert( p->inTrans>0 ); + SHARED_LOCK_TRACE(pBt, "clearAllLocks", 0, 0); + while( *ppIter ){ BtLock *pLock = *ppIter; assert( (pBt->btsFlags & BTS_EXCLUSIVE)==0 || pBt->pWriter==pLock->pBtree ); @@ -70616,6 +72947,9 @@ static void clearAllSharedCacheTableLocks(Btree *p){ */ static void downgradeAllSharedCacheTableLocks(Btree *p){ BtShared *pBt = p->pBt; + + SHARED_LOCK_TRACE(pBt, "downgradeLocks", 0, 0); + if( pBt->pWriter==p ){ BtLock *pLock; pBt->pWriter = 0; @@ -70830,7 +73164,7 @@ static int saveCursorKey(BtCursor *pCur){ ** below. */ void *pKey; pCur->nKey = sqlite3BtreePayloadSize(pCur); - pKey = sqlite3Malloc( pCur->nKey + 9 + 8 ); + pKey = sqlite3Malloc( ((i64)pCur->nKey) + 9 + 8 ); if( pKey ){ rc = sqlite3BtreePayload(pCur, 0, (int)pCur->nKey, pKey); if( rc==SQLITE_OK ){ @@ -70973,7 +73307,7 @@ static int btreeMoveto( assert( nKey==(i64)(int)nKey ); pIdxKey = sqlite3VdbeAllocUnpackedRecord(pKeyInfo); if( pIdxKey==0 ) return SQLITE_NOMEM_BKPT; - sqlite3VdbeRecordUnpack(pKeyInfo, (int)nKey, pKey, pIdxKey); + sqlite3VdbeRecordUnpack((int)nKey, pKey, pIdxKey); if( pIdxKey->nField==0 || pIdxKey->nField>pKeyInfo->nAllField ){ rc = SQLITE_CORRUPT_BKPT; }else{ @@ -71120,7 +73454,7 @@ SQLITE_PRIVATE void sqlite3BtreeCursorHint(BtCursor *pCur, int eHintType, ...){ */ SQLITE_PRIVATE void sqlite3BtreeCursorHintFlags(BtCursor *pCur, unsigned x){ assert( x==BTREE_SEEK_EQ || x==BTREE_BULKLOAD || x==0 ); - pCur->hints = x; + pCur->hints = (u8)x; } @@ -71314,14 +73648,15 @@ static SQLITE_NOINLINE void btreeParseCellAdjustSizeForOverflow( static int btreePayloadToLocal(MemPage *pPage, i64 nPayload){ int maxLocal; /* Maximum amount of payload held locally */ maxLocal = pPage->maxLocal; + assert( nPayload>=0 ); if( nPayload<=maxLocal ){ - return nPayload; + return (int)nPayload; }else{ int minLocal; /* Minimum amount of payload held locally */ int surplus; /* Overflow payload available for local storage */ minLocal = pPage->minLocal; - surplus = minLocal + (nPayload - minLocal)%(pPage->pBt->usableSize-4); - return ( surplus <= maxLocal ) ? surplus : minLocal; + surplus = (int)(minLocal +(nPayload - minLocal)%(pPage->pBt->usableSize-4)); + return (surplus <= maxLocal) ? surplus : minLocal; } } @@ -71431,11 +73766,13 @@ static void btreeParseCellPtr( pInfo->pPayload = pIter; testcase( nPayload==pPage->maxLocal ); testcase( nPayload==(u32)pPage->maxLocal+1 ); + assert( nPayload>=0 ); + assert( pPage->maxLocal <= BT_MAX_LOCAL ); if( nPayload<=pPage->maxLocal ){ /* This is the (easy) common case where the entire payload fits ** on the local page. No overflow is required. */ - pInfo->nSize = nPayload + (u16)(pIter - pCell); + pInfo->nSize = (u16)nPayload + (u16)(pIter - pCell); if( pInfo->nSize<4 ) pInfo->nSize = 4; pInfo->nLocal = (u16)nPayload; }else{ @@ -71468,11 +73805,13 @@ static void btreeParseCellPtrIndex( pInfo->pPayload = pIter; testcase( nPayload==pPage->maxLocal ); testcase( nPayload==(u32)pPage->maxLocal+1 ); + assert( nPayload>=0 ); + assert( pPage->maxLocal <= BT_MAX_LOCAL ); if( nPayload<=pPage->maxLocal ){ /* This is the (easy) common case where the entire payload fits ** on the local page. No overflow is required. */ - pInfo->nSize = nPayload + (u16)(pIter - pCell); + pInfo->nSize = (u16)nPayload + (u16)(pIter - pCell); if( pInfo->nSize<4 ) pInfo->nSize = 4; pInfo->nLocal = (u16)nPayload; }else{ @@ -72011,24 +74350,24 @@ static SQLITE_INLINE int allocateSpace(MemPage *pPage, int nByte, int *pIdx){ ** at the end of the page. So do additional corruption checks inside this ** routine and return SQLITE_CORRUPT if any problems are found. */ -static int freeSpace(MemPage *pPage, u16 iStart, u16 iSize){ - u16 iPtr; /* Address of ptr to next freeblock */ - u16 iFreeBlk; /* Address of the next freeblock */ +static int freeSpace(MemPage *pPage, int iStart, int iSize){ + int iPtr; /* Address of ptr to next freeblock */ + int iFreeBlk; /* Address of the next freeblock */ u8 hdr; /* Page header size. 0 or 100 */ - u8 nFrag = 0; /* Reduction in fragmentation */ - u16 iOrigSize = iSize; /* Original value of iSize */ - u16 x; /* Offset to cell content area */ - u32 iEnd = iStart + iSize; /* First byte past the iStart buffer */ + int nFrag = 0; /* Reduction in fragmentation */ + int iOrigSize = iSize; /* Original value of iSize */ + int x; /* Offset to cell content area */ + int iEnd = iStart + iSize; /* First byte past the iStart buffer */ unsigned char *data = pPage->aData; /* Page content */ u8 *pTmp; /* Temporary ptr into data[] */ assert( pPage->pBt!=0 ); assert( sqlite3PagerIswriteable(pPage->pDbPage) ); assert( CORRUPT_DB || iStart>=pPage->hdrOffset+6+pPage->childPtrSize ); - assert( CORRUPT_DB || iEnd <= pPage->pBt->usableSize ); + assert( CORRUPT_DB || iEnd <= (int)pPage->pBt->usableSize ); assert( sqlite3_mutex_held(pPage->pBt->mutex) ); assert( iSize>=4 ); /* Minimum cell size is 4 */ - assert( CORRUPT_DB || iStart<=pPage->pBt->usableSize-4 ); + assert( CORRUPT_DB || iStart<=(int)pPage->pBt->usableSize-4 ); /* The list of freeblocks must be in ascending order. Find the ** spot on the list where iStart should be inserted. @@ -72045,7 +74384,7 @@ static int freeSpace(MemPage *pPage, u16 iStart, u16 iSize){ } iPtr = iFreeBlk; } - if( iFreeBlk>pPage->pBt->usableSize-4 ){ /* TH3: corrupt081.100 */ + if( iFreeBlk>(int)pPage->pBt->usableSize-4 ){ /* TH3: corrupt081.100 */ return SQLITE_CORRUPT_PAGE(pPage); } assert( iFreeBlk>iPtr || iFreeBlk==0 || CORRUPT_DB ); @@ -72060,7 +74399,7 @@ static int freeSpace(MemPage *pPage, u16 iStart, u16 iSize){ nFrag = iFreeBlk - iEnd; if( iEnd>iFreeBlk ) return SQLITE_CORRUPT_PAGE(pPage); iEnd = iFreeBlk + get2byte(&data[iFreeBlk+2]); - if( iEnd > pPage->pBt->usableSize ){ + if( iEnd > (int)pPage->pBt->usableSize ){ return SQLITE_CORRUPT_PAGE(pPage); } iSize = iEnd - iStart; @@ -72081,7 +74420,7 @@ static int freeSpace(MemPage *pPage, u16 iStart, u16 iSize){ } } if( nFrag>data[hdr+7] ) return SQLITE_CORRUPT_PAGE(pPage); - data[hdr+7] -= nFrag; + data[hdr+7] -= (u8)nFrag; } pTmp = &data[hdr+5]; x = get2byte(pTmp); @@ -72102,7 +74441,8 @@ static int freeSpace(MemPage *pPage, u16 iStart, u16 iSize){ /* Insert the new freeblock into the freelist */ put2byte(&data[iPtr], iStart); put2byte(&data[iStart], iFreeBlk); - put2byte(&data[iStart+2], iSize); + assert( iSize>=0 && iSize<=0xffff ); + put2byte(&data[iStart+2], (u16)iSize); } pPage->nFree += iOrigSize; return SQLITE_OK; @@ -72328,7 +74668,7 @@ static int btreeInitPage(MemPage *pPage){ assert( pBt->pageSize>=512 && pBt->pageSize<=65536 ); pPage->maskPage = (u16)(pBt->pageSize - 1); pPage->nOverflow = 0; - pPage->cellOffset = pPage->hdrOffset + 8 + pPage->childPtrSize; + pPage->cellOffset = (u16)(pPage->hdrOffset + 8 + pPage->childPtrSize); pPage->aCellIdx = data + pPage->childPtrSize + 8; pPage->aDataEnd = pPage->aData + pBt->pageSize; pPage->aDataOfst = pPage->aData + pPage->childPtrSize; @@ -72362,8 +74702,8 @@ static int btreeInitPage(MemPage *pPage){ static void zeroPage(MemPage *pPage, int flags){ unsigned char *data = pPage->aData; BtShared *pBt = pPage->pBt; - u8 hdr = pPage->hdrOffset; - u16 first; + int hdr = pPage->hdrOffset; + int first; assert( sqlite3PagerPagenumber(pPage->pDbPage)==pPage->pgno || CORRUPT_DB ); assert( sqlite3PagerGetExtra(pPage->pDbPage) == (void*)pPage ); @@ -72380,7 +74720,7 @@ static void zeroPage(MemPage *pPage, int flags){ put2byte(&data[hdr+5], pBt->usableSize); pPage->nFree = (u16)(pBt->usableSize - first); decodeFlags(pPage, flags); - pPage->cellOffset = first; + pPage->cellOffset = (u16)first; pPage->aDataEnd = &data[pBt->pageSize]; pPage->aCellIdx = &data[first]; pPage->aDataOfst = &data[pPage->childPtrSize]; @@ -72951,6 +75291,7 @@ static int removeFromSharingList(BtShared *pBt){ sqlite3_mutex_leave(pMainMtx); return removed; #else + UNUSED_PARAMETER( pBt ); return 1; #endif } @@ -73166,8 +75507,12 @@ SQLITE_PRIVATE int sqlite3BtreeSetPageSize(Btree *p, int pageSize, int nReserve, BtShared *pBt = p->pBt; assert( nReserve>=0 && nReserve<=255 ); sqlite3BtreeEnter(p); - pBt->nReserveWanted = nReserve; + pBt->nReserveWanted = (u8)nReserve; x = pBt->pageSize - pBt->usableSize; + if( x==nReserve && (pageSize==0 || (u32)pageSize==pBt->pageSize) ){ + sqlite3BtreeLeave(p); + return SQLITE_OK; + } if( nReservebtsFlags & BTS_PAGESIZE_FIXED ){ sqlite3BtreeLeave(p); @@ -73272,7 +75617,7 @@ SQLITE_PRIVATE int sqlite3BtreeSecureDelete(Btree *p, int newFlag){ assert( BTS_FAST_SECURE==(BTS_OVERWRITE|BTS_SECURE_DELETE) ); if( newFlag>=0 ){ p->pBt->btsFlags &= ~BTS_FAST_SECURE; - p->pBt->btsFlags |= BTS_SECURE_DELETE*newFlag; + p->pBt->btsFlags |= (u16)(BTS_SECURE_DELETE*newFlag); } b = (p->pBt->btsFlags & BTS_FAST_SECURE)/BTS_SECURE_DELETE; sqlite3BtreeLeave(p); @@ -73792,6 +76137,13 @@ static SQLITE_NOINLINE int btreeBeginTrans( (void)sqlite3PagerWalWriteLock(pPager, 0); unlockBtreeIfUnused(pBt); } +#if defined(SQLITE_ENABLE_SETLK_TIMEOUT) + if( rc==SQLITE_BUSY_TIMEOUT ){ + /* If a blocking lock timed out, break out of the loop here so that + ** the busy-handler is not invoked. */ + break; + } +#endif }while( (rc&0xFF)==SQLITE_BUSY && pBt->inTransaction==TRANS_NONE && btreeInvokeBusyHandler(pBt) ); sqlite3PagerWalDb(pPager, 0); @@ -74847,6 +77199,25 @@ SQLITE_PRIVATE int sqlite3BtreeCursorSize(void){ return ROUND8(sizeof(BtCursor)); } +#ifdef SQLITE_DEBUG +/* +** Return true if and only if the Btree object will be automatically +** closed with the BtCursor closes. This is used within assert() statements +** only. +*/ +SQLITE_PRIVATE int sqlite3BtreeClosesWithCursor( + Btree *pBtree, /* the btree object */ + BtCursor *pCur /* Corresponding cursor */ +){ + BtShared *pBt = pBtree->pBt; + if( (pBt->openFlags & BTREE_SINGLE)==0 ) return 0; + if( pBt->pCursor!=pCur ) return 0; + if( pCur->pNext!=0 ) return 0; + if( pCur->pBtree!=pBtree ) return 0; + return 1; +} +#endif + /* ** Initialize memory that will be converted into a BtCursor object. ** @@ -75229,9 +77600,12 @@ static int accessPayload( if( pCur->aOverflow==0 || nOvfl*(int)sizeof(Pgno) > sqlite3MallocSize(pCur->aOverflow) ){ - Pgno *aNew = (Pgno*)sqlite3Realloc( - pCur->aOverflow, nOvfl*2*sizeof(Pgno) - ); + Pgno *aNew; + if( sqlite3FaultSim(413) ){ + aNew = 0; + }else{ + aNew = (Pgno*)sqlite3Realloc(pCur->aOverflow, nOvfl*2*sizeof(Pgno)); + } if( aNew==0 ){ return SQLITE_NOMEM_BKPT; }else{ @@ -75241,6 +77615,12 @@ static int accessPayload( memset(pCur->aOverflow, 0, nOvfl*sizeof(Pgno)); pCur->curFlags |= BTCF_ValidOvfl; }else{ + /* Sanity check the validity of the overflow page cache */ + assert( pCur->aOverflow[0]==nextPage + || pCur->aOverflow[0]==0 + || CORRUPT_DB ); + assert( pCur->aOverflow[0]!=0 || pCur->aOverflow[offset/ovflSize]==0 ); + /* If the overflow page-list cache has been allocated and the ** entry for the first required overflow page is valid, skip ** directly to it. @@ -75722,6 +78102,47 @@ SQLITE_PRIVATE int sqlite3BtreeFirst(BtCursor *pCur, int *pRes){ return rc; } +/* Set *pRes to 1 (true) if the BTree pointed to by cursor pCur contains zero +** rows of content. Set *pRes to 0 (false) if the table contains content. +** Return SQLITE_OK on success or some error code (ex: SQLITE_NOMEM) if +** something goes wrong. +*/ +SQLITE_PRIVATE int sqlite3BtreeIsEmpty(BtCursor *pCur, int *pRes){ + int rc; + + assert( cursorOwnsBtShared(pCur) ); + assert( sqlite3_mutex_held(pCur->pBtree->db->mutex) ); + if( pCur->eState==CURSOR_VALID ){ + *pRes = 0; + return SQLITE_OK; + } + rc = moveToRoot(pCur); + if( rc==SQLITE_EMPTY ){ + *pRes = 1; + rc = SQLITE_OK; + }else{ + *pRes = 0; + } + return rc; +} + +#ifdef SQLITE_DEBUG +/* The cursors is CURSOR_VALID and has BTCF_AtLast set. Verify that +** this flags are true for a consistent database. +** +** This routine is is called from within assert() statements only. +** It is an internal verification routine and does not appear in production +** builds. +*/ +static int cursorIsAtLastEntry(BtCursor *pCur){ + int ii; + for(ii=0; iiiPage; ii++){ + if( pCur->aiIdx[ii]!=pCur->apPage[ii]->nCell ) return 0; + } + return pCur->ix==pCur->pPage->nCell-1 && pCur->pPage->leaf!=0; +} +#endif + /* Move the cursor to the last entry in the table. Return SQLITE_OK ** on success. Set *pRes to 0 if the cursor actually points to something ** or set *pRes to 1 if the table is empty. @@ -75750,18 +78171,7 @@ SQLITE_PRIVATE int sqlite3BtreeLast(BtCursor *pCur, int *pRes){ /* If the cursor already points to the last entry, this is a no-op. */ if( CURSOR_VALID==pCur->eState && (pCur->curFlags & BTCF_AtLast)!=0 ){ -#ifdef SQLITE_DEBUG - /* This block serves to assert() that the cursor really does point - ** to the last entry in the b-tree. */ - int ii; - for(ii=0; iiiPage; ii++){ - assert( pCur->aiIdx[ii]==pCur->apPage[ii]->nCell ); - } - assert( pCur->ix==pCur->pPage->nCell-1 || CORRUPT_DB ); - testcase( pCur->ix!=pCur->pPage->nCell-1 ); - /* ^-- dbsqlfuzz b92b72e4de80b5140c30ab71372ca719b8feb618 */ - assert( pCur->pPage->leaf ); -#endif + assert( cursorIsAtLastEntry(pCur) || CORRUPT_DB ); *pRes = 0; return SQLITE_OK; } @@ -75814,6 +78224,7 @@ SQLITE_PRIVATE int sqlite3BtreeTableMoveto( } if( pCur->info.nKeycurFlags & BTCF_AtLast)!=0 ){ + assert( cursorIsAtLastEntry(pCur) || CORRUPT_DB ); *pRes = -1; return SQLITE_OK; } @@ -75934,8 +78345,8 @@ SQLITE_PRIVATE int sqlite3BtreeTableMoveto( } /* -** Compare the "idx"-th cell on the page the cursor pCur is currently -** pointing to to pIdxKey using xRecordCompare. Return negative or +** Compare the "idx"-th cell on the page pPage against the key +** pointing to by pIdxKey using xRecordCompare. Return negative or ** zero if the cell is less than or equal pIdxKey. Return positive ** if unknown. ** @@ -75950,12 +78361,11 @@ SQLITE_PRIVATE int sqlite3BtreeTableMoveto( ** a positive value as that will cause the optimization to be skipped. */ static int indexCellCompare( - BtCursor *pCur, + MemPage *pPage, int idx, UnpackedRecord *pIdxKey, RecordCompare xRecordCompare ){ - MemPage *pPage = pCur->pPage; int c; int nCell; /* Size of the pCell cell in bytes */ u8 *pCell = findCellPastPtr(pPage, idx); @@ -76064,17 +78474,17 @@ SQLITE_PRIVATE int sqlite3BtreeIndexMoveto( ){ int c; if( pCur->ix==pCur->pPage->nCell-1 - && (c = indexCellCompare(pCur, pCur->ix, pIdxKey, xRecordCompare))<=0 + && (c = indexCellCompare(pCur->pPage,pCur->ix,pIdxKey,xRecordCompare))<=0 && pIdxKey->errCode==SQLITE_OK ){ *pRes = c; return SQLITE_OK; /* Cursor already pointing at the correct spot */ } if( pCur->iPage>0 - && indexCellCompare(pCur, 0, pIdxKey, xRecordCompare)<=0 + && indexCellCompare(pCur->pPage, 0, pIdxKey, xRecordCompare)<=0 && pIdxKey->errCode==SQLITE_OK ){ - pCur->curFlags &= ~BTCF_ValidOvfl; + pCur->curFlags &= ~(BTCF_ValidOvfl|BTCF_AtLast); if( !pCur->pPage->isInit ){ return SQLITE_CORRUPT_BKPT; } @@ -76166,7 +78576,7 @@ SQLITE_PRIVATE int sqlite3BtreeIndexMoveto( rc = SQLITE_CORRUPT_PAGE(pPage); goto moveto_index_finish; } - pCellKey = sqlite3Malloc( nCell+nOverrun ); + pCellKey = sqlite3Malloc( (u64)nCell+(u64)nOverrun ); if( pCellKey==0 ){ rc = SQLITE_NOMEM_BKPT; goto moveto_index_finish; @@ -76280,15 +78690,15 @@ SQLITE_PRIVATE i64 sqlite3BtreeRowCountEst(BtCursor *pCur){ assert( cursorOwnsBtShared(pCur) ); assert( sqlite3_mutex_held(pCur->pBtree->db->mutex) ); - /* Currently this interface is only called by the OP_IfSmaller - ** opcode, and it that case the cursor will always be valid and - ** will always point to a leaf node. */ - if( NEVER(pCur->eState!=CURSOR_VALID) ) return -1; + /* Currently this interface is only called by the OP_IfSizeBetween + ** opcode and the OP_Count opcode with P3=1. In either case, + ** the cursor will always be valid unless the btree is empty. */ + if( pCur->eState!=CURSOR_VALID ) return 0; if( NEVER(pCur->pPage->leaf==0) ) return -1; n = pCur->pPage->nCell; for(i=0; iiPage; i++){ - n *= pCur->apPage[i]->nCell; + n *= pCur->apPage[i]->nCell+1; } return n; } @@ -77652,7 +80062,8 @@ static int rebuildPage( if( j>(u32)usableSize ){ j = 0; } memcpy(&pTmp[j], &aData[j], usableSize - j); - for(k=0; ALWAYS(kixNx[k]<=i; k++){} + assert( pCArray->ixNx[NB*2-1]>i ); + for(k=0; pCArray->ixNx[k]<=i; k++){} pSrcEnd = pCArray->apEnd[k]; pData = pEnd; @@ -77684,7 +80095,8 @@ static int rebuildPage( } /* The pPg->nFree field is now set incorrectly. The caller will fix it. */ - pPg->nCell = nCell; + assert( nCell < 10922 ); + pPg->nCell = (u16)nCell; pPg->nOverflow = 0; put2byte(&aData[hdr+1], 0); @@ -77735,7 +80147,8 @@ static int pageInsertArray( u8 *pEnd; /* Maximum extent of cell data */ assert( CORRUPT_DB || pPg->hdrOffset==0 ); /* Never called on page 1 */ if( iEnd<=iFirst ) return 0; - for(k=0; ALWAYS(kixNx[k]<=i ; k++){} + assert( pCArray->ixNx[NB*2-1]>i ); + for(k=0; pCArray->ixNx[k]<=i ; k++){} pEnd = pCArray->apEnd[k]; while( 1 /*Exit by break*/ ){ int sz, rc; @@ -77930,9 +80343,13 @@ static int editPage( if( pageInsertArray( pPg, pBegin, &pData, pCellptr, iNew+nCell, nNew-nCell, pCArray - ) ) goto editpage_fail; + ) + ){ + goto editpage_fail; + } - pPg->nCell = nNew; + assert( nNew < 10922 ); + pPg->nCell = (u16)nNew; pPg->nOverflow = 0; put2byte(&aData[hdr+3], pPg->nCell); @@ -78020,6 +80437,7 @@ static int balance_quick(MemPage *pParent, MemPage *pPage, u8 *pSpace){ b.szCell = &szCell; b.apEnd[0] = pPage->aDataEnd; b.ixNx[0] = 2; + b.ixNx[NB*2-1] = 0x7fffffff; rc = rebuildPage(&b, 0, 1, pNew); if( NEVER(rc) ){ releasePage(pNew); @@ -78240,7 +80658,7 @@ static int balance_nonroot( int pageFlags; /* Value of pPage->aData[0] */ int iSpace1 = 0; /* First unused byte of aSpace1[] */ int iOvflSpace = 0; /* First unused byte of aOvflSpace[] */ - int szScratch; /* Size of scratch memory requested */ + u64 szScratch; /* Size of scratch memory requested */ MemPage *apOld[NB]; /* pPage and up to two siblings */ MemPage *apNew[NB+2]; /* pPage and up to NB siblings after balancing */ u8 *pRight; /* Location in parent of right-sibling pointer */ @@ -78255,7 +80673,9 @@ static int balance_nonroot( CellArray b; /* Parsed information on cells being balanced */ memset(abDone, 0, sizeof(abDone)); - memset(&b, 0, sizeof(b)); + assert( sizeof(b) - sizeof(b.ixNx) == offsetof(CellArray,ixNx) ); + memset(&b, 0, sizeof(b)-sizeof(b.ixNx[0])); + b.ixNx[NB*2-1] = 0x7fffffff; pBt = pParent->pBt; assert( sqlite3_mutex_held(pBt->mutex) ); assert( sqlite3PagerIswriteable(pParent->pDbPage) ); @@ -78414,7 +80834,7 @@ static int balance_nonroot( ** table-interior, index-leaf, or index-interior). */ if( pOld->aData[0]!=apOld[0]->aData[0] ){ - rc = SQLITE_CORRUPT_BKPT; + rc = SQLITE_CORRUPT_PAGE(pOld); goto balance_cleanup; } @@ -78438,7 +80858,7 @@ static int balance_nonroot( memset(&b.szCell[b.nCell], 0, sizeof(b.szCell[0])*(limit+pOld->nOverflow)); if( pOld->nOverflow>0 ){ if( NEVER(limitaiOvfl[0]) ){ - rc = SQLITE_CORRUPT_BKPT; + rc = SQLITE_CORRUPT_PAGE(pOld); goto balance_cleanup; } limit = pOld->aiOvfl[0]; @@ -78735,7 +81155,12 @@ static int balance_nonroot( ** of the right-most new sibling page is set to the value that was ** originally in the same field of the right-most old sibling page. */ if( (pageFlags & PTF_LEAF)==0 && nOld!=nNew ){ - MemPage *pOld = (nNew>nOld ? apNew : apOld)[nOld-1]; + MemPage *pOld; + if( nNew>nOld ){ + pOld = apNew[nOld-1]; + }else{ + pOld = apOld[nOld-1]; + } memcpy(&apNew[nNew-1]->aData[8], &pOld->aData[8], 4); } @@ -78846,7 +81271,8 @@ static int balance_nonroot( iOvflSpace += sz; assert( sz<=pBt->maxLocal+23 ); assert( iOvflSpace <= (int)pBt->pageSize ); - for(k=0; ALWAYS(kj ); + for(k=0; b.ixNx[k]<=j; k++){} pSrcEnd = b.apEnd[k]; if( SQLITE_OVERFLOW(pSrcEnd, pCell, pCell+sz) ){ rc = SQLITE_CORRUPT_BKPT; @@ -79081,7 +81507,7 @@ static int anotherValidCursor(BtCursor *pCur){ && pOther->eState==CURSOR_VALID && pOther->pPage==pCur->pPage ){ - return SQLITE_CORRUPT_BKPT; + return SQLITE_CORRUPT_PAGE(pCur->pPage); } } return SQLITE_OK; @@ -79141,7 +81567,7 @@ static int balance(BtCursor *pCur){ /* The page being written is not a root page, and there is currently ** more than one reference to it. This only happens if the page is one ** of its own ancestor pages. Corruption. */ - rc = SQLITE_CORRUPT_BKPT; + rc = SQLITE_CORRUPT_PAGE(pPage); }else{ MemPage * const pParent = pCur->apPage[iPage-1]; int const iIdx = pCur->aiIdx[iPage-1]; @@ -79305,7 +81731,7 @@ static SQLITE_NOINLINE int btreeOverwriteOverflowCell( rc = btreeGetPage(pBt, ovflPgno, &pPage, 0); if( rc ) return rc; if( sqlite3PagerPageRefcount(pPage->pDbPage)!=1 || pPage->isInit ){ - rc = SQLITE_CORRUPT_BKPT; + rc = SQLITE_CORRUPT_PAGE(pPage); }else{ if( iOffset+ovflPageSize<(u32)nTotal ){ ovflPgno = get4byte(pPage->aData); @@ -79333,7 +81759,7 @@ static int btreeOverwriteCell(BtCursor *pCur, const BtreePayload *pX){ if( pCur->info.pPayload + pCur->info.nLocal > pPage->aDataEnd || pCur->info.pPayload < pPage->aData + pPage->cellOffset ){ - return SQLITE_CORRUPT_BKPT; + return SQLITE_CORRUPT_PAGE(pPage); } if( pCur->info.nLocal==nTotal ){ /* The entire cell is local */ @@ -79414,7 +81840,7 @@ SQLITE_PRIVATE int sqlite3BtreeInsert( ** Which can only happen if the SQLITE_NoSchemaError flag was set when ** the schema was loaded. This cannot be asserted though, as a user might ** set the flag, load the schema, and then unset the flag. */ - return SQLITE_CORRUPT_BKPT; + return SQLITE_CORRUPT_PGNO(pCur->pgnoRoot); } } @@ -79522,7 +81948,7 @@ SQLITE_PRIVATE int sqlite3BtreeInsert( if( pCur->info.nKey==pX->nKey ){ BtreePayload x2; x2.pData = pX->pKey; - x2.nData = pX->nKey; + x2.nData = (int)pX->nKey; assert( pX->nKey<=0x7fffffff ); x2.nZero = 0; return btreeOverwriteCell(pCur, &x2); } @@ -79537,7 +81963,7 @@ SQLITE_PRIVATE int sqlite3BtreeInsert( if( pPage->nFree<0 ){ if( NEVER(pCur->eState>CURSOR_INVALID) ){ /* ^^^^^--- due to the moveToRoot() call above */ - rc = SQLITE_CORRUPT_BKPT; + rc = SQLITE_CORRUPT_PAGE(pPage); }else{ rc = btreeComputeFreeSpace(pPage); } @@ -79579,7 +82005,7 @@ SQLITE_PRIVATE int sqlite3BtreeInsert( CellInfo info; assert( idx>=0 ); if( idx>=pPage->nCell ){ - return SQLITE_CORRUPT_BKPT; + return SQLITE_CORRUPT_PAGE(pPage); } rc = sqlite3PagerWrite(pPage->pDbPage); if( rc ){ @@ -79606,10 +82032,10 @@ SQLITE_PRIVATE int sqlite3BtreeInsert( ** necessary to add the PTRMAP_OVERFLOW1 pointer-map entry. */ assert( rc==SQLITE_OK ); /* clearCell never fails when nLocal==nPayload */ if( oldCell < pPage->aData+pPage->hdrOffset+10 ){ - return SQLITE_CORRUPT_BKPT; + return SQLITE_CORRUPT_PAGE(pPage); } if( oldCell+szNew > pPage->aDataEnd ){ - return SQLITE_CORRUPT_BKPT; + return SQLITE_CORRUPT_PAGE(pPage); } memcpy(oldCell, newCell, szNew); return SQLITE_OK; @@ -79619,7 +82045,7 @@ SQLITE_PRIVATE int sqlite3BtreeInsert( }else if( loc<0 && pPage->nCell>0 ){ assert( pPage->leaf ); idx = ++pCur->ix; - pCur->curFlags &= ~BTCF_ValidNKey; + pCur->curFlags &= ~(BTCF_ValidNKey|BTCF_ValidOvfl); }else{ assert( pPage->leaf ); } @@ -79649,7 +82075,7 @@ SQLITE_PRIVATE int sqlite3BtreeInsert( */ if( pPage->nOverflow ){ assert( rc==SQLITE_OK ); - pCur->curFlags &= ~(BTCF_ValidNKey); + pCur->curFlags &= ~(BTCF_ValidNKey|BTCF_ValidOvfl); rc = balance(pCur); /* Must make sure nOverflow is reset to zero even if the balance() @@ -79703,7 +82129,7 @@ SQLITE_PRIVATE int sqlite3BtreeTransferRow(BtCursor *pDest, BtCursor *pSrc, i64 getCellInfo(pSrc); if( pSrc->info.nPayload<0x80 ){ - *(aOut++) = pSrc->info.nPayload; + *(aOut++) = (u8)pSrc->info.nPayload; }else{ aOut += sqlite3PutVarint(aOut, pSrc->info.nPayload); } @@ -79711,12 +82137,12 @@ SQLITE_PRIVATE int sqlite3BtreeTransferRow(BtCursor *pDest, BtCursor *pSrc, i64 nIn = pSrc->info.nLocal; aIn = pSrc->info.pPayload; if( aIn+nIn>pSrc->pPage->aDataEnd ){ - return SQLITE_CORRUPT_BKPT; + return SQLITE_CORRUPT_PAGE(pSrc->pPage); } nRem = pSrc->info.nPayload; if( nIn==nRem && nInpPage->maxLocal ){ memcpy(aOut, aIn, nIn); - pBt->nPreformatSize = nIn + (aOut - pBt->pTmpSpace); + pBt->nPreformatSize = nIn + (int)(aOut - pBt->pTmpSpace); return SQLITE_OK; }else{ int rc = SQLITE_OK; @@ -79728,7 +82154,7 @@ SQLITE_PRIVATE int sqlite3BtreeTransferRow(BtCursor *pDest, BtCursor *pSrc, i64 u32 nOut; /* Size of output buffer aOut[] */ nOut = btreePayloadToLocal(pDest->pPage, pSrc->info.nPayload); - pBt->nPreformatSize = nOut + (aOut - pBt->pTmpSpace); + pBt->nPreformatSize = (int)nOut + (int)(aOut - pBt->pTmpSpace); if( nOutinfo.nPayload ){ pPgnoOut = &aOut[nOut]; pBt->nPreformatSize += 4; @@ -79736,7 +82162,7 @@ SQLITE_PRIVATE int sqlite3BtreeTransferRow(BtCursor *pDest, BtCursor *pSrc, i64 if( nRem>nIn ){ if( aIn+nIn+4>pSrc->pPage->aDataEnd ){ - return SQLITE_CORRUPT_BKPT; + return SQLITE_CORRUPT_PAGE(pSrc->pPage); } ovflIn = get4byte(&pSrc->info.pPayload[nIn]); } @@ -79832,7 +82258,7 @@ SQLITE_PRIVATE int sqlite3BtreeDelete(BtCursor *pCur, u8 flags){ assert( rc!=SQLITE_OK || CORRUPT_DB || pCur->eState==CURSOR_VALID ); if( rc || pCur->eState!=CURSOR_VALID ) return rc; }else{ - return SQLITE_CORRUPT_BKPT; + return SQLITE_CORRUPT_PGNO(pCur->pgnoRoot); } } assert( pCur->eState==CURSOR_VALID ); @@ -79841,14 +82267,14 @@ SQLITE_PRIVATE int sqlite3BtreeDelete(BtCursor *pCur, u8 flags){ iCellIdx = pCur->ix; pPage = pCur->pPage; if( pPage->nCell<=iCellIdx ){ - return SQLITE_CORRUPT_BKPT; + return SQLITE_CORRUPT_PAGE(pPage); } pCell = findCell(pPage, iCellIdx); if( pPage->nFree<0 && btreeComputeFreeSpace(pPage) ){ - return SQLITE_CORRUPT_BKPT; + return SQLITE_CORRUPT_PAGE(pPage); } if( pCell<&pPage->aCellIdx[pPage->nCell] ){ - return SQLITE_CORRUPT_BKPT; + return SQLITE_CORRUPT_PAGE(pPage); } /* If the BTREE_SAVEPOSITION bit is on, then the cursor position must @@ -79939,7 +82365,7 @@ SQLITE_PRIVATE int sqlite3BtreeDelete(BtCursor *pCur, u8 flags){ n = pCur->pPage->pgno; } pCell = findCell(pLeaf, pLeaf->nCell-1); - if( pCell<&pLeaf->aData[4] ) return SQLITE_CORRUPT_BKPT; + if( pCell<&pLeaf->aData[4] ) return SQLITE_CORRUPT_PAGE(pLeaf); nCell = pLeaf->xCellSize(pLeaf, pCell); assert( MX_CELL_SIZE(pBt) >= nCell ); pTmp = pBt->pTmpSpace; @@ -80055,7 +82481,7 @@ static int btreeCreateTable(Btree *p, Pgno *piTable, int createTabFlags){ */ sqlite3BtreeGetMeta(p, BTREE_LARGEST_ROOT_PAGE, &pgnoRoot); if( pgnoRoot>btreePagecount(pBt) ){ - return SQLITE_CORRUPT_BKPT; + return SQLITE_CORRUPT_PGNO(pgnoRoot); } pgnoRoot++; @@ -80103,7 +82529,7 @@ static int btreeCreateTable(Btree *p, Pgno *piTable, int createTabFlags){ } rc = ptrmapGet(pBt, pgnoRoot, &eType, &iPtrPage); if( eType==PTRMAP_ROOTPAGE || eType==PTRMAP_FREEPAGE ){ - rc = SQLITE_CORRUPT_BKPT; + rc = SQLITE_CORRUPT_PGNO(pgnoRoot); } if( rc!=SQLITE_OK ){ releasePage(pRoot); @@ -80193,14 +82619,14 @@ static int clearDatabasePage( assert( sqlite3_mutex_held(pBt->mutex) ); if( pgno>btreePagecount(pBt) ){ - return SQLITE_CORRUPT_BKPT; + return SQLITE_CORRUPT_PGNO(pgno); } rc = getAndInitPage(pBt, pgno, &pPage, 0); if( rc ) return rc; if( (pBt->openFlags & BTREE_SINGLE)==0 && sqlite3PagerPageRefcount(pPage->pDbPage) != (1 + (pgno==1)) ){ - rc = SQLITE_CORRUPT_BKPT; + rc = SQLITE_CORRUPT_PAGE(pPage); goto cleardatabasepage_out; } hdr = pPage->hdrOffset; @@ -80304,7 +82730,7 @@ static int btreeDropTable(Btree *p, Pgno iTable, int *piMoved){ assert( p->inTrans==TRANS_WRITE ); assert( iTable>=2 ); if( iTable>btreePagecount(pBt) ){ - return SQLITE_CORRUPT_BKPT; + return SQLITE_CORRUPT_PGNO(iTable); } rc = sqlite3BtreeClearTable(p, iTable, 0); @@ -80898,6 +83324,9 @@ static int checkTreePage( ** number of cells on the page. */ nCell = get2byte(&data[hdr+3]); assert( pPage->nCell==nCell ); + if( pPage->leaf || pPage->intKey==0 ){ + pCheck->nRow += nCell; + } /* EVIDENCE-OF: R-23882-45353 The cell pointer array of a b-tree page ** immediately follows the b-tree page header. */ @@ -81009,6 +83438,7 @@ static int checkTreePage( btreeHeapInsert(heap, (pc<<16)|(pc+size-1)); } } + assert( heap!=0 ); /* Add the freeblocks to the min-heap ** ** EVIDENCE-OF: R-20690-50594 The second field of the b-tree page header @@ -81108,6 +83538,7 @@ SQLITE_PRIVATE int sqlite3BtreeIntegrityCheck( sqlite3 *db, /* Database connection that is running the check */ Btree *p, /* The btree to be checked */ Pgno *aRoot, /* An array of root pages numbers for individual trees */ + Mem *aCnt, /* Memory cells to write counts for each tree to */ int nRoot, /* Number of entries in aRoot[] */ int mxErr, /* Stop reporting errors after this many */ int *pnErr, /* OUT: Write number of errors seen to this variable */ @@ -81121,7 +83552,9 @@ SQLITE_PRIVATE int sqlite3BtreeIntegrityCheck( int bPartial = 0; /* True if not checking all btrees */ int bCkFreelist = 1; /* True to scan the freelist */ VVA_ONLY( int nRef ); + assert( nRoot>0 ); + assert( aCnt!=0 ); /* aRoot[0]==0 means this is a partial check */ if( aRoot[0]==0 ){ @@ -81194,15 +83627,18 @@ SQLITE_PRIVATE int sqlite3BtreeIntegrityCheck( testcase( pBt->db->flags & SQLITE_CellSizeCk ); pBt->db->flags &= ~(u64)SQLITE_CellSizeCk; for(i=0; (int)iautoVacuum && aRoot[i]>1 && !bPartial ){ - checkPtrmap(&sCheck, aRoot[i], PTRMAP_ROOTPAGE, 0); - } + if( pBt->autoVacuum && aRoot[i]>1 && !bPartial ){ + checkPtrmap(&sCheck, aRoot[i], PTRMAP_ROOTPAGE, 0); + } #endif - sCheck.v0 = aRoot[i]; - checkTreePage(&sCheck, aRoot[i], ¬Used, LARGEST_INT64); + sCheck.v0 = aRoot[i]; + checkTreePage(&sCheck, aRoot[i], ¬Used, LARGEST_INT64); + } + sqlite3MemSetArrayInt64(aCnt, i, sCheck.nRow); } pBt->db->flags = savedDbFlags; @@ -81339,6 +83775,7 @@ SQLITE_PRIVATE int sqlite3BtreeIsInBackup(Btree *p){ */ SQLITE_PRIVATE void *sqlite3BtreeSchema(Btree *p, int nBytes, void(*xFree)(void *)){ BtShared *pBt = p->pBt; + assert( nBytes==0 || nBytes==sizeof(Schema) ); sqlite3BtreeEnter(p); if( !pBt->pSchema && nBytes ){ pBt->pSchema = sqlite3DbMallocZero(0, nBytes); @@ -81355,6 +83792,7 @@ SQLITE_PRIVATE void *sqlite3BtreeSchema(Btree *p, int nBytes, void(*xFree)(void */ SQLITE_PRIVATE int sqlite3BtreeSchemaLocked(Btree *p){ int rc; + UNUSED_PARAMETER(p); /* only used in DEBUG builds */ assert( sqlite3_mutex_held(p->db->mutex) ); sqlite3BtreeEnter(p); rc = querySharedCacheTableLock(p, SCHEMA_ROOT, READ_LOCK); @@ -82455,7 +84893,7 @@ static void vdbeMemRenderNum(int sz, char *zBuf, Mem *p){ ** corresponding string value, then it is important that the string be ** derived from the numeric value, not the other way around, to ensure ** that the index and table are consistent. See ticket -** https://www.sqlite.org/src/info/343634942dd54ab (2018-01-31) for +** https://sqlite.org/src/info/343634942dd54ab (2018-01-31) for ** an example. ** ** This routine looks at pMem to verify that if it has both a numeric @@ -82641,7 +85079,7 @@ SQLITE_PRIVATE void sqlite3VdbeMemZeroTerminateIfAble(Mem *pMem){ return; } if( pMem->enc!=SQLITE_UTF8 ) return; - if( NEVER(pMem->z==0) ) return; + assert( pMem->z!=0 ); if( pMem->flags & MEM_Dyn ){ if( pMem->xDel==sqlite3_free && sqlite3_msize(pMem->z) >= (u64)(pMem->n+1) @@ -83257,6 +85695,13 @@ SQLITE_PRIVATE void sqlite3VdbeMemSetInt64(Mem *pMem, i64 val){ } } +/* +** Set the iIdx'th entry of array aMem[] to contain integer value val. +*/ +SQLITE_PRIVATE void sqlite3MemSetArrayInt64(sqlite3_value *aMem, int iIdx, i64 val){ + sqlite3VdbeMemSetInt64(&aMem[iIdx], val); +} + /* A no-op destructor */ SQLITE_PRIVATE void sqlite3NoopDestructor(void *p){ UNUSED_PARAMETER(p); } @@ -83353,27 +85798,30 @@ SQLITE_PRIVATE int sqlite3VdbeMemTooBig(Mem *p){ SQLITE_PRIVATE void sqlite3VdbeMemAboutToChange(Vdbe *pVdbe, Mem *pMem){ int i; Mem *pX; - for(i=1, pX=pVdbe->aMem+1; inMem; i++, pX++){ - if( pX->pScopyFrom==pMem ){ - u16 mFlags; - if( pVdbe->db->flags & SQLITE_VdbeTrace ){ - sqlite3DebugPrintf("Invalidate R[%d] due to change in R[%d]\n", - (int)(pX - pVdbe->aMem), (int)(pMem - pVdbe->aMem)); - } - /* If pX is marked as a shallow copy of pMem, then try to verify that - ** no significant changes have been made to pX since the OP_SCopy. - ** A significant change would indicated a missed call to this - ** function for pX. Minor changes, such as adding or removing a - ** dual type, are allowed, as long as the underlying value is the - ** same. */ - mFlags = pMem->flags & pX->flags & pX->mScopyFlags; - assert( (mFlags&(MEM_Int|MEM_IntReal))==0 || pMem->u.i==pX->u.i ); - - /* pMem is the register that is changing. But also mark pX as - ** undefined so that we can quickly detect the shallow-copy error */ - pX->flags = MEM_Undefined; - pX->pScopyFrom = 0; - } + if( pMem->bScopy ){ + for(i=1, pX=pVdbe->aMem+1; inMem; i++, pX++){ + if( pX->pScopyFrom==pMem ){ + u16 mFlags; + if( pVdbe->db->flags & SQLITE_VdbeTrace ){ + sqlite3DebugPrintf("Invalidate R[%d] due to change in R[%d]\n", + (int)(pX - pVdbe->aMem), (int)(pMem - pVdbe->aMem)); + } + /* If pX is marked as a shallow copy of pMem, then try to verify that + ** no significant changes have been made to pX since the OP_SCopy. + ** A significant change would indicated a missed call to this + ** function for pX. Minor changes, such as adding or removing a + ** dual type, are allowed, as long as the underlying value is the + ** same. */ + mFlags = pMem->flags & pX->flags & pX->mScopyFlags; + assert( (mFlags&(MEM_Int|MEM_IntReal))==0 || pMem->u.i==pX->u.i ); + + /* pMem is the register that is changing. But also mark pX as + ** undefined so that we can quickly detect the shallow-copy error */ + pX->flags = MEM_Undefined; + pX->pScopyFrom = 0; + } + } + pMem->bScopy = 0; } pMem->pScopyFrom = 0; } @@ -83530,6 +85978,7 @@ SQLITE_PRIVATE int sqlite3VdbeMemSetStr( if( sqlite3VdbeMemClearAndResize(pMem, (int)MAX(nAlloc,32)) ){ return SQLITE_NOMEM_BKPT; } + assert( pMem->z!=0 ); memcpy(pMem->z, z, nAlloc); }else{ sqlite3VdbeMemRelease(pMem); @@ -83744,7 +86193,7 @@ static sqlite3_value *valueNew(sqlite3 *db, struct ValueNewStat4Ctx *p){ if( pRec==0 ){ Index *pIdx = p->pIdx; /* Index being probed */ - int nByte; /* Bytes of space to allocate */ + i64 nByte; /* Bytes of space to allocate */ int i; /* Counter variable */ int nCol = pIdx->nColumn; /* Number of index columns including rowid */ @@ -83810,7 +86259,7 @@ static int valueFromFunction( ){ sqlite3_context ctx; /* Context object for function invocation */ sqlite3_value **apVal = 0; /* Function arguments */ - int nVal = 0; /* Size of apVal[] array */ + int nVal = 0; /* Number of function arguments */ FuncDef *pFunc = 0; /* Function definition */ sqlite3_value *pVal = 0; /* New value */ int rc = SQLITE_OK; /* Return code */ @@ -83841,7 +86290,8 @@ static int valueFromFunction( goto value_from_function_out; } for(i=0; ia[i].pExpr, enc, aff, &apVal[i]); + rc = sqlite3Stat4ValueFromExpr(pCtx->pParse, pList->a[i].pExpr, aff, + &apVal[i]); if( apVal[i]==0 || rc!=SQLITE_OK ) goto value_from_function_out; } } @@ -83945,14 +86395,20 @@ static int valueFromExpr( } /* Handle negative integers in a single step. This is needed in the - ** case when the value is -9223372036854775808. - */ - if( op==TK_UMINUS - && (pExpr->pLeft->op==TK_INTEGER || pExpr->pLeft->op==TK_FLOAT) ){ - pExpr = pExpr->pLeft; - op = pExpr->op; - negInt = -1; - zNeg = "-"; + ** case when the value is -9223372036854775808. Except - do not do this + ** for hexadecimal literals. */ + if( op==TK_UMINUS ){ + Expr *pLeft = pExpr->pLeft; + if( (pLeft->op==TK_INTEGER || pLeft->op==TK_FLOAT) ){ + if( ExprHasProperty(pLeft, EP_IntValue) + || pLeft->u.zToken[0]!='0' || (pLeft->u.zToken[1] & ~0x20)!='X' + ){ + pExpr = pLeft; + op = pExpr->op; + negInt = -1; + zNeg = "-"; + } + } } if( op==TK_STRING || op==TK_FLOAT || op==TK_INTEGER ){ @@ -83961,12 +86417,26 @@ static int valueFromExpr( if( ExprHasProperty(pExpr, EP_IntValue) ){ sqlite3VdbeMemSetInt64(pVal, (i64)pExpr->u.iValue*negInt); }else{ - zVal = sqlite3MPrintf(db, "%s%s", zNeg, pExpr->u.zToken); - if( zVal==0 ) goto no_mem; - sqlite3ValueSetStr(pVal, -1, zVal, SQLITE_UTF8, SQLITE_DYNAMIC); + i64 iVal; + if( op==TK_INTEGER && 0==sqlite3DecOrHexToI64(pExpr->u.zToken, &iVal) ){ + sqlite3VdbeMemSetInt64(pVal, iVal*negInt); + }else{ + zVal = sqlite3MPrintf(db, "%s%s", zNeg, pExpr->u.zToken); + if( zVal==0 ) goto no_mem; + sqlite3ValueSetStr(pVal, -1, zVal, SQLITE_UTF8, SQLITE_DYNAMIC); + } } - if( (op==TK_INTEGER || op==TK_FLOAT ) && affinity==SQLITE_AFF_BLOB ){ - sqlite3ValueApplyAffinity(pVal, SQLITE_AFF_NUMERIC, SQLITE_UTF8); + if( affinity==SQLITE_AFF_BLOB ){ + if( op==TK_FLOAT ){ + assert( pVal && pVal->z && pVal->flags==(MEM_Str|MEM_Term) ); + sqlite3AtoF(pVal->z, &pVal->u.r, pVal->n, SQLITE_UTF8); + pVal->flags = MEM_Real; + }else if( op==TK_INTEGER ){ + /* This case is required by -9223372036854775808 and other strings + ** that look like integers but cannot be handled by the + ** sqlite3DecOrHexToI64() call above. */ + sqlite3ValueApplyAffinity(pVal, SQLITE_AFF_NUMERIC, SQLITE_UTF8); + } }else{ sqlite3ValueApplyAffinity(pVal, affinity, SQLITE_UTF8); } @@ -84236,17 +86706,17 @@ SQLITE_PRIVATE int sqlite3Stat4Column( sqlite3_value **ppVal /* OUT: Extracted value */ ){ u32 t = 0; /* a column type code */ - int nHdr; /* Size of the header in the record */ - int iHdr; /* Next unread header byte */ - int iField; /* Next unread data byte */ - int szField = 0; /* Size of the current data field */ + u32 nHdr; /* Size of the header in the record */ + u32 iHdr; /* Next unread header byte */ + i64 iField; /* Next unread data byte */ + u32 szField = 0; /* Size of the current data field */ int i; /* Column index */ u8 *a = (u8*)pRec; /* Typecast byte array */ Mem *pMem = *ppVal; /* Write result into this Mem object */ assert( iCol>0 ); iHdr = getVarint32(a, nHdr); - if( nHdr>nRec || iHdr>=nHdr ) return SQLITE_CORRUPT_BKPT; + if( nHdr>(u32)nRec || iHdr>=nHdr ) return SQLITE_CORRUPT_BKPT; iField = nHdr; for(i=0; i<=iCol; i++){ iHdr += getVarint32(&a[iHdr], t); @@ -84787,12 +87257,10 @@ SQLITE_PRIVATE int sqlite3VdbeAddFunctionCall( int eCallCtx /* Calling context */ ){ Vdbe *v = pParse->pVdbe; - int nByte; int addr; sqlite3_context *pCtx; assert( v ); - nByte = sizeof(*pCtx) + (nArg-1)*sizeof(sqlite3_value*); - pCtx = sqlite3DbMallocRawNN(pParse->db, nByte); + pCtx = sqlite3DbMallocRawNN(pParse->db, SZ_CONTEXT(nArg)); if( pCtx==0 ){ assert( pParse->db->mallocFailed ); freeEphemeralFunction(pParse->db, (FuncDef*)pFunc); @@ -85068,7 +87536,7 @@ static Op *opIterNext(VdbeOpIter *p){ } if( pRet->p4type==P4_SUBPROGRAM ){ - int nByte = (p->nSub+1)*sizeof(SubProgram*); + i64 nByte = (1+(u64)p->nSub)*sizeof(SubProgram*); int j; for(j=0; jnSub; j++){ if( p->apSub[j]==pRet->p4.pProgram ) break; @@ -85198,8 +87666,8 @@ SQLITE_PRIVATE void sqlite3VdbeAssertAbortable(Vdbe *p){ ** (1) For each jump instruction with a negative P2 value (a label) ** resolve the P2 value to an actual address. ** -** (2) Compute the maximum number of arguments used by any SQL function -** and store that value in *pMaxFuncArgs. +** (2) Compute the maximum number of arguments used by the xUpdate/xFilter +** methods of any virtual table and store that value in *pMaxVtabArgs. ** ** (3) Update the Vdbe.readOnly and Vdbe.bIsReader flags to accurately ** indicate what the prepared statement actually does. @@ -85212,8 +87680,8 @@ SQLITE_PRIVATE void sqlite3VdbeAssertAbortable(Vdbe *p){ ** script numbers the opcodes correctly. Changes to this routine must be ** coordinated with changes to mkopcodeh.tcl. */ -static void resolveP2Values(Vdbe *p, int *pMaxFuncArgs){ - int nMaxArgs = *pMaxFuncArgs; +static void resolveP2Values(Vdbe *p, int *pMaxVtabArgs){ + int nMaxVtabArgs = *pMaxVtabArgs; Op *pOp; Parse *pParse = p->pParse; int *aLabel = pParse->aLabel; @@ -85258,15 +87726,19 @@ static void resolveP2Values(Vdbe *p, int *pMaxFuncArgs){ } #ifndef SQLITE_OMIT_VIRTUALTABLE case OP_VUpdate: { - if( pOp->p2>nMaxArgs ) nMaxArgs = pOp->p2; + if( pOp->p2>nMaxVtabArgs ) nMaxVtabArgs = pOp->p2; break; } case OP_VFilter: { int n; + /* The instruction immediately prior to VFilter will be an + ** OP_Integer that sets the "argc" value for the VFilter. See + ** the code where OP_VFilter is generated at tag-20250207a. */ assert( (pOp - p->aOp) >= 3 ); assert( pOp[-1].opcode==OP_Integer ); + assert( pOp[-1].p2==pOp->p3+1 ); n = pOp[-1].p1; - if( n>nMaxArgs ) nMaxArgs = n; + if( n>nMaxVtabArgs ) nMaxVtabArgs = n; /* Fall through into the default case */ /* no break */ deliberate_fall_through } @@ -85281,6 +87753,15 @@ static void resolveP2Values(Vdbe *p, int *pMaxFuncArgs){ assert( aLabel!=0 ); /* True because of tag-20230419-1 */ pOp->p2 = aLabel[ADDR(pOp->p2)]; } + + /* OPFLG_JUMP opcodes never have P2==0, though OPFLG_JUMP0 opcodes + ** might */ + assert( pOp->p2>0 + || (sqlite3OpcodeProperty[pOp->opcode] & OPFLG_JUMP0)!=0 ); + + /* Jumps never go off the end of the bytecode array */ + assert( pOp->p2nOp + || (sqlite3OpcodeProperty[pOp->opcode] & OPFLG_JUMP)==0 ); break; } } @@ -85298,7 +87779,7 @@ static void resolveP2Values(Vdbe *p, int *pMaxFuncArgs){ pParse->aLabel = 0; } pParse->nLabel = 0; - *pMaxFuncArgs = nMaxArgs; + *pMaxVtabArgs = nMaxVtabArgs; assert( p->bIsReader!=0 || DbMaskAllZero(p->btreeMask) ); } @@ -85527,7 +88008,7 @@ SQLITE_PRIVATE void sqlite3VdbeScanStatus( const char *zName /* Name of table or index being scanned */ ){ if( IS_STMT_SCANSTATUS(p->db) ){ - sqlite3_int64 nByte = (p->nScan+1) * sizeof(ScanStatus); + i64 nByte = (1+(i64)p->nScan) * sizeof(ScanStatus); ScanStatus *aNew; aNew = (ScanStatus*)sqlite3DbRealloc(p->db, p->aScan, nByte); if( aNew ){ @@ -85637,6 +88118,9 @@ SQLITE_PRIVATE void sqlite3VdbeChangeP5(Vdbe *p, u16 p5){ */ SQLITE_PRIVATE void sqlite3VdbeTypeofColumn(Vdbe *p, int iDest){ VdbeOp *pOp = sqlite3VdbeGetLastOp(p); +#ifdef SQLITE_DEBUG + while( pOp->opcode==OP_ReleaseReg ) pOp--; +#endif if( pOp->p3==iDest && pOp->opcode==OP_Column ){ pOp->p5 |= OPFLAG_TYPEOFARG; } @@ -85746,6 +88230,12 @@ static void freeP4(sqlite3 *db, int p4type, void *p4){ if( db->pnBytesFreed==0 ) sqlite3DeleteTable(db, (Table*)p4); break; } + case P4_SUBRTNSIG: { + SubrtnSig *pSig = (SubrtnSig*)p4; + sqlite3DbFree(db, pSig->zAff); + sqlite3DbFree(db, pSig); + break; + } } } @@ -86325,6 +88815,11 @@ SQLITE_PRIVATE char *sqlite3VdbeDisplayP4(sqlite3 *db, Op *pOp){ zP4 = pOp->p4.pTab->zName; break; } + case P4_SUBRTNSIG: { + SubrtnSig *pSig = pOp->p4.pSubrtnSig; + sqlite3_str_appendf(&x, "subrtnsig:%d,%s", pSig->selId, pSig->zAff); + break; + } default: { zP4 = pOp->p4.z; } @@ -86466,6 +88961,7 @@ SQLITE_PRIVATE void sqlite3VdbePrintOp(FILE *pOut, int pc, VdbeOp *pOp){ ** will be initialized before use. */ static void initMemArray(Mem *p, int N, sqlite3 *db, u16 flags){ + assert( db!=0 ); if( N>0 ){ do{ p->flags = flags; @@ -86473,6 +88969,7 @@ static void initMemArray(Mem *p, int N, sqlite3 *db, u16 flags){ p->szMalloc = 0; #ifdef SQLITE_DEBUG p->pScopyFrom = 0; + p->bScopy = 0; #endif p++; }while( (--N)>0 ); @@ -86491,6 +88988,7 @@ static void releaseMemArray(Mem *p, int N){ if( p && N ){ Mem *pEnd = &p[N]; sqlite3 *db = p->db; + assert( db!=0 ); if( db->pnBytesFreed ){ do{ if( p->szMalloc ) sqlite3DbFree(db, p->zMalloc); @@ -86962,7 +89460,7 @@ SQLITE_PRIVATE void sqlite3VdbeMakeReady( int nVar; /* Number of parameters */ int nMem; /* Number of VM memory registers */ int nCursor; /* Number of cursors required */ - int nArg; /* Number of arguments in subprograms */ + int nArg; /* Max number args to xFilter or xUpdate */ int n; /* Loop counter */ struct ReusableSpace x; /* Reusable bulk memory */ @@ -86971,6 +89469,7 @@ SQLITE_PRIVATE void sqlite3VdbeMakeReady( assert( pParse!=0 ); assert( p->eVdbeState==VDBE_INIT_STATE ); assert( pParse==p->pParse ); + assert( pParse->db==p->db ); p->pVList = pParse->pVList; pParse->pVList = 0; db = p->db; @@ -87033,6 +89532,9 @@ SQLITE_PRIVATE void sqlite3VdbeMakeReady( p->apCsr = allocSpace(&x, p->apCsr, nCursor*sizeof(VdbeCursor*)); } } +#ifdef SQLITE_DEBUG + p->napArg = nArg; +#endif if( db->mallocFailed ){ p->nVar = 0; @@ -87304,10 +89806,12 @@ static int vdbeCommit(sqlite3 *db, Vdbe *p){ if( 0==sqlite3Strlen30(sqlite3BtreeGetFilename(db->aDb[0].pBt)) || nTrans<=1 ){ - for(i=0; rc==SQLITE_OK && inDb; i++){ - Btree *pBt = db->aDb[i].pBt; - if( pBt ){ - rc = sqlite3BtreeCommitPhaseOne(pBt, 0); + if( needXcommit ){ + for(i=0; rc==SQLITE_OK && inDb; i++){ + Btree *pBt = db->aDb[i].pBt; + if( sqlite3BtreeTxnState(pBt)>=SQLITE_TXN_WRITE ){ + rc = sqlite3BtreeCommitPhaseOne(pBt, 0); + } } } @@ -87318,7 +89822,9 @@ static int vdbeCommit(sqlite3 *db, Vdbe *p){ */ for(i=0; rc==SQLITE_OK && inDb; i++){ Btree *pBt = db->aDb[i].pBt; - if( pBt ){ + int txn = sqlite3BtreeTxnState(pBt); + if( txn!=SQLITE_TXN_NONE ){ + assert( needXcommit || txn==SQLITE_TXN_READ ); rc = sqlite3BtreeCommitPhaseTwo(pBt, 0); } } @@ -87573,28 +90079,31 @@ SQLITE_PRIVATE int sqlite3VdbeCloseStatement(Vdbe *p, int eOp){ /* -** This function is called when a transaction opened by the database +** These functions are called when a transaction opened by the database ** handle associated with the VM passed as an argument is about to be -** committed. If there are outstanding deferred foreign key constraint -** violations, return SQLITE_ERROR. Otherwise, SQLITE_OK. +** committed. If there are outstanding foreign key constraint violations +** return an error code. Otherwise, SQLITE_OK. ** ** If there are outstanding FK violations and this function returns -** SQLITE_ERROR, set the result of the VM to SQLITE_CONSTRAINT_FOREIGNKEY -** and write an error message to it. Then return SQLITE_ERROR. +** non-zero, set the result of the VM to SQLITE_CONSTRAINT_FOREIGNKEY +** and write an error message to it. */ #ifndef SQLITE_OMIT_FOREIGN_KEY -SQLITE_PRIVATE int sqlite3VdbeCheckFk(Vdbe *p, int deferred){ +static SQLITE_NOINLINE int vdbeFkError(Vdbe *p){ + p->rc = SQLITE_CONSTRAINT_FOREIGNKEY; + p->errorAction = OE_Abort; + sqlite3VdbeError(p, "FOREIGN KEY constraint failed"); + if( (p->prepFlags & SQLITE_PREPARE_SAVESQL)==0 ) return SQLITE_ERROR; + return SQLITE_CONSTRAINT_FOREIGNKEY; +} +SQLITE_PRIVATE int sqlite3VdbeCheckFkImmediate(Vdbe *p){ + if( p->nFkConstraint==0 ) return SQLITE_OK; + return vdbeFkError(p); +} +SQLITE_PRIVATE int sqlite3VdbeCheckFkDeferred(Vdbe *p){ sqlite3 *db = p->db; - if( (deferred && (db->nDeferredCons+db->nDeferredImmCons)>0) - || (!deferred && p->nFkConstraint>0) - ){ - p->rc = SQLITE_CONSTRAINT_FOREIGNKEY; - p->errorAction = OE_Abort; - sqlite3VdbeError(p, "FOREIGN KEY constraint failed"); - if( (p->prepFlags & SQLITE_PREPARE_SAVESQL)==0 ) return SQLITE_ERROR; - return SQLITE_CONSTRAINT_FOREIGNKEY; - } - return SQLITE_OK; + if( (db->nDeferredCons+db->nDeferredImmCons)==0 ) return SQLITE_OK; + return vdbeFkError(p); } #endif @@ -87688,7 +90197,7 @@ SQLITE_PRIVATE int sqlite3VdbeHalt(Vdbe *p){ /* Check for immediate foreign key violations. */ if( p->rc==SQLITE_OK || (p->errorAction==OE_Fail && !isSpecialError) ){ - sqlite3VdbeCheckFk(p, 0); + (void)sqlite3VdbeCheckFkImmediate(p); } /* If the auto-commit flag is set and this is the only active writer @@ -87702,7 +90211,7 @@ SQLITE_PRIVATE int sqlite3VdbeHalt(Vdbe *p){ && db->nVdbeWrite==(p->readOnly==0) ){ if( p->rc==SQLITE_OK || (p->errorAction==OE_Fail && !isSpecialError) ){ - rc = sqlite3VdbeCheckFk(p, 1); + rc = sqlite3VdbeCheckFkDeferred(p); if( rc!=SQLITE_OK ){ if( NEVER(p->readOnly) ){ sqlite3VdbeLeave(p); @@ -88512,29 +91021,22 @@ SQLITE_PRIVATE void sqlite3VdbeSerialGet( return; } /* -** This routine is used to allocate sufficient space for an UnpackedRecord -** structure large enough to be used with sqlite3VdbeRecordUnpack() if -** the first argument is a pointer to KeyInfo structure pKeyInfo. -** -** The space is either allocated using sqlite3DbMallocRaw() or from within -** the unaligned buffer passed via the second and third arguments (presumably -** stack space). If the former, then *ppFree is set to a pointer that should -** be eventually freed by the caller using sqlite3DbFree(). Or, if the -** allocation comes from the pSpace/szSpace buffer, *ppFree is set to NULL -** before returning. +** Allocate sufficient space for an UnpackedRecord structure large enough +** to hold a decoded index record for pKeyInfo. ** -** If an OOM error occurs, NULL is returned. +** The space is allocated using sqlite3DbMallocRaw(). If an OOM error +** occurs, NULL is returned. */ SQLITE_PRIVATE UnpackedRecord *sqlite3VdbeAllocUnpackedRecord( KeyInfo *pKeyInfo /* Description of the record */ ){ UnpackedRecord *p; /* Unpacked record to return */ - int nByte; /* Number of bytes required for *p */ + u64 nByte; /* Number of bytes required for *p */ + assert( sizeof(UnpackedRecord) + sizeof(Mem)*65536 < 0x7fffffff ); nByte = ROUND8P(sizeof(UnpackedRecord)) + sizeof(Mem)*(pKeyInfo->nKeyField+1); p = (UnpackedRecord *)sqlite3DbMallocRaw(pKeyInfo->db, nByte); if( !p ) return 0; p->aMem = (Mem*)&((char*)p)[ROUND8P(sizeof(UnpackedRecord))]; - assert( pKeyInfo->aSortFlags!=0 ); p->pKeyInfo = pKeyInfo; p->nField = pKeyInfo->nKeyField + 1; return p; @@ -88546,7 +91048,6 @@ SQLITE_PRIVATE UnpackedRecord *sqlite3VdbeAllocUnpackedRecord( ** contents of the decoded record. */ SQLITE_PRIVATE void sqlite3VdbeRecordUnpack( - KeyInfo *pKeyInfo, /* Information about the record format */ int nKey, /* Size of the binary record */ const void *pKey, /* The binary record */ UnpackedRecord *p /* Populate this structure before returning. */ @@ -88557,6 +91058,7 @@ SQLITE_PRIVATE void sqlite3VdbeRecordUnpack( u16 u; /* Unsigned loop counter */ u32 szHdr; Mem *pMem = p->aMem; + KeyInfo *pKeyInfo = p->pKeyInfo; p->default_rc = 0; assert( EIGHT_BYTE_ALIGNMENT(pMem) ); @@ -88574,16 +91076,18 @@ SQLITE_PRIVATE void sqlite3VdbeRecordUnpack( pMem->z = 0; sqlite3VdbeSerialGet(&aKey[d], serial_type, pMem); d += sqlite3VdbeSerialTypeLen(serial_type); - pMem++; if( (++u)>=p->nField ) break; + pMem++; } if( d>(u32)nKey && u ){ assert( CORRUPT_DB ); /* In a corrupt record entry, the last pMem might have been set up using ** uninitialized memory. Overwrite its value with NULL, to prevent ** warnings from MSAN. */ - sqlite3VdbeMemSetNull(pMem-1); + sqlite3VdbeMemSetNull(pMem-(unField)); } + testcase( u == pKeyInfo->nKeyField + 1 ); + testcase( u < pKeyInfo->nKeyField + 1 ); assert( u<=pKeyInfo->nKeyField + 1 ); p->nField = u; } @@ -88751,6 +91255,32 @@ static void vdbeAssertFieldCountWithinLimits( ** or positive value if *pMem1 is less than, equal to or greater than ** *pMem2, respectively. Similar in spirit to "rc = (*pMem1) - (*pMem2);". */ +static SQLITE_NOINLINE int vdbeCompareMemStringWithEncodingChange( + const Mem *pMem1, + const Mem *pMem2, + const CollSeq *pColl, + u8 *prcErr /* If an OOM occurs, set to SQLITE_NOMEM */ +){ + int rc; + const void *v1, *v2; + Mem c1; + Mem c2; + sqlite3VdbeMemInit(&c1, pMem1->db, MEM_Null); + sqlite3VdbeMemInit(&c2, pMem1->db, MEM_Null); + sqlite3VdbeMemShallowCopy(&c1, pMem1, MEM_Ephem); + sqlite3VdbeMemShallowCopy(&c2, pMem2, MEM_Ephem); + v1 = sqlite3ValueText((sqlite3_value*)&c1, pColl->enc); + v2 = sqlite3ValueText((sqlite3_value*)&c2, pColl->enc); + if( (v1==0 || v2==0) ){ + if( prcErr ) *prcErr = SQLITE_NOMEM_BKPT; + rc = 0; + }else{ + rc = pColl->xCmp(pColl->pUser, c1.n, v1, c2.n, v2); + } + sqlite3VdbeMemReleaseMalloc(&c1); + sqlite3VdbeMemReleaseMalloc(&c2); + return rc; +} static int vdbeCompareMemString( const Mem *pMem1, const Mem *pMem2, @@ -88762,25 +91292,7 @@ static int vdbeCompareMemString( ** comparison function directly */ return pColl->xCmp(pColl->pUser,pMem1->n,pMem1->z,pMem2->n,pMem2->z); }else{ - int rc; - const void *v1, *v2; - Mem c1; - Mem c2; - sqlite3VdbeMemInit(&c1, pMem1->db, MEM_Null); - sqlite3VdbeMemInit(&c2, pMem1->db, MEM_Null); - sqlite3VdbeMemShallowCopy(&c1, pMem1, MEM_Ephem); - sqlite3VdbeMemShallowCopy(&c2, pMem2, MEM_Ephem); - v1 = sqlite3ValueText((sqlite3_value*)&c1, pColl->enc); - v2 = sqlite3ValueText((sqlite3_value*)&c2, pColl->enc); - if( (v1==0 || v2==0) ){ - if( prcErr ) *prcErr = SQLITE_NOMEM_BKPT; - rc = 0; - }else{ - rc = pColl->xCmp(pColl->pUser, c1.n, v1, c2.n, v2); - } - sqlite3VdbeMemReleaseMalloc(&c1); - sqlite3VdbeMemReleaseMalloc(&c2); - return rc; + return vdbeCompareMemStringWithEncodingChange(pMem1,pMem2,pColl,prcErr); } } @@ -88834,7 +91346,7 @@ SQLITE_PRIVATE SQLITE_NOINLINE int sqlite3BlobCompare(const Mem *pB1, const Mem ** We must use separate SQLITE_NOINLINE functions here, since otherwise ** optimizer code movement causes gcov to become very confused. */ -#if defined(SQLITE_COVERAGE_TEST) || defined(SQLITE_DEBUG) +#if defined(SQLITE_COVERAGE_TEST) || defined(SQLITE_DEBUG) static int SQLITE_NOINLINE doubleLt(double a, double b){ return ar ); - testcase( x==r ); - return (xr); }else{ i64 y; - double s; if( r<-9223372036854775808.0 ) return +1; if( r>=9223372036854775808.0 ) return -1; y = (i64)r; if( iy ) return +1; - s = (double)i; - testcase( doubleLt(s,r) ); - testcase( doubleLt(r,s) ); - testcase( doubleEq(r,s) ); - return (sr); + testcase( doubleLt(((double)i),r) ); + testcase( doubleLt(r,((double)i)) ); + testcase( doubleEq(r,((double)i)) ); + return (((double)i)r); } } @@ -89452,6 +91955,7 @@ SQLITE_PRIVATE RecordCompare sqlite3VdbeFindCompare(UnpackedRecord *p){ ** The easiest way to enforce this limit is to consider only records with ** 13 fields or less. If the first field is an integer, the maximum legal ** header size is (12*5 + 1 + 1) bytes. */ + assert( p->pKeyInfo->aSortFlags!=0 ); if( p->pKeyInfo->nAllField<=13 ){ int flags = p->aMem[0].flags; if( p->pKeyInfo->aSortFlags[0] ){ @@ -89671,7 +92175,8 @@ SQLITE_PRIVATE sqlite3_value *sqlite3VdbeGetBoundValue(Vdbe *v, int iVar, u8 aff assert( iVar>0 ); if( v ){ Mem *pMem = &v->aVar[iVar-1]; - assert( (v->db->flags & SQLITE_EnableQPSG)==0 ); + assert( (v->db->flags & SQLITE_EnableQPSG)==0 + || (v->db->mDbFlags & DBFLAG_InternalFunc)!=0 ); if( 0==(pMem->flags & MEM_Null) ){ sqlite3_value *pRet = sqlite3ValueNew(v->db); if( pRet ){ @@ -89691,7 +92196,8 @@ SQLITE_PRIVATE sqlite3_value *sqlite3VdbeGetBoundValue(Vdbe *v, int iVar, u8 aff */ SQLITE_PRIVATE void sqlite3VdbeSetVarmask(Vdbe *v, int iVar){ assert( iVar>0 ); - assert( (v->db->flags & SQLITE_EnableQPSG)==0 ); + assert( (v->db->flags & SQLITE_EnableQPSG)==0 + || (v->db->mDbFlags & DBFLAG_InternalFunc)!=0 ); if( iVar>=32 ){ v->expmask |= 0x80000000; }else{ @@ -89699,6 +92205,7 @@ SQLITE_PRIVATE void sqlite3VdbeSetVarmask(Vdbe *v, int iVar){ } } +#ifndef SQLITE_OMIT_DATETIME_FUNCS /* ** Cause a function to throw an error if it was call from OP_PureFunc ** rather than OP_Function. @@ -89732,6 +92239,7 @@ SQLITE_PRIVATE int sqlite3NotPureFunc(sqlite3_context *pCtx){ } return 1; } +#endif /* SQLITE_OMIT_DATETIME_FUNCS */ #if defined(SQLITE_ENABLE_CURSOR_HINTS) && defined(SQLITE_DEBUG) /* @@ -89808,7 +92316,6 @@ SQLITE_PRIVATE void sqlite3VdbePreUpdateHook( i64 iKey2; PreUpdate preupdate; const char *zTbl = pTab->zName; - static const u8 fakeSortOrder = 0; #ifdef SQLITE_DEBUG int nRealCol; if( pTab->tabFlags & TF_WithoutRowid ){ @@ -89843,10 +92350,11 @@ SQLITE_PRIVATE void sqlite3VdbePreUpdateHook( preupdate.pCsr = pCsr; preupdate.op = op; preupdate.iNewReg = iReg; - preupdate.keyinfo.db = db; - preupdate.keyinfo.enc = ENC(db); - preupdate.keyinfo.nKeyField = pTab->nCol; - preupdate.keyinfo.aSortFlags = (u8*)&fakeSortOrder; + preupdate.pKeyinfo = (KeyInfo*)&preupdate.uKey; + preupdate.pKeyinfo->db = db; + preupdate.pKeyinfo->enc = ENC(db); + preupdate.pKeyinfo->nKeyField = pTab->nCol; + preupdate.pKeyinfo->aSortFlags = 0; /* Indicate .aColl, .nAllField uninit */ preupdate.iKey1 = iKey1; preupdate.iKey2 = iKey2; preupdate.pTab = pTab; @@ -89856,8 +92364,9 @@ SQLITE_PRIVATE void sqlite3VdbePreUpdateHook( db->xPreUpdateCallback(db->pPreUpdateArg, db, op, zDb, zTbl, iKey1, iKey2); db->pPreUpdate = 0; sqlite3DbFree(db, preupdate.aRecord); - vdbeFreeUnpacked(db, preupdate.keyinfo.nKeyField+1, preupdate.pUnpacked); - vdbeFreeUnpacked(db, preupdate.keyinfo.nKeyField+1, preupdate.pNewUnpacked); + vdbeFreeUnpacked(db, preupdate.pKeyinfo->nKeyField+1,preupdate.pUnpacked); + vdbeFreeUnpacked(db, preupdate.pKeyinfo->nKeyField+1,preupdate.pNewUnpacked); + sqlite3VdbeMemRelease(&preupdate.oldipk); if( preupdate.aNew ){ int i; for(i=0; inField; i++){ @@ -89865,9 +92374,27 @@ SQLITE_PRIVATE void sqlite3VdbePreUpdateHook( } sqlite3DbNNFreeNN(db, preupdate.aNew); } + if( preupdate.apDflt ){ + int i; + for(i=0; inCol; i++){ + sqlite3ValueFree(preupdate.apDflt[i]); + } + sqlite3DbFree(db, preupdate.apDflt); + } } #endif /* SQLITE_ENABLE_PREUPDATE_HOOK */ +#ifdef SQLITE_ENABLE_PERCENTILE +/* +** Return the name of an SQL function associated with the sqlite3_context. +*/ +SQLITE_PRIVATE const char *sqlite3VdbeFuncName(const sqlite3_context *pCtx){ + assert( pCtx!=0 ); + assert( pCtx->pFunc!=0 ); + return pCtx->pFunc->zName; +} +#endif /* SQLITE_ENABLE_PERCENTILE */ + /************** End of vdbeaux.c *********************************************/ /************** Begin file vdbeapi.c *****************************************/ /* @@ -89935,7 +92462,6 @@ static SQLITE_NOINLINE void invokeProfileCallback(sqlite3 *db, Vdbe *p){ sqlite3_int64 iNow; sqlite3_int64 iElapse; assert( p->startTime>0 ); - assert( (db->mTrace & (SQLITE_TRACE_PROFILE|SQLITE_TRACE_XPROFILE))!=0 ); assert( db->init.busy==0 ); assert( p->zSql!=0 ); sqlite3OsCurrentTimeInt64(db->pVfs, &iNow); @@ -90655,7 +93181,7 @@ static int sqlite3Step(Vdbe *p){ } assert( db->nVdbeWrite>0 || db->autoCommit==0 - || (db->nDeferredCons==0 && db->nDeferredImmCons==0) + || ((db->nDeferredCons + db->nDeferredImmCons)==0) ); #ifndef SQLITE_OMIT_TRACE @@ -91166,6 +93692,7 @@ static const Mem *columnNullValue(void){ #ifdef SQLITE_DEBUG /* .pScopyFrom = */ (Mem*)0, /* .mScopyFlags= */ 0, + /* .bScopy = */ 0, #endif }; return &nullMem; @@ -91207,7 +93734,7 @@ static Mem *columnMem(sqlite3_stmt *pStmt, int i){ ** sqlite3_column_int64() ** sqlite3_column_text() ** sqlite3_column_text16() -** sqlite3_column_real() +** sqlite3_column_double() ** sqlite3_column_bytes() ** sqlite3_column_bytes16() ** sqlite3_column_blob() @@ -91493,6 +94020,17 @@ SQLITE_API const void *sqlite3_column_origin_name16(sqlite3_stmt *pStmt, int N){ ** ** The error code stored in database p->db is overwritten with the return ** value in any case. +** +** (tag-20240917-01) If vdbeUnbind(p,(u32)(i-1)) returns SQLITE_OK, +** that means all of the the following will be true: +** +** p!=0 +** p->pVar!=0 +** i>0 +** i<=p->nVar +** +** An assert() is normally added after vdbeUnbind() to help static analyzers +** realize this. */ static int vdbeUnbind(Vdbe *p, unsigned int i){ Mem *pVar; @@ -91550,11 +94088,16 @@ static int bindText( rc = vdbeUnbind(p, (u32)(i-1)); if( rc==SQLITE_OK ){ + assert( p!=0 && p->aVar!=0 && i>0 && i<=p->nVar ); /* tag-20240917-01 */ if( zData!=0 ){ pVar = &p->aVar[i-1]; rc = sqlite3VdbeMemSetStr(pVar, zData, nData, encoding, xDel); - if( rc==SQLITE_OK && encoding!=0 ){ - rc = sqlite3VdbeChangeEncoding(pVar, ENC(p->db)); + if( rc==SQLITE_OK ){ + if( encoding==0 ){ + pVar->enc = ENC(p->db); + }else{ + rc = sqlite3VdbeChangeEncoding(pVar, ENC(p->db)); + } } if( rc ){ sqlite3Error(p->db, rc); @@ -91599,6 +94142,7 @@ SQLITE_API int sqlite3_bind_double(sqlite3_stmt *pStmt, int i, double rValue){ Vdbe *p = (Vdbe *)pStmt; rc = vdbeUnbind(p, (u32)(i-1)); if( rc==SQLITE_OK ){ + assert( p!=0 && p->aVar!=0 && i>0 && i<=p->nVar ); /* tag-20240917-01 */ sqlite3VdbeMemSetDouble(&p->aVar[i-1], rValue); sqlite3_mutex_leave(p->db->mutex); } @@ -91612,6 +94156,7 @@ SQLITE_API int sqlite3_bind_int64(sqlite3_stmt *pStmt, int i, sqlite_int64 iValu Vdbe *p = (Vdbe *)pStmt; rc = vdbeUnbind(p, (u32)(i-1)); if( rc==SQLITE_OK ){ + assert( p!=0 && p->aVar!=0 && i>0 && i<=p->nVar ); /* tag-20240917-01 */ sqlite3VdbeMemSetInt64(&p->aVar[i-1], iValue); sqlite3_mutex_leave(p->db->mutex); } @@ -91622,6 +94167,7 @@ SQLITE_API int sqlite3_bind_null(sqlite3_stmt *pStmt, int i){ Vdbe *p = (Vdbe*)pStmt; rc = vdbeUnbind(p, (u32)(i-1)); if( rc==SQLITE_OK ){ + assert( p!=0 && p->aVar!=0 && i>0 && i<=p->nVar ); /* tag-20240917-01 */ sqlite3_mutex_leave(p->db->mutex); } return rc; @@ -91637,6 +94183,7 @@ SQLITE_API int sqlite3_bind_pointer( Vdbe *p = (Vdbe*)pStmt; rc = vdbeUnbind(p, (u32)(i-1)); if( rc==SQLITE_OK ){ + assert( p!=0 && p->aVar!=0 && i>0 && i<=p->nVar ); /* tag-20240917-01 */ sqlite3VdbeMemSetPointer(&p->aVar[i-1], pPtr, zPTtype, xDestructor); sqlite3_mutex_leave(p->db->mutex); }else if( xDestructor ){ @@ -91664,7 +94211,7 @@ SQLITE_API int sqlite3_bind_text64( assert( xDel!=SQLITE_DYNAMIC ); if( enc!=SQLITE_UTF8 ){ if( enc==SQLITE_UTF16 ) enc = SQLITE_UTF16NATIVE; - nData &= ~(u16)1; + nData &= ~(u64)1; } return bindText(pStmt, i, zData, nData, xDel, enc); } @@ -91718,6 +94265,7 @@ SQLITE_API int sqlite3_bind_zeroblob(sqlite3_stmt *pStmt, int i, int n){ Vdbe *p = (Vdbe *)pStmt; rc = vdbeUnbind(p, (u32)(i-1)); if( rc==SQLITE_OK ){ + assert( p!=0 && p->aVar!=0 && i>0 && i<=p->nVar ); /* tag-20240917-01 */ #ifndef SQLITE_OMIT_INCRBLOB sqlite3VdbeMemSetZeroBlob(&p->aVar[i-1], n); #else @@ -92018,7 +94566,7 @@ static UnpackedRecord *vdbeUnpackRecord( pRet = sqlite3VdbeAllocUnpackedRecord(pKeyInfo); if( pRet ){ memset(pRet->aMem, 0, sizeof(Mem)*(pKeyInfo->nKeyField+1)); - sqlite3VdbeRecordUnpack(pKeyInfo, nKey, pKey, pRet); + sqlite3VdbeRecordUnpack(nKey, pKey, pRet); } return pRet; } @@ -92031,6 +94579,7 @@ SQLITE_API int sqlite3_preupdate_old(sqlite3 *db, int iIdx, sqlite3_value **ppVa PreUpdate *p; Mem *pMem; int rc = SQLITE_OK; + int iStore = 0; #ifdef SQLITE_ENABLE_API_ARMOR if( db==0 || ppValue==0 ){ @@ -92045,44 +94594,78 @@ SQLITE_API int sqlite3_preupdate_old(sqlite3 *db, int iIdx, sqlite3_value **ppVa goto preupdate_old_out; } if( p->pPk ){ - iIdx = sqlite3TableColumnToIndex(p->pPk, iIdx); + iStore = sqlite3TableColumnToIndex(p->pPk, iIdx); + }else if( iIdx >= p->pTab->nCol ){ + rc = SQLITE_MISUSE_BKPT; + goto preupdate_old_out; + }else{ + iStore = sqlite3TableColumnToStorage(p->pTab, iIdx); } - if( iIdx>=p->pCsr->nField || iIdx<0 ){ + if( iStore>=p->pCsr->nField || iStore<0 ){ rc = SQLITE_RANGE; goto preupdate_old_out; } - /* If the old.* record has not yet been loaded into memory, do so now. */ - if( p->pUnpacked==0 ){ - u32 nRec; - u8 *aRec; + if( iIdx==p->pTab->iPKey ){ + *ppValue = pMem = &p->oldipk; + sqlite3VdbeMemSetInt64(pMem, p->iKey1); + }else{ + + /* If the old.* record has not yet been loaded into memory, do so now. */ + if( p->pUnpacked==0 ){ + u32 nRec; + u8 *aRec; - assert( p->pCsr->eCurType==CURTYPE_BTREE ); - nRec = sqlite3BtreePayloadSize(p->pCsr->uc.pCursor); - aRec = sqlite3DbMallocRaw(db, nRec); - if( !aRec ) goto preupdate_old_out; - rc = sqlite3BtreePayload(p->pCsr->uc.pCursor, 0, nRec, aRec); - if( rc==SQLITE_OK ){ - p->pUnpacked = vdbeUnpackRecord(&p->keyinfo, nRec, aRec); - if( !p->pUnpacked ) rc = SQLITE_NOMEM; - } - if( rc!=SQLITE_OK ){ - sqlite3DbFree(db, aRec); - goto preupdate_old_out; + assert( p->pCsr->eCurType==CURTYPE_BTREE ); + nRec = sqlite3BtreePayloadSize(p->pCsr->uc.pCursor); + aRec = sqlite3DbMallocRaw(db, nRec); + if( !aRec ) goto preupdate_old_out; + rc = sqlite3BtreePayload(p->pCsr->uc.pCursor, 0, nRec, aRec); + if( rc==SQLITE_OK ){ + p->pUnpacked = vdbeUnpackRecord(p->pKeyinfo, nRec, aRec); + if( !p->pUnpacked ) rc = SQLITE_NOMEM; + } + if( rc!=SQLITE_OK ){ + sqlite3DbFree(db, aRec); + goto preupdate_old_out; + } + p->aRecord = aRec; } - p->aRecord = aRec; - } - pMem = *ppValue = &p->pUnpacked->aMem[iIdx]; - if( iIdx==p->pTab->iPKey ){ - sqlite3VdbeMemSetInt64(pMem, p->iKey1); - }else if( iIdx>=p->pUnpacked->nField ){ - *ppValue = (sqlite3_value *)columnNullValue(); - }else if( p->pTab->aCol[iIdx].affinity==SQLITE_AFF_REAL ){ - if( pMem->flags & (MEM_Int|MEM_IntReal) ){ - testcase( pMem->flags & MEM_Int ); - testcase( pMem->flags & MEM_IntReal ); - sqlite3VdbeMemRealify(pMem); + pMem = *ppValue = &p->pUnpacked->aMem[iStore]; + if( iStore>=p->pUnpacked->nField ){ + /* This occurs when the table has been extended using ALTER TABLE + ** ADD COLUMN. The value to return is the default value of the column. */ + Column *pCol = &p->pTab->aCol[iIdx]; + if( pCol->iDflt>0 ){ + if( p->apDflt==0 ){ + int nByte; + assert( sizeof(sqlite3_value*)*UMXV(p->pTab->nCol) < 0x7fffffff ); + nByte = sizeof(sqlite3_value*)*p->pTab->nCol; + p->apDflt = (sqlite3_value**)sqlite3DbMallocZero(db, nByte); + if( p->apDflt==0 ) goto preupdate_old_out; + } + if( p->apDflt[iIdx]==0 ){ + sqlite3_value *pVal = 0; + Expr *pDflt; + assert( p->pTab!=0 && IsOrdinaryTable(p->pTab) ); + pDflt = p->pTab->u.tab.pDfltList->a[pCol->iDflt-1].pExpr; + rc = sqlite3ValueFromExpr(db, pDflt, ENC(db), pCol->affinity, &pVal); + if( rc==SQLITE_OK && pVal==0 ){ + rc = SQLITE_CORRUPT_BKPT; + } + p->apDflt[iIdx] = pVal; + } + *ppValue = p->apDflt[iIdx]; + }else{ + *ppValue = (sqlite3_value *)columnNullValue(); + } + }else if( p->pTab->aCol[iIdx].affinity==SQLITE_AFF_REAL ){ + if( pMem->flags & (MEM_Int|MEM_IntReal) ){ + testcase( pMem->flags & MEM_Int ); + testcase( pMem->flags & MEM_IntReal ); + sqlite3VdbeMemRealify(pMem); + } } } @@ -92104,7 +94687,7 @@ SQLITE_API int sqlite3_preupdate_count(sqlite3 *db){ #else p = db->pPreUpdate; #endif - return (p ? p->keyinfo.nKeyField : 0); + return (p ? p->pKeyinfo->nKeyField : 0); } #endif /* SQLITE_ENABLE_PREUPDATE_HOOK */ @@ -92156,6 +94739,7 @@ SQLITE_API int sqlite3_preupdate_new(sqlite3 *db, int iIdx, sqlite3_value **ppVa PreUpdate *p; int rc = SQLITE_OK; Mem *pMem; + int iStore = 0; #ifdef SQLITE_ENABLE_API_ARMOR if( db==0 || ppValue==0 ){ @@ -92168,9 +94752,14 @@ SQLITE_API int sqlite3_preupdate_new(sqlite3 *db, int iIdx, sqlite3_value **ppVa goto preupdate_new_out; } if( p->pPk && p->op!=SQLITE_UPDATE ){ - iIdx = sqlite3TableColumnToIndex(p->pPk, iIdx); + iStore = sqlite3TableColumnToIndex(p->pPk, iIdx); + }else if( iIdx >= p->pTab->nCol ){ + return SQLITE_MISUSE_BKPT; + }else{ + iStore = sqlite3TableColumnToStorage(p->pTab, iIdx); } - if( iIdx>=p->pCsr->nField || iIdx<0 ){ + + if( iStore>=p->pCsr->nField || iStore<0 ){ rc = SQLITE_RANGE; goto preupdate_new_out; } @@ -92183,40 +94772,41 @@ SQLITE_API int sqlite3_preupdate_new(sqlite3 *db, int iIdx, sqlite3_value **ppVa Mem *pData = &p->v->aMem[p->iNewReg]; rc = ExpandBlob(pData); if( rc!=SQLITE_OK ) goto preupdate_new_out; - pUnpack = vdbeUnpackRecord(&p->keyinfo, pData->n, pData->z); + pUnpack = vdbeUnpackRecord(p->pKeyinfo, pData->n, pData->z); if( !pUnpack ){ rc = SQLITE_NOMEM; goto preupdate_new_out; } p->pNewUnpacked = pUnpack; } - pMem = &pUnpack->aMem[iIdx]; + pMem = &pUnpack->aMem[iStore]; if( iIdx==p->pTab->iPKey ){ sqlite3VdbeMemSetInt64(pMem, p->iKey2); - }else if( iIdx>=pUnpack->nField ){ + }else if( iStore>=pUnpack->nField ){ pMem = (sqlite3_value *)columnNullValue(); } }else{ - /* For an UPDATE, memory cell (p->iNewReg+1+iIdx) contains the required + /* For an UPDATE, memory cell (p->iNewReg+1+iStore) contains the required ** value. Make a copy of the cell contents and return a pointer to it. ** It is not safe to return a pointer to the memory cell itself as the ** caller may modify the value text encoding. */ assert( p->op==SQLITE_UPDATE ); if( !p->aNew ){ - p->aNew = (Mem *)sqlite3DbMallocZero(db, sizeof(Mem) * p->pCsr->nField); + assert( sizeof(Mem)*UMXV(p->pCsr->nField) < 0x7fffffff ); + p->aNew = (Mem *)sqlite3DbMallocZero(db, sizeof(Mem)*p->pCsr->nField); if( !p->aNew ){ rc = SQLITE_NOMEM; goto preupdate_new_out; } } - assert( iIdx>=0 && iIdxpCsr->nField ); - pMem = &p->aNew[iIdx]; + assert( iStore>=0 && iStorepCsr->nField ); + pMem = &p->aNew[iStore]; if( pMem->flags==0 ){ if( iIdx==p->pTab->iPKey ){ sqlite3VdbeMemSetInt64(pMem, p->iKey2); }else{ - rc = sqlite3VdbeMemCopy(pMem, &p->v->aMem[p->iNewReg+1+iIdx]); + rc = sqlite3VdbeMemCopy(pMem, &p->v->aMem[p->iNewReg+1+iStore]); if( rc!=SQLITE_OK ) goto preupdate_new_out; } } @@ -92276,7 +94866,6 @@ SQLITE_API int sqlite3_stmt_scanstatus_v2( } if( flags & SQLITE_SCANSTAT_COMPLEX ){ idx = iScan; - pScan = &p->aScan[idx]; }else{ /* If the COMPLEX flag is clear, then this function must ignore any ** ScanStatus structures with ScanStatus.addrLoop set to 0. */ @@ -92289,6 +94878,8 @@ SQLITE_API int sqlite3_stmt_scanstatus_v2( } } if( idx>=p->nScan ) return 1; + assert( pScan==0 || pScan==&p->aScan[idx] ); + pScan = &p->aScan[idx]; switch( iScanStatusOp ){ case SQLITE_SCANSTAT_NLOOP: { @@ -92439,10 +95030,10 @@ SQLITE_API void sqlite3_stmt_scanstatus_reset(sqlite3_stmt *pStmt){ ** a host parameter. If the text contains no host parameters, return ** the total number of bytes in the text. */ -static int findNextHostParameter(const char *zSql, int *pnToken){ +static i64 findNextHostParameter(const char *zSql, i64 *pnToken){ int tokenType; - int nTotal = 0; - int n; + i64 nTotal = 0; + i64 n; *pnToken = 0; while( zSql[0] ){ @@ -92489,8 +95080,8 @@ SQLITE_PRIVATE char *sqlite3VdbeExpandSql( sqlite3 *db; /* The database connection */ int idx = 0; /* Index of a host parameter */ int nextIndex = 1; /* Index of next ? host parameter */ - int n; /* Length of a token prefix */ - int nToken; /* Length of the parameter token */ + i64 n; /* Length of a token prefix */ + i64 nToken; /* Length of the parameter token */ int i; /* Loop counter */ Mem *pVar; /* Value of a host parameter */ StrAccum out; /* Accumulate the output here */ @@ -92629,6 +95220,104 @@ SQLITE_PRIVATE char *sqlite3VdbeExpandSql( /* #include "sqliteInt.h" */ /* #include "vdbeInt.h" */ +/* +** High-resolution hardware timer used for debugging and testing only. +*/ +#if defined(VDBE_PROFILE) \ + || defined(SQLITE_PERFORMANCE_TRACE) \ + || defined(SQLITE_ENABLE_STMT_SCANSTATUS) +/************** Include hwtime.h in the middle of vdbe.c *********************/ +/************** Begin file hwtime.h ******************************************/ +/* +** 2008 May 27 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This file contains inline asm code for retrieving "high-performance" +** counters for x86 and x86_64 class CPUs. +*/ +#ifndef SQLITE_HWTIME_H +#define SQLITE_HWTIME_H + +/* +** The following routine only works on Pentium-class (or newer) processors. +** It uses the RDTSC opcode to read the cycle count value out of the +** processor and returns that value. This can be used for high-res +** profiling. +*/ +#if !defined(__STRICT_ANSI__) && \ + (defined(__GNUC__) || defined(_MSC_VER)) && \ + (defined(i386) || defined(__i386__) || defined(_M_IX86)) + + #if defined(__GNUC__) + + __inline__ sqlite_uint64 sqlite3Hwtime(void){ + unsigned int lo, hi; + __asm__ __volatile__ ("rdtsc" : "=a" (lo), "=d" (hi)); + return (sqlite_uint64)hi << 32 | lo; + } + + #elif defined(_MSC_VER) + + __declspec(naked) __inline sqlite_uint64 __cdecl sqlite3Hwtime(void){ + __asm { + rdtsc + ret ; return value at EDX:EAX + } + } + + #endif + +#elif !defined(__STRICT_ANSI__) && (defined(__GNUC__) && defined(__x86_64__)) + + __inline__ sqlite_uint64 sqlite3Hwtime(void){ + unsigned int lo, hi; + __asm__ __volatile__ ("rdtsc" : "=a" (lo), "=d" (hi)); + return (sqlite_uint64)hi << 32 | lo; + } + +#elif !defined(__STRICT_ANSI__) && (defined(__GNUC__) && defined(__ppc__)) + + __inline__ sqlite_uint64 sqlite3Hwtime(void){ + unsigned long long retval; + unsigned long junk; + __asm__ __volatile__ ("\n\ + 1: mftbu %1\n\ + mftb %L0\n\ + mftbu %0\n\ + cmpw %0,%1\n\ + bne 1b" + : "=r" (retval), "=r" (junk)); + return retval; + } + +#else + + /* + ** asm() is needed for hardware timing support. Without asm(), + ** disable the sqlite3Hwtime() routine. + ** + ** sqlite3Hwtime() is only used for some obscure debugging + ** and analysis configurations, not in any deliverable, so this + ** should not be a great loss. + */ +SQLITE_PRIVATE sqlite_uint64 sqlite3Hwtime(void){ return ((sqlite_uint64)0); } + +#endif + +#endif /* !defined(SQLITE_HWTIME_H) */ + +/************** End of hwtime.h **********************************************/ +/************** Continuing where we left off in vdbe.c ***********************/ +#endif + /* ** Invoke this macro on memory cells just prior to changing the ** value of the cell. This macro verifies that shallow copies are @@ -92875,11 +95564,11 @@ static VdbeCursor *allocateCursor( */ Mem *pMem = iCur>0 ? &p->aMem[p->nMem-iCur] : p->aMem; - int nByte; + i64 nByte; VdbeCursor *pCx = 0; - nByte = - ROUND8P(sizeof(VdbeCursor)) + 2*sizeof(u32)*nField + - (eCurType==CURTYPE_BTREE?sqlite3BtreeCursorSize():0); + nByte = SZ_VDBECURSOR(nField); + assert( ROUND8(nByte)==nByte ); + if( eCurType==CURTYPE_BTREE ) nByte += sqlite3BtreeCursorSize(); assert( iCur>=0 && iCurnCursor ); if( p->apCsr[iCur] ){ /*OPTIMIZATION-IF-FALSE*/ @@ -92903,7 +95592,7 @@ static VdbeCursor *allocateCursor( pMem->szMalloc = 0; return 0; } - pMem->szMalloc = nByte; + pMem->szMalloc = (int)nByte; } p->apCsr[iCur] = pCx = (VdbeCursor*)pMem->zMalloc; @@ -92912,8 +95601,8 @@ static VdbeCursor *allocateCursor( pCx->nField = nField; pCx->aOffset = &pCx->aType[nField]; if( eCurType==CURTYPE_BTREE ){ - pCx->uc.pCursor = (BtCursor*) - &pMem->z[ROUND8P(sizeof(VdbeCursor))+2*sizeof(u32)*nField]; + assert( ROUND8(SZ_VDBECURSOR(nField))==SZ_VDBECURSOR(nField) ); + pCx->uc.pCursor = (BtCursor*)&pMem->z[SZ_VDBECURSOR(nField)]; sqlite3BtreeCursorZero(pCx->uc.pCursor); } return pCx; @@ -93206,6 +95895,7 @@ static void registerTrace(int iReg, Mem *p){ printf("R[%d] = ", iReg); memTracePrint(p); if( p->pScopyFrom ){ + assert( p->pScopyFrom->bScopy ); printf(" <== R[%d]", (int)(p->pScopyFrom - &p[-iReg])); } printf("\n"); @@ -93315,7 +96005,7 @@ static u64 filterHash(const Mem *aMem, const Op *pOp){ static SQLITE_NOINLINE int vdbeColumnFromOverflow( VdbeCursor *pC, /* The BTree cursor from which we are reading */ int iCol, /* The column to read */ - int t, /* The serial-type code for the column value */ + u32 t, /* The serial-type code for the column value */ i64 iOffset, /* Offset to the start of the content value */ u32 cacheStatus, /* Current Vdbe.cacheCtr value */ u32 colCacheCtr, /* Current value of the column cache counter */ @@ -93390,6 +96080,36 @@ static SQLITE_NOINLINE int vdbeColumnFromOverflow( return rc; } +/* +** Send a "statement aborts" message to the error log. +*/ +static SQLITE_NOINLINE void sqlite3VdbeLogAbort( + Vdbe *p, /* The statement that is running at the time of failure */ + int rc, /* Error code */ + Op *pOp, /* Opcode that filed */ + Op *aOp /* All opcodes */ +){ + const char *zSql = p->zSql; /* Original SQL text */ + const char *zPrefix = ""; /* Prefix added to SQL text */ + int pc; /* Opcode address */ + char zXtra[100]; /* Buffer space to store zPrefix */ + + if( p->pFrame ){ + assert( aOp[0].opcode==OP_Init ); + if( aOp[0].p4.z!=0 ){ + assert( aOp[0].p4.z[0]=='-' + && aOp[0].p4.z[1]=='-' + && aOp[0].p4.z[2]==' ' ); + sqlite3_snprintf(sizeof(zXtra), zXtra,"/* %s */ ",aOp[0].p4.z+3); + zPrefix = zXtra; + }else{ + zPrefix = "/* unknown trigger */ "; + } + } + pc = (int)(pOp - aOp); + sqlite3_log(rc, "statement aborts at %d: %s; [%s%s]", + pc, p->zErrMsg, zPrefix, zSql); +} /* ** Return the symbolic name for the data type of a pMem @@ -93737,7 +96457,7 @@ case OP_Return: { /* in1 */ ** ** See also: EndCoroutine */ -case OP_InitCoroutine: { /* jump */ +case OP_InitCoroutine: { /* jump0 */ assert( pOp->p1>0 && pOp->p1<=(p->nMem+1 - p->nCursor) ); assert( pOp->p2>=0 && pOp->p2nOp ); assert( pOp->p3>=0 && pOp->p3nOp ); @@ -93760,7 +96480,9 @@ case OP_InitCoroutine: { /* jump */ ** ** The instruction at the address in register P1 is a Yield. ** Jump to the P2 parameter of that Yield. -** After the jump, register P1 becomes undefined. +** After the jump, the value register P1 is left with a value +** such that subsequent OP_Yields go back to the this same +** OP_EndCoroutine instruction. ** ** See also: InitCoroutine */ @@ -93772,8 +96494,8 @@ case OP_EndCoroutine: { /* in1 */ pCaller = &aOp[pIn1->u.i]; assert( pCaller->opcode==OP_Yield ); assert( pCaller->p2>=0 && pCaller->p2nOp ); + pIn1->u.i = (int)(pOp - p->aOp) - 1; pOp = &aOp[pCaller->p2 - 1]; - pIn1->flags = MEM_Undefined; break; } @@ -93790,7 +96512,7 @@ case OP_EndCoroutine: { /* in1 */ ** ** See also: InitCoroutine */ -case OP_Yield: { /* in1, jump */ +case OP_Yield: { /* in1, jump0 */ int pcDest; pIn1 = &aMem[pOp->p1]; assert( VdbeMemDynamic(pIn1)==0 ); @@ -93820,7 +96542,7 @@ case OP_HaltIfNull: { /* in3 */ /* no break */ deliberate_fall_through } -/* Opcode: Halt P1 P2 * P4 P5 +/* Opcode: Halt P1 P2 P3 P4 P5 ** ** Exit immediately. All open cursors, etc are closed ** automatically. @@ -93833,18 +96555,22 @@ case OP_HaltIfNull: { /* in3 */ ** then back out all changes that have occurred during this execution of the ** VDBE, but do not rollback the transaction. ** -** If P4 is not null then it is an error message string. +** If P3 is not zero and P4 is NULL, then P3 is a register that holds the +** text of an error message. ** -** P5 is a value between 0 and 4, inclusive, that modifies the P4 string. +** If P3 is zero and P4 is not null then the error message string is held +** in P4. +** +** P5 is a value between 1 and 4, inclusive, then the P4 error message +** string is modified as follows: ** -** 0: (no change) ** 1: NOT NULL constraint failed: P4 ** 2: UNIQUE constraint failed: P4 ** 3: CHECK constraint failed: P4 ** 4: FOREIGN KEY constraint failed: P4 ** -** If P5 is not zero and P4 is NULL, then everything after the ":" is -** omitted. +** If P3 is zero and P5 is not zero and P4 is NULL, then everything after +** the ":" is omitted. ** ** There is an implied "Halt 0 0 0" instruction inserted at the very end of ** every program. So a jump past the last instruction of the program @@ -93857,6 +96583,9 @@ case OP_Halt: { #ifdef SQLITE_DEBUG if( pOp->p2==OE_Abort ){ sqlite3VdbeAssertAbortable(p); } #endif + assert( pOp->p4type==P4_NOTUSED + || pOp->p4type==P4_STATIC + || pOp->p4type==P4_DYNAMIC ); /* A deliberately coded "OP_Halt SQLITE_INTERNAL * * * *" opcode indicates ** something is wrong with the code generator. Raise an assertion in order @@ -93887,7 +96616,12 @@ case OP_Halt: { p->errorAction = (u8)pOp->p2; assert( pOp->p5<=4 ); if( p->rc ){ - if( pOp->p5 ){ + if( pOp->p3>0 && pOp->p4type==P4_NOTUSED ){ + const char *zErr; + assert( pOp->p3<=(p->nMem + 1 - p->nCursor) ); + zErr = sqlite3ValueText(&aMem[pOp->p3], SQLITE_UTF8); + sqlite3VdbeError(p, "%s", zErr); + }else if( pOp->p5 ){ static const char * const azType[] = { "NOT NULL", "UNIQUE", "CHECK", "FOREIGN KEY" }; testcase( pOp->p5==1 ); @@ -93901,8 +96635,7 @@ case OP_Halt: { }else{ sqlite3VdbeError(p, "%s", pOp->p4.z); } - pcx = (int)(pOp - aOp); - sqlite3_log(pOp->p1, "abort at %d in [%s]: %s", pcx, p->zSql, p->zErrMsg); + sqlite3VdbeLogAbort(p, pOp->p1, pOp, aOp); } rc = sqlite3VdbeHalt(p); assert( rc==SQLITE_BUSY || rc==SQLITE_OK || rc==SQLITE_ERROR ); @@ -94120,19 +96853,15 @@ case OP_Blob: { /* out2 */ break; } -/* Opcode: Variable P1 P2 * P4 * -** Synopsis: r[P2]=parameter(P1,P4) +/* Opcode: Variable P1 P2 * * * +** Synopsis: r[P2]=parameter(P1) ** ** Transfer the values of bound parameter P1 into register P2 -** -** If the parameter is named, then its name appears in P4. -** The P4 value is used by sqlite3_bind_parameter_name(). */ case OP_Variable: { /* out2 */ Mem *pVar; /* Value being transferred */ assert( pOp->p1>0 && pOp->p1<=p->nVar ); - assert( pOp->p4.z==0 || pOp->p4.z==sqlite3VListNumToName(p->pVList,pOp->p1) ); pVar = &p->aVar[pOp->p1 - 1]; if( sqlite3VdbeMemTooBig(pVar) ){ goto too_big; @@ -94179,6 +96908,7 @@ case OP_Move: { { int i; for(i=1; inMem; i++){ if( aMem[i].pScopyFrom==pIn1 ){ + assert( aMem[i].bScopy ); aMem[i].pScopyFrom = pOut; } } @@ -94251,6 +96981,7 @@ case OP_SCopy: { /* out2 */ #ifdef SQLITE_DEBUG pOut->pScopyFrom = pIn1; pOut->mScopyFlags = pIn1->flags; + pIn1->bScopy = 1; #endif break; } @@ -94283,7 +97014,7 @@ case OP_IntCopy: { /* out2 */ ** RETURNING clause. */ case OP_FkCheck: { - if( (rc = sqlite3VdbeCheckFk(p,0))!=SQLITE_OK ){ + if( (rc = sqlite3VdbeCheckFkImmediate(p))!=SQLITE_OK ){ goto abort_due_to_error; } break; @@ -94375,10 +97106,14 @@ case OP_Concat: { /* same as TK_CONCAT, in1, in2, out3 */ if( sqlite3VdbeMemExpandBlob(pIn2) ) goto no_mem; flags2 = pIn2->flags & ~MEM_Str; } - nByte = pIn1->n + pIn2->n; + nByte = pIn1->n; + nByte += pIn2->n; if( nByte>db->aLimit[SQLITE_LIMIT_LENGTH] ){ goto too_big; } +#if SQLITE_MAX_LENGTH>2147483645 + if( nByte>2147483645 ){ goto too_big; } +#endif if( sqlite3VdbeMemGrow(pOut, (int)nByte+2, pOut==pIn2) ){ goto no_mem; } @@ -94653,7 +97388,7 @@ case OP_AddImm: { /* in1 */ ** without data loss, then jump immediately to P2, or if P2==0 ** raise an SQLITE_MISMATCH exception. */ -case OP_MustBeInt: { /* jump, in1 */ +case OP_MustBeInt: { /* jump0, in1 */ pIn1 = &aMem[pOp->p1]; if( (pIn1->flags & MEM_Int)==0 ){ applyAffinity(pIn1, SQLITE_AFF_NUMERIC, encoding); @@ -94694,7 +97429,7 @@ case OP_RealAffinity: { /* in1 */ } #endif -#ifndef SQLITE_OMIT_CAST +#if !defined(SQLITE_OMIT_CAST) || !defined(SQLITE_OMIT_ANALYZE) /* Opcode: Cast P1 P2 * * * ** Synopsis: affinity(r[P1]) ** @@ -95062,6 +97797,7 @@ case OP_Compare: { pKeyInfo = pOp->p4.pKeyInfo; assert( n>0 ); assert( pKeyInfo!=0 ); + assert( pKeyInfo->aSortFlags!=0 ); p1 = pOp->p1; p2 = pOp->p2; #ifdef SQLITE_DEBUG @@ -95230,7 +97966,7 @@ case OP_BitNot: { /* same as TK_BITNOT, in1, out2 */ break; } -/* Opcode: Once P1 P2 * * * +/* Opcode: Once P1 P2 P3 * * ** ** Fall through to the next instruction the first time this opcode is ** encountered on each invocation of the byte-code program. Jump to P2 @@ -95246,6 +97982,12 @@ case OP_BitNot: { /* same as TK_BITNOT, in1, out2 */ ** whether or not the jump should be taken. The bitmask is necessary ** because the self-altering code trick does not work for recursive ** triggers. +** +** The P3 operand is not used directly by this opcode. However P3 is +** used by the code generator as follows: If this opcode is the start +** of a subroutine and that subroutine uses a Bloom filter, then P3 will +** be the register that holds that Bloom filter. See tag-202407032019 +** in the source code for implementation details. */ case OP_Once: { /* jump */ u32 iAddr; /* Address of this instruction */ @@ -95818,6 +98560,15 @@ case OP_Column: { /* ncycle */ ** Take the affinities from the Table object in P4. If any value ** cannot be coerced into the correct type, then raise an error. ** +** If P3==0, then omit checking of VIRTUAL columns. +** +** If P3==1, then omit checking of all generated column, both VIRTUAL +** and STORED. +** +** If P3>=2, then only check column number P3-2 in the table (which will +** be a VIRTUAL column) against the value in reg[P1]. In this case, +** P2 will be 1. +** ** This opcode is similar to OP_Affinity except that this opcode ** forces the register type to the Table column type. This is used ** to implement "strict affinity". @@ -95831,8 +98582,8 @@ case OP_Column: { /* ncycle */ ** **
            **
          • P2 should be the number of non-virtual columns in the -** table of P4. -**
          • Table P4 should be a STRICT table. +** table of P4 unless P3>1, in which case P2 will be 1. +**
          • Table P4 is a STRICT table. **
          ** ** If any precondition is false, an assertion fault occurs. @@ -95841,16 +98592,28 @@ case OP_TypeCheck: { Table *pTab; Column *aCol; int i; + int nCol; assert( pOp->p4type==P4_TABLE ); pTab = pOp->p4.pTab; assert( pTab->tabFlags & TF_Strict ); - assert( pTab->nNVCol==pOp->p2 ); + assert( pOp->p3>=0 && pOp->p3nCol+2 ); aCol = pTab->aCol; pIn1 = &aMem[pOp->p1]; - for(i=0; inCol; i++){ - if( aCol[i].colFlags & COLFLAG_GENERATED ){ - if( aCol[i].colFlags & COLFLAG_VIRTUAL ) continue; + if( pOp->p3<2 ){ + assert( pTab->nNVCol==pOp->p2 ); + i = 0; + nCol = pTab->nCol; + }else{ + i = pOp->p3-2; + nCol = i+1; + assert( inCol ); + assert( aCol[i].colFlags & COLFLAG_VIRTUAL ); + assert( pOp->p2==1 ); + } + for(; ip3<2 ){ + if( (aCol[i].colFlags & COLFLAG_VIRTUAL)!=0 ) continue; if( pOp->p3 ){ pIn1++; continue; } } assert( pIn1 < &aMem[pOp->p1+pOp->p2] ); @@ -96172,7 +98935,7 @@ case OP_MakeRecord: { len = (u32)pRec->n; serial_type = (len*2) + 12 + ((pRec->flags & MEM_Str)!=0); if( pRec->flags & MEM_Zero ){ - serial_type += pRec->u.nZero*2; + serial_type += (u32)pRec->u.nZero*2; if( nData ){ if( sqlite3VdbeMemExpandBlob(pRec) ) goto no_mem; len += pRec->u.nZero; @@ -96266,11 +99029,16 @@ case OP_MakeRecord: { switch( len ){ default: zPayload[7] = (u8)(v&0xff); v >>= 8; zPayload[6] = (u8)(v&0xff); v >>= 8; + /* no break */ deliberate_fall_through case 6: zPayload[5] = (u8)(v&0xff); v >>= 8; zPayload[4] = (u8)(v&0xff); v >>= 8; + /* no break */ deliberate_fall_through case 4: zPayload[3] = (u8)(v&0xff); v >>= 8; + /* no break */ deliberate_fall_through case 3: zPayload[2] = (u8)(v&0xff); v >>= 8; + /* no break */ deliberate_fall_through case 2: zPayload[1] = (u8)(v&0xff); v >>= 8; + /* no break */ deliberate_fall_through case 1: zPayload[0] = (u8)(v&0xff); } zPayload += len; @@ -96286,6 +99054,7 @@ case OP_MakeRecord: { zHdr += sqlite3PutVarint(zHdr, serial_type); if( pRec->n ){ assert( pRec->z!=0 ); + assert( pRec->z!=(const char*)sqlite3CtypeMap ); memcpy(zPayload, pRec->z, pRec->n); zPayload += pRec->n; } @@ -96433,7 +99202,7 @@ case OP_Savepoint: { */ int isTransaction = pSavepoint->pNext==0 && db->isTransactionSavepoint; if( isTransaction && p1==SAVEPOINT_RELEASE ){ - if( (rc = sqlite3VdbeCheckFk(p, 1))!=SQLITE_OK ){ + if( (rc = sqlite3VdbeCheckFkDeferred(p))!=SQLITE_OK ){ goto vdbe_return; } db->autoCommit = 1; @@ -96551,7 +99320,7 @@ case OP_AutoCommit: { "SQL statements in progress"); rc = SQLITE_BUSY; goto abort_due_to_error; - }else if( (rc = sqlite3VdbeCheckFk(p, 1))!=SQLITE_OK ){ + }else if( (rc = sqlite3VdbeCheckFkDeferred(p))!=SQLITE_OK ){ goto vdbe_return; }else{ db->autoCommit = (u8)desiredAutoCommit; @@ -96929,23 +99698,23 @@ case OP_OpenWrite: if( pDb->pSchema->file_format < p->minWriteFileFormat ){ p->minWriteFileFormat = pDb->pSchema->file_format; } + if( pOp->p5 & OPFLAG_P2ISREG ){ + assert( p2>0 ); + assert( p2<=(u32)(p->nMem+1 - p->nCursor) ); + pIn2 = &aMem[p2]; + assert( memIsValid(pIn2) ); + assert( (pIn2->flags & MEM_Int)!=0 ); + sqlite3VdbeMemIntegerify(pIn2); + p2 = (int)pIn2->u.i; + /* The p2 value always comes from a prior OP_CreateBtree opcode and + ** that opcode will always set the p2 value to 2 or more or else fail. + ** If there were a failure, the prepared statement would have halted + ** before reaching this instruction. */ + assert( p2>=2 ); + } }else{ wrFlag = 0; - } - if( pOp->p5 & OPFLAG_P2ISREG ){ - assert( p2>0 ); - assert( p2<=(u32)(p->nMem+1 - p->nCursor) ); - assert( pOp->opcode==OP_OpenWrite ); - pIn2 = &aMem[p2]; - assert( memIsValid(pIn2) ); - assert( (pIn2->flags & MEM_Int)!=0 ); - sqlite3VdbeMemIntegerify(pIn2); - p2 = (int)pIn2->u.i; - /* The p2 value always comes from a prior OP_CreateBtree opcode and - ** that opcode will always set the p2 value to 2 or more or else fail. - ** If there were a failure, the prepared statement would have halted - ** before reaching this instruction. */ - assert( p2>=2 ); + assert( (pOp->p5 & OPFLAG_P2ISREG)==0 ); } if( pOp->p4type==P4_KEYINFO ){ pKeyInfo = pOp->p4.pKeyInfo; @@ -97122,8 +99891,13 @@ case OP_OpenEphemeral: { /* ncycle */ } } pCx->isOrdered = (pOp->p5!=BTREE_UNORDERED); + assert( p->apCsr[pOp->p1]==pCx ); if( rc ){ + assert( !sqlite3BtreeClosesWithCursor(pCx->ub.pBtx, pCx->uc.pCursor) ); sqlite3BtreeClose(pCx->ub.pBtx); + p->apCsr[pOp->p1] = 0; /* Not required; helps with static analysis */ + }else{ + assert( sqlite3BtreeClosesWithCursor(pCx->ub.pBtx, pCx->uc.pCursor) ); } } } @@ -97189,7 +99963,8 @@ case OP_SequenceTest: { ** is the only cursor opcode that works with a pseudo-table. ** ** P3 is the number of fields in the records that will be stored by -** the pseudo-table. +** the pseudo-table. If P2 is 0 or negative then the pseudo-cursor +** will return NULL for every column. */ case OP_OpenPseudo: { VdbeCursor *pCx; @@ -97332,10 +100107,10 @@ case OP_ColumnsUsed: { ** ** See also: Found, NotFound, SeekGt, SeekGe, SeekLt */ -case OP_SeekLT: /* jump, in3, group, ncycle */ -case OP_SeekLE: /* jump, in3, group, ncycle */ -case OP_SeekGE: /* jump, in3, group, ncycle */ -case OP_SeekGT: { /* jump, in3, group, ncycle */ +case OP_SeekLT: /* jump0, in3, group, ncycle */ +case OP_SeekLE: /* jump0, in3, group, ncycle */ +case OP_SeekGE: /* jump0, in3, group, ncycle */ +case OP_SeekGT: { /* jump0, in3, group, ncycle */ int res; /* Comparison result */ int oc; /* Opcode */ VdbeCursor *pC; /* The cursor to seek */ @@ -97900,6 +100675,7 @@ case OP_Found: { /* jump, in3, ncycle */ r.pKeyInfo = pC->pKeyInfo; r.default_rc = 0; #ifdef SQLITE_DEBUG + (void)sqlite3FaultSim(50); /* For use by --counter in TH3 */ for(ii=0; iipKeyInfo); if( pIdxKey==0 ) goto no_mem; - sqlite3VdbeRecordUnpack(pC->pKeyInfo, r.aMem->n, r.aMem->z, pIdxKey); + sqlite3VdbeRecordUnpack(r.aMem->n, r.aMem->z, pIdxKey); pIdxKey->default_rc = 0; rc = sqlite3BtreeIndexMoveto(pC->uc.pCursor, pIdxKey, &pC->seekResult); sqlite3DbFreeNN(db, pIdxKey); @@ -98002,7 +100778,7 @@ case OP_Found: { /* jump, in3, ncycle */ ** ** See also: Found, NotFound, NoConflict, SeekRowid */ -case OP_SeekRowid: { /* jump, in3, ncycle */ +case OP_SeekRowid: { /* jump0, in3, ncycle */ VdbeCursor *pC; BtCursor *pCrsr; int res; @@ -98630,7 +101406,7 @@ case OP_RowData: { /* The OP_RowData opcodes always follow OP_NotExists or ** OP_SeekRowid or OP_Rewind/Op_Next with no intervening instructions ** that might invalidate the cursor. - ** If this where not the case, on of the following assert()s + ** If this were not the case, one of the following assert()s ** would fail. Should this ever change (because of changes in the code ** generator) then the fix would be to insert a call to ** sqlite3VdbeCursorMoveto(). @@ -98761,7 +101537,7 @@ case OP_NullRow: { ** configured to use Prev, not Next. */ case OP_SeekEnd: /* ncycle */ -case OP_Last: { /* jump, ncycle */ +case OP_Last: { /* jump0, ncycle */ VdbeCursor *pC; BtCursor *pCrsr; int res; @@ -98795,28 +101571,38 @@ case OP_Last: { /* jump, ncycle */ break; } -/* Opcode: IfSmaller P1 P2 P3 * * +/* Opcode: IfSizeBetween P1 P2 P3 P4 * ** -** Estimate the number of rows in the table P1. Jump to P2 if that -** estimate is less than approximately 2**(0.1*P3). +** Let N be the approximate number of rows in the table or index +** with cursor P1 and let X be 10*log2(N) if N is positive or -1 +** if N is zero. +** +** Jump to P2 if X is in between P3 and P4, inclusive. */ -case OP_IfSmaller: { /* jump */ +case OP_IfSizeBetween: { /* jump */ VdbeCursor *pC; BtCursor *pCrsr; int res; i64 sz; assert( pOp->p1>=0 && pOp->p1nCursor ); + assert( pOp->p4type==P4_INT32 ); + assert( pOp->p3>=-1 && pOp->p3<=640*2 ); + assert( pOp->p4.i>=-1 && pOp->p4.i<=640*2 ); pC = p->apCsr[pOp->p1]; assert( pC!=0 ); pCrsr = pC->uc.pCursor; assert( pCrsr ); rc = sqlite3BtreeFirst(pCrsr, &res); if( rc ) goto abort_due_to_error; - if( res==0 ){ + if( res!=0 ){ + sz = -1; /* -Infinity encoding */ + }else{ sz = sqlite3BtreeRowCountEst(pCrsr); - if( ALWAYS(sz>=0) && sqlite3LogEst((u64)sz)p3 ) res = 1; + assert( sz>0 ); + sz = sqlite3LogEst((u64)sz); } + res = sz>=pOp->p3 && sz<=pOp->p4.i; VdbeBranchTaken(res!=0,2); if( res ) goto jump_to_p2; break; @@ -98869,7 +101655,7 @@ case OP_Sort: { /* jump ncycle */ ** from the beginning toward the end. In other words, the cursor is ** configured to use Next, not Prev. */ -case OP_Rewind: { /* jump, ncycle */ +case OP_Rewind: { /* jump0, ncycle */ VdbeCursor *pC; BtCursor *pCrsr; int res; @@ -98904,6 +101690,32 @@ case OP_Rewind: { /* jump, ncycle */ break; } +/* Opcode: IfEmpty P1 P2 * * * +** Synopsis: if( empty(P1) ) goto P2 +** +** Check to see if the b-tree table that cursor P1 references is empty +** and jump to P2 if it is. +*/ +case OP_IfEmpty: { /* jump */ + VdbeCursor *pC; + BtCursor *pCrsr; + int res; + + assert( pOp->p1>=0 && pOp->p1nCursor ); + assert( pOp->p2>=0 && pOp->p2nOp ); + + pC = p->apCsr[pOp->p1]; + assert( pC!=0 ); + assert( pC->eCurType==CURTYPE_BTREE ); + pCrsr = pC->uc.pCursor; + assert( pCrsr ); + rc = sqlite3BtreeIsEmpty(pCrsr, &res); + if( rc ) goto abort_due_to_error; + VdbeBranchTaken(res!=0,2); + if( res ) goto jump_to_p2; + break; +} + /* Opcode: Next P1 P2 P3 * P5 ** ** Advance cursor P1 so that it points to the next key/data pair in its @@ -99516,11 +102328,18 @@ case OP_CreateBtree: { /* out2 */ break; } -/* Opcode: SqlExec * * * P4 * +/* Opcode: SqlExec P1 P2 * P4 * ** ** Run the SQL statement or statements specified in the P4 string. -** Disable Auth and Trace callbacks while those statements are running if -** P1 is true. +** +** The P1 parameter is a bitmask of options: +** +** 0x0001 Disable Auth and Trace callbacks while the statements +** in P4 are running. +** +** 0x0002 Set db->nAnalysisLimit to P2 while the statements in +** P4 are running. +** */ case OP_SqlExec: { char *zErr; @@ -99528,6 +102347,7 @@ case OP_SqlExec: { sqlite3_xauth xAuth; #endif u8 mTrace; + int savedAnalysisLimit; sqlite3VdbeIncrWriteCounter(p, 0); db->nSqlExec++; @@ -99536,18 +102356,23 @@ case OP_SqlExec: { xAuth = db->xAuth; #endif mTrace = db->mTrace; - if( pOp->p1 ){ + savedAnalysisLimit = db->nAnalysisLimit; + if( pOp->p1 & 0x0001 ){ #ifndef SQLITE_OMIT_AUTHORIZATION db->xAuth = 0; #endif db->mTrace = 0; } + if( pOp->p1 & 0x0002 ){ + db->nAnalysisLimit = pOp->p2; + } rc = sqlite3_exec(db, pOp->p4.z, 0, 0, &zErr); db->nSqlExec--; #ifndef SQLITE_OMIT_AUTHORIZATION db->xAuth = xAuth; #endif db->mTrace = mTrace; + db->nAnalysisLimit = savedAnalysisLimit; if( zErr || rc ){ sqlite3VdbeError(p, "%s", zErr); sqlite3_free(zErr); @@ -99699,11 +102524,11 @@ case OP_DropTrigger: { /* Opcode: IntegrityCk P1 P2 P3 P4 P5 ** ** Do an analysis of the currently open database. Store in -** register P1 the text of an error message describing any problems. -** If no problems are found, store a NULL in register P1. +** register (P1+1) the text of an error message describing any problems. +** If no problems are found, store a NULL in register (P1+1). ** -** The register P3 contains one less than the maximum number of allowed errors. -** At most reg(P3) errors will be reported. +** The register (P1) contains one less than the maximum number of allowed +** errors. At most reg(P1) errors will be reported. ** In other words, the analysis stops as soon as reg(P1) errors are ** seen. Reg(P1) is updated with the number of errors remaining. ** @@ -99723,19 +102548,21 @@ case OP_IntegrityCk: { Mem *pnErr; /* Register keeping track of errors remaining */ assert( p->bIsReader ); + assert( pOp->p4type==P4_INTARRAY ); nRoot = pOp->p2; aRoot = pOp->p4.ai; assert( nRoot>0 ); + assert( aRoot!=0 ); assert( aRoot[0]==(Pgno)nRoot ); - assert( pOp->p3>0 && pOp->p3<=(p->nMem+1 - p->nCursor) ); - pnErr = &aMem[pOp->p3]; + assert( pOp->p1>0 && (pOp->p1+1)<=(p->nMem+1 - p->nCursor) ); + pnErr = &aMem[pOp->p1]; assert( (pnErr->flags & MEM_Int)!=0 ); assert( (pnErr->flags & (MEM_Str|MEM_Blob))==0 ); - pIn1 = &aMem[pOp->p1]; + pIn1 = &aMem[pOp->p1+1]; assert( pOp->p5nDb ); assert( DbMaskTest(p->btreeMask, pOp->p5) ); - rc = sqlite3BtreeIntegrityCheck(db, db->aDb[pOp->p5].pBt, &aRoot[1], nRoot, - (int)pnErr->u.i+1, &nErr, &z); + rc = sqlite3BtreeIntegrityCheck(db, db->aDb[pOp->p5].pBt, &aRoot[1], + &aMem[pOp->p3], nRoot, (int)pnErr->u.i+1, &nErr, &z); sqlite3VdbeMemSetNull(pIn1); if( nErr==0 ){ assert( z==0 ); @@ -99862,7 +102689,9 @@ case OP_RowSetTest: { /* jump, in1, in3 */ ** P1 contains the address of the memory cell that contains the first memory ** cell in an array of values used as arguments to the sub-program. P2 ** contains the address to jump to if the sub-program throws an IGNORE -** exception using the RAISE() function. Register P3 contains the address +** exception using the RAISE() function. P2 might be zero, if there is +** no possibility that an IGNORE exception will be raised. +** Register P3 contains the address ** of a memory cell in this (the parent) VM that is used to allocate the ** memory required by the sub-vdbe at runtime. ** @@ -99870,9 +102699,9 @@ case OP_RowSetTest: { /* jump, in1, in3 */ ** ** If P5 is non-zero, then recursive program invocation is enabled. */ -case OP_Program: { /* jump */ +case OP_Program: { /* jump0 */ int nMem; /* Number of memory registers for sub-program */ - int nByte; /* Bytes of runtime space required for sub-program */ + i64 nByte; /* Bytes of runtime space required for sub-program */ Mem *pRt; /* Register to allocate runtime space */ Mem *pMem; /* Used to iterate through memory cells */ Mem *pEnd; /* Last memory cell in new array */ @@ -99923,7 +102752,7 @@ case OP_Program: { /* jump */ nByte = ROUND8(sizeof(VdbeFrame)) + nMem * sizeof(Mem) + pProgram->nCsr * sizeof(VdbeCursor*) - + (pProgram->nOp + 7)/8; + + (7 + (i64)pProgram->nOp)/8; pFrame = sqlite3DbMallocZero(db, nByte); if( !pFrame ){ goto no_mem; @@ -99931,7 +102760,7 @@ case OP_Program: { /* jump */ sqlite3VdbeMemRelease(pRt); pRt->flags = MEM_Blob|MEM_Dyn; pRt->z = (char*)pFrame; - pRt->n = nByte; + pRt->n = (int)nByte; pRt->xDel = sqlite3VdbeFrameMemDel; pFrame->v = p; @@ -100030,12 +102859,14 @@ case OP_Param: { /* out2 */ ** statement counter is incremented (immediate foreign key constraints). */ case OP_FkCounter: { - if( db->flags & SQLITE_DeferFKs ){ - db->nDeferredImmCons += pOp->p2; - }else if( pOp->p1 ){ + if( pOp->p1 ){ db->nDeferredCons += pOp->p2; }else{ - p->nFkConstraint += pOp->p2; + if( db->flags & SQLITE_DeferFKs ){ + db->nDeferredImmCons += pOp->p2; + }else{ + p->nFkConstraint += pOp->p2; + } } break; } @@ -100235,18 +103066,29 @@ case OP_AggInverse: case OP_AggStep: { int n; sqlite3_context *pCtx; + u64 nAlloc; assert( pOp->p4type==P4_FUNCDEF ); n = pOp->p5; assert( pOp->p3>0 && pOp->p3<=(p->nMem+1 - p->nCursor) ); assert( n==0 || (pOp->p2>0 && pOp->p2+n<=(p->nMem+1 - p->nCursor)+1) ); assert( pOp->p3p2 || pOp->p3>=pOp->p2+n ); - pCtx = sqlite3DbMallocRawNN(db, n*sizeof(sqlite3_value*) + - (sizeof(pCtx[0]) + sizeof(Mem) - sizeof(sqlite3_value*))); + + /* Allocate space for (a) the context object and (n-1) extra pointers + ** to append to the sqlite3_context.argv[1] array, and (b) a memory + ** cell in which to store the accumulation. Be careful that the memory + ** cell is 8-byte aligned, even on platforms where a pointer is 32-bits. + ** + ** Note: We could avoid this by using a regular memory cell from aMem[] for + ** the accumulator, instead of allocating one here. */ + nAlloc = ROUND8P( SZ_CONTEXT(n) ); + pCtx = sqlite3DbMallocRawNN(db, nAlloc + sizeof(Mem)); if( pCtx==0 ) goto no_mem; - pCtx->pMem = 0; - pCtx->pOut = (Mem*)&(pCtx->argv[n]); + pCtx->pOut = (Mem*)((u8*)pCtx + nAlloc); + assert( EIGHT_BYTE_ALIGNMENT(pCtx->pOut) ); + sqlite3VdbeMemInit(pCtx->pOut, db, MEM_Null); + pCtx->pMem = 0; pCtx->pFunc = pOp->p4.pFunc; pCtx->iOp = (int)(pOp - aOp); pCtx->pVdbe = p; @@ -100410,6 +103252,7 @@ case OP_Checkpoint: { || pOp->p2==SQLITE_CHECKPOINT_FULL || pOp->p2==SQLITE_CHECKPOINT_RESTART || pOp->p2==SQLITE_CHECKPOINT_TRUNCATE + || pOp->p2==SQLITE_CHECKPOINT_NOOP ); rc = sqlite3Checkpoint(db, pOp->p1, pOp->p2, &aRes[1], &aRes[2]); if( rc ){ @@ -100745,7 +103588,14 @@ case OP_VOpen: { /* ncycle */ const sqlite3_module *pModule; assert( p->bIsReader ); - pCur = 0; + pCur = p->apCsr[pOp->p1]; + if( pCur!=0 + && ALWAYS( pCur->eCurType==CURTYPE_VTAB ) + && ALWAYS( pCur->uc.pVCur->pVtab==pOp->p4.pVtab->pVtab ) + ){ + /* This opcode is a no-op if the cursor is already open */ + break; + } pVCur = 0; pVtab = pOp->p4.pVtab->pVtab; if( pVtab==0 || NEVER(pVtab->pModule==0) ){ @@ -100899,6 +103749,7 @@ case OP_VFilter: { /* jump, ncycle */ /* Invoke the xFilter method */ apArg = p->apArg; + assert( nArg<=p->napArg ); for(i = 0; ivtabOnConflict; apArg = p->apArg; pX = &aMem[pOp->p3]; + assert( nArg<=p->napArg ); for(i=0; iopcode==OP_Noop || pOp->opcode==OP_Explain ); @@ -101670,8 +104537,7 @@ default: { /* This is really OP_Noop, OP_Explain */ p->rc = rc; sqlite3SystemError(db, rc); testcase( sqlite3GlobalConfig.xLog!=0 ); - sqlite3_log(rc, "statement aborts at %d: [%s] %s", - (int)(pOp - aOp), p->zSql, p->zErrMsg); + sqlite3VdbeLogAbort(p, rc, pOp, aOp); if( p->eVdbeState==VDBE_RUN_STATE ) sqlite3VdbeHalt(p); if( rc==SQLITE_IOERR_NOMEM ) sqlite3OomFault(db); if( rc==SQLITE_CORRUPT && db->autoCommit==0 ){ @@ -101880,6 +104746,7 @@ SQLITE_API int sqlite3_blob_open( char *zErr = 0; Table *pTab; Incrblob *pBlob = 0; + int iDb; Parse sParse; #ifdef SQLITE_ENABLE_API_ARMOR @@ -101914,13 +104781,21 @@ SQLITE_API int sqlite3_blob_open( pTab = 0; sqlite3ErrorMsg(&sParse, "cannot open table without rowid: %s", zTable); } + if( pTab && (pTab->tabFlags&TF_HasGenerated)!=0 ){ + pTab = 0; + sqlite3ErrorMsg(&sParse, "cannot open table with generated columns: %s", + zTable); + } #ifndef SQLITE_OMIT_VIEW if( pTab && IsView(pTab) ){ pTab = 0; sqlite3ErrorMsg(&sParse, "cannot open view: %s", zTable); } #endif - if( !pTab ){ + if( pTab==0 + || ((iDb = sqlite3SchemaToIndex(db, pTab->pSchema))==1 && + sqlite3OpenTempDatabase(&sParse)) + ){ if( sParse.zErrMsg ){ sqlite3DbFree(db, zErr); zErr = sParse.zErrMsg; @@ -101931,15 +104806,11 @@ SQLITE_API int sqlite3_blob_open( goto blob_open_out; } pBlob->pTab = pTab; - pBlob->zDb = db->aDb[sqlite3SchemaToIndex(db, pTab->pSchema)].zDbSName; + pBlob->zDb = db->aDb[iDb].zDbSName; /* Now search pTab for the exact column. */ - for(iCol=0; iColnCol; iCol++) { - if( sqlite3StrICmp(pTab->aCol[iCol].zCnName, zColumn)==0 ){ - break; - } - } - if( iCol==pTab->nCol ){ + iCol = sqlite3ColumnIndex(pTab, zColumn); + if( iCol<0 ){ sqlite3DbFree(db, zErr); zErr = sqlite3MPrintf(db, "no such column: \"%s\"", zColumn); rc = SQLITE_ERROR; @@ -102019,7 +104890,6 @@ SQLITE_API int sqlite3_blob_open( {OP_Halt, 0, 0, 0}, /* 5 */ }; Vdbe *v = (Vdbe *)pBlob->pStmt; - int iDb = sqlite3SchemaToIndex(db, pTab->pSchema); VdbeOp *aOp; sqlite3VdbeAddOp4Int(v, OP_Transaction, iDb, wrFlag, @@ -102128,7 +104998,7 @@ static int blobReadWrite( int iOffset, int (*xCall)(BtCursor*, u32, u32, void*) ){ - int rc; + int rc = SQLITE_OK; Incrblob *p = (Incrblob *)pBlob; Vdbe *v; sqlite3 *db; @@ -102168,17 +105038,32 @@ static int blobReadWrite( ** using the incremental-blob API, this works. For the sessions module ** anyhow. */ - sqlite3_int64 iKey; - iKey = sqlite3BtreeIntegerKey(p->pCsr); - assert( v->apCsr[0]!=0 ); - assert( v->apCsr[0]->eCurType==CURTYPE_BTREE ); - sqlite3VdbePreUpdateHook( - v, v->apCsr[0], SQLITE_DELETE, p->zDb, p->pTab, iKey, -1, p->iCol - ); + if( sqlite3BtreeCursorIsValidNN(p->pCsr)==0 ){ + /* If the cursor is not currently valid, try to reseek it. This + ** always either fails or finds the correct row - the cursor will + ** have been marked permanently CURSOR_INVALID if the open row has + ** been deleted. */ + int bDiff = 0; + rc = sqlite3BtreeCursorRestore(p->pCsr, &bDiff); + assert( bDiff==0 || sqlite3BtreeCursorIsValidNN(p->pCsr)==0 ); + } + if( sqlite3BtreeCursorIsValidNN(p->pCsr) ){ + sqlite3_int64 iKey; + iKey = sqlite3BtreeIntegerKey(p->pCsr); + assert( v->apCsr[0]!=0 ); + assert( v->apCsr[0]->eCurType==CURTYPE_BTREE ); + sqlite3VdbePreUpdateHook( + v, v->apCsr[0], SQLITE_DELETE, p->zDb, p->pTab, iKey, -1, p->iCol + ); + } } + if( rc==SQLITE_OK ){ + rc = xCall(p->pCsr, iOffset+p->iOffset, n, z); + } +#else + rc = xCall(p->pCsr, iOffset+p->iOffset, n, z); #endif - rc = xCall(p->pCsr, iOffset+p->iOffset, n, z); sqlite3BtreeLeaveCursor(p->pCsr); if( rc==SQLITE_ABORT ){ sqlite3VdbeFinalize(v); @@ -102567,6 +105452,7 @@ struct SortSubtask { SorterCompare xCompare; /* Compare function to use */ SorterFile file; /* Temp file for level-0 PMAs */ SorterFile file2; /* Space for other PMAs */ + u64 nSpill; /* Total bytes written by this task */ }; @@ -102597,9 +105483,12 @@ struct VdbeSorter { u8 iPrev; /* Previous thread used to flush PMA */ u8 nTask; /* Size of aTask[] array */ u8 typeMask; - SortSubtask aTask[1]; /* One or more subtasks */ + SortSubtask aTask[FLEXARRAY]; /* One or more subtasks */ }; +/* Size (in bytes) of a VdbeSorter object that works with N or fewer subtasks */ +#define SZ_VDBESORTER(N) (offsetof(VdbeSorter,aTask)+(N)*sizeof(SortSubtask)) + #define SORTER_TYPE_INTEGER 0x01 #define SORTER_TYPE_TEXT 0x02 @@ -102684,6 +105573,7 @@ struct PmaWriter { int iBufEnd; /* Last byte of buffer to write */ i64 iWriteOff; /* Offset of start of buffer in file */ sqlite3_file *pFd; /* File handle to write to */ + u64 nPmaSpill; /* Total number of bytes written */ }; /* @@ -102821,13 +105711,14 @@ static int vdbePmaReadBlob( while( nRem>0 ){ int rc; /* vdbePmaReadBlob() return code */ int nCopy; /* Number of bytes to copy */ - u8 *aNext; /* Pointer to buffer to copy data from */ + u8 *aNext = 0; /* Pointer to buffer to copy data from */ nCopy = nRem; if( nRem>p->nBuffer ) nCopy = p->nBuffer; rc = vdbePmaReadBlob(p, nCopy, &aNext); if( rc!=SQLITE_OK ) return rc; assert( aNext!=p->aAlloc ); + assert( aNext!=0 ); memcpy(&p->aAlloc[nByte - nRem], aNext, nCopy); nRem -= nCopy; } @@ -103027,7 +105918,7 @@ static int vdbeSorterCompareTail( ){ UnpackedRecord *r2 = pTask->pUnpacked; if( *pbKey2Cached==0 ){ - sqlite3VdbeRecordUnpack(pTask->pSorter->pKeyInfo, nKey2, pKey2, r2); + sqlite3VdbeRecordUnpack(nKey2, pKey2, r2); *pbKey2Cached = 1; } return sqlite3VdbeRecordCompareWithSkip(nKey1, pKey1, r2, 1); @@ -103054,7 +105945,7 @@ static int vdbeSorterCompare( ){ UnpackedRecord *r2 = pTask->pUnpacked; if( !*pbKey2Cached ){ - sqlite3VdbeRecordUnpack(pTask->pSorter->pKeyInfo, nKey2, pKey2, r2); + sqlite3VdbeRecordUnpack(nKey2, pKey2, r2); *pbKey2Cached = 1; } return sqlite3VdbeRecordCompare(nKey1, pKey1, r2); @@ -103094,6 +105985,7 @@ static int vdbeSorterCompareText( ); } }else{ + assert( pTask->pSorter->pKeyInfo->aSortFlags!=0 ); assert( !(pTask->pSorter->pKeyInfo->aSortFlags[0]&KEYINFO_ORDER_BIGNULL) ); if( pTask->pSorter->pKeyInfo->aSortFlags[0] ){ res = res * -1; @@ -103157,6 +106049,7 @@ static int vdbeSorterCompareInt( } } + assert( pTask->pSorter->pKeyInfo->aSortFlags!=0 ); if( res==0 ){ if( pTask->pSorter->pKeyInfo->nKeyField>1 ){ res = vdbeSorterCompareTail( @@ -103200,7 +106093,7 @@ SQLITE_PRIVATE int sqlite3VdbeSorterInit( VdbeSorter *pSorter; /* The new sorter */ KeyInfo *pKeyInfo; /* Copy of pCsr->pKeyInfo with db==0 */ int szKeyInfo; /* Size of pCsr->pKeyInfo in bytes */ - int sz; /* Size of pSorter in bytes */ + i64 sz; /* Size of pSorter in bytes */ int rc = SQLITE_OK; #if SQLITE_MAX_WORKER_THREADS==0 # define nWorker 0 @@ -103228,8 +106121,11 @@ SQLITE_PRIVATE int sqlite3VdbeSorterInit( assert( pCsr->pKeyInfo ); assert( !pCsr->isEphemeral ); assert( pCsr->eCurType==CURTYPE_SORTER ); - szKeyInfo = sizeof(KeyInfo) + (pCsr->pKeyInfo->nKeyField-1)*sizeof(CollSeq*); - sz = sizeof(VdbeSorter) + nWorker * sizeof(SortSubtask); + assert( sizeof(KeyInfo) + UMXV(pCsr->pKeyInfo->nKeyField)*sizeof(CollSeq*) + < 0x7fffffff ); + assert( pCsr->pKeyInfo->nKeyField<=pCsr->pKeyInfo->nAllField ); + szKeyInfo = SZ_KEYINFO(pCsr->pKeyInfo->nAllField); + sz = SZ_VDBESORTER(nWorker+1); pSorter = (VdbeSorter*)sqlite3DbMallocZero(db, sz + szKeyInfo); pCsr->uc.pSorter = pSorter; @@ -103242,7 +106138,12 @@ SQLITE_PRIVATE int sqlite3VdbeSorterInit( pKeyInfo->db = 0; if( nField && nWorker==0 ){ pKeyInfo->nKeyField = nField; + assert( nField<=pCsr->pKeyInfo->nAllField ); } + /* It is OK that pKeyInfo reuses the aSortFlags field from pCsr->pKeyInfo, + ** since the pCsr->pKeyInfo->aSortFlags[] array is invariant and lives + ** longer that pSorter. */ + assert( pKeyInfo->aSortFlags==pCsr->pKeyInfo->aSortFlags ); sqlite3BtreeEnter(pBt); pSorter->pgsz = pgsz = sqlite3BtreeGetPageSize(pBt); sqlite3BtreeLeave(pBt); @@ -103441,7 +106342,7 @@ static int vdbeSorterJoinAll(VdbeSorter *pSorter, int rcin){ */ static MergeEngine *vdbeMergeEngineNew(int nReader){ int N = 2; /* Smallest power of two >= nReader */ - int nByte; /* Total bytes of space to allocate */ + i64 nByte; /* Total bytes of space to allocate */ MergeEngine *pNew; /* Pointer to allocated object to return */ assert( nReader<=SORTER_MAX_MERGE_COUNT ); @@ -103531,6 +106432,12 @@ SQLITE_PRIVATE void sqlite3VdbeSorterClose(sqlite3 *db, VdbeCursor *pCsr){ assert( pCsr->eCurType==CURTYPE_SORTER ); pSorter = pCsr->uc.pSorter; if( pSorter ){ + /* Increment db->nSpill by the total number of bytes of data written + ** to temp files by this sort operation. */ + int ii; + for(ii=0; iinTask; ii++){ + db->nSpill += pSorter->aTask[ii].nSpill; + } sqlite3VdbeSorterReset(db, pSorter); sqlite3_free(pSorter->list.aMemory); sqlite3DbFree(db, pSorter); @@ -103693,6 +106600,10 @@ static int vdbeSorterSort(SortSubtask *pTask, SorterList *pList){ p->u.pNext = 0; for(i=0; aSlot[i]; i++){ p = vdbeSorterMerge(pTask, p, aSlot[i]); + /* ,--Each aSlot[] holds twice as much as the previous. So we cannot use + ** | up all 64 aSlots[] with only a 64-bit address space. + ** v */ + assert( iaBuffer[p->iBufStart], p->iBufEnd - p->iBufStart, p->iWriteOff + p->iBufStart ); + p->nPmaSpill += (p->iBufEnd - p->iBufStart); p->iBufStart = p->iBufEnd = 0; p->iWriteOff += p->nBuffer; } @@ -103768,17 +106680,20 @@ static void vdbePmaWriteBlob(PmaWriter *p, u8 *pData, int nData){ ** required. Otherwise, return an SQLite error code. ** ** Before returning, set *piEof to the offset immediately following the -** last byte written to the file. +** last byte written to the file. Also, increment (*pnSpill) by the total +** number of bytes written to the file. */ -static int vdbePmaWriterFinish(PmaWriter *p, i64 *piEof){ +static int vdbePmaWriterFinish(PmaWriter *p, i64 *piEof, u64 *pnSpill){ int rc; if( p->eFWErr==0 && ALWAYS(p->aBuffer) && p->iBufEnd>p->iBufStart ){ p->eFWErr = sqlite3OsWrite(p->pFd, &p->aBuffer[p->iBufStart], p->iBufEnd - p->iBufStart, p->iWriteOff + p->iBufStart ); + p->nPmaSpill += (p->iBufEnd - p->iBufStart); } *piEof = (p->iWriteOff + p->iBufEnd); + *pnSpill += p->nPmaSpill; sqlite3_free(p->aBuffer); rc = p->eFWErr; memset(p, 0, sizeof(PmaWriter)); @@ -103858,7 +106773,7 @@ static int vdbeSorterListToPMA(SortSubtask *pTask, SorterList *pList){ if( pList->aMemory==0 ) sqlite3_free(p); } pList->pList = p; - rc = vdbePmaWriterFinish(&writer, &pTask->file.iEof); + rc = vdbePmaWriterFinish(&writer, &pTask->file.iEof, &pTask->nSpill); } vdbeSorterWorkDebug(pTask, "exit"); @@ -104172,7 +107087,7 @@ static int vdbeIncrPopulate(IncrMerger *pIncr){ rc = vdbeMergeEngineStep(pIncr->pMerger, &dummy); } - rc2 = vdbePmaWriterFinish(&writer, &pOut->iEof); + rc2 = vdbePmaWriterFinish(&writer, &pOut->iEof, &pTask->nSpill); if( rc==SQLITE_OK ) rc = rc2; vdbeSorterPopulateDebug(pTask, "exit"); return rc; @@ -105018,7 +107933,7 @@ SQLITE_PRIVATE int sqlite3VdbeSorterCompare( assert( r2->nField==nKeyCol ); pKey = vdbeSorterRowkey(pSorter, &nKey); - sqlite3VdbeRecordUnpack(pKeyInfo, nKey, pKey, r2); + sqlite3VdbeRecordUnpack(nKey, pKey, r2); for(i=0; iaMem[i].flags & MEM_Null ){ *pRes = -1; @@ -105320,10 +108235,10 @@ static int bytecodevtabColumn( #ifdef SQLITE_ENABLE_STMT_SCANSTATUS case 9: /* nexec */ - sqlite3_result_int(ctx, pOp->nExec); + sqlite3_result_int64(ctx, pOp->nExec); break; case 10: /* ncycle */ - sqlite3_result_int(ctx, pOp->nCycle); + sqlite3_result_int64(ctx, pOp->nCycle); break; #else case 9: /* nexec */ @@ -106097,7 +109012,9 @@ SQLITE_PRIVATE int sqlite3WalkSelectFrom(Walker *pWalker, Select *p){ pSrc = p->pSrc; if( ALWAYS(pSrc) ){ for(i=pSrc->nSrc, pItem=pSrc->a; i>0; i--, pItem++){ - if( pItem->pSelect && sqlite3WalkSelect(pWalker, pItem->pSelect) ){ + if( pItem->fg.isSubquery + && sqlite3WalkSelect(pWalker, pItem->u4.pSubq->pSelect) + ){ return WRC_Abort; } if( pItem->fg.isTabFunc @@ -106403,7 +109320,7 @@ static void extendFJMatch( if( pNew ){ pNew->iTable = pMatch->iCursor; pNew->iColumn = iColumn; - pNew->y.pTab = pMatch->pTab; + pNew->y.pTab = pMatch->pSTab; assert( (pMatch->fg.jointype & (JT_LEFT|JT_LTORJ))!=0 ); ExprSetProperty(pNew, EP_CanBeNull); *ppList = sqlite3ExprListAppend(pParse, *ppList, pNew); @@ -106416,7 +109333,7 @@ static void extendFJMatch( static SQLITE_NOINLINE int isValidSchemaTableName( const char *zTab, /* Name as it appears in the SQL */ Table *pTab, /* The schema table we are trying to match */ - Schema *pSchema /* non-NULL if a database qualifier is present */ + const char *zDb /* non-NULL if a database qualifier is present */ ){ const char *zLegacy; assert( pTab!=0 ); @@ -106427,7 +109344,7 @@ static SQLITE_NOINLINE int isValidSchemaTableName( if( sqlite3StrICmp(zTab+7, &PREFERRED_TEMP_SCHEMA_TABLE[7])==0 ){ return 1; } - if( pSchema==0 ) return 0; + if( zDb==0 ) return 0; if( sqlite3StrICmp(zTab+7, &LEGACY_SCHEMA_TABLE[7])==0 ) return 1; if( sqlite3StrICmp(zTab+7, &PREFERRED_SCHEMA_TABLE[7])==0 ) return 1; }else{ @@ -106467,7 +109384,7 @@ static int lookupName( Parse *pParse, /* The parsing context */ const char *zDb, /* Name of the database containing table, or NULL */ const char *zTab, /* Name of table containing column, or NULL */ - const char *zCol, /* Name of the column. */ + const Expr *pRight, /* Name of the column. */ NameContext *pNC, /* The name context used to resolve the name */ Expr *pExpr /* Make this EXPR node point to the selected column */ ){ @@ -106482,8 +109399,8 @@ static int lookupName( Schema *pSchema = 0; /* Schema of the expression */ int eNewExprOp = TK_COLUMN; /* New value for pExpr->op on success */ Table *pTab = 0; /* Table holding the row */ - Column *pCol; /* A column of pTab */ ExprList *pFJMatch = 0; /* Matches for FULL JOIN .. USING */ + const char *zCol = pRight->u.zToken; assert( pNC ); /* the name context cannot be NULL. */ assert( zCol ); /* The Z in X.Y.Z cannot be NULL */ @@ -106532,11 +109449,10 @@ static int lookupName( if( pSrcList ){ for(i=0, pItem=pSrcList->a; inSrc; i++, pItem++){ - u8 hCol; - pTab = pItem->pTab; + pTab = pItem->pSTab; assert( pTab!=0 && pTab->zName!=0 ); assert( pTab->nCol>0 || pParse->nErr ); - assert( (int)pItem->fg.isNestedFrom == IsNestedFrom(pItem->pSelect) ); + assert( (int)pItem->fg.isNestedFrom == IsNestedFrom(pItem)); if( pItem->fg.isNestedFrom ){ /* In this case, pItem is a subquery that has been formed from a ** parenthesized subset of the FROM clause terms. Example: @@ -106545,8 +109461,12 @@ static int lookupName( ** This pItem -------------^ */ int hit = 0; - assert( pItem->pSelect!=0 ); - pEList = pItem->pSelect->pEList; + Select *pSel; + assert( pItem->fg.isSubquery ); + assert( pItem->u4.pSubq!=0 ); + pSel = pItem->u4.pSubq->pSelect; + assert( pSel!=0 ); + pEList = pSel->pEList; assert( pEList!=0 ); assert( pEList->nExpr==pTab->nCol ); for(j=0; jnExpr; j++){ @@ -106558,10 +109478,13 @@ static int lookupName( if( cnt>0 ){ if( pItem->fg.isUsing==0 || sqlite3IdListIndex(pItem->u3.pUsing, zCol)<0 + || pMatch==pItem ){ /* Two or more tables have the same column name which is - ** not joined by USING. This is an error. Signal as much - ** by clearing pFJMatch and letting cnt go above 1. */ + ** not joined by USING. Or, a single table has two columns + ** that match a USING term (if pMatch==pItem). These are both + ** "ambiguous column name" errors. Signal as much by clearing + ** pFJMatch and letting cnt go above 1. */ sqlite3ExprListDelete(db, pFJMatch); pFJMatch = 0; }else @@ -106609,61 +109532,85 @@ static int lookupName( } }else if( sqlite3StrICmp(zTab, pTab->zName)!=0 ){ if( pTab->tnum!=1 ) continue; - if( !isValidSchemaTableName(zTab, pTab, pSchema) ) continue; + if( !isValidSchemaTableName(zTab, pTab, zDb) ) continue; } assert( ExprUseYTab(pExpr) ); if( IN_RENAME_OBJECT && pItem->zAlias ){ sqlite3RenameTokenRemap(pParse, 0, (void*)&pExpr->y.pTab); } } - hCol = sqlite3StrIHash(zCol); - for(j=0, pCol=pTab->aCol; jnCol; j++, pCol++){ - if( pCol->hName==hCol - && sqlite3StrICmp(pCol->zCnName, zCol)==0 - ){ - if( cnt>0 ){ - if( pItem->fg.isUsing==0 - || sqlite3IdListIndex(pItem->u3.pUsing, zCol)<0 - ){ - /* Two or more tables have the same column name which is - ** not joined by USING. This is an error. Signal as much - ** by clearing pFJMatch and letting cnt go above 1. */ - sqlite3ExprListDelete(db, pFJMatch); - pFJMatch = 0; - }else - if( (pItem->fg.jointype & JT_RIGHT)==0 ){ - /* An INNER or LEFT JOIN. Use the left-most table */ - continue; - }else - if( (pItem->fg.jointype & JT_LEFT)==0 ){ - /* A RIGHT JOIN. Use the right-most table */ - cnt = 0; - sqlite3ExprListDelete(db, pFJMatch); - pFJMatch = 0; - }else{ - /* For a FULL JOIN, we must construct a coalesce() func */ - extendFJMatch(pParse, &pFJMatch, pMatch, pExpr->iColumn); - } - } - cnt++; - pMatch = pItem; - /* Substitute the rowid (column -1) for the INTEGER PRIMARY KEY */ - pExpr->iColumn = j==pTab->iPKey ? -1 : (i16)j; - if( pItem->fg.isNestedFrom ){ - sqlite3SrcItemColumnUsed(pItem, j); + j = sqlite3ColumnIndex(pTab, zCol); + if( j>=0 ){ + if( cnt>0 ){ + if( pItem->fg.isUsing==0 + || sqlite3IdListIndex(pItem->u3.pUsing, zCol)<0 + ){ + /* Two or more tables have the same column name which is + ** not joined by USING. This is an error. Signal as much + ** by clearing pFJMatch and letting cnt go above 1. */ + sqlite3ExprListDelete(db, pFJMatch); + pFJMatch = 0; + }else + if( (pItem->fg.jointype & JT_RIGHT)==0 ){ + /* An INNER or LEFT JOIN. Use the left-most table */ + continue; + }else + if( (pItem->fg.jointype & JT_LEFT)==0 ){ + /* A RIGHT JOIN. Use the right-most table */ + cnt = 0; + sqlite3ExprListDelete(db, pFJMatch); + pFJMatch = 0; + }else{ + /* For a FULL JOIN, we must construct a coalesce() func */ + extendFJMatch(pParse, &pFJMatch, pMatch, pExpr->iColumn); } - break; + } + cnt++; + pMatch = pItem; + /* Substitute the rowid (column -1) for the INTEGER PRIMARY KEY */ + pExpr->iColumn = j==pTab->iPKey ? -1 : (i16)j; + if( pItem->fg.isNestedFrom ){ + sqlite3SrcItemColumnUsed(pItem, j); } } if( 0==cnt && VisibleRowid(pTab) ){ + /* pTab is a potential ROWID match. Keep track of it and match + ** the ROWID later if that seems appropriate. (Search for "cntTab" + ** to find related code.) Only allow a ROWID match if there is + ** a single ROWID match candidate. + */ +#ifdef SQLITE_ALLOW_ROWID_IN_VIEW + /* In SQLITE_ALLOW_ROWID_IN_VIEW mode, allow a ROWID match + ** if there is a single VIEW candidate or if there is a single + ** non-VIEW candidate plus multiple VIEW candidates. In other + ** words non-VIEW candidate terms take precedence over VIEWs. + */ + if( cntTab==0 + || (cntTab==1 + && pMatch!=0 + && ALWAYS(pMatch->pSTab!=0) + && (pMatch->pSTab->tabFlags & TF_Ephemeral)!=0 + && (pTab->tabFlags & TF_Ephemeral)==0) + ){ + cntTab = 1; + pMatch = pItem; + }else{ + cntTab++; + } +#else + /* The (much more common) non-SQLITE_ALLOW_ROWID_IN_VIEW case is + ** simpler since we require exactly one candidate, which will + ** always be a non-VIEW + */ cntTab++; pMatch = pItem; +#endif } } if( pMatch ){ pExpr->iTable = pMatch->iCursor; assert( ExprUseYTab(pExpr) ); - pExpr->y.pTab = pMatch->pTab; + pExpr->y.pTab = pMatch->pSTab; if( (pMatch->fg.jointype & (JT_LEFT|JT_LTORJ))!=0 ){ ExprSetProperty(pExpr, EP_CanBeNull); } @@ -106686,7 +109633,8 @@ static int lookupName( if( pParse->bReturning ){ if( (pNC->ncFlags & NC_UBaseReg)!=0 && ALWAYS(zTab==0 - || sqlite3StrICmp(zTab,pParse->pTriggerTab->zName)==0) + || sqlite3StrICmp(zTab,pParse->pTriggerTab->zName)==0 + || isValidSchemaTableName(zTab, pParse->pTriggerTab, 0)) ){ pExpr->iTable = op!=TK_DELETE; pTab = pParse->pTriggerTab; @@ -106704,7 +109652,7 @@ static int lookupName( if( (pNC->ncFlags & NC_UUpsert)!=0 && zTab!=0 ){ Upsert *pUpsert = pNC->uNC.pUpsert; if( pUpsert && sqlite3StrICmp("excluded",zTab)==0 ){ - pTab = pUpsert->pUpsertSrc->a[0].pTab; + pTab = pUpsert->pUpsertSrc->a[0].pSTab; pExpr->iTable = EXCLUDED_TABLE_NUMBER; } } @@ -106712,23 +109660,18 @@ static int lookupName( if( pTab ){ int iCol; - u8 hCol = sqlite3StrIHash(zCol); pSchema = pTab->pSchema; cntTab++; - for(iCol=0, pCol=pTab->aCol; iColnCol; iCol++, pCol++){ - if( pCol->hName==hCol - && sqlite3StrICmp(pCol->zCnName, zCol)==0 - ){ - if( iCol==pTab->iPKey ){ - iCol = -1; - } - break; + iCol = sqlite3ColumnIndex(pTab, zCol); + if( iCol>=0 ){ + if( pTab->iPKey==iCol ) iCol = -1; + }else{ + if( sqlite3IsRowid(zCol) && VisibleRowid(pTab) ){ + iCol = -1; + }else{ + iCol = pTab->nCol; } } - if( iCol>=pTab->nCol && sqlite3IsRowid(zCol) && VisibleRowid(pTab) ){ - /* IMP: R-51414-32910 */ - iCol = -1; - } if( iColnCol ){ cnt++; pMatch = 0; @@ -106783,13 +109726,18 @@ static int lookupName( ** Perhaps the name is a reference to the ROWID */ if( cnt==0 - && cntTab==1 + && cntTab>=1 && pMatch && (pNC->ncFlags & (NC_IdxExpr|NC_GenCol))==0 && sqlite3IsRowid(zCol) - && ALWAYS(VisibleRowid(pMatch->pTab) || pMatch->fg.isNestedFrom) + && ALWAYS(VisibleRowid(pMatch->pSTab) || pMatch->fg.isNestedFrom) ){ - cnt = 1; + cnt = cntTab; +#if SQLITE_ALLOW_ROWID_IN_VIEW+0==2 + if( pMatch->pSTab!=0 && IsView(pMatch->pSTab) ){ + eNewExprOp = TK_NULL; + } +#endif if( pMatch->fg.isNestedFrom==0 ) pExpr->iColumn = -1; pExpr->affExpr = SQLITE_AFF_INTEGER; } @@ -106943,6 +109891,10 @@ static int lookupName( sqlite3ErrorMsg(pParse, "%s: %s.%s.%s", zErr, zDb, zTab, zCol); }else if( zTab ){ sqlite3ErrorMsg(pParse, "%s: %s.%s", zErr, zTab, zCol); + }else if( cnt==0 && ExprHasProperty(pRight,EP_DblQuoted) ){ + sqlite3ErrorMsg(pParse, "%s: \"%s\" - should this be a" + " string literal in single-quotes?", + zErr, zCol); }else{ sqlite3ErrorMsg(pParse, "%s: %s", zErr, zCol); } @@ -106976,8 +109928,12 @@ static int lookupName( ** If a generated column is referenced, set bits for every column ** of the table. */ - if( pExpr->iColumn>=0 && cnt==1 && pMatch!=0 ){ - pMatch->colUsed |= sqlite3ExprColUsed(pExpr); + if( pMatch ){ + if( pExpr->iColumn>=0 ){ + pMatch->colUsed |= sqlite3ExprColUsed(pExpr); + }else{ + pMatch->fg.rowidUsed = 1; + } } pExpr->op = eNewExprOp; @@ -107015,7 +109971,7 @@ SQLITE_PRIVATE Expr *sqlite3CreateColumnExpr(sqlite3 *db, SrcList *pSrc, int iSr SrcItem *pItem = &pSrc->a[iSrc]; Table *pTab; assert( ExprUseYTab(p) ); - pTab = p->y.pTab = pItem->pTab; + pTab = p->y.pTab = pItem->pSTab; p->iTable = pItem->iCursor; if( p->y.pTab->iPKey==iCol ){ p->iColumn = -1; @@ -107078,8 +110034,8 @@ static void notValidImpl( /* ** Expression p should encode a floating point value between 1.0 and 0.0. -** Return 1024 times this value. Or return -1 if p is not a floating point -** value between 1.0 and 0.0. +** Return 134,217,728 (2^27) times this value. Or return -1 if p is not +** a floating point value between 1.0 and 0.0. */ static int exprProbability(Expr *p){ double r = -1.0; @@ -107134,7 +110090,7 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ pItem = pSrcList->a; pExpr->op = TK_COLUMN; assert( ExprUseYTab(pExpr) ); - pExpr->y.pTab = pItem->pTab; + pExpr->y.pTab = pItem->pSTab; pExpr->iTable = pItem->iCursor; pExpr->iColumn--; pExpr->affExpr = SQLITE_AFF_INTEGER; @@ -107220,7 +110176,6 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ */ case TK_ID: case TK_DOT: { - const char *zColumn; const char *zTable; const char *zDb; Expr *pRight; @@ -107229,7 +110184,7 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ zDb = 0; zTable = 0; assert( !ExprHasProperty(pExpr, EP_IntValue) ); - zColumn = pExpr->u.zToken; + pRight = pExpr; }else{ Expr *pLeft = pExpr->pLeft; testcase( pNC->ncFlags & NC_IdxExpr ); @@ -107248,21 +110203,20 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ } assert( ExprUseUToken(pLeft) && ExprUseUToken(pRight) ); zTable = pLeft->u.zToken; - zColumn = pRight->u.zToken; assert( ExprUseYTab(pExpr) ); if( IN_RENAME_OBJECT ){ sqlite3RenameTokenRemap(pParse, (void*)pExpr, (void*)pRight); sqlite3RenameTokenRemap(pParse, (void*)&pExpr->y.pTab, (void*)pLeft); } } - return lookupName(pParse, zDb, zTable, zColumn, pNC, pExpr); + return lookupName(pParse, zDb, zTable, pRight, pNC, pExpr); } /* Resolve function names */ case TK_FUNCTION: { - ExprList *pList = pExpr->x.pList; /* The argument list */ - int n = pList ? pList->nExpr : 0; /* Number of arguments */ + ExprList *pList; /* The argument list */ + int n; /* Number of arguments */ int no_such_func = 0; /* True if no such function exists */ int wrong_num_args = 0; /* True if wrong number of arguments */ int is_agg = 0; /* True if is an aggregate function */ @@ -107275,6 +110229,8 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ #endif assert( !ExprHasProperty(pExpr, EP_xIsSelect|EP_IntValue) ); assert( pExpr->pLeft==0 || pExpr->pLeft->op==TK_ORDER ); + pList = pExpr->x.pList; + n = pList ? pList->nExpr : 0; zId = pExpr->u.zToken; pDef = sqlite3FindFunction(pParse->db, zId, n, enc, 0); if( pDef==0 ){ @@ -107323,6 +110279,24 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ } } #endif + + /* If the function may call sqlite3_value_subtype(), then set the + ** EP_SubtArg flag on all of its argument expressions. This prevents + ** where.c from replacing the expression with a value read from an + ** index on the same expression, which will not have the correct + ** subtype. Also set the flag if the function expression itself is + ** an EP_SubtArg expression. In this case subtypes are required as + ** the function may return a value with a subtype back to its + ** caller using sqlite3_result_value(). */ + if( (pDef->funcFlags & SQLITE_SUBTYPE) + || ExprHasProperty(pExpr, EP_SubtArg) + ){ + int ii; + for(ii=0; iia[ii].pExpr, EP_SubtArg); + } + } + if( pDef->funcFlags & (SQLITE_FUNC_CONSTANT|SQLITE_FUNC_SLOCHNG) ){ /* For the purposes of the EP_ConstFunc flag, date and time ** functions and other functions that change slowly are considered @@ -107336,13 +110310,12 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ ** sqlite_version() that might change over time cannot be used ** in an index or generated column. Curiously, they can be used ** in a CHECK constraint. SQLServer, MySQL, and PostgreSQL all - ** all this. */ + ** allow this. */ sqlite3ResolveNotValid(pParse, pNC, "non-deterministic functions", NC_IdxExpr|NC_PartIdx|NC_GenCol, 0, pExpr); }else{ assert( (NC_SelfRef & 0xff)==NC_SelfRef ); /* Must fit in 8 bits */ pExpr->op2 = pNC->ncFlags & NC_SelfRef; - if( pNC->ncFlags & NC_FromDDL ) ExprSetProperty(pExpr, EP_FromDDL); } if( (pDef->funcFlags & SQLITE_FUNC_INTERNAL)!=0 && pParse->nested==0 @@ -107358,6 +110331,7 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ if( (pDef->funcFlags & (SQLITE_FUNC_DIRECT|SQLITE_FUNC_UNSAFE))!=0 && !IN_RENAME_OBJECT ){ + if( pNC->ncFlags & NC_FromDDL ) ExprSetProperty(pExpr, EP_FromDDL); sqlite3ExprFunctionUsable(pParse, pExpr, pDef); } } @@ -107431,11 +110405,9 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ #endif } } -#ifndef SQLITE_OMIT_WINDOWFUNC - else if( ExprHasProperty(pExpr, EP_WinFunc) ){ + else if( ExprHasProperty(pExpr, EP_WinFunc) || pExpr->pLeft ){ is_agg = 1; } -#endif sqlite3WalkExprList(pWalker, pList); if( is_agg ){ if( pExpr->pLeft ){ @@ -107444,9 +110416,9 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ sqlite3WalkExprList(pWalker, pExpr->pLeft->x.pList); } #ifndef SQLITE_OMIT_WINDOWFUNC - if( pWin ){ + if( pWin && pParse->nErr==0 ){ Select *pSel = pNC->pWinSelect; - assert( pWin==0 || (ExprUseYWin(pExpr) && pWin==pExpr->y.pWin) ); + assert( ExprUseYWin(pExpr) && pWin==pExpr->y.pWin ); if( IN_RENAME_OBJECT==0 ){ sqlite3WindowUpdate(pParse, pSel ? pSel->pWinDefn : 0, pWin, pDef); if( pParse->db->mallocFailed ) break; @@ -107494,17 +110466,21 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ return WRC_Prune; } #ifndef SQLITE_OMIT_SUBQUERY + case TK_EXISTS: case TK_SELECT: - case TK_EXISTS: testcase( pExpr->op==TK_EXISTS ); #endif case TK_IN: { testcase( pExpr->op==TK_IN ); + testcase( pExpr->op==TK_EXISTS ); + testcase( pExpr->op==TK_SELECT ); if( ExprUseXSelect(pExpr) ){ int nRef = pNC->nRef; testcase( pNC->ncFlags & NC_IsCheck ); testcase( pNC->ncFlags & NC_PartIdx ); testcase( pNC->ncFlags & NC_IdxExpr ); testcase( pNC->ncFlags & NC_GenCol ); + assert( pExpr->x.pSelect ); + if( pExpr->op==TK_EXISTS ) pParse->bHasExists = 1; if( pNC->ncFlags & NC_SelfRef ){ notValidImpl(pParse, pNC, "subqueries", pExpr, pExpr); }else{ @@ -107513,6 +110489,7 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ assert( pNC->nRef>=nRef ); if( nRef!=pNC->nRef ){ ExprSetProperty(pExpr, EP_VarSelect); + pExpr->x.pSelect->selFlags |= SF_Correlated; } pNC->ncFlags |= NC_Subquery; } @@ -107651,7 +110628,7 @@ static int resolveOrderByTermToExprList( int rc; /* Return code from subprocedures */ u8 savedSuppErr; /* Saved value of db->suppressErr */ - assert( sqlite3ExprIsInteger(pE, &i)==0 ); + assert( sqlite3ExprIsInteger(pE, &i, 0)==0 ); pEList = pSelect->pEList; /* Resolve all names in the ORDER BY term expression @@ -107750,7 +110727,7 @@ static int resolveCompoundOrderBy( if( pItem->fg.done ) continue; pE = sqlite3ExprSkipCollateAndLikely(pItem->pExpr); if( NEVER(pE==0) ) continue; - if( sqlite3ExprIsInteger(pE, &iCol) ){ + if( sqlite3ExprIsInteger(pE, &iCol, 0) ){ if( iCol<=0 || iCol>pEList->nExpr ){ resolveOutOfRangeError(pParse, "ORDER", i+1, pEList->nExpr, pE); return 1; @@ -107935,7 +110912,7 @@ static int resolveOrderGroupBy( continue; } } - if( sqlite3ExprIsInteger(pE2, &iCol) ){ + if( sqlite3ExprIsInteger(pE2, &iCol, 0) ){ /* The ORDER BY term is an integer constant. Again, set the column ** number so that sqlite3ResolveOrderGroupBy() will convert the ** order-by term to a copy of the result-set expression */ @@ -108026,7 +111003,11 @@ static int resolveSelectStep(Walker *pWalker, Select *p){ ** moves the pOrderBy down to the sub-query. It will be moved back ** after the names have been resolved. */ if( p->selFlags & SF_Converted ){ - Select *pSub = p->pSrc->a[0].pSelect; + Select *pSub; + assert( p->pSrc->a[0].fg.isSubquery ); + assert( p->pSrc->a[0].u4.pSubq!=0 ); + pSub = p->pSrc->a[0].u4.pSubq->pSelect; + assert( pSub!=0 ); assert( p->pSrc->nSrc==1 && p->pOrderBy ); assert( pSub->pPrior && pSub->pOrderBy==0 ); pSub->pOrderBy = p->pOrderBy; @@ -108038,12 +111019,16 @@ static int resolveSelectStep(Walker *pWalker, Select *p){ if( pOuterNC ) pOuterNC->nNestedSelect++; for(i=0; ipSrc->nSrc; i++){ SrcItem *pItem = &p->pSrc->a[i]; - if( pItem->pSelect && (pItem->pSelect->selFlags & SF_Resolved)==0 ){ + assert( pItem->zName!=0 + || pItem->fg.isSubquery ); /* Test of tag-20240424-1*/ + if( pItem->fg.isSubquery + && (pItem->u4.pSubq->pSelect->selFlags & SF_Resolved)==0 + ){ int nRef = pOuterNC ? pOuterNC->nRef : 0; const char *zSavedContext = pParse->zAuthContext; if( pItem->zName ) pParse->zAuthContext = pItem->zName; - sqlite3ResolveSelectNames(pParse, pItem->pSelect, pOuterNC); + sqlite3ResolveSelectNames(pParse, pItem->u4.pSubq->pSelect, pOuterNC); pParse->zAuthContext = zSavedContext; if( pParse->nErr ) return WRC_Abort; assert( db->mallocFailed==0 ); @@ -108145,7 +111130,10 @@ static int resolveSelectStep(Walker *pWalker, Select *p){ ** These integers will be replaced by copies of the corresponding result ** set expressions by the call to resolveOrderGroupBy() below. */ if( p->selFlags & SF_Converted ){ - Select *pSub = p->pSrc->a[0].pSelect; + Select *pSub; + assert( p->pSrc->a[0].fg.isSubquery ); + pSub = p->pSrc->a[0].u4.pSubq->pSelect; + assert( pSub!=0 ); p->pOrderBy = pSub->pOrderBy; pSub->pOrderBy = 0; } @@ -108299,6 +111287,9 @@ SQLITE_PRIVATE int sqlite3ResolveExprNames( ** Resolve all names for all expression in an expression list. This is ** just like sqlite3ResolveExprNames() except that it works for an expression ** list rather than a single expression. +** +** The return value is SQLITE_OK (0) for success or SQLITE_ERROR (1) for a +** failure. */ SQLITE_PRIVATE int sqlite3ResolveExprListNames( NameContext *pNC, /* Namespace to resolve expressions in. */ @@ -108307,7 +111298,7 @@ SQLITE_PRIVATE int sqlite3ResolveExprListNames( int i; int savedHasAgg = 0; Walker w; - if( pList==0 ) return WRC_Continue; + if( pList==0 ) return SQLITE_OK; w.pParse = pNC->pParse; w.xExprCallback = resolveExprStep; w.xSelectCallback = resolveSelectStep; @@ -108321,7 +111312,7 @@ SQLITE_PRIVATE int sqlite3ResolveExprListNames( #if SQLITE_MAX_EXPR_DEPTH>0 w.pParse->nHeight += pExpr->nHeight; if( sqlite3ExprCheckHeight(w.pParse, w.pParse->nHeight) ){ - return WRC_Abort; + return SQLITE_ERROR; } #endif sqlite3WalkExprNN(&w, pExpr); @@ -108338,10 +111329,10 @@ SQLITE_PRIVATE int sqlite3ResolveExprListNames( (NC_HasAgg|NC_MinMaxAgg|NC_HasWin|NC_OrderAgg); pNC->ncFlags &= ~(NC_HasAgg|NC_MinMaxAgg|NC_HasWin|NC_OrderAgg); } - if( w.pParse->nErr>0 ) return WRC_Abort; + if( w.pParse->nErr>0 ) return SQLITE_ERROR; } pNC->ncFlags |= savedHasAgg; - return WRC_Continue; + return SQLITE_OK; } /* @@ -108397,20 +111388,25 @@ SQLITE_PRIVATE int sqlite3ResolveSelfReference( Expr *pExpr, /* Expression to resolve. May be NULL. */ ExprList *pList /* Expression list to resolve. May be NULL. */ ){ - SrcList sSrc; /* Fake SrcList for pParse->pNewTable */ + SrcList *pSrc; /* Fake SrcList for pParse->pNewTable */ NameContext sNC; /* Name context for pParse->pNewTable */ int rc; + union { + SrcList sSrc; + u8 srcSpace[SZ_SRCLIST_1]; /* Memory space for the fake SrcList */ + } uSrc; assert( type==0 || pTab!=0 ); assert( type==NC_IsCheck || type==NC_PartIdx || type==NC_IdxExpr || type==NC_GenCol || pTab==0 ); memset(&sNC, 0, sizeof(sNC)); - memset(&sSrc, 0, sizeof(sSrc)); + memset(&uSrc, 0, sizeof(uSrc)); + pSrc = &uSrc.sSrc; if( pTab ){ - sSrc.nSrc = 1; - sSrc.a[0].zName = pTab->zName; - sSrc.a[0].pTab = pTab; - sSrc.a[0].iCursor = -1; + pSrc->nSrc = 1; + pSrc->a[0].zName = pTab->zName; + pSrc->a[0].pSTab = pTab; + pSrc->a[0].iCursor = -1; if( pTab->pSchema!=pParse->db->aDb[1].pSchema ){ /* Cause EP_FromDDL to be set on TK_FUNCTION nodes of non-TEMP ** schema elements */ @@ -108418,7 +111414,7 @@ SQLITE_PRIVATE int sqlite3ResolveSelfReference( } } sNC.pParse = pParse; - sNC.pSrcList = &sSrc; + sNC.pSrcList = pSrc; sNC.ncFlags = type | NC_IsDDL; if( (rc = sqlite3ResolveExprNames(&sNC, pExpr))!=SQLITE_OK ) return rc; if( pList ) rc = sqlite3ResolveExprListNames(&sNC, pList); @@ -108502,7 +111498,9 @@ SQLITE_PRIVATE char sqlite3ExprAffinity(const Expr *pExpr){ pExpr->pLeft->x.pSelect->pEList->a[pExpr->iColumn].pExpr ); } - if( op==TK_VECTOR ){ + if( op==TK_VECTOR + || (op==TK_FUNCTION && pExpr->affExpr==SQLITE_AFF_DEFER) + ){ assert( ExprUseXList(pExpr) ); return sqlite3ExprAffinity(pExpr->x.pList->a[0].pExpr); } @@ -108514,7 +111512,9 @@ SQLITE_PRIVATE char sqlite3ExprAffinity(const Expr *pExpr){ op = pExpr->op; continue; } - if( op!=TK_REGISTER || (op = pExpr->op2)==TK_REGISTER ) break; + if( op!=TK_REGISTER ) break; + op = pExpr->op2; + if( NEVER( op==TK_REGISTER ) ) break; } return pExpr->affExpr; } @@ -108647,9 +111647,10 @@ SQLITE_PRIVATE Expr *sqlite3ExprSkipCollateAndLikely(Expr *pExpr){ assert( pExpr->x.pList->nExpr>0 ); assert( pExpr->op==TK_FUNCTION ); pExpr = pExpr->x.pList->a[0].pExpr; - }else{ - assert( pExpr->op==TK_COLLATE ); + }else if( pExpr->op==TK_COLLATE ){ pExpr = pExpr->pLeft; + }else{ + break; } } return pExpr; @@ -108692,7 +111693,9 @@ SQLITE_PRIVATE CollSeq *sqlite3ExprCollSeq(Parse *pParse, const Expr *pExpr){ p = p->pLeft; continue; } - if( op==TK_VECTOR ){ + if( op==TK_VECTOR + || (op==TK_FUNCTION && p->affExpr==SQLITE_AFF_DEFER) + ){ assert( ExprUseXList(p) ); p = p->x.pList->a[0].pExpr; continue; @@ -108905,7 +111908,7 @@ static int codeCompare( p5 = binaryCompareP5(pLeft, pRight, jumpIfNull); addr = sqlite3VdbeAddOp4(pParse->pVdbe, opcode, in2, dest, in1, (void*)p4, P4_COLLSEQ); - sqlite3VdbeChangeP5(pParse->pVdbe, (u8)p5); + sqlite3VdbeChangeP5(pParse->pVdbe, (u16)p5); return addr; } @@ -109343,11 +112346,12 @@ SQLITE_PRIVATE void sqlite3ExprSetErrorOffset(Expr *pExpr, int iOfst){ ** appear to be quoted. If the quotes were of the form "..." (double-quotes) ** then the EP_DblQuoted flag is set on the expression node. ** -** Special case: If op==TK_INTEGER and pToken points to a string that -** can be translated into a 32-bit integer, then the token is not -** stored in u.zToken. Instead, the integer values is written -** into u.iValue and the EP_IntValue flag is set. No extra storage +** Special case (tag-20240227-a): If op==TK_INTEGER and pToken points to +** a string that can be translated into a 32-bit integer, then the token is +** not stored in u.zToken. Instead, the integer values is written +** into u.iValue and the EP_IntValue flag is set. No extra storage ** is allocated to hold the integer text and the dequote flag is ignored. +** See also tag-20240227-b. */ SQLITE_PRIVATE Expr *sqlite3ExprAlloc( sqlite3 *db, /* Handle for sqlite3DbMallocRawNN() */ @@ -109363,7 +112367,7 @@ SQLITE_PRIVATE Expr *sqlite3ExprAlloc( if( pToken ){ if( op!=TK_INTEGER || pToken->z==0 || sqlite3GetInt32(pToken->z, &iValue)==0 ){ - nExtra = pToken->n+1; + nExtra = pToken->n+1; /* tag-20240227-a */ assert( iValue>=0 ); } } @@ -109565,7 +112569,7 @@ SQLITE_PRIVATE Expr *sqlite3ExprAnd(Parse *pParse, Expr *pLeft, Expr *pRight){ return pLeft; }else{ u32 f = pLeft->flags | pRight->flags; - if( (f&(EP_OuterON|EP_InnerON|EP_IsFalse))==EP_IsFalse + if( (f&(EP_OuterON|EP_InnerON|EP_IsFalse|EP_HasFunc))==EP_IsFalse && !IN_RENAME_OBJECT ){ sqlite3ExprDeferredDelete(pParse, pLeft); @@ -109660,6 +112664,11 @@ SQLITE_PRIVATE void sqlite3ExprAddFunctionOrderBy( sqlite3ExprListDelete(db, pOrderBy); return; } + if( pOrderBy->nExpr>db->aLimit[SQLITE_LIMIT_COLUMN] ){ + sqlite3ErrorMsg(pParse, "too many terms in ORDER BY clause"); + sqlite3ExprListDelete(db, pOrderBy); + return; + } pOB = sqlite3ExprAlloc(db, TK_ORDER, 0, 0); if( pOB==0 ){ @@ -109795,6 +112804,7 @@ SQLITE_PRIVATE void sqlite3ExprAssignVarNumber(Parse *pParse, Expr *pExpr, u32 n static SQLITE_NOINLINE void sqlite3ExprDeleteNN(sqlite3 *db, Expr *p){ assert( p!=0 ); assert( db!=0 ); +exprDeleteRestart: assert( !ExprUseUValue(p) || p->u.iValue>=0 ); assert( !ExprUseYWin(p) || !ExprUseYSub(p) ); assert( !ExprUseYWin(p) || p->y.pWin!=0 || db->mallocFailed ); @@ -109810,7 +112820,6 @@ static SQLITE_NOINLINE void sqlite3ExprDeleteNN(sqlite3 *db, Expr *p){ if( !ExprHasProperty(p, (EP_TokenOnly|EP_Leaf)) ){ /* The Expr.x union is never used at the same time as Expr.pRight */ assert( (ExprUseXList(p) && p->x.pList==0) || p->pRight==0 ); - if( p->pLeft && p->op!=TK_SELECT_COLUMN ) sqlite3ExprDeleteNN(db, p->pLeft); if( p->pRight ){ assert( !ExprHasProperty(p, EP_WinFunc) ); sqlite3ExprDeleteNN(db, p->pRight); @@ -109825,6 +112834,19 @@ static SQLITE_NOINLINE void sqlite3ExprDeleteNN(sqlite3 *db, Expr *p){ } #endif } + if( p->pLeft && p->op!=TK_SELECT_COLUMN ){ + Expr *pLeft = p->pLeft; + if( !ExprHasProperty(p, EP_Static) + && !ExprHasProperty(pLeft, EP_Static) + ){ + /* Avoid unnecessary recursion on unary operators */ + sqlite3DbNNFreeNN(db, p); + p = pLeft; + goto exprDeleteRestart; + }else{ + sqlite3ExprDeleteNN(db, pLeft); + } + } } if( !ExprHasProperty(p, EP_Static) ){ sqlite3DbNNFreeNN(db, p); @@ -109857,11 +112879,11 @@ SQLITE_PRIVATE void sqlite3ClearOnOrUsing(sqlite3 *db, OnOrUsing *p){ ** ** The pExpr might be deleted immediately on an OOM error. ** -** The deferred delete is (currently) implemented by adding the -** pExpr to the pParse->pConstExpr list with a register number of 0. +** Return 0 if the delete was successfully deferred. Return non-zero +** if the delete happened immediately because of an OOM. */ -SQLITE_PRIVATE void sqlite3ExprDeferredDelete(Parse *pParse, Expr *pExpr){ - sqlite3ParserAddCleanup(pParse, sqlite3ExprDeleteGeneric, pExpr); +SQLITE_PRIVATE int sqlite3ExprDeferredDelete(Parse *pParse, Expr *pExpr){ + return 0==sqlite3ParserAddCleanup(pParse, sqlite3ExprDeleteGeneric, pExpr); } /* Invoke sqlite3RenameExprUnmap() and sqlite3ExprDelete() on the @@ -110150,7 +113172,7 @@ static Expr *exprDup( SQLITE_PRIVATE With *sqlite3WithDup(sqlite3 *db, With *p){ With *pRet = 0; if( p ){ - sqlite3_int64 nByte = sizeof(*p) + sizeof(p->a[0]) * (p->nCte-1); + sqlite3_int64 nByte = SZ_WITH(p->nCte); pRet = sqlite3DbMallocZero(db, nByte); if( pRet ){ int i; @@ -110261,7 +113283,6 @@ SQLITE_PRIVATE ExprList *sqlite3ExprListDup(sqlite3 *db, const ExprList *p, int } pItem->zEName = sqlite3DbStrDup(db, pOldItem->zEName); pItem->fg = pOldItem->fg; - pItem->fg.done = 0; pItem->u = pOldItem->u; } return pNew; @@ -110278,41 +113299,55 @@ SQLITE_PRIVATE ExprList *sqlite3ExprListDup(sqlite3 *db, const ExprList *p, int SQLITE_PRIVATE SrcList *sqlite3SrcListDup(sqlite3 *db, const SrcList *p, int flags){ SrcList *pNew; int i; - int nByte; assert( db!=0 ); if( p==0 ) return 0; - nByte = sizeof(*p) + (p->nSrc>0 ? sizeof(p->a[0]) * (p->nSrc-1) : 0); - pNew = sqlite3DbMallocRawNN(db, nByte ); + pNew = sqlite3DbMallocRawNN(db, SZ_SRCLIST(p->nSrc) ); if( pNew==0 ) return 0; pNew->nSrc = pNew->nAlloc = p->nSrc; for(i=0; inSrc; i++){ SrcItem *pNewItem = &pNew->a[i]; const SrcItem *pOldItem = &p->a[i]; Table *pTab; - pNewItem->pSchema = pOldItem->pSchema; - pNewItem->zDatabase = sqlite3DbStrDup(db, pOldItem->zDatabase); + pNewItem->fg = pOldItem->fg; + if( pOldItem->fg.isSubquery ){ + Subquery *pNewSubq = sqlite3DbMallocRaw(db, sizeof(Subquery)); + if( pNewSubq==0 ){ + assert( db->mallocFailed ); + pNewItem->fg.isSubquery = 0; + }else{ + memcpy(pNewSubq, pOldItem->u4.pSubq, sizeof(*pNewSubq)); + pNewSubq->pSelect = sqlite3SelectDup(db, pNewSubq->pSelect, flags); + if( pNewSubq->pSelect==0 ){ + sqlite3DbFree(db, pNewSubq); + pNewSubq = 0; + pNewItem->fg.isSubquery = 0; + } + } + pNewItem->u4.pSubq = pNewSubq; + }else if( pOldItem->fg.fixedSchema ){ + pNewItem->u4.pSchema = pOldItem->u4.pSchema; + }else{ + pNewItem->u4.zDatabase = sqlite3DbStrDup(db, pOldItem->u4.zDatabase); + } pNewItem->zName = sqlite3DbStrDup(db, pOldItem->zName); pNewItem->zAlias = sqlite3DbStrDup(db, pOldItem->zAlias); - pNewItem->fg = pOldItem->fg; pNewItem->iCursor = pOldItem->iCursor; - pNewItem->addrFillSub = pOldItem->addrFillSub; - pNewItem->regReturn = pOldItem->regReturn; if( pNewItem->fg.isIndexedBy ){ pNewItem->u1.zIndexedBy = sqlite3DbStrDup(db, pOldItem->u1.zIndexedBy); + }else if( pNewItem->fg.isTabFunc ){ + pNewItem->u1.pFuncArg = + sqlite3ExprListDup(db, pOldItem->u1.pFuncArg, flags); + }else{ + pNewItem->u1.nRow = pOldItem->u1.nRow; } pNewItem->u2 = pOldItem->u2; if( pNewItem->fg.isCte ){ pNewItem->u2.pCteUse->nUse++; } - if( pNewItem->fg.isTabFunc ){ - pNewItem->u1.pFuncArg = - sqlite3ExprListDup(db, pOldItem->u1.pFuncArg, flags); - } - pTab = pNewItem->pTab = pOldItem->pTab; + pTab = pNewItem->pSTab = pOldItem->pSTab; if( pTab ){ pTab->nTabRef++; } - pNewItem->pSelect = sqlite3SelectDup(db, pOldItem->pSelect, flags); if( pOldItem->fg.isUsing ){ assert( pNewItem->fg.isUsing ); pNewItem->u3.pUsing = sqlite3IdListDup(db, pOldItem->u3.pUsing); @@ -110328,16 +113363,13 @@ SQLITE_PRIVATE IdList *sqlite3IdListDup(sqlite3 *db, const IdList *p){ int i; assert( db!=0 ); if( p==0 ) return 0; - assert( p->eU4!=EU4_EXPR ); - pNew = sqlite3DbMallocRawNN(db, sizeof(*pNew)+(p->nId-1)*sizeof(p->a[0]) ); + pNew = sqlite3DbMallocRawNN(db, SZ_IDLIST(p->nId)); if( pNew==0 ) return 0; pNew->nId = p->nId; - pNew->eU4 = p->eU4; for(i=0; inId; i++){ struct IdList_item *pNewItem = &pNew->a[i]; const struct IdList_item *pOldItem = &p->a[i]; pNewItem->zName = sqlite3DbStrDup(db, pOldItem->zName); - pNewItem->u4 = pOldItem->u4; } return pNew; } @@ -110363,7 +113395,7 @@ SQLITE_PRIVATE Select *sqlite3SelectDup(sqlite3 *db, const Select *pDup, int fla pNew->pLimit = sqlite3ExprDup(db, p->pLimit, flags); pNew->iLimit = 0; pNew->iOffset = 0; - pNew->selFlags = p->selFlags & ~SF_UsesEphemeral; + pNew->selFlags = p->selFlags & ~(u32)SF_UsesEphemeral; pNew->addrOpenEphm[0] = -1; pNew->addrOpenEphm[1] = -1; pNew->nSelectRow = p->nSelectRow; @@ -110386,7 +113418,6 @@ SQLITE_PRIVATE Select *sqlite3SelectDup(sqlite3 *db, const Select *pDup, int fla pp = &pNew->pPrior; pNext = pNew; } - return pRet; } #else @@ -110416,7 +113447,7 @@ SQLITE_PRIVATE SQLITE_NOINLINE ExprList *sqlite3ExprListAppendNew( struct ExprList_item *pItem; ExprList *pList; - pList = sqlite3DbMallocRawNN(db, sizeof(ExprList)+sizeof(pList->a[0])*4 ); + pList = sqlite3DbMallocRawNN(db, SZ_EXPRLIST(4)); if( pList==0 ){ sqlite3ExprDelete(db, pExpr); return 0; @@ -110436,8 +113467,7 @@ SQLITE_PRIVATE SQLITE_NOINLINE ExprList *sqlite3ExprListAppendGrow( struct ExprList_item *pItem; ExprList *pNew; pList->nAlloc *= 2; - pNew = sqlite3DbRealloc(db, pList, - sizeof(*pList)+(pList->nAlloc-1)*sizeof(pList->a[0])); + pNew = sqlite3DbRealloc(db, pList, SZ_EXPRLIST(pList->nAlloc)); if( pNew==0 ){ sqlite3ExprListDelete(db, pList); sqlite3ExprDelete(db, pExpr); @@ -110773,6 +113803,133 @@ SQLITE_PRIVATE Expr *sqlite3ExprSimplifiedAndOr(Expr *pExpr){ return pExpr; } +/* +** Return true if it might be advantageous to compute the right operand +** of expression pExpr first, before the left operand. +** +** Normally the left operand is computed before the right operand. But if +** the left operand contains a subquery and the right does not, then it +** might be more efficient to compute the right operand first. +*/ +static int exprEvalRhsFirst(Expr *pExpr){ + if( ExprHasProperty(pExpr->pLeft, EP_Subquery) + && !ExprHasProperty(pExpr->pRight, EP_Subquery) + ){ + return 1; + }else{ + return 0; + } +} + +/* +** Compute the two operands of a binary operator. +** +** If either operand contains a subquery, then the code strives to +** compute the operand containing the subquery second. If the other +** operand evalutes to NULL, then a jump is made. The address of the +** IsNull operand that does this jump is returned. The caller can use +** this to optimize the computation so as to avoid doing the potentially +** expensive subquery. +** +** If no optimization opportunities exist, return 0. +*/ +static int exprComputeOperands( + Parse *pParse, /* Parsing context */ + Expr *pExpr, /* The comparison expression */ + int *pR1, /* OUT: Register holding the left operand */ + int *pR2, /* OUT: Register holding the right operand */ + int *pFree1, /* OUT: Temp register to free if not zero */ + int *pFree2 /* OUT: Another temp register to free if not zero */ +){ + int addrIsNull; + int r1, r2; + Vdbe *v = pParse->pVdbe; + + assert( v!=0 ); + /* + ** If the left operand contains a (possibly expensive) subquery and the + ** right operand does not and the right operation might be NULL, + ** then compute the right operand first and do an IsNull jump if the + ** right operand evalutes to NULL. + */ + if( exprEvalRhsFirst(pExpr) && sqlite3ExprCanBeNull(pExpr->pRight) ){ + r2 = sqlite3ExprCodeTemp(pParse, pExpr->pRight, pFree2); + addrIsNull = sqlite3VdbeAddOp1(v, OP_IsNull, r2); + VdbeComment((v, "skip left operand")); + VdbeCoverage(v); + }else{ + r2 = 0; /* Silence a false-positive uninit-var warning in MSVC */ + addrIsNull = 0; + } + r1 = sqlite3ExprCodeTemp(pParse, pExpr->pLeft, pFree1); + if( addrIsNull==0 ){ + /* + ** If the right operand contains a subquery and the left operand does not + ** and the left operand might be NULL, then do an IsNull check + ** check on the left operand before computing the right operand. + */ + if( ExprHasProperty(pExpr->pRight, EP_Subquery) + && sqlite3ExprCanBeNull(pExpr->pLeft) + ){ + addrIsNull = sqlite3VdbeAddOp1(v, OP_IsNull, r1); + VdbeComment((v, "skip right operand")); + VdbeCoverage(v); + } + r2 = sqlite3ExprCodeTemp(pParse, pExpr->pRight, pFree2); + } + *pR1 = r1; + *pR2 = r2; + return addrIsNull; +} + +/* +** pExpr is a TK_FUNCTION node. Try to determine whether or not the +** function is a constant function. A function is constant if all of +** the following are true: +** +** (1) It is a scalar function (not an aggregate or window function) +** (2) It has either the SQLITE_FUNC_CONSTANT or SQLITE_FUNC_SLOCHNG +** property. +** (3) All of its arguments are constants +** +** This routine sets pWalker->eCode to 0 if pExpr is not a constant. +** It makes no changes to pWalker->eCode if pExpr is constant. In +** every case, it returns WRC_Abort. +** +** Called as a service subroutine from exprNodeIsConstant(). +*/ +static SQLITE_NOINLINE int exprNodeIsConstantFunction( + Walker *pWalker, + Expr *pExpr +){ + int n; /* Number of arguments */ + ExprList *pList; /* List of arguments */ + FuncDef *pDef; /* The function */ + sqlite3 *db; /* The database */ + + assert( pExpr->op==TK_FUNCTION ); + if( ExprHasProperty(pExpr, EP_TokenOnly) + || (pList = pExpr->x.pList)==0 + ){; + n = 0; + }else{ + n = pList->nExpr; + sqlite3WalkExprList(pWalker, pList); + if( pWalker->eCode==0 ) return WRC_Abort; + } + db = pWalker->pParse->db; + pDef = sqlite3FindFunction(db, pExpr->u.zToken, n, ENC(db), 0); + if( pDef==0 + || pDef->xFinalize!=0 + || (pDef->funcFlags & (SQLITE_FUNC_CONSTANT|SQLITE_FUNC_SLOCHNG))==0 + || ExprHasProperty(pExpr, EP_WinFunc) + ){ + pWalker->eCode = 0; + return WRC_Abort; + } + return WRC_Prune; +} + /* ** These routines are Walker callbacks used to check expressions to @@ -110801,6 +113958,7 @@ SQLITE_PRIVATE Expr *sqlite3ExprSimplifiedAndOr(Expr *pExpr){ ** malformed schema error. */ static int exprNodeIsConstant(Walker *pWalker, Expr *pExpr){ + assert( pWalker->eCode>0 ); /* If pWalker->eCode is 2 then any term of the expression that comes from ** the ON or USING clauses of an outer join disqualifies the expression @@ -110820,6 +113978,8 @@ static int exprNodeIsConstant(Walker *pWalker, Expr *pExpr){ ){ if( pWalker->eCode==5 ) ExprSetProperty(pExpr, EP_FromDDL); return WRC_Continue; + }else if( pWalker->pParse ){ + return exprNodeIsConstantFunction(pWalker, pExpr); }else{ pWalker->eCode = 0; return WRC_Abort; @@ -110848,9 +114008,11 @@ static int exprNodeIsConstant(Walker *pWalker, Expr *pExpr){ case TK_IF_NULL_ROW: case TK_REGISTER: case TK_DOT: + case TK_RAISE: testcase( pExpr->op==TK_REGISTER ); testcase( pExpr->op==TK_IF_NULL_ROW ); testcase( pExpr->op==TK_DOT ); + testcase( pExpr->op==TK_RAISE ); pWalker->eCode = 0; return WRC_Abort; case TK_VARIABLE: @@ -110872,15 +114034,15 @@ static int exprNodeIsConstant(Walker *pWalker, Expr *pExpr){ return WRC_Continue; } } -static int exprIsConst(Expr *p, int initFlag, int iCur){ +static int exprIsConst(Parse *pParse, Expr *p, int initFlag){ Walker w; w.eCode = initFlag; + w.pParse = pParse; w.xExprCallback = exprNodeIsConstant; w.xSelectCallback = sqlite3SelectWalkFail; #ifdef SQLITE_DEBUG w.xSelectCallback2 = sqlite3SelectWalkAssert2; #endif - w.u.iCur = iCur; sqlite3WalkExpr(&w, p); return w.eCode; } @@ -110892,9 +114054,15 @@ static int exprIsConst(Expr *p, int initFlag, int iCur){ ** For the purposes of this function, a double-quoted string (ex: "abc") ** is considered a variable but a single-quoted string (ex: 'abc') is ** a constant. +** +** The pParse parameter may be NULL. But if it is NULL, there is no way +** to determine if function calls are constant or not, and hence all +** function calls will be considered to be non-constant. If pParse is +** not NULL, then a function call might be constant, depending on the +** function and on its parameters. */ -SQLITE_PRIVATE int sqlite3ExprIsConstant(Expr *p){ - return exprIsConst(p, 1, 0); +SQLITE_PRIVATE int sqlite3ExprIsConstant(Parse *pParse, Expr *p){ + return exprIsConst(pParse, p, 1); } /* @@ -110910,8 +114078,24 @@ SQLITE_PRIVATE int sqlite3ExprIsConstant(Expr *p){ ** can be added to the pParse->pConstExpr list and evaluated once when ** the prepared statement starts up. See sqlite3ExprCodeRunJustOnce(). */ -SQLITE_PRIVATE int sqlite3ExprIsConstantNotJoin(Expr *p){ - return exprIsConst(p, 2, 0); +static int sqlite3ExprIsConstantNotJoin(Parse *pParse, Expr *p){ + return exprIsConst(pParse, p, 2); +} + +/* +** This routine examines sub-SELECT statements as an expression is being +** walked as part of sqlite3ExprIsTableConstant(). Sub-SELECTs are considered +** constant as long as they are uncorrelated - meaning that they do not +** contain any terms from outer contexts. +*/ +static int exprSelectWalkTableConstant(Walker *pWalker, Select *pSelect){ + assert( pSelect!=0 ); + assert( pWalker->eCode==3 || pWalker->eCode==0 ); + if( (pSelect->selFlags & SF_Correlated)!=0 ){ + pWalker->eCode = 0; + return WRC_Abort; + } + return WRC_Prune; } /* @@ -110919,9 +114103,26 @@ SQLITE_PRIVATE int sqlite3ExprIsConstantNotJoin(Expr *p){ ** for any single row of the table with cursor iCur. In other words, the ** expression must not refer to any non-deterministic function nor any ** table other than iCur. +** +** Consider uncorrelated subqueries to be constants if the bAllowSubq +** parameter is true. */ -SQLITE_PRIVATE int sqlite3ExprIsTableConstant(Expr *p, int iCur){ - return exprIsConst(p, 3, iCur); +static int sqlite3ExprIsTableConstant(Expr *p, int iCur, int bAllowSubq){ + Walker w; + w.eCode = 3; + w.pParse = 0; + w.xExprCallback = exprNodeIsConstant; + if( bAllowSubq ){ + w.xSelectCallback = exprSelectWalkTableConstant; + }else{ + w.xSelectCallback = sqlite3SelectWalkFail; +#ifdef SQLITE_DEBUG + w.xSelectCallback2 = sqlite3SelectWalkAssert2; +#endif + } + w.u.iCur = iCur; + sqlite3WalkExpr(&w, p); + return w.eCode; } /* @@ -110939,7 +114140,10 @@ SQLITE_PRIVATE int sqlite3ExprIsTableConstant(Expr *p, int iCur){ ** ** (1) pExpr cannot refer to any table other than pSrc->iCursor. ** -** (2) pExpr cannot use subqueries or non-deterministic functions. +** (2a) pExpr cannot use subqueries unless the bAllowSubq parameter is +** true and the subquery is non-correlated +** +** (2b) pExpr cannot use non-deterministic functions. ** ** (3) pSrc cannot be part of the left operand for a RIGHT JOIN. ** (Is there some way to relax this constraint?) @@ -110948,7 +114152,7 @@ SQLITE_PRIVATE int sqlite3ExprIsTableConstant(Expr *p, int iCur){ ** (4a) pExpr must come from an ON clause.. ** (4b) and specifically the ON clause associated with the LEFT JOIN. ** -** (5) If pSrc is not the right operand of a LEFT JOIN or the left +** (5) If pSrc is the right operand of a LEFT JOIN or the left ** operand of a RIGHT JOIN, then pExpr must be from the WHERE ** clause, not an ON clause. ** @@ -110968,7 +114172,8 @@ SQLITE_PRIVATE int sqlite3ExprIsTableConstant(Expr *p, int iCur){ SQLITE_PRIVATE int sqlite3ExprIsSingleTableConstraint( Expr *pExpr, /* The constraint */ const SrcList *pSrcList, /* Complete FROM clause */ - int iSrc /* Which element of pSrcList to use */ + int iSrc, /* Which element of pSrcList to use */ + int bAllowSubq /* Allow non-correlated subqueries */ ){ const SrcItem *pSrc = &pSrcList->a[iSrc]; if( pSrc->fg.jointype & JT_LTORJ ){ @@ -110993,7 +114198,8 @@ SQLITE_PRIVATE int sqlite3ExprIsSingleTableConstraint( } } } - return sqlite3ExprIsTableConstant(pExpr, pSrc->iCursor); /* rules (1), (2) */ + /* Rules (1), (2a), and (2b) handled by the following: */ + return sqlite3ExprIsTableConstant(pExpr, pSrc->iCursor, bAllowSubq); } @@ -111078,7 +114284,7 @@ SQLITE_PRIVATE int sqlite3ExprIsConstantOrGroupBy(Parse *pParse, Expr *p, ExprLi */ SQLITE_PRIVATE int sqlite3ExprIsConstantOrFunction(Expr *p, u8 isInit){ assert( isInit==0 || isInit==1 ); - return exprIsConst(p, 4+isInit, 0); + return exprIsConst(0, p, 4+isInit); } #ifdef SQLITE_ENABLE_CURSOR_HINTS @@ -111104,8 +114310,12 @@ SQLITE_PRIVATE int sqlite3ExprContainsSubquery(Expr *p){ ** to fit in a 32-bit integer, return 1 and put the value of the integer ** in *pValue. If the expression is not an integer or if it is too big ** to fit in a signed 32-bit integer, return 0 and leave *pValue unchanged. +** +** If the pParse pointer is provided, then allow the expression p to be +** a parameter (TK_VARIABLE) that is bound to an integer. +** But if pParse is NULL, then p must be a pure integer literal. */ -SQLITE_PRIVATE int sqlite3ExprIsInteger(const Expr *p, int *pValue){ +SQLITE_PRIVATE int sqlite3ExprIsInteger(const Expr *p, int *pValue, Parse *pParse){ int rc = 0; if( NEVER(p==0) ) return 0; /* Used to only happen following on OOM */ @@ -111120,18 +114330,38 @@ SQLITE_PRIVATE int sqlite3ExprIsInteger(const Expr *p, int *pValue){ } switch( p->op ){ case TK_UPLUS: { - rc = sqlite3ExprIsInteger(p->pLeft, pValue); + rc = sqlite3ExprIsInteger(p->pLeft, pValue, 0); break; } case TK_UMINUS: { int v = 0; - if( sqlite3ExprIsInteger(p->pLeft, &v) ){ + if( sqlite3ExprIsInteger(p->pLeft, &v, 0) ){ assert( ((unsigned int)v)!=0x80000000 ); *pValue = -v; rc = 1; } break; } + case TK_VARIABLE: { + sqlite3_value *pVal; + if( pParse==0 ) break; + if( NEVER(pParse->pVdbe==0) ) break; + if( (pParse->db->flags & SQLITE_EnableQPSG)!=0 ) break; + sqlite3VdbeSetVarmask(pParse->pVdbe, p->iColumn); + pVal = sqlite3VdbeGetBoundValue(pParse->pReprepare, p->iColumn, + SQLITE_AFF_BLOB); + if( pVal ){ + if( sqlite3_value_type(pVal)==SQLITE_INTEGER ){ + sqlite3_int64 vv = sqlite3_value_int64(pVal); + if( vv == (vv & 0x7fffffff) ){ /* non-negative numbers only */ + *pValue = (int)vv; + rc = 1; + } + } + sqlite3ValueFree(pVal); + } + break; + } default: break; } return rc; @@ -111168,9 +114398,12 @@ SQLITE_PRIVATE int sqlite3ExprCanBeNull(const Expr *p){ return 0; case TK_COLUMN: assert( ExprUseYTab(p) ); - return ExprHasProperty(p, EP_CanBeNull) || - NEVER(p->y.pTab==0) || /* Reference to column of index on expr */ - (p->iColumn>=0 + return ExprHasProperty(p, EP_CanBeNull) + || NEVER(p->y.pTab==0) /* Reference to column of index on expr */ +#ifdef SQLITE_ALLOW_ROWID_IN_VIEW + || (p->iColumn==XN_ROWID && IsView(p->y.pTab)) +#endif + || (p->iColumn>=0 && p->y.pTab->aCol!=0 /* Possible due to prior error */ && ALWAYS(p->iColumny.pTab->nCol) && p->y.pTab->aCol[p->iColumn].notNull==0); @@ -111242,13 +114475,7 @@ SQLITE_PRIVATE const char *sqlite3RowidAlias(Table *pTab){ int ii; assert( VisibleRowid(pTab) ); for(ii=0; iinCol; iCol++){ - if( sqlite3_stricmp(azOpt[ii], pTab->aCol[iCol].zCnName)==0 ) break; - } - if( iCol==pTab->nCol ){ - return azOpt[ii]; - } + if( sqlite3ColumnIndex(pTab, azOpt[ii])<0 ) return azOpt[ii]; } return 0; } @@ -111282,8 +114509,8 @@ static Select *isCandidateForInOpt(const Expr *pX){ pSrc = p->pSrc; assert( pSrc!=0 ); if( pSrc->nSrc!=1 ) return 0; /* Single term in FROM clause */ - if( pSrc->a[0].pSelect ) return 0; /* FROM is not a subquery or view */ - pTab = pSrc->a[0].pTab; + if( pSrc->a[0].fg.isSubquery) return 0;/* FROM is not a subquery or view */ + pTab = pSrc->a[0].pSTab; assert( pTab!=0 ); assert( !IsView(pTab) ); /* FROM clause is not a view */ if( IsVirtual(pTab) ) return 0; /* FROM clause not a virtual table */ @@ -111323,13 +114550,13 @@ static void sqlite3SetHasNullFlag(Vdbe *v, int iCur, int regHasNull){ ** The argument is an IN operator with a list (not a subquery) on the ** right-hand side. Return TRUE if that list is constant. */ -static int sqlite3InRhsIsConstant(Expr *pIn){ +static int sqlite3InRhsIsConstant(Parse *pParse, Expr *pIn){ Expr *pLHS; int res; assert( !ExprHasProperty(pIn, EP_xIsSelect) ); pLHS = pIn->pLeft; pIn->pLeft = 0; - res = sqlite3ExprIsConstant(pIn); + res = sqlite3ExprIsConstant(pParse, pIn); pIn->pLeft = pLHS; return res; } @@ -111466,7 +114693,7 @@ SQLITE_PRIVATE int sqlite3FindInIndex( assert( p->pEList!=0 ); /* Because of isCandidateForInOpt(p) */ assert( p->pEList->a[0].pExpr!=0 ); /* Because of isCandidateForInOpt(p) */ assert( p->pSrc!=0 ); /* Because of isCandidateForInOpt(p) */ - pTab = p->pSrc->a[0].pTab; + pTab = p->pSrc->a[0].pSTab; /* Code an OP_Transaction and OP_TableLock for . */ iDb = sqlite3SchemaToIndex(db, pTab->pSchema); @@ -111558,6 +114785,7 @@ SQLITE_PRIVATE int sqlite3FindInIndex( if( aiMap ) aiMap[i] = j; } + assert( nExpr>0 && nExprx.pList->nExpr<=2) + && (!sqlite3InRhsIsConstant(pParse,pX) || pX->x.pList->nExpr<=2) ){ pParse->nTab--; /* Back out the allocation of the unused cursor */ iTab = -1; /* Cursor is not allocated */ @@ -111651,7 +114879,7 @@ static char *exprINAffinity(Parse *pParse, const Expr *pExpr){ char *zRet; assert( pExpr->op==TK_IN ); - zRet = sqlite3DbMallocRaw(pParse->db, nVal+1); + zRet = sqlite3DbMallocRaw(pParse->db, 1+(i64)nVal); if( zRet ){ int i; for(i=0; imSubrtnSig & (1<<(pNewSig->selId&7)))==0 ) return 0; + assert( pExpr->op==TK_IN ); + assert( !ExprUseYSub(pExpr) ); + assert( ExprUseXSelect(pExpr) ); + assert( pExpr->x.pSelect!=0 ); + assert( (pExpr->x.pSelect->selFlags & SF_All)==0 ); + v = pParse->pVdbe; + assert( v!=0 ); + pOp = sqlite3VdbeGetOp(v, 1); + pEnd = sqlite3VdbeGetLastOp(v); + for(; pOpp4type!=P4_SUBRTNSIG ) continue; + assert( pOp->opcode==OP_BeginSubrtn ); + pSig = pOp->p4.pSubrtnSig; + assert( pSig!=0 ); + if( !pSig->bComplete ) continue; + if( pNewSig->selId!=pSig->selId ) continue; + if( strcmp(pNewSig->zAff,pSig->zAff)!=0 ) continue; + pExpr->y.sub.iAddr = pSig->iAddr; + pExpr->y.sub.regReturn = pSig->regReturn; + pExpr->iTable = pSig->iTable; + ExprSetProperty(pExpr, EP_Subrtn); + return 1; + } + return 0; +} +#endif /* SQLITE_OMIT_SUBQUERY */ + #ifndef SQLITE_OMIT_SUBQUERY /* ** Generate code that will construct an ephemeral table containing all terms @@ -111740,6 +115012,7 @@ SQLITE_PRIVATE void sqlite3CodeRhsOfIN( KeyInfo *pKeyInfo = 0; /* Key information */ int nVal; /* Size of vector pLeft */ Vdbe *v; /* The prepared statement under construction */ + SubrtnSig *pSig = 0; /* Signature for this subroutine */ v = pParse->pVdbe; assert( v!=0 ); @@ -111755,11 +115028,27 @@ SQLITE_PRIVATE void sqlite3CodeRhsOfIN( ** and reuse it many names. */ if( !ExprHasProperty(pExpr, EP_VarSelect) && pParse->iSelfTab==0 ){ - /* Reuse of the RHS is allowed */ - /* If this routine has already been coded, but the previous code - ** might not have been invoked yet, so invoke it now as a subroutine. + /* Reuse of the RHS is allowed + ** + ** Compute a signature for the RHS of the IN operator to facility + ** finding and reusing prior instances of the same IN operator. + */ + assert( !ExprUseXSelect(pExpr) || pExpr->x.pSelect!=0 ); + if( ExprUseXSelect(pExpr) && (pExpr->x.pSelect->selFlags & SF_All)==0 ){ + pSig = sqlite3DbMallocRawNN(pParse->db, sizeof(pSig[0])); + if( pSig ){ + pSig->selId = pExpr->x.pSelect->selId; + pSig->zAff = exprINAffinity(pParse, pExpr); + } + } + + /* Check to see if there is a prior materialization of the RHS of + ** this IN operator. If there is, then make use of that prior + ** materialization rather than recomputing it. */ - if( ExprHasProperty(pExpr, EP_Subrtn) ){ + if( ExprHasProperty(pExpr, EP_Subrtn) + || findCompatibleInRhsSubrtn(pParse, pExpr, pSig) + ){ addrOnce = sqlite3VdbeAddOp0(v, OP_Once); VdbeCoverage(v); if( ExprUseXSelect(pExpr) ){ ExplainQueryPlan((pParse, 0, "REUSE LIST SUBQUERY %d", @@ -111771,6 +115060,10 @@ SQLITE_PRIVATE void sqlite3CodeRhsOfIN( assert( iTab!=pExpr->iTable ); sqlite3VdbeAddOp2(v, OP_OpenDup, iTab, pExpr->iTable); sqlite3VdbeJumpHere(v, addrOnce); + if( pSig ){ + sqlite3DbFree(pParse->db, pSig->zAff); + sqlite3DbFree(pParse->db, pSig); + } return; } @@ -111781,7 +115074,14 @@ SQLITE_PRIVATE void sqlite3CodeRhsOfIN( pExpr->y.sub.regReturn = ++pParse->nMem; pExpr->y.sub.iAddr = sqlite3VdbeAddOp2(v, OP_BeginSubrtn, 0, pExpr->y.sub.regReturn) + 1; - + if( pSig ){ + pSig->bComplete = 0; + pSig->iAddr = pExpr->y.sub.iAddr; + pSig->regReturn = pExpr->y.sub.regReturn; + pSig->iTable = iTab; + pParse->mSubrtnSig = 1 << (pSig->selId&7); + sqlite3VdbeChangeP4(v, -1, (const char*)pSig, P4_SUBRTNSIG); + } addrOnce = sqlite3VdbeAddOp0(v, OP_Once); VdbeCoverage(v); } @@ -111822,15 +115122,31 @@ SQLITE_PRIVATE void sqlite3CodeRhsOfIN( SelectDest dest; int i; int rc; + int addrBloom = 0; sqlite3SelectDestInit(&dest, SRT_Set, iTab); dest.zAffSdst = exprINAffinity(pParse, pExpr); pSelect->iLimit = 0; + if( addrOnce && OptimizationEnabled(pParse->db, SQLITE_BloomFilter) ){ + int regBloom = ++pParse->nMem; + addrBloom = sqlite3VdbeAddOp2(v, OP_Blob, 10000, regBloom); + VdbeComment((v, "Bloom filter")); + dest.iSDParm2 = regBloom; + } testcase( pSelect->selFlags & SF_Distinct ); testcase( pKeyInfo==0 ); /* Caused by OOM in sqlite3KeyInfoAlloc() */ pCopy = sqlite3SelectDup(pParse->db, pSelect, 0); rc = pParse->db->mallocFailed ? 1 :sqlite3Select(pParse, pCopy, &dest); sqlite3SelectDelete(pParse->db, pCopy); sqlite3DbFree(pParse->db, dest.zAffSdst); + if( addrBloom ){ + /* Remember that location of the Bloom filter in the P3 operand + ** of the OP_Once that began this subroutine. tag-202407032019 */ + sqlite3VdbeGetOp(v, addrOnce)->p3 = dest.iSDParm2; + if( dest.iSDParm2==0 ){ + /* If the Bloom filter won't actually be used, keep it small */ + sqlite3VdbeGetOp(v, addrBloom)->p1 = 10; + } + } if( rc ){ sqlite3KeyInfoUnref(pKeyInfo); return; @@ -111881,7 +115197,7 @@ SQLITE_PRIVATE void sqlite3CodeRhsOfIN( ** this code only executes once. Because for a non-constant ** expression we need to rerun this code each time. */ - if( addrOnce && !sqlite3ExprIsConstant(pE2) ){ + if( addrOnce && !sqlite3ExprIsConstant(pParse, pE2) ){ sqlite3VdbeChangeToNoop(v, addrOnce-1); sqlite3VdbeChangeToNoop(v, addrOnce); ExprClearProperty(pExpr, EP_Subrtn); @@ -111896,6 +115212,7 @@ SQLITE_PRIVATE void sqlite3CodeRhsOfIN( sqlite3ReleaseTempReg(pParse, r1); sqlite3ReleaseTempReg(pParse, r2); } + if( pSig ) pSig->bComplete = 1; if( pKeyInfo ){ sqlite3VdbeChangeP4(v, addr, (void *)pKeyInfo, P4_KEYINFO); } @@ -112009,17 +115326,23 @@ SQLITE_PRIVATE int sqlite3CodeSubselect(Parse *pParse, Expr *pExpr){ VdbeComment((v, "Init EXISTS result")); } if( pSel->pLimit ){ - /* The subquery already has a limit. If the pre-existing limit is X - ** then make the new limit X<>0 so that the new limit is either 1 or 0 */ - sqlite3 *db = pParse->db; - pLimit = sqlite3Expr(db, TK_INTEGER, "0"); - if( pLimit ){ - pLimit->affExpr = SQLITE_AFF_NUMERIC; - pLimit = sqlite3PExpr(pParse, TK_NE, - sqlite3ExprDup(db, pSel->pLimit->pLeft, 0), pLimit); + /* The subquery already has a limit. If the pre-existing limit X is + ** not already integer value 1 or 0, then make the new limit X<>0 so that + ** the new limit is either 1 or 0 */ + Expr *pLeft = pSel->pLimit->pLeft; + if( ExprHasProperty(pLeft, EP_IntValue)==0 + || (pLeft->u.iValue!=1 && pLeft->u.iValue!=0) + ){ + sqlite3 *db = pParse->db; + pLimit = sqlite3Expr(db, TK_INTEGER, "0"); + if( pLimit ){ + pLimit->affExpr = SQLITE_AFF_NUMERIC; + pLimit = sqlite3PExpr(pParse, TK_NE, + sqlite3ExprDup(db, pLeft, 0), pLimit); + } + sqlite3ExprDeferredDelete(pParse, pLeft); + pSel->pLimit->pLeft = pLimit; } - sqlite3ExprDeferredDelete(pParse, pSel->pLimit->pLeft); - pSel->pLimit->pLeft = pLimit; }else{ /* If there is no pre-existing limit add a limit of 1 */ pLimit = sqlite3Expr(pParse->db, TK_INTEGER, "1"); @@ -112107,7 +115430,6 @@ static void sqlite3ExprCodeIN( int rRhsHasNull = 0; /* Register that is true if RHS contains NULL values */ int eType; /* Type of the RHS */ int rLhs; /* Register(s) holding the LHS values */ - int rLhsOrig; /* LHS values prior to reordering by aiMap[] */ Vdbe *v; /* Statement under construction */ int *aiMap = 0; /* Map from vector field to index column */ char *zAff = 0; /* Affinity string for comparisons */ @@ -112128,9 +115450,7 @@ static void sqlite3ExprCodeIN( if( sqlite3ExprCheckIN(pParse, pExpr) ) return; zAff = exprINAffinity(pParse, pExpr); nVector = sqlite3ExprVectorSize(pExpr->pLeft); - aiMap = (int*)sqlite3DbMallocZero( - pParse->db, nVector*(sizeof(int) + sizeof(char)) + 1 - ); + aiMap = (int*)sqlite3DbMallocZero(pParse->db, nVector*sizeof(int)); if( pParse->db->mallocFailed ) goto sqlite3ExprCodeIN_oom_error; /* Attempt to compute the RHS. After this step, if anything other than @@ -112172,19 +115492,8 @@ static void sqlite3ExprCodeIN( ** by code generated below. */ assert( pParse->okConstFactor==okConstFactor ); pParse->okConstFactor = 0; - rLhsOrig = exprCodeVector(pParse, pLeft, &iDummy); + rLhs = exprCodeVector(pParse, pLeft, &iDummy); pParse->okConstFactor = okConstFactor; - for(i=0; ix.pList; pColl = sqlite3ExprCollSeq(pParse, pExpr->pLeft); @@ -112240,6 +115550,26 @@ static void sqlite3ExprCodeIN( goto sqlite3ExprCodeIN_finished; } + if( eType!=IN_INDEX_ROWID ){ + /* If this IN operator will use an index, then the order of columns in the + ** vector might be different from the order in the index. In that case, + ** we need to reorder the LHS values to be in index order. Run Affinity + ** before reordering the columns, so that the affinity is correct. + */ + sqlite3VdbeAddOp4(v, OP_Affinity, rLhs, nVector, 0, zAff, nVector); + for(i=0; iy.sub.iAddr); + assert( pOp->opcode==OP_Once || pParse->nErr ); + if( pOp->opcode==OP_Once && pOp->p3>0 ){ /* tag-202407032019 */ + assert( OptimizationEnabled(pParse->db, SQLITE_BloomFilter) ); + sqlite3VdbeAddOp4Int(v, OP_Filter, pOp->p3, destIfFalse, + rLhs, nVector); VdbeCoverage(v); + } + } sqlite3VdbeAddOp4Int(v, OP_NotFound, iTab, destIfFalse, rLhs, nVector); VdbeCoverage(v); goto sqlite3ExprCodeIN_finished; @@ -112339,7 +115678,6 @@ static void sqlite3ExprCodeIN( sqlite3VdbeJumpHere(v, addrTruthOp); sqlite3ExprCodeIN_finished: - if( rLhs!=rLhsOrig ) sqlite3ReleaseTempReg(pParse, rLhs); VdbeComment((v, "end IN expr")); sqlite3ExprCodeIN_oom_error: sqlite3DbFree(pParse->db, aiMap); @@ -112454,7 +115792,12 @@ SQLITE_PRIVATE void sqlite3ExprCodeGeneratedColumn( iAddr = 0; } sqlite3ExprCodeCopy(pParse, sqlite3ColumnExpr(pTab,pCol), regOut); - if( pCol->affinity>=SQLITE_AFF_TEXT ){ + if( (pCol->colFlags & COLFLAG_VIRTUAL)!=0 + && (pTab->tabFlags & TF_Strict)!=0 + ){ + int p3 = 2+(int)(pCol - pTab->aCol); + sqlite3VdbeAddOp4(v, OP_TypeCheck, regOut, 1, p3, (char*)pTab, P4_TABLE); + }else if( pCol->affinity>=SQLITE_AFF_TEXT ){ sqlite3VdbeAddOp4(v, OP_Affinity, regOut, 1, 0, &pCol->affinity, 1); } if( iAddr ) sqlite3VdbeJumpHere(v, iAddr); @@ -112555,13 +115898,17 @@ SQLITE_PRIVATE void sqlite3ExprCodeMove(Parse *pParse, int iFrom, int iTo, int n ** register iReg. The caller must ensure that iReg already contains ** the correct value for the expression. */ -static void exprToRegister(Expr *pExpr, int iReg){ +SQLITE_PRIVATE void sqlite3ExprToRegister(Expr *pExpr, int iReg){ Expr *p = sqlite3ExprSkipCollateAndLikely(pExpr); if( NEVER(p==0) ) return; - p->op2 = p->op; - p->op = TK_REGISTER; - p->iTable = iReg; - ExprClearProperty(p, EP_Skip); + if( p->op==TK_REGISTER ){ + assert( p->iTable==iReg ); + }else{ + p->op2 = p->op; + p->op = TK_REGISTER; + p->iTable = iReg; + ExprClearProperty(p, EP_Skip); + } } /* @@ -112731,6 +116078,59 @@ static int exprCodeInlineFunction( return target; } +/* +** Expression Node callback for sqlite3ExprCanReturnSubtype(). +** +** Only a function call is able to return a subtype. So if the node +** is not a function call, return WRC_Prune immediately. +** +** A function call is able to return a subtype if it has the +** SQLITE_RESULT_SUBTYPE property. +** +** Assume that every function is able to pass-through a subtype from +** one of its argument (using sqlite3_result_value()). Most functions +** are not this way, but we don't have a mechanism to distinguish those +** that are from those that are not, so assume they all work this way. +** That means that if one of its arguments is another function and that +** other function is able to return a subtype, then this function is +** able to return a subtype. +*/ +static int exprNodeCanReturnSubtype(Walker *pWalker, Expr *pExpr){ + int n; + FuncDef *pDef; + sqlite3 *db; + if( pExpr->op!=TK_FUNCTION ){ + return WRC_Prune; + } + assert( ExprUseXList(pExpr) ); + db = pWalker->pParse->db; + n = ALWAYS(pExpr->x.pList) ? pExpr->x.pList->nExpr : 0; + pDef = sqlite3FindFunction(db, pExpr->u.zToken, n, ENC(db), 0); + if( NEVER(pDef==0) || (pDef->funcFlags & SQLITE_RESULT_SUBTYPE)!=0 ){ + pWalker->eCode = 1; + return WRC_Prune; + } + return WRC_Continue; +} + +/* +** Return TRUE if expression pExpr is able to return a subtype. +** +** A TRUE return does not guarantee that a subtype will be returned. +** It only indicates that a subtype return is possible. False positives +** are acceptable as they only disable an optimization. False negatives, +** on the other hand, can lead to incorrect answers. +*/ +static int sqlite3ExprCanReturnSubtype(Parse *pParse, Expr *pExpr){ + Walker w; + memset(&w, 0, sizeof(w)); + w.pParse = pParse; + w.xExprCallback = exprNodeCanReturnSubtype; + sqlite3WalkExpr(&w, pExpr); + return w.eCode; +} + + /* ** Check to see if pExpr is one of the indexed expressions on pParse->pIdxEpr. ** If it is, then resolve the expression by reading from the index and @@ -112763,6 +116163,17 @@ static SQLITE_NOINLINE int sqlite3IndexedExprLookup( continue; } + + /* Functions that might set a subtype should not be replaced by the + ** value taken from an expression index if they are themselves an + ** argument to another scalar function or aggregate. + ** https://sqlite.org/forum/forumpost/68d284c86b082c3e */ + if( ExprHasProperty(pExpr, EP_SubtArg) + && sqlite3ExprCanReturnSubtype(pParse, pExpr) + ){ + continue; + } + v = pParse->pVdbe; assert( v!=0 ); if( p->bMaybeNullRow ){ @@ -112791,7 +116202,7 @@ static SQLITE_NOINLINE int sqlite3IndexedExprLookup( /* -** Expresion pExpr is guaranteed to be a TK_COLUMN or equivalent. This +** Expression pExpr is guaranteed to be a TK_COLUMN or equivalent. This ** function checks the Parse.pIdxPartExpr list to see if this column ** can be replaced with a constant value. If so, it generates code to ** put the constant value in a register (ideally, but not necessarily, @@ -112824,6 +116235,80 @@ static int exprPartidxExprLookup(Parse *pParse, Expr *pExpr, int iTarget){ return 0; } +/* +** Generate code that evaluates an AND or OR operator leaving a +** boolean result in a register. pExpr is the AND/OR expression. +** Store the result in the "target" register. Use short-circuit +** evaluation to avoid computing both operands, if possible. +** +** The code generated might require the use of a temporary register. +** If it does, then write the number of that temporary register +** into *pTmpReg. If not, leave *pTmpReg unchanged. +*/ +static SQLITE_NOINLINE int exprCodeTargetAndOr( + Parse *pParse, /* Parsing context */ + Expr *pExpr, /* AND or OR expression to be coded */ + int target, /* Put result in this register, guaranteed */ + int *pTmpReg /* Write a temporary register here */ +){ + int op; /* The opcode. TK_AND or TK_OR */ + int skipOp; /* Opcode for the branch that skips one operand */ + int addrSkip; /* Branch instruction that skips one of the operands */ + int regSS = 0; /* Register holding computed operand when other omitted */ + int r1, r2; /* Registers for left and right operands, respectively */ + Expr *pAlt; /* Alternative, simplified expression */ + Vdbe *v; /* statement being coded */ + + assert( pExpr!=0 ); + op = pExpr->op; + assert( op==TK_AND || op==TK_OR ); + assert( TK_AND==OP_And ); testcase( op==TK_AND ); + assert( TK_OR==OP_Or ); testcase( op==TK_OR ); + assert( pParse->pVdbe!=0 ); + v = pParse->pVdbe; + pAlt = sqlite3ExprSimplifiedAndOr(pExpr); + if( pAlt!=pExpr ){ + r1 = sqlite3ExprCodeTarget(pParse, pAlt, target); + sqlite3VdbeAddOp3(v, OP_And, r1, r1, target); + return target; + } + skipOp = op==TK_AND ? OP_IfNot : OP_If; + if( exprEvalRhsFirst(pExpr) ){ + /* Compute the right operand first. Skip the computation of the left + ** operand if the right operand fully determines the result */ + r2 = regSS = sqlite3ExprCodeTarget(pParse, pExpr->pRight, target); + addrSkip = sqlite3VdbeAddOp1(v, skipOp, r2); + VdbeComment((v, "skip left operand")); + VdbeCoverage(v); + r1 = sqlite3ExprCodeTemp(pParse, pExpr->pLeft, pTmpReg); + }else{ + /* Compute the left operand first */ + r1 = sqlite3ExprCodeTarget(pParse, pExpr->pLeft, target); + if( ExprHasProperty(pExpr->pRight, EP_Subquery) ){ + /* Skip over the computation of the right operand if the right + ** operand is a subquery and the left operand completely determines + ** the result */ + regSS = r1; + addrSkip = sqlite3VdbeAddOp1(v, skipOp, r1); + VdbeComment((v, "skip right operand")); + VdbeCoverage(v); + }else{ + addrSkip = regSS = 0; + } + r2 = sqlite3ExprCodeTemp(pParse, pExpr->pRight, pTmpReg); + } + sqlite3VdbeAddOp3(v, op, r2, r1, target); + testcase( (*pTmpReg)==0 ); + if( addrSkip ){ + sqlite3VdbeAddOp2(v, OP_Goto, 0, sqlite3VdbeCurrentAddr(v)+2); + sqlite3VdbeJumpHere(v, addrSkip); + sqlite3VdbeAddOp3(v, OP_Or, regSS, regSS, target); + VdbeComment((v, "short-circut value")); + } + return target; +} + + /* ** Generate code into the current Vdbe to evaluate the given @@ -113015,6 +116500,12 @@ SQLITE_PRIVATE int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target) sqlite3VdbeLoadString(v, target, pExpr->u.zToken); return target; } + case TK_NULLS: { + /* Set a range of registers to NULL. pExpr->y.nReg registers starting + ** with target */ + sqlite3VdbeAddOp3(v, OP_Null, 0, target, target + pExpr->y.nReg - 1); + return target; + } default: { /* Make NULL the default case so that if a bug causes an illegal ** Expr node to be passed into this function, it will be handled @@ -113045,12 +116536,6 @@ SQLITE_PRIVATE int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target) assert( pExpr->u.zToken!=0 ); assert( pExpr->u.zToken[0]!=0 ); sqlite3VdbeAddOp2(v, OP_Variable, pExpr->iColumn, target); - if( pExpr->u.zToken[1]!=0 ){ - const char *z = sqlite3VListNumToName(pParse->pVList, pExpr->iColumn); - assert( pExpr->u.zToken[0]=='?' || (z && !strcmp(pExpr->u.zToken, z)) ); - pParse->pVList[0] = 0; /* Indicate VList may no longer be enlarged */ - sqlite3VdbeAppendP4(v, (char*)z, P4_STATIC); - } return target; } case TK_REGISTER: { @@ -113079,11 +116564,17 @@ SQLITE_PRIVATE int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target) case TK_NE: case TK_EQ: { Expr *pLeft = pExpr->pLeft; + int addrIsNull = 0; if( sqlite3ExprIsVector(pLeft) ){ codeVectorCompare(pParse, pExpr, target, op, p5); }else{ - r1 = sqlite3ExprCodeTemp(pParse, pLeft, ®Free1); - r2 = sqlite3ExprCodeTemp(pParse, pExpr->pRight, ®Free2); + if( ExprHasProperty(pExpr, EP_Subquery) && p5!=SQLITE_NULLEQ ){ + addrIsNull = exprComputeOperands(pParse, pExpr, + &r1, &r2, ®Free1, ®Free2); + }else{ + r1 = sqlite3ExprCodeTemp(pParse, pExpr->pLeft, ®Free1); + r2 = sqlite3ExprCodeTemp(pParse, pExpr->pRight, ®Free2); + } sqlite3VdbeAddOp2(v, OP_Integer, 1, inReg); codeCompare(pParse, pLeft, pExpr->pRight, op, r1, r2, sqlite3VdbeCurrentAddr(v)+2, p5, @@ -113098,6 +116589,11 @@ SQLITE_PRIVATE int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target) sqlite3VdbeAddOp2(v, OP_Integer, 0, inReg); }else{ sqlite3VdbeAddOp3(v, OP_ZeroOrNull, r1, inReg, r2); + if( addrIsNull ){ + sqlite3VdbeAddOp2(v, OP_Goto, 0, sqlite3VdbeCurrentAddr(v)+2); + sqlite3VdbeJumpHere(v, addrIsNull); + sqlite3VdbeAddOp2(v, OP_Null, 0, inReg); + } } testcase( regFree1==0 ); testcase( regFree2==0 ); @@ -113105,7 +116601,10 @@ SQLITE_PRIVATE int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target) break; } case TK_AND: - case TK_OR: + case TK_OR: { + inReg = exprCodeTargetAndOr(pParse, pExpr, target, ®Free1); + break; + } case TK_PLUS: case TK_STAR: case TK_MINUS: @@ -113116,8 +116615,7 @@ SQLITE_PRIVATE int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target) case TK_LSHIFT: case TK_RSHIFT: case TK_CONCAT: { - assert( TK_AND==OP_And ); testcase( op==TK_AND ); - assert( TK_OR==OP_Or ); testcase( op==TK_OR ); + int addrIsNull; assert( TK_PLUS==OP_Add ); testcase( op==TK_PLUS ); assert( TK_MINUS==OP_Subtract ); testcase( op==TK_MINUS ); assert( TK_REM==OP_Remainder ); testcase( op==TK_REM ); @@ -113127,11 +116625,23 @@ SQLITE_PRIVATE int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target) assert( TK_LSHIFT==OP_ShiftLeft ); testcase( op==TK_LSHIFT ); assert( TK_RSHIFT==OP_ShiftRight ); testcase( op==TK_RSHIFT ); assert( TK_CONCAT==OP_Concat ); testcase( op==TK_CONCAT ); - r1 = sqlite3ExprCodeTemp(pParse, pExpr->pLeft, ®Free1); - r2 = sqlite3ExprCodeTemp(pParse, pExpr->pRight, ®Free2); + if( ExprHasProperty(pExpr, EP_Subquery) ){ + addrIsNull = exprComputeOperands(pParse, pExpr, + &r1, &r2, ®Free1, ®Free2); + }else{ + r1 = sqlite3ExprCodeTemp(pParse, pExpr->pLeft, ®Free1); + r2 = sqlite3ExprCodeTemp(pParse, pExpr->pRight, ®Free2); + addrIsNull = 0; + } sqlite3VdbeAddOp3(v, op, r2, r1, target); testcase( regFree1==0 ); testcase( regFree2==0 ); + if( addrIsNull ){ + sqlite3VdbeAddOp2(v, OP_Goto, 0, sqlite3VdbeCurrentAddr(v)+2); + sqlite3VdbeJumpHere(v, addrIsNull); + sqlite3VdbeAddOp2(v, OP_Null, 0, target); + VdbeComment((v, "short-circut value")); + } break; } case TK_UMINUS: { @@ -113224,7 +116734,9 @@ SQLITE_PRIVATE int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target) } #endif - if( ConstFactorOk(pParse) && sqlite3ExprIsConstantNotJoin(pExpr) ){ + if( ConstFactorOk(pParse) + && sqlite3ExprIsConstantNotJoin(pParse,pExpr) + ){ /* SQL functions can be expensive. So try to avoid running them ** multiple times if we know they always give the same result */ return sqlite3ExprCodeRunJustOnce(pParse, pExpr, -1); @@ -113255,7 +116767,7 @@ SQLITE_PRIVATE int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target) } for(i=0; ia[i].pExpr) ){ + if( i<32 && sqlite3ExprIsConstant(pParse, pFarg->a[i].pExpr) ){ testcase( i==31 ); constMask |= MASKBIT32(i); } @@ -113397,8 +116909,9 @@ SQLITE_PRIVATE int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target) if( !ExprHasProperty(pExpr, EP_Collate) ){ /* A TK_COLLATE Expr node without the EP_Collate tag is a so-called ** "SOFT-COLLATE" that is added to constraints that are pushed down - ** from outer queries into sub-queries by the push-down optimization. - ** Clear subtypes as subtypes may not cross a subquery boundary. + ** from outer queries into sub-queries by the WHERE-clause push-down + ** optimization. Clear subtypes as subtypes may not cross a subquery + ** boundary. */ assert( pExpr->pLeft ); sqlite3ExprCode(pParse, pExpr->pLeft, target); @@ -113567,7 +117080,7 @@ SQLITE_PRIVATE int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target) break; } testcase( pX->op==TK_COLUMN ); - exprToRegister(pDel, exprCodeVector(pParse, pDel, ®Free1)); + sqlite3ExprToRegister(pDel, exprCodeVector(pParse, pDel, ®Free1)); testcase( regFree1==0 ); memset(&opCompare, 0, sizeof(opCompare)); opCompare.op = TK_EQ; @@ -113621,15 +117134,14 @@ SQLITE_PRIVATE int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target) } assert( !ExprHasProperty(pExpr, EP_IntValue) ); if( pExpr->affExpr==OE_Ignore ){ - sqlite3VdbeAddOp4( - v, OP_Halt, SQLITE_OK, OE_Ignore, 0, pExpr->u.zToken,0); + sqlite3VdbeAddOp2(v, OP_Halt, SQLITE_OK, OE_Ignore); VdbeCoverage(v); }else{ - sqlite3HaltConstraint(pParse, + r1 = sqlite3ExprCodeTemp(pParse, pExpr->pLeft, ®Free1); + sqlite3VdbeAddOp3(v, OP_Halt, pParse->pTriggerTab ? SQLITE_CONSTRAINT_TRIGGER : SQLITE_ERROR, - pExpr->affExpr, pExpr->u.zToken, 0, 0); + pExpr->affExpr, r1); } - break; } #endif @@ -113703,6 +117215,25 @@ SQLITE_PRIVATE int sqlite3ExprCodeRunJustOnce( return regDest; } +/* +** Make arrangements to invoke OP_Null on a range of registers +** during initialization. +*/ +SQLITE_PRIVATE SQLITE_NOINLINE void sqlite3ExprNullRegisterRange( + Parse *pParse, /* Parsing context */ + int iReg, /* First register to set to NULL */ + int nReg /* Number of sequential registers to NULL out */ +){ + u8 okConstFactor = pParse->okConstFactor; + Expr t; + memset(&t, 0, sizeof(t)); + t.op = TK_NULLS; + t.y.nReg = nReg; + pParse->okConstFactor = 1; + sqlite3ExprCodeRunJustOnce(pParse, &t, iReg); + pParse->okConstFactor = okConstFactor; +} + /* ** Generate code to evaluate an expression and store the results ** into a register. Return the register number where the results @@ -113722,7 +117253,7 @@ SQLITE_PRIVATE int sqlite3ExprCodeTemp(Parse *pParse, Expr *pExpr, int *pReg){ if( ConstFactorOk(pParse) && ALWAYS(pExpr!=0) && pExpr->op!=TK_REGISTER - && sqlite3ExprIsConstantNotJoin(pExpr) + && sqlite3ExprIsConstantNotJoin(pParse, pExpr) ){ *pReg = 0; r2 = sqlite3ExprCodeRunJustOnce(pParse, pExpr, -1); @@ -113786,7 +117317,7 @@ SQLITE_PRIVATE void sqlite3ExprCodeCopy(Parse *pParse, Expr *pExpr, int target){ ** might choose to code the expression at initialization time. */ SQLITE_PRIVATE void sqlite3ExprCodeFactorable(Parse *pParse, Expr *pExpr, int target){ - if( pParse->okConstFactor && sqlite3ExprIsConstantNotJoin(pExpr) ){ + if( pParse->okConstFactor && sqlite3ExprIsConstantNotJoin(pParse,pExpr) ){ sqlite3ExprCodeRunJustOnce(pParse, pExpr, target); }else{ sqlite3ExprCodeCopy(pParse, pExpr, target); @@ -113845,7 +117376,7 @@ SQLITE_PRIVATE int sqlite3ExprCodeExprList( sqlite3VdbeAddOp2(v, copyOp, j+srcReg-1, target+i); } }else if( (flags & SQLITE_ECEL_FACTOR)!=0 - && sqlite3ExprIsConstantNotJoin(pExpr) + && sqlite3ExprIsConstantNotJoin(pParse,pExpr) ){ sqlite3ExprCodeRunJustOnce(pParse, pExpr, target+i); }else{ @@ -113918,7 +117449,7 @@ static void exprCodeBetween( compRight.op = TK_LE; compRight.pLeft = pDel; compRight.pRight = pExpr->x.pList->a[1].pExpr; - exprToRegister(pDel, exprCodeVector(pParse, pDel, ®Free1)); + sqlite3ExprToRegister(pDel, exprCodeVector(pParse, pDel, ®Free1)); if( xJump ){ xJump(pParse, &exprAnd, dest, jumpIfNull); }else{ @@ -113978,17 +117509,27 @@ SQLITE_PRIVATE void sqlite3ExprIfTrue(Parse *pParse, Expr *pExpr, int dest, int Expr *pAlt = sqlite3ExprSimplifiedAndOr(pExpr); if( pAlt!=pExpr ){ sqlite3ExprIfTrue(pParse, pAlt, dest, jumpIfNull); - }else if( op==TK_AND ){ - int d2 = sqlite3VdbeMakeLabel(pParse); - testcase( jumpIfNull==0 ); - sqlite3ExprIfFalse(pParse, pExpr->pLeft, d2, - jumpIfNull^SQLITE_JUMPIFNULL); - sqlite3ExprIfTrue(pParse, pExpr->pRight, dest, jumpIfNull); - sqlite3VdbeResolveLabel(v, d2); }else{ - testcase( jumpIfNull==0 ); - sqlite3ExprIfTrue(pParse, pExpr->pLeft, dest, jumpIfNull); - sqlite3ExprIfTrue(pParse, pExpr->pRight, dest, jumpIfNull); + Expr *pFirst, *pSecond; + if( exprEvalRhsFirst(pExpr) ){ + pFirst = pExpr->pRight; + pSecond = pExpr->pLeft; + }else{ + pFirst = pExpr->pLeft; + pSecond = pExpr->pRight; + } + if( op==TK_AND ){ + int d2 = sqlite3VdbeMakeLabel(pParse); + testcase( jumpIfNull==0 ); + sqlite3ExprIfFalse(pParse, pFirst, d2, + jumpIfNull^SQLITE_JUMPIFNULL); + sqlite3ExprIfTrue(pParse, pSecond, dest, jumpIfNull); + sqlite3VdbeResolveLabel(v, d2); + }else{ + testcase( jumpIfNull==0 ); + sqlite3ExprIfTrue(pParse, pFirst, dest, jumpIfNull); + sqlite3ExprIfTrue(pParse, pSecond, dest, jumpIfNull); + } } break; } @@ -114027,10 +117568,16 @@ SQLITE_PRIVATE void sqlite3ExprIfTrue(Parse *pParse, Expr *pExpr, int dest, int case TK_GE: case TK_NE: case TK_EQ: { + int addrIsNull; if( sqlite3ExprIsVector(pExpr->pLeft) ) goto default_expr; - testcase( jumpIfNull==0 ); - r1 = sqlite3ExprCodeTemp(pParse, pExpr->pLeft, ®Free1); - r2 = sqlite3ExprCodeTemp(pParse, pExpr->pRight, ®Free2); + if( ExprHasProperty(pExpr, EP_Subquery) && jumpIfNull!=SQLITE_NULLEQ ){ + addrIsNull = exprComputeOperands(pParse, pExpr, + &r1, &r2, ®Free1, ®Free2); + }else{ + r1 = sqlite3ExprCodeTemp(pParse, pExpr->pLeft, ®Free1); + r2 = sqlite3ExprCodeTemp(pParse, pExpr->pRight, ®Free2); + addrIsNull = 0; + } codeCompare(pParse, pExpr->pLeft, pExpr->pRight, op, r1, r2, dest, jumpIfNull, ExprHasProperty(pExpr,EP_Commuted)); assert(TK_LT==OP_Lt); testcase(op==OP_Lt); VdbeCoverageIf(v,op==OP_Lt); @@ -114045,6 +117592,13 @@ SQLITE_PRIVATE void sqlite3ExprIfTrue(Parse *pParse, Expr *pExpr, int dest, int VdbeCoverageIf(v, op==OP_Ne && jumpIfNull!=SQLITE_NULLEQ); testcase( regFree1==0 ); testcase( regFree2==0 ); + if( addrIsNull ){ + if( jumpIfNull ){ + sqlite3VdbeChangeP2(v, addrIsNull, dest); + }else{ + sqlite3VdbeJumpHere(v, addrIsNull); + } + } break; } case TK_ISNULL: @@ -114052,11 +117606,11 @@ SQLITE_PRIVATE void sqlite3ExprIfTrue(Parse *pParse, Expr *pExpr, int dest, int assert( TK_ISNULL==OP_IsNull ); testcase( op==TK_ISNULL ); assert( TK_NOTNULL==OP_NotNull ); testcase( op==TK_NOTNULL ); r1 = sqlite3ExprCodeTemp(pParse, pExpr->pLeft, ®Free1); - sqlite3VdbeTypeofColumn(v, r1); + assert( regFree1==0 || regFree1==r1 ); + if( regFree1 ) sqlite3VdbeTypeofColumn(v, r1); sqlite3VdbeAddOp2(v, op, r1, dest); VdbeCoverageIf(v, op==TK_ISNULL); VdbeCoverageIf(v, op==TK_NOTNULL); - testcase( regFree1==0 ); break; } case TK_BETWEEN: { @@ -114152,17 +117706,27 @@ SQLITE_PRIVATE void sqlite3ExprIfFalse(Parse *pParse, Expr *pExpr, int dest, int Expr *pAlt = sqlite3ExprSimplifiedAndOr(pExpr); if( pAlt!=pExpr ){ sqlite3ExprIfFalse(pParse, pAlt, dest, jumpIfNull); - }else if( pExpr->op==TK_AND ){ - testcase( jumpIfNull==0 ); - sqlite3ExprIfFalse(pParse, pExpr->pLeft, dest, jumpIfNull); - sqlite3ExprIfFalse(pParse, pExpr->pRight, dest, jumpIfNull); }else{ - int d2 = sqlite3VdbeMakeLabel(pParse); - testcase( jumpIfNull==0 ); - sqlite3ExprIfTrue(pParse, pExpr->pLeft, d2, - jumpIfNull^SQLITE_JUMPIFNULL); - sqlite3ExprIfFalse(pParse, pExpr->pRight, dest, jumpIfNull); - sqlite3VdbeResolveLabel(v, d2); + Expr *pFirst, *pSecond; + if( exprEvalRhsFirst(pExpr) ){ + pFirst = pExpr->pRight; + pSecond = pExpr->pLeft; + }else{ + pFirst = pExpr->pLeft; + pSecond = pExpr->pRight; + } + if( pExpr->op==TK_AND ){ + testcase( jumpIfNull==0 ); + sqlite3ExprIfFalse(pParse, pFirst, dest, jumpIfNull); + sqlite3ExprIfFalse(pParse, pSecond, dest, jumpIfNull); + }else{ + int d2 = sqlite3VdbeMakeLabel(pParse); + testcase( jumpIfNull==0 ); + sqlite3ExprIfTrue(pParse, pFirst, d2, + jumpIfNull^SQLITE_JUMPIFNULL); + sqlite3ExprIfFalse(pParse, pSecond, dest, jumpIfNull); + sqlite3VdbeResolveLabel(v, d2); + } } break; } @@ -114204,10 +117768,16 @@ SQLITE_PRIVATE void sqlite3ExprIfFalse(Parse *pParse, Expr *pExpr, int dest, int case TK_GE: case TK_NE: case TK_EQ: { + int addrIsNull; if( sqlite3ExprIsVector(pExpr->pLeft) ) goto default_expr; - testcase( jumpIfNull==0 ); - r1 = sqlite3ExprCodeTemp(pParse, pExpr->pLeft, ®Free1); - r2 = sqlite3ExprCodeTemp(pParse, pExpr->pRight, ®Free2); + if( ExprHasProperty(pExpr, EP_Subquery) && jumpIfNull!=SQLITE_NULLEQ ){ + addrIsNull = exprComputeOperands(pParse, pExpr, + &r1, &r2, ®Free1, ®Free2); + }else{ + r1 = sqlite3ExprCodeTemp(pParse, pExpr->pLeft, ®Free1); + r2 = sqlite3ExprCodeTemp(pParse, pExpr->pRight, ®Free2); + addrIsNull = 0; + } codeCompare(pParse, pExpr->pLeft, pExpr->pRight, op, r1, r2, dest, jumpIfNull,ExprHasProperty(pExpr,EP_Commuted)); assert(TK_LT==OP_Lt); testcase(op==OP_Lt); VdbeCoverageIf(v,op==OP_Lt); @@ -114222,16 +117792,23 @@ SQLITE_PRIVATE void sqlite3ExprIfFalse(Parse *pParse, Expr *pExpr, int dest, int VdbeCoverageIf(v, op==OP_Ne && jumpIfNull==SQLITE_NULLEQ); testcase( regFree1==0 ); testcase( regFree2==0 ); + if( addrIsNull ){ + if( jumpIfNull ){ + sqlite3VdbeChangeP2(v, addrIsNull, dest); + }else{ + sqlite3VdbeJumpHere(v, addrIsNull); + } + } break; } case TK_ISNULL: case TK_NOTNULL: { r1 = sqlite3ExprCodeTemp(pParse, pExpr->pLeft, ®Free1); - sqlite3VdbeTypeofColumn(v, r1); + assert( regFree1==0 || regFree1==r1 ); + if( regFree1 ) sqlite3VdbeTypeofColumn(v, r1); sqlite3VdbeAddOp2(v, op, r1, dest); testcase( op==TK_ISNULL ); VdbeCoverageIf(v, op==TK_ISNULL); testcase( op==TK_NOTNULL ); VdbeCoverageIf(v, op==TK_NOTNULL); - testcase( regFree1==0 ); break; } case TK_BETWEEN: { @@ -114297,16 +117874,23 @@ SQLITE_PRIVATE void sqlite3ExprIfFalseDup(Parse *pParse, Expr *pExpr, int dest,i ** same as that currently bound to variable pVar, non-zero is returned. ** Otherwise, if the values are not the same or if pExpr is not a simple ** SQL value, zero is returned. +** +** If the SQLITE_EnableQPSG flag is set on the database connection, then +** this routine always returns false. */ -static int exprCompareVariable( +static SQLITE_NOINLINE int exprCompareVariable( const Parse *pParse, const Expr *pVar, const Expr *pExpr ){ - int res = 0; + int res = 2; int iVar; sqlite3_value *pL, *pR = 0; + if( pExpr->op==TK_VARIABLE && pVar->iColumn==pExpr->iColumn ){ + return 0; + } + if( (pParse->db->flags & SQLITE_EnableQPSG)!=0 ) return 2; sqlite3ValueFromExpr(pParse->db, pExpr, SQLITE_UTF8, SQLITE_AFF_BLOB, &pR); if( pR ){ iVar = pVar->iColumn; @@ -114316,12 +117900,11 @@ static int exprCompareVariable( if( sqlite3_value_type(pL)==SQLITE_TEXT ){ sqlite3_value_text(pL); /* Make sure the encoding is UTF-8 */ } - res = 0==sqlite3MemCompare(pL, pR, 0); + res = sqlite3MemCompare(pL, pR, 0) ? 2 : 0; } sqlite3ValueFree(pR); sqlite3ValueFree(pL); } - return res; } @@ -114347,12 +117930,10 @@ static int exprCompareVariable( ** just might result in some slightly slower code. But returning ** an incorrect 0 or 1 could lead to a malfunction. ** -** If pParse is not NULL then TK_VARIABLE terms in pA with bindings in -** pParse->pReprepare can be matched against literals in pB. The -** pParse->pVdbe->expmask bitmask is updated for each variable referenced. -** If pParse is NULL (the normal case) then any TK_VARIABLE term in -** Argument pParse should normally be NULL. If it is not NULL and pA or -** pB causes a return value of 2. +** If pParse is not NULL and SQLITE_EnableQPSG is off then TK_VARIABLE +** terms in pA with bindings in pParse->pReprepare can be matched against +** literals in pB. The pParse->pVdbe->expmask bitmask is updated for +** each variable referenced. */ SQLITE_PRIVATE int sqlite3ExprCompare( const Parse *pParse, @@ -114364,8 +117945,8 @@ SQLITE_PRIVATE int sqlite3ExprCompare( if( pA==0 || pB==0 ){ return pB==pA ? 0 : 2; } - if( pParse && pA->op==TK_VARIABLE && exprCompareVariable(pParse, pA, pB) ){ - return 0; + if( pParse && pA->op==TK_VARIABLE ){ + return exprCompareVariable(pParse, pA, pB); } combinedFlags = pA->flags | pB->flags; if( combinedFlags & EP_IntValue ){ @@ -114560,18 +118141,70 @@ static int exprImpliesNotNull( return 0; } +/* +** Return true if the boolean value of the expression is always either +** FALSE or NULL. +*/ +static int sqlite3ExprIsNotTrue(Expr *pExpr){ + int v; + if( pExpr->op==TK_NULL ) return 1; + if( pExpr->op==TK_TRUEFALSE && sqlite3ExprTruthValue(pExpr)==0 ) return 1; + v = 1; + if( sqlite3ExprIsInteger(pExpr, &v, 0) && v==0 ) return 1; + return 0; +} + +/* +** Return true if the expression is one of the following: +** +** CASE WHEN x THEN y END +** CASE WHEN x THEN y ELSE NULL END +** CASE WHEN x THEN y ELSE false END +** iif(x,y) +** iif(x,y,NULL) +** iif(x,y,false) +*/ +static int sqlite3ExprIsIIF(sqlite3 *db, const Expr *pExpr){ + ExprList *pList; + if( pExpr->op==TK_FUNCTION ){ + const char *z = pExpr->u.zToken; + FuncDef *pDef; + if( (z[0]!='i' && z[0]!='I') ) return 0; + if( pExpr->x.pList==0 ) return 0; + pDef = sqlite3FindFunction(db, z, pExpr->x.pList->nExpr, ENC(db), 0); +#ifdef SQLITE_ENABLE_UNKNOWN_SQL_FUNCTION + if( pDef==0 ) return 0; +#else + if( NEVER(pDef==0) ) return 0; +#endif + if( (pDef->funcFlags & SQLITE_FUNC_INLINE)==0 ) return 0; + if( SQLITE_PTR_TO_INT(pDef->pUserData)!=INLINEFUNC_iif ) return 0; + }else if( pExpr->op==TK_CASE ){ + if( pExpr->pLeft!=0 ) return 0; + }else{ + return 0; + } + pList = pExpr->x.pList; + assert( pList!=0 ); + if( pList->nExpr==2 ) return 1; + if( pList->nExpr==3 && sqlite3ExprIsNotTrue(pList->a[2].pExpr) ) return 1; + return 0; +} + /* ** Return true if we can prove the pE2 will always be true if pE1 is ** true. Return false if we cannot complete the proof or if pE2 might ** be false. Examples: ** -** pE1: x==5 pE2: x==5 Result: true -** pE1: x>0 pE2: x==5 Result: false -** pE1: x=21 pE2: x=21 OR y=43 Result: true -** pE1: x!=123 pE2: x IS NOT NULL Result: true -** pE1: x!=?1 pE2: x IS NOT NULL Result: true -** pE1: x IS NULL pE2: x IS NOT NULL Result: false -** pE1: x IS ?2 pE2: x IS NOT NULL Result: false +** pE1: x==5 pE2: x==5 Result: true +** pE1: x>0 pE2: x==5 Result: false +** pE1: x=21 pE2: x=21 OR y=43 Result: true +** pE1: x!=123 pE2: x IS NOT NULL Result: true +** pE1: x!=?1 pE2: x IS NOT NULL Result: true +** pE1: x IS NULL pE2: x IS NOT NULL Result: false +** pE1: x IS ?2 pE2: x IS NOT NULL Result: false +** pE1: iif(x,y) pE2: x Result: true +** PE1: iif(x,y,0) pE2: x Result: true ** ** When comparing TK_COLUMN nodes between pE1 and pE2, if pE2 has ** Expr.iTable<0 then assume a table number given by iTab. @@ -114605,6 +118238,9 @@ SQLITE_PRIVATE int sqlite3ExprImpliesExpr( ){ return 1; } + if( sqlite3ExprIsIIF(pParse->db, pE1) ){ + return sqlite3ExprImpliesExpr(pParse,pE1->x.pList->a[0].pExpr,pE2,iTab); + } return 0; } @@ -114996,9 +118632,8 @@ static int agginfoPersistExprCb(Walker *pWalker, Expr *pExpr){ && pAggInfo->aCol[iAgg].pCExpr==pExpr ){ pExpr = sqlite3ExprDup(db, pExpr, 0); - if( pExpr ){ + if( pExpr && !sqlite3ExprDeferredDelete(pParse, pExpr) ){ pAggInfo->aCol[iAgg].pCExpr = pExpr; - sqlite3ExprDeferredDelete(pParse, pExpr); } } }else{ @@ -115007,9 +118642,8 @@ static int agginfoPersistExprCb(Walker *pWalker, Expr *pExpr){ && pAggInfo->aFunc[iAgg].pFExpr==pExpr ){ pExpr = sqlite3ExprDup(db, pExpr, 0); - if( pExpr ){ + if( pExpr && !sqlite3ExprDeferredDelete(pParse, pExpr) ){ pAggInfo->aFunc[iAgg].pFExpr = pExpr; - sqlite3ExprDeferredDelete(pParse, pExpr); } } } @@ -115074,7 +118708,9 @@ static void findOrCreateAggInfoColumn( ){ struct AggInfo_col *pCol; int k; + int mxTerm = pParse->db->aLimit[SQLITE_LIMIT_COLUMN]; + assert( mxTerm <= SMXV(i16) ); assert( pAggInfo->iFirstReg==0 ); pCol = pAggInfo->aCol; for(k=0; knColumn; k++, pCol++){ @@ -115092,6 +118728,10 @@ static void findOrCreateAggInfoColumn( assert( pParse->db->mallocFailed ); return; } + if( k>mxTerm ){ + sqlite3ErrorMsg(pParse, "more than %d aggregate terms", mxTerm); + k = mxTerm; + } pCol = &pAggInfo->aCol[k]; assert( ExprUseYTab(pExpr) ); pCol->pTab = pExpr->y.pTab; @@ -115125,6 +118765,7 @@ static void findOrCreateAggInfoColumn( if( pExpr->op==TK_COLUMN ){ pExpr->op = TK_AGG_COLUMN; } + assert( k <= SMXV(pExpr->iAgg) ); pExpr->iAgg = (i16)k; } @@ -115209,13 +118850,19 @@ static int analyzeAggregate(Walker *pWalker, Expr *pExpr){ ** function that is already in the pAggInfo structure */ struct AggInfo_func *pItem = pAggInfo->aFunc; + int mxTerm = pParse->db->aLimit[SQLITE_LIMIT_COLUMN]; + assert( mxTerm <= SMXV(i16) ); for(i=0; inFunc; i++, pItem++){ if( NEVER(pItem->pFExpr==pExpr) ) break; if( sqlite3ExprCompare(0, pItem->pFExpr, pExpr, -1)==0 ){ break; } } - if( i>=pAggInfo->nFunc ){ + if( i>mxTerm ){ + sqlite3ErrorMsg(pParse, "more than %d aggregate terms", mxTerm); + i = mxTerm; + assert( inFunc ); + }else if( i>=pAggInfo->nFunc ){ /* pExpr is original. Make a new entry in pAggInfo->aFunc[] */ u8 enc = ENC(pParse->db); @@ -115269,6 +118916,7 @@ static int analyzeAggregate(Walker *pWalker, Expr *pExpr){ */ assert( !ExprHasProperty(pExpr, EP_TokenOnly|EP_Reduced) ); ExprSetVVAProperty(pExpr, EP_NoReduce); + assert( i <= SMXV(pExpr->iAgg) ); pExpr->iAgg = (i16)i; pExpr->pAggInfo = pAggInfo; return WRC_Prune; @@ -115979,13 +119627,13 @@ SQLITE_PRIVATE void sqlite3AlterBeginAddColumn(Parse *pParse, SrcList *pSrc){ assert( pNew->nCol>0 ); nAlloc = (((pNew->nCol-1)/8)*8)+8; assert( nAlloc>=pNew->nCol && nAlloc%8==0 && nAlloc-pNew->nCol<8 ); - pNew->aCol = (Column*)sqlite3DbMallocZero(db, sizeof(Column)*nAlloc); + pNew->aCol = (Column*)sqlite3DbMallocZero(db, sizeof(Column)*(u32)nAlloc); pNew->zName = sqlite3MPrintf(db, "sqlite_altertab_%s", pTab->zName); if( !pNew->aCol || !pNew->zName ){ assert( db->mallocFailed ); goto exit_begin_add_column; } - memcpy(pNew->aCol, pTab->aCol, sizeof(Column)*pNew->nCol); + memcpy(pNew->aCol, pTab->aCol, sizeof(Column)*(size_t)pNew->nCol); for(i=0; inCol; i++){ Column *pCol = &pNew->aCol[i]; pCol->zCnName = sqlite3DbStrDup(db, pCol->zCnName); @@ -116080,10 +119728,8 @@ SQLITE_PRIVATE void sqlite3AlterRenameColumn( ** altered. Set iCol to be the index of the column being renamed */ zOld = sqlite3NameFromToken(db, pOld); if( !zOld ) goto exit_rename_column; - for(iCol=0; iColnCol; iCol++){ - if( 0==sqlite3StrICmp(pTab->aCol[iCol].zCnName, zOld) ) break; - } - if( iCol==pTab->nCol ){ + iCol = sqlite3ColumnIndex(pTab, zOld); + if( iCol<0 ){ sqlite3ErrorMsg(pParse, "no such column: \"%T\"", pOld); goto exit_rename_column; } @@ -116586,6 +120232,7 @@ static int renameParseSql( int bTemp /* True if SQL is from temp schema */ ){ int rc; + u64 flags; sqlite3ParseObjectInit(p, db); if( zSql==0 ){ @@ -116594,11 +120241,21 @@ static int renameParseSql( if( sqlite3StrNICmp(zSql,"CREATE ",7)!=0 ){ return SQLITE_CORRUPT_BKPT; } - db->init.iDb = bTemp ? 1 : sqlite3FindDbName(db, zDb); + if( bTemp ){ + db->init.iDb = 1; + }else{ + int iDb = sqlite3FindDbName(db, zDb); + assert( iDb>=0 && iDb<=0xff ); + db->init.iDb = (u8)iDb; + } p->eParseMode = PARSE_MODE_RENAME; p->db = db; p->nQueryLoop = 1; + flags = db->flags; + testcase( (db->flags & SQLITE_Comments)==0 && strstr(zSql," /* ")!=0 ); + db->flags |= SQLITE_Comments; rc = sqlite3RunParser(p, zSql); + db->flags = flags; if( db->mallocFailed ) rc = SQLITE_NOMEM; if( rc==SQLITE_OK && NEVER(p->pNewTable==0 && p->pNewIndex==0 && p->pNewTrigger==0) @@ -116661,10 +120318,11 @@ static int renameEditSql( nQuot = sqlite3Strlen30(zQuot)-1; } - assert( nQuot>=nNew ); - zOut = sqlite3DbMallocZero(db, nSql + pRename->nList*nQuot + 1); + assert( nQuot>=nNew && nSql>=0 && nNew>=0 ); + zOut = sqlite3DbMallocZero(db, (u64)nSql + pRename->nList*(u64)nQuot + 1); }else{ - zOut = (char*)sqlite3DbMallocZero(db, (nSql*2+1) * 3); + assert( nSql>0 ); + zOut = (char*)sqlite3DbMallocZero(db, (2*(u64)nSql + 1) * 3); if( zOut ){ zBuf1 = &zOut[nSql*2+1]; zBuf2 = &zOut[nSql*4+2]; @@ -116676,16 +120334,17 @@ static int renameEditSql( ** with the new column name, or with single-quoted versions of themselves. ** All that remains is to construct and return the edited SQL string. */ if( zOut ){ - int nOut = nSql; - memcpy(zOut, zSql, nSql); + i64 nOut = nSql; + assert( nSql>0 ); + memcpy(zOut, zSql, (size_t)nSql); while( pRename->pList ){ int iOff; /* Offset of token to replace in zOut */ - u32 nReplace; + i64 nReplace; const char *zReplace; RenameToken *pBest = renameColumnTokenNext(pRename); if( zNew ){ - if( bQuote==0 && sqlite3IsIdChar(*pBest->t.z) ){ + if( bQuote==0 && sqlite3IsIdChar(*(u8*)pBest->t.z) ){ nReplace = nNew; zReplace = zNew; }else{ @@ -116703,14 +120362,15 @@ static int renameEditSql( memcpy(zBuf1, pBest->t.z, pBest->t.n); zBuf1[pBest->t.n] = 0; sqlite3Dequote(zBuf1); - sqlite3_snprintf(nSql*2, zBuf2, "%Q%s", zBuf1, + assert( nSql < 0x15555554 /* otherwise malloc would have failed */ ); + sqlite3_snprintf((int)(nSql*2), zBuf2, "%Q%s", zBuf1, pBest->t.z[pBest->t.n]=='\'' ? " " : "" ); zReplace = zBuf2; nReplace = sqlite3Strlen30(zReplace); } - iOff = pBest->t.z - zSql; + iOff = (int)(pBest->t.z - zSql); if( pBest->t.n!=nReplace ){ memmove(&zOut[iOff + nReplace], &zOut[iOff + pBest->t.n], nOut - (iOff + pBest->t.n) @@ -116736,11 +120396,12 @@ static int renameEditSql( ** Set all pEList->a[].fg.eEName fields in the expression-list to val. */ static void renameSetENames(ExprList *pEList, int val){ + assert( val==ENAME_NAME || val==ENAME_TAB || val==ENAME_SPAN ); if( pEList ){ int i; for(i=0; inExpr; i++){ assert( val==ENAME_NAME || pEList->a[i].fg.eEName==ENAME_NAME ); - pEList->a[i].fg.eEName = val; + pEList->a[i].fg.eEName = val&0x3; } } } @@ -116768,7 +120429,7 @@ static int renameResolveTrigger(Parse *pParse){ /* ALWAYS() because if the table of the trigger does not exist, the ** error would have been hit before this point */ if( ALWAYS(pParse->pTriggerTab) ){ - rc = sqlite3ViewGetColumnNames(pParse, pParse->pTriggerTab); + rc = sqlite3ViewGetColumnNames(pParse, pParse->pTriggerTab)!=0; } /* Resolve symbols in WHEN clause */ @@ -116814,8 +120475,9 @@ static int renameResolveTrigger(Parse *pParse){ int i; for(i=0; ipFrom->nSrc && rc==SQLITE_OK; i++){ SrcItem *p = &pStep->pFrom->a[i]; - if( p->pSelect ){ - sqlite3SelectPrep(pParse, p->pSelect, 0); + if( p->fg.isSubquery ){ + assert( p->u4.pSubq!=0 ); + sqlite3SelectPrep(pParse, p->u4.pSubq->pSelect, 0); } } } @@ -116883,8 +120545,12 @@ static void renameWalkTrigger(Walker *pWalker, Trigger *pTrigger){ } if( pStep->pFrom ){ int i; - for(i=0; ipFrom->nSrc; i++){ - sqlite3WalkSelect(pWalker, pStep->pFrom->a[i].pSelect); + SrcList *pFrom = pStep->pFrom; + for(i=0; inSrc; i++){ + if( pFrom->a[i].fg.isSubquery ){ + assert( pFrom->a[i].u4.pSubq!=0 ); + sqlite3WalkSelect(pWalker, pFrom->a[i].u4.pSubq->pSelect); + } } } } @@ -116992,7 +120658,7 @@ static void renameColumnFunc( if( sParse.pNewTable ){ if( IsView(sParse.pNewTable) ){ Select *pSelect = sParse.pNewTable->u.view.pSelect; - pSelect->selFlags &= ~SF_View; + pSelect->selFlags &= ~(u32)SF_View; sParse.rc = SQLITE_OK; sqlite3SelectPrep(&sParse, pSelect, 0); rc = (db->mallocFailed ? SQLITE_NOMEM : sParse.rc); @@ -117131,7 +120797,7 @@ static int renameTableSelectCb(Walker *pWalker, Select *pSelect){ } for(i=0; inSrc; i++){ SrcItem *pItem = &pSrc->a[i]; - if( pItem->pTab==p->pTab ){ + if( pItem->pSTab==p->pTab ){ renameTokenFind(pWalker->pParse, p, pItem->zName); } } @@ -117210,7 +120876,7 @@ static void renameTableFunc( sNC.pParse = &sParse; assert( pSelect->selFlags & SF_View ); - pSelect->selFlags &= ~SF_View; + pSelect->selFlags &= ~(u32)SF_View; sqlite3SelectPrep(&sParse, pTab->u.view.pSelect, &sNC); if( sParse.nErr ){ rc = sParse.rc; @@ -117383,7 +121049,7 @@ static void renameQuotefixFunc( if( sParse.pNewTable ){ if( IsView(sParse.pNewTable) ){ Select *pSelect = sParse.pNewTable->u.view.pSelect; - pSelect->selFlags &= ~SF_View; + pSelect->selFlags &= ~(u32)SF_View; sParse.rc = SQLITE_OK; sqlite3SelectPrep(&sParse, pSelect, 0); rc = (db->mallocFailed ? SQLITE_NOMEM : sParse.rc); @@ -117482,10 +121148,10 @@ static void renameTableTest( if( zDb && zInput ){ int rc; Parse sParse; - int flags = db->flags; + u64 flags = db->flags; if( bNoDQS ) db->flags &= ~(SQLITE_DqsDML|SQLITE_DqsDDL); rc = renameParseSql(&sParse, zDb, db, zInput, bTemp); - db->flags |= (flags & (SQLITE_DqsDML|SQLITE_DqsDDL)); + db->flags = flags; if( rc==SQLITE_OK ){ if( isLegacy==0 && sParse.pNewTable && IsView(sParse.pNewTable) ){ NameContext sNC; @@ -117710,7 +121376,12 @@ SQLITE_PRIVATE void sqlite3AlterDropColumn(Parse *pParse, SrcList *pSrc, const T if( i==pTab->iPKey ){ sqlite3VdbeAddOp2(v, OP_Null, 0, regOut); }else{ + char aff = pTab->aCol[i].affinity; + if( aff==SQLITE_AFF_REAL ){ + pTab->aCol[i].affinity = SQLITE_AFF_NUMERIC; + } sqlite3ExprCodeGetColumnOfTable(v, pTab, iCur, i, regOut); + pTab->aCol[i].affinity = aff; } nField++; } @@ -117972,7 +121643,8 @@ static void openStatTable( sqlite3NestedParse(pParse, "CREATE TABLE %Q.%s(%s)", pDb->zDbSName, zTab, aTable[i].zCols ); - aRoot[i] = (u32)pParse->regRoot; + assert( pParse->isCreate || pParse->nErr ); + aRoot[i] = (u32)pParse->u1.cr.regRoot; aCreateTbl[i] = OPFLAG_P2ISREG; } }else{ @@ -118163,7 +121835,7 @@ static void statInit( int nCol; /* Number of columns in index being sampled */ int nKeyCol; /* Number of key columns */ int nColUp; /* nCol rounded up for alignment */ - int n; /* Bytes of space to allocate */ + i64 n; /* Bytes of space to allocate */ sqlite3 *db = sqlite3_context_db_handle(context); /* Database connection */ #ifdef SQLITE_ENABLE_STAT4 /* Maximum number of samples. 0 if STAT4 data is not collected */ @@ -118199,7 +121871,7 @@ static void statInit( p->db = db; p->nEst = sqlite3_value_int64(argv[2]); p->nRow = 0; - p->nLimit = sqlite3_value_int64(argv[3]); + p->nLimit = sqlite3_value_int(argv[3]); p->nCol = nCol; p->nKeyCol = nKeyCol; p->nSkipAhead = 0; @@ -118629,7 +122301,7 @@ static void statGet( if( iVal==2 && p->nRow*10 <= nDistinct*11 ) iVal = 1; sqlite3_str_appendf(&sStat, " %llu", iVal); #ifdef SQLITE_ENABLE_STAT4 - assert( p->current.anEq[i] ); + assert( p->current.anEq[i] || p->nRow==0 ); #endif } sqlite3ResultStrAccum(context, &sStat); @@ -118814,7 +122486,7 @@ static void analyzeOneTable( for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ int nCol; /* Number of columns in pIdx. "N" */ - int addrRewind; /* Address of "OP_Rewind iIdxCur" */ + int addrGotoEnd; /* Address of "OP_Rewind iIdxCur" */ int addrNextRow; /* Address of "next_row:" */ const char *zIdxName; /* Name of the index */ int nColTest; /* Number of columns to test for changes */ @@ -118838,9 +122510,14 @@ static void analyzeOneTable( /* ** Pseudo-code for loop that calls stat_push(): ** - ** Rewind csr - ** if eof(csr) goto end_of_scan; ** regChng = 0 + ** Rewind csr + ** if eof(csr){ + ** stat_init() with count = 0; + ** goto end_of_scan; + ** } + ** count() + ** stat_init() ** goto chng_addr_0; ** ** next_row: @@ -118879,41 +122556,36 @@ static void analyzeOneTable( sqlite3VdbeSetP4KeyInfo(pParse, pIdx); VdbeComment((v, "%s", pIdx->zName)); - /* Invoke the stat_init() function. The arguments are: + /* Implementation of the following: ** + ** regChng = 0 + ** Rewind csr + ** if eof(csr){ + ** stat_init() with count = 0; + ** goto end_of_scan; + ** } + ** count() + ** stat_init() + ** goto chng_addr_0; + */ + assert( regTemp2==regStat+4 ); + sqlite3VdbeAddOp2(v, OP_Integer, db->nAnalysisLimit, regTemp2); + + /* Arguments to stat_init(): ** (1) the number of columns in the index including the rowid ** (or for a WITHOUT ROWID table, the number of PK columns), ** (2) the number of columns in the key without the rowid/pk - ** (3) estimated number of rows in the index, - */ + ** (3) estimated number of rows in the index. */ sqlite3VdbeAddOp2(v, OP_Integer, nCol, regStat+1); assert( regRowid==regStat+2 ); sqlite3VdbeAddOp2(v, OP_Integer, pIdx->nKeyCol, regRowid); -#ifdef SQLITE_ENABLE_STAT4 - if( OptimizationEnabled(db, SQLITE_Stat4) ){ - sqlite3VdbeAddOp2(v, OP_Count, iIdxCur, regTemp); - addrRewind = sqlite3VdbeAddOp1(v, OP_Rewind, iIdxCur); - VdbeCoverage(v); - }else -#endif - { - addrRewind = sqlite3VdbeAddOp1(v, OP_Rewind, iIdxCur); - VdbeCoverage(v); - sqlite3VdbeAddOp3(v, OP_Count, iIdxCur, regTemp, 1); - } - assert( regTemp2==regStat+4 ); - sqlite3VdbeAddOp2(v, OP_Integer, db->nAnalysisLimit, regTemp2); + sqlite3VdbeAddOp3(v, OP_Count, iIdxCur, regTemp, + OptimizationDisabled(db, SQLITE_Stat4)); sqlite3VdbeAddFunctionCall(pParse, 0, regStat+1, regStat, 4, &statInitFuncdef, 0); + addrGotoEnd = sqlite3VdbeAddOp1(v, OP_Rewind, iIdxCur); + VdbeCoverage(v); - /* Implementation of the following: - ** - ** Rewind csr - ** if eof(csr) goto end_of_scan; - ** regChng = 0 - ** goto next_push_0; - ** - */ sqlite3VdbeAddOp2(v, OP_Integer, 0, regChng); addrNextRow = sqlite3VdbeCurrentAddr(v); @@ -119020,6 +122692,12 @@ static void analyzeOneTable( } /* Add the entry to the stat1 table. */ + if( pIdx->pPartIdxWhere ){ + /* Partial indexes might get a zero-entry in sqlite_stat1. But + ** an empty table is omitted from sqlite_stat1. */ + sqlite3VdbeJumpHere(v, addrGotoEnd); + addrGotoEnd = 0; + } callStatGet(pParse, regStat, STAT_GET_STAT1, regStat1); assert( "BBB"[0]==SQLITE_AFF_TEXT ); sqlite3VdbeAddOp4(v, OP_MakeRecord, regTabname, 3, regTemp, "BBB", 0); @@ -119043,6 +122721,13 @@ static void analyzeOneTable( int addrIsNull; u8 seekOp = HasRowid(pTab) ? OP_NotExists : OP_NotFound; + /* No STAT4 data is generated if the number of rows is zero */ + if( addrGotoEnd==0 ){ + sqlite3VdbeAddOp2(v, OP_Cast, regStat1, SQLITE_AFF_INTEGER); + addrGotoEnd = sqlite3VdbeAddOp1(v, OP_IfNot, regStat1); + VdbeCoverage(v); + } + if( doOnce ){ int mxCol = nCol; Index *pX; @@ -119095,7 +122780,7 @@ static void analyzeOneTable( #endif /* SQLITE_ENABLE_STAT4 */ /* End of analysis */ - sqlite3VdbeJumpHere(v, addrRewind); + if( addrGotoEnd ) sqlite3VdbeJumpHere(v, addrGotoEnd); } @@ -119319,16 +123004,6 @@ static void decodeIntArray( while( z[0]!=0 && z[0]!=' ' ) z++; while( z[0]==' ' ) z++; } - - /* Set the bLowQual flag if the peak number of rows obtained - ** from a full equality match is so large that a full table scan - ** seems likely to be faster than using the index. - */ - if( aLog[0] > 66 /* Index has more than 100 rows */ - && aLog[0] <= aLog[nOut-1] /* And only a single value seen */ - ){ - pIndex->bLowQual = 1; - } } } @@ -119541,12 +123216,13 @@ static int loadStatTbl( while( sqlite3_step(pStmt)==SQLITE_ROW ){ int nIdxCol = 1; /* Number of columns in stat4 records */ - char *zIndex; /* Index name */ - Index *pIdx; /* Pointer to the index object */ - int nSample; /* Number of samples */ - int nByte; /* Bytes of space required */ - int i; /* Bytes of space required */ - tRowcnt *pSpace; + char *zIndex; /* Index name */ + Index *pIdx; /* Pointer to the index object */ + int nSample; /* Number of samples */ + i64 nByte; /* Bytes of space required */ + i64 i; /* Bytes of space required */ + tRowcnt *pSpace; /* Available allocated memory space */ + u8 *pPtr; /* Available memory as a u8 for easier manipulation */ zIndex = (char *)sqlite3_column_text(pStmt, 0); if( zIndex==0 ) continue; @@ -119566,7 +123242,7 @@ static int loadStatTbl( } pIdx->nSampleCol = nIdxCol; pIdx->mxSample = nSample; - nByte = sizeof(IndexSample) * nSample; + nByte = ROUND8(sizeof(IndexSample) * nSample); nByte += sizeof(tRowcnt) * nIdxCol * 3 * nSample; nByte += nIdxCol * sizeof(tRowcnt); /* Space for Index.aAvgEq[] */ @@ -119575,7 +123251,10 @@ static int loadStatTbl( sqlite3_finalize(pStmt); return SQLITE_NOMEM_BKPT; } - pSpace = (tRowcnt*)&pIdx->aSample[nSample]; + pPtr = (u8*)pIdx->aSample; + pPtr += ROUND8(nSample*sizeof(pIdx->aSample[0])); + pSpace = (tRowcnt*)pPtr; + assert( EIGHT_BYTE_ALIGNMENT( pSpace ) ); pIdx->aAvgEq = pSpace; pSpace += nIdxCol; pIdx->pTable->tabFlags |= TF_HasStat4; for(i=0; iaDb, sizeof(db->aDb[0])*2); }else{ - aNew = sqlite3DbRealloc(db, db->aDb, sizeof(db->aDb[0])*(db->nDb+1) ); + aNew = sqlite3DbRealloc(db, db->aDb, sizeof(db->aDb[0])*(1+(i64)db->nDb)); if( aNew==0 ) return; } db->aDb = aNew; @@ -119939,6 +123618,12 @@ static void attachFunc( sqlite3_free(zErr); return; } + if( (db->flags & SQLITE_AttachWrite)==0 ){ + flags &= ~(SQLITE_OPEN_CREATE|SQLITE_OPEN_READWRITE); + flags |= SQLITE_OPEN_READONLY; + }else if( (db->flags & SQLITE_AttachCreate)==0 ){ + flags &= ~SQLITE_OPEN_CREATE; + } assert( pVfs ); flags |= SQLITE_OPEN_MAIN_DB; rc = sqlite3BtreeOpen(pVfs, zPath, db, &pNew->pBt, 0, flags); @@ -119985,21 +123670,19 @@ static void attachFunc( sqlite3BtreeEnterAll(db); db->init.iDb = 0; db->mDbFlags &= ~(DBFLAG_SchemaKnownOk); +#ifdef SQLITE_ENABLE_SETLK_TIMEOUT + if( db->setlkFlags & SQLITE_SETLK_BLOCK_ON_CONNECT ){ + int val = 1; + sqlite3_file *fd = sqlite3PagerFile(sqlite3BtreePager(pNew->pBt)); + sqlite3OsFileControlHint(fd, SQLITE_FCNTL_BLOCK_ON_CONNECT, &val); + } +#endif if( !REOPEN_AS_MEMDB(db) ){ rc = sqlite3Init(db, &zErrDyn); } sqlite3BtreeLeaveAll(db); assert( zErrDyn==0 || rc!=SQLITE_OK ); } -#ifdef SQLITE_USER_AUTHENTICATION - if( rc==SQLITE_OK && !REOPEN_AS_MEMDB(db) ){ - u8 newAuth = 0; - rc = sqlite3UserAuthCheckLogin(db, zName, &newAuth); - if( newAuthauth.authLevel ){ - rc = SQLITE_AUTH_USER; - } - } -#endif if( rc ){ if( ALWAYS(!REOPEN_AS_MEMDB(db)) ){ int iDb = db->nDb - 1; @@ -120243,20 +123926,21 @@ static int fixSelectCb(Walker *p, Select *pSelect){ if( NEVER(pList==0) ) return WRC_Continue; for(i=0, pItem=pList->a; inSrc; i++, pItem++){ - if( pFix->bTemp==0 ){ - if( pItem->zDatabase ){ - if( iDb!=sqlite3FindDbName(db, pItem->zDatabase) ){ + if( pFix->bTemp==0 && pItem->fg.isSubquery==0 ){ + if( pItem->fg.fixedSchema==0 && pItem->u4.zDatabase!=0 ){ + if( iDb!=sqlite3FindDbName(db, pItem->u4.zDatabase) ){ sqlite3ErrorMsg(pFix->pParse, "%s %T cannot reference objects in database %s", - pFix->zType, pFix->pName, pItem->zDatabase); + pFix->zType, pFix->pName, pItem->u4.zDatabase); return WRC_Abort; } - sqlite3DbFree(db, pItem->zDatabase); - pItem->zDatabase = 0; + sqlite3DbFree(db, pItem->u4.zDatabase); pItem->fg.notCte = 1; + pItem->fg.hadSchema = 1; } - pItem->pSchema = pFix->pSchema; + pItem->u4.pSchema = pFix->pSchema; pItem->fg.fromDDL = 1; + pItem->fg.fixedSchema = 1; } #if !defined(SQLITE_OMIT_VIEW) || !defined(SQLITE_OMIT_TRIGGER) if( pList->a[i].fg.isUsing==0 @@ -120496,11 +124180,7 @@ SQLITE_PRIVATE int sqlite3AuthReadCol( int rc; /* Auth callback return code */ if( db->init.busy ) return SQLITE_OK; - rc = db->xAuth(db->pAuthArg, SQLITE_READ, zTab,zCol,zDb,pParse->zAuthContext -#ifdef SQLITE_USER_AUTHENTICATION - ,db->auth.zAuthUser -#endif - ); + rc = db->xAuth(db->pAuthArg, SQLITE_READ, zTab,zCol,zDb,pParse->zAuthContext); if( rc==SQLITE_DENY ){ char *z = sqlite3_mprintf("%s.%s", zTab, zCol); if( db->nDb>2 || iDb!=0 ) z = sqlite3_mprintf("%s.%z", zDb, z); @@ -120549,7 +124229,7 @@ SQLITE_PRIVATE void sqlite3AuthRead( assert( pTabList ); for(iSrc=0; iSrcnSrc; iSrc++){ if( pExpr->iTable==pTabList->a[iSrc].iCursor ){ - pTab = pTabList->a[iSrc].pTab; + pTab = pTabList->a[iSrc].pSTab; break; } } @@ -120607,11 +124287,7 @@ SQLITE_PRIVATE int sqlite3AuthCheck( testcase( zArg3==0 ); testcase( pParse->zAuthContext==0 ); - rc = db->xAuth(db->pAuthArg, code, zArg1, zArg2, zArg3, pParse->zAuthContext -#ifdef SQLITE_USER_AUTHENTICATION - ,db->auth.zAuthUser -#endif - ); + rc = db->xAuth(db->pAuthArg,code,zArg1,zArg2,zArg3,pParse->zAuthContext); if( rc==SQLITE_DENY ){ sqlite3ErrorMsg(pParse, "not authorized"); pParse->rc = SQLITE_AUTH; @@ -120723,6 +124399,7 @@ static SQLITE_NOINLINE void lockTable( } } + assert( pToplevel->nTableLock < 0x7fff0000 ); nBytes = sizeof(TableLock) * (pToplevel->nTableLock+1); pToplevel->aTableLock = sqlite3DbReallocOrFree(pToplevel->db, pToplevel->aTableLock, nBytes); @@ -120823,10 +124500,12 @@ SQLITE_PRIVATE void sqlite3FinishCoding(Parse *pParse){ || sqlite3VdbeAssertMayAbort(v, pParse->mayAbort)); if( v ){ if( pParse->bReturning ){ - Returning *pReturning = pParse->u1.pReturning; + Returning *pReturning; int addrRewind; int reg; + assert( !pParse->isCreate ); + pReturning = pParse->u1.d.pReturning; if( pReturning->nRetCol ){ sqlite3VdbeAddOp0(v, OP_FkCheck); addrRewind = @@ -120844,17 +124523,6 @@ SQLITE_PRIVATE void sqlite3FinishCoding(Parse *pParse){ } sqlite3VdbeAddOp0(v, OP_Halt); -#if SQLITE_USER_AUTHENTICATION - if( pParse->nTableLock>0 && db->init.busy==0 ){ - sqlite3UserAuthInit(db); - if( db->auth.authLevelrc = SQLITE_AUTH_USER; - return; - } - } -#endif - /* The cookie mask contains one bit for each database file open. ** (Bit 0 is for main, bit 1 is for temp, and so forth.) Bits are ** set for each database that is used. Generate code to start a @@ -120913,7 +124581,9 @@ SQLITE_PRIVATE void sqlite3FinishCoding(Parse *pParse){ } if( pParse->bReturning ){ - Returning *pRet = pParse->u1.pReturning; + Returning *pRet; + assert( !pParse->isCreate ); + pRet = pParse->u1.d.pReturning; if( pRet->nRetCol ){ sqlite3VdbeAddOp2(v, OP_OpenEphemeral, pRet->iRetCur, pRet->nRetCol); } @@ -120983,16 +124653,6 @@ SQLITE_PRIVATE void sqlite3NestedParse(Parse *pParse, const char *zFormat, ...){ pParse->nested--; } -#if SQLITE_USER_AUTHENTICATION -/* -** Return TRUE if zTable is the name of the system table that stores the -** list of users and their access credentials. -*/ -SQLITE_PRIVATE int sqlite3UserAuthTable(const char *zTable){ - return sqlite3_stricmp(zTable, "sqlite_user")==0; -} -#endif - /* ** Locate the in-memory structure that describes a particular database ** table given the name of that table and (optionally) the name of the @@ -121011,13 +124671,6 @@ SQLITE_PRIVATE Table *sqlite3FindTable(sqlite3 *db, const char *zName, const cha /* All mutexes are required for schema access. Make sure we hold them. */ assert( zDatabase!=0 || sqlite3BtreeHoldsAllMutexes(db) ); -#if SQLITE_USER_AUTHENTICATION - /* Only the admin user is allowed to know that the sqlite_user table - ** exists */ - if( db->auth.authLevelnDb; i++){ if( sqlite3StrICmp(zDatabase, db->aDb[i].zDbSName)==0 ) break; @@ -121111,6 +124764,16 @@ SQLITE_PRIVATE Table *sqlite3LocateTable( if( pMod==0 && sqlite3_strnicmp(zName, "pragma_", 7)==0 ){ pMod = sqlite3PragmaVtabRegister(db, zName); } +#ifndef SQLITE_OMIT_JSON + if( pMod==0 && sqlite3_strnicmp(zName, "json", 4)==0 ){ + pMod = sqlite3JsonVtabRegister(db, zName); + } +#endif +#ifdef SQLITE_ENABLE_CARRAY + if( pMod==0 && sqlite3_stricmp(zName, "carray")==0 ){ + pMod = sqlite3CarrayRegister(db); + } +#endif if( pMod && sqlite3VtabEponymousTableInit(pParse, pMod) ){ testcase( pMod->pEpoTab==0 ); return pMod->pEpoTab; @@ -121152,12 +124815,12 @@ SQLITE_PRIVATE Table *sqlite3LocateTableItem( SrcItem *p ){ const char *zDb; - assert( p->pSchema==0 || p->zDatabase==0 ); - if( p->pSchema ){ - int iDb = sqlite3SchemaToIndex(pParse->db, p->pSchema); + if( p->fg.fixedSchema ){ + int iDb = sqlite3SchemaToIndex(pParse->db, p->u4.pSchema); zDb = pParse->db->aDb[iDb].zDbSName; }else{ - zDb = p->zDatabase; + assert( !p->fg.isSubquery ); + zDb = p->u4.zDatabase; } return sqlite3LocateTable(pParse, flags, p->zName, zDb); } @@ -121745,10 +125408,16 @@ SQLITE_PRIVATE Index *sqlite3PrimaryKeyIndex(Table *pTab){ ** find the (first) offset of that column in index pIdx. Or return -1 ** if column iCol is not used in index pIdx. */ -SQLITE_PRIVATE i16 sqlite3TableColumnToIndex(Index *pIdx, i16 iCol){ +SQLITE_PRIVATE int sqlite3TableColumnToIndex(Index *pIdx, int iCol){ int i; + i16 iCol16; + assert( iCol>=(-1) && iCol<=SQLITE_MAX_COLUMN ); + assert( pIdx->nColumn<=SQLITE_MAX_COLUMN*2 ); + iCol16 = iCol; for(i=0; inColumn; i++){ - if( iCol==pIdx->aiColumn[i] ) return i; + if( iCol16==pIdx->aiColumn[i] ){ + return i; + } } return -1; } @@ -122002,8 +125671,9 @@ SQLITE_PRIVATE void sqlite3StartTable( /* If the file format and encoding in the database have not been set, ** set them now. */ - reg1 = pParse->regRowid = ++pParse->nMem; - reg2 = pParse->regRoot = ++pParse->nMem; + assert( pParse->isCreate ); + reg1 = pParse->u1.cr.regRowid = ++pParse->nMem; + reg2 = pParse->u1.cr.regRoot = ++pParse->nMem; reg3 = ++pParse->nMem; sqlite3VdbeAddOp3(v, OP_ReadCookie, iDb, reg3, BTREE_FILE_FORMAT); sqlite3VdbeUsesBtree(v, iDb); @@ -122018,8 +125688,8 @@ SQLITE_PRIVATE void sqlite3StartTable( ** The record created does not contain anything yet. It will be replaced ** by the real entry in code generated at sqlite3EndTable(). ** - ** The rowid for the new entry is left in register pParse->regRowid. - ** The root page number of the new table is left in reg pParse->regRoot. + ** The rowid for the new entry is left in register pParse->u1.cr.regRowid. + ** The root page of the new table is left in reg pParse->u1.cr.regRoot. ** The rowid and root page number values are needed by the code that ** sqlite3EndTable will generate. */ @@ -122030,7 +125700,7 @@ SQLITE_PRIVATE void sqlite3StartTable( #endif { assert( !pParse->bReturning ); - pParse->u1.addrCrTab = + pParse->u1.cr.addrCrTab = sqlite3VdbeAddOp3(v, OP_CreateBtree, iDb, reg2, BTREE_INTKEY); } sqlite3OpenSchemaTable(pParse, iDb); @@ -122039,6 +125709,9 @@ SQLITE_PRIVATE void sqlite3StartTable( sqlite3VdbeAddOp3(v, OP_Insert, 0, reg3, reg1); sqlite3VdbeChangeP5(v, OPFLAG_APPEND); sqlite3VdbeAddOp0(v, OP_Close); + }else if( db->init.imposterTable ){ + pTable->tabFlags |= TF_Imposter; + if( db->init.imposterTable>=2 ) pTable->tabFlags |= TF_Readonly; } /* Normal (non-error) return. */ @@ -122108,7 +125781,8 @@ SQLITE_PRIVATE void sqlite3AddReturning(Parse *pParse, ExprList *pList){ sqlite3ExprListDelete(db, pList); return; } - pParse->u1.pReturning = pRet; + assert( !pParse->isCreate ); + pParse->u1.d.pReturning = pRet; pRet->pParse = pParse; pRet->pReturnEL = pList; sqlite3ParserAddCleanup(pParse, sqlite3DeleteReturning, pRet); @@ -122150,7 +125824,6 @@ SQLITE_PRIVATE void sqlite3AddColumn(Parse *pParse, Token sName, Token sType){ char *zType; Column *pCol; sqlite3 *db = pParse->db; - u8 hName; Column *aNew; u8 eType = COLTYPE_CUSTOM; u8 szEst = 1; @@ -122204,13 +125877,10 @@ SQLITE_PRIVATE void sqlite3AddColumn(Parse *pParse, Token sName, Token sType){ memcpy(z, sName.z, sName.n); z[sName.n] = 0; sqlite3Dequote(z); - hName = sqlite3StrIHash(z); - for(i=0; inCol; i++){ - if( p->aCol[i].hName==hName && sqlite3StrICmp(z, p->aCol[i].zCnName)==0 ){ - sqlite3ErrorMsg(pParse, "duplicate column name: %s", z); - sqlite3DbFree(db, z); - return; - } + if( p->nCol && sqlite3ColumnIndex(p, z)>=0 ){ + sqlite3ErrorMsg(pParse, "duplicate column name: %s", z); + sqlite3DbFree(db, z); + return; } aNew = sqlite3DbRealloc(db,p->aCol,((i64)p->nCol+1)*sizeof(p->aCol[0])); if( aNew==0 ){ @@ -122221,7 +125891,7 @@ SQLITE_PRIVATE void sqlite3AddColumn(Parse *pParse, Token sName, Token sType){ pCol = &p->aCol[p->nCol]; memset(pCol, 0, sizeof(p->aCol[0])); pCol->zCnName = z; - pCol->hName = hName; + pCol->hName = sqlite3StrIHash(z); sqlite3ColumnPropertiesFromName(p, pCol); if( sType.n==0 ){ @@ -122245,9 +125915,14 @@ SQLITE_PRIVATE void sqlite3AddColumn(Parse *pParse, Token sName, Token sType){ pCol->affinity = sqlite3AffinityType(zType, pCol); pCol->colFlags |= COLFLAG_HASTYPE; } + if( p->nCol<=0xff ){ + u8 h = pCol->hName % sizeof(p->aHx); + p->aHx[h] = p->nCol; + } p->nCol++; p->nNVCol++; - pParse->constraintName.n = 0; + assert( pParse->isCreate ); + pParse->u1.cr.constraintName.n = 0; } /* @@ -122511,15 +126186,11 @@ SQLITE_PRIVATE void sqlite3AddPrimaryKey( assert( pCExpr!=0 ); sqlite3StringToId(pCExpr); if( pCExpr->op==TK_ID ){ - const char *zCName; assert( !ExprHasProperty(pCExpr, EP_IntValue) ); - zCName = pCExpr->u.zToken; - for(iCol=0; iColnCol; iCol++){ - if( sqlite3StrICmp(zCName, pTab->aCol[iCol].zCnName)==0 ){ - pCol = &pTab->aCol[iCol]; - makeColumnPartOfPrimaryKey(pParse, pCol); - break; - } + iCol = sqlite3ColumnIndex(pTab, pCExpr->u.zToken); + if( iCol>=0 ){ + pCol = &pTab->aCol[iCol]; + makeColumnPartOfPrimaryKey(pParse, pCol); } } } @@ -122571,8 +126242,10 @@ SQLITE_PRIVATE void sqlite3AddCheckConstraint( && !sqlite3BtreeIsReadonly(db->aDb[db->init.iDb].pBt) ){ pTab->pCheck = sqlite3ExprListAppend(pParse, pTab->pCheck, pCheckExpr); - if( pParse->constraintName.n ){ - sqlite3ExprListSetName(pParse, pTab->pCheck, &pParse->constraintName, 1); + assert( pParse->isCreate ); + if( pParse->u1.cr.constraintName.n ){ + sqlite3ExprListSetName(pParse, pTab->pCheck, + &pParse->u1.cr.constraintName, 1); }else{ Token t; for(zStart++; sqlite3Isspace(zStart[0]); zStart++){} @@ -122767,7 +126440,8 @@ static void identPut(char *z, int *pIdx, char *zSignedIdent){ ** from sqliteMalloc() and must be freed by the calling function. */ static char *createTableStmt(sqlite3 *db, Table *p){ - int i, k, n; + int i, k, len; + i64 n; char *zStmt; char *zSep, *zSep2, *zEnd; Column *pCol; @@ -122791,8 +126465,9 @@ static char *createTableStmt(sqlite3 *db, Table *p){ sqlite3OomFault(db); return 0; } - sqlite3_snprintf(n, zStmt, "CREATE TABLE "); - k = sqlite3Strlen30(zStmt); + assert( n>14 && n<=0x7fffffff ); + memcpy(zStmt, "CREATE TABLE ", 13); + k = 13; identPut(zStmt, &k, p->zName); zStmt[k++] = '('; for(pCol=p->aCol, i=0; inCol; i++, pCol++){ @@ -122804,13 +126479,15 @@ static char *createTableStmt(sqlite3 *db, Table *p){ /* SQLITE_AFF_REAL */ " REAL", /* SQLITE_AFF_FLEXNUM */ " NUM", }; - int len; const char *zType; - sqlite3_snprintf(n-k, &zStmt[k], zSep); - k += sqlite3Strlen30(&zStmt[k]); + len = sqlite3Strlen30(zSep); + assert( k+lenzCnName); + assert( kaffinity-SQLITE_AFF_BLOB >= 0 ); assert( pCol->affinity-SQLITE_AFF_BLOB < ArraySize(azType) ); testcase( pCol->affinity==SQLITE_AFF_BLOB ); @@ -122825,11 +126502,14 @@ static char *createTableStmt(sqlite3 *db, Table *p){ assert( pCol->affinity==SQLITE_AFF_BLOB || pCol->affinity==SQLITE_AFF_FLEXNUM || pCol->affinity==sqlite3AffinityType(zType, 0) ); + assert( k+lennColumn>=N ) return SQLITE_OK; + db = pParse->db; + assert( N>0 ); + assert( N <= SQLITE_MAX_COLUMN*2 /* tag-20250221-1 */ ); + testcase( N==2*pParse->db->aLimit[SQLITE_LIMIT_COLUMN] ); assert( pIdx->isResized==0 ); - nByte = (sizeof(char*) + sizeof(LogEst) + sizeof(i16) + 1)*N; + nByte = (sizeof(char*) + sizeof(LogEst) + sizeof(i16) + 1)*(u64)N; zExtra = sqlite3DbMallocZero(db, nByte); if( zExtra==0 ) return SQLITE_NOMEM_BKPT; memcpy(zExtra, pIdx->azColl, sizeof(char*)*pIdx->nColumn); @@ -122856,7 +126541,7 @@ static int resizeIndexObject(sqlite3 *db, Index *pIdx, int N){ zExtra += sizeof(i16)*N; memcpy(zExtra, pIdx->aSortOrder, pIdx->nColumn); pIdx->aSortOrder = (u8*)zExtra; - pIdx->nColumn = N; + pIdx->nColumn = (u16)N; /* See tag-20250221-1 above for proof of safety */ pIdx->isResized = 1; return SQLITE_OK; } @@ -123022,9 +126707,9 @@ static void convertToWithoutRowidTable(Parse *pParse, Table *pTab){ ** into BTREE_BLOBKEY. */ assert( !pParse->bReturning ); - if( pParse->u1.addrCrTab ){ + if( pParse->u1.cr.addrCrTab ){ assert( v ); - sqlite3VdbeChangeP3(v, pParse->u1.addrCrTab, BTREE_BLOBKEY); + sqlite3VdbeChangeP3(v, pParse->u1.cr.addrCrTab, BTREE_BLOBKEY); } /* Locate the PRIMARY KEY index. Or, if this table was originally @@ -123110,14 +126795,14 @@ static void convertToWithoutRowidTable(Parse *pParse, Table *pTab){ pIdx->nColumn = pIdx->nKeyCol; continue; } - if( resizeIndexObject(db, pIdx, pIdx->nKeyCol+n) ) return; + if( resizeIndexObject(pParse, pIdx, pIdx->nKeyCol+n) ) return; for(i=0, j=pIdx->nKeyCol; inKeyCol, pPk, i) ){ testcase( hasColumn(pIdx->aiColumn, pIdx->nKeyCol, pPk->aiColumn[i]) ); pIdx->aiColumn[j] = pPk->aiColumn[i]; pIdx->azColl[j] = pPk->azColl[i]; if( pPk->aSortOrder[i] ){ - /* See ticket https://www.sqlite.org/src/info/bba7b69f9849b5bf */ + /* See ticket https://sqlite.org/src/info/bba7b69f9849b5bf */ pIdx->bAscKeyBug = 1; } j++; @@ -123134,7 +126819,7 @@ static void convertToWithoutRowidTable(Parse *pParse, Table *pTab){ if( !hasColumn(pPk->aiColumn, nPk, i) && (pTab->aCol[i].colFlags & COLFLAG_VIRTUAL)==0 ) nExtra++; } - if( resizeIndexObject(db, pPk, nPk+nExtra) ) return; + if( resizeIndexObject(pParse, pPk, nPk+nExtra) ) return; for(i=0, j=nPk; inCol; i++){ if( !hasColumn(pPk->aiColumn, j, i) && (pTab->aCol[i].colFlags & COLFLAG_VIRTUAL)==0 @@ -123464,7 +127149,7 @@ SQLITE_PRIVATE void sqlite3EndTable( /* If this is a CREATE TABLE xx AS SELECT ..., execute the SELECT ** statement to populate the new table. The root-page number for the - ** new table is in register pParse->regRoot. + ** new table is in register pParse->u1.cr.regRoot. ** ** Once the SELECT has been coded by sqlite3Select(), it is in a ** suitable state to query for the column names and types to be used @@ -123483,20 +127168,21 @@ SQLITE_PRIVATE void sqlite3EndTable( int regRowid; /* Rowid of the next row to insert */ int addrInsLoop; /* Top of the loop for inserting rows */ Table *pSelTab; /* A table that describes the SELECT results */ + int iCsr; /* Write cursor on the new table */ if( IN_SPECIAL_PARSE ){ pParse->rc = SQLITE_ERROR; pParse->nErr++; return; } + iCsr = pParse->nTab++; regYield = ++pParse->nMem; regRec = ++pParse->nMem; regRowid = ++pParse->nMem; - assert(pParse->nTab==1); sqlite3MayAbort(pParse); - sqlite3VdbeAddOp3(v, OP_OpenWrite, 1, pParse->regRoot, iDb); + assert( pParse->isCreate ); + sqlite3VdbeAddOp3(v, OP_OpenWrite, iCsr, pParse->u1.cr.regRoot, iDb); sqlite3VdbeChangeP5(v, OPFLAG_P2ISREG); - pParse->nTab = 2; addrTop = sqlite3VdbeCurrentAddr(v) + 1; sqlite3VdbeAddOp3(v, OP_InitCoroutine, regYield, 0, addrTop); if( pParse->nErr ) return; @@ -123517,11 +127203,11 @@ SQLITE_PRIVATE void sqlite3EndTable( VdbeCoverage(v); sqlite3VdbeAddOp3(v, OP_MakeRecord, dest.iSdst, dest.nSdst, regRec); sqlite3TableAffinity(v, p, 0); - sqlite3VdbeAddOp2(v, OP_NewRowid, 1, regRowid); - sqlite3VdbeAddOp3(v, OP_Insert, 1, regRec, regRowid); + sqlite3VdbeAddOp2(v, OP_NewRowid, iCsr, regRowid); + sqlite3VdbeAddOp3(v, OP_Insert, iCsr, regRec, regRowid); sqlite3VdbeGoto(v, addrInsLoop); sqlite3VdbeJumpHere(v, addrInsLoop); - sqlite3VdbeAddOp1(v, OP_Close, 1); + sqlite3VdbeAddOp1(v, OP_Close, iCsr); } /* Compute the complete text of the CREATE statement */ @@ -123540,6 +127226,7 @@ SQLITE_PRIVATE void sqlite3EndTable( ** schema table. We just need to update that slot with all ** the information we've collected. */ + assert( pParse->isCreate ); sqlite3NestedParse(pParse, "UPDATE %Q." LEGACY_SCHEMA_TABLE " SET type='%s', name=%Q, tbl_name=%Q, rootpage=#%d, sql=%Q" @@ -123548,9 +127235,9 @@ SQLITE_PRIVATE void sqlite3EndTable( zType, p->zName, p->zName, - pParse->regRoot, + pParse->u1.cr.regRoot, zStmt, - pParse->regRowid + pParse->u1.cr.regRowid ); sqlite3DbFree(db, zStmt); sqlite3ChangeCookie(pParse, iDb); @@ -123578,13 +127265,10 @@ SQLITE_PRIVATE void sqlite3EndTable( /* Test for cycles in generated columns and illegal expressions ** in CHECK constraints and in DEFAULT clauses. */ if( p->tabFlags & TF_HasGenerated ){ - sqlite3VdbeAddOp4(v, OP_SqlExec, 1, 0, 0, + sqlite3VdbeAddOp4(v, OP_SqlExec, 0x0001, 0, 0, sqlite3MPrintf(db, "SELECT*FROM\"%w\".\"%w\"", db->aDb[iDb].zDbSName, p->zName), P4_DYNAMIC); } - sqlite3VdbeAddOp4(v, OP_SqlExec, 1, 0, 0, - sqlite3MPrintf(db, "PRAGMA \"%w\".integrity_check(%Q)", - db->aDb[iDb].zDbSName, p->zName), P4_DYNAMIC); } /* Add the table to the in-memory representation of the database. @@ -123661,9 +127345,12 @@ SQLITE_PRIVATE void sqlite3CreateView( ** on a view, even though views do not have rowids. The following flag ** setting fixes this problem. But the fix can be disabled by compiling ** with -DSQLITE_ALLOW_ROWID_IN_VIEW in case there are legacy apps that - ** depend upon the old buggy behavior. */ -#ifndef SQLITE_ALLOW_ROWID_IN_VIEW - p->tabFlags |= TF_NoVisibleRowid; + ** depend upon the old buggy behavior. The ability can also be toggled + ** using sqlite3_config(SQLITE_CONFIG_ROWID_IN_VIEW,...) */ +#ifdef SQLITE_ALLOW_ROWID_IN_VIEW + p->tabFlags |= sqlite3Config.mNoVisibleRowid; /* Optional. Allow by default */ +#else + p->tabFlags |= TF_NoVisibleRowid; /* Never allow rowid in view */ #endif sqlite3TwoPartName(pParse, pName1, pName2, &pName); @@ -123719,8 +127406,9 @@ SQLITE_PRIVATE void sqlite3CreateView( #if !defined(SQLITE_OMIT_VIEW) || !defined(SQLITE_OMIT_VIRTUALTABLE) /* ** The Table structure pTable is really a VIEW. Fill in the names of -** the columns of the view in the pTable structure. Return the number -** of errors. If an error is seen leave an error message in pParse->zErrMsg. +** the columns of the view in the pTable structure. Return non-zero if +** there are errors. If an error is seen an error message is left +** in pParse->zErrMsg. */ static SQLITE_NOINLINE int viewGetColumnNames(Parse *pParse, Table *pTable){ Table *pSelTab; /* A fake table from which we get the result set */ @@ -123843,7 +127531,7 @@ static SQLITE_NOINLINE int viewGetColumnNames(Parse *pParse, Table *pTable){ sqlite3DeleteColumnNames(db, pTable); } #endif /* SQLITE_OMIT_VIEW */ - return nErr; + return nErr + pParse->nErr; } SQLITE_PRIVATE int sqlite3ViewGetColumnNames(Parse *pParse, Table *pTable){ assert( pTable!=0 ); @@ -124141,6 +127829,8 @@ SQLITE_PRIVATE void sqlite3DropTable(Parse *pParse, SrcList *pName, int isView, } assert( pParse->nErr==0 ); assert( pName->nSrc==1 ); + assert( pName->a[0].fg.fixedSchema==0 ); + assert( pName->a[0].fg.isSubquery==0 ); if( sqlite3ReadSchema(pParse) ) goto exit_drop_table; if( noErr ) db->suppressErr++; assert( isView==0 || isView==LOCATE_VIEW ); @@ -124149,7 +127839,7 @@ SQLITE_PRIVATE void sqlite3DropTable(Parse *pParse, SrcList *pName, int isView, if( pTab==0 ){ if( noErr ){ - sqlite3CodeVerifyNamedSchema(pParse, pName->a[0].zDatabase); + sqlite3CodeVerifyNamedSchema(pParse, pName->a[0].u4.zDatabase); sqlite3ForceNotReadOnly(pParse); } goto exit_drop_table; @@ -124287,7 +127977,7 @@ SQLITE_PRIVATE void sqlite3CreateForeignKey( }else{ nCol = pFromCol->nExpr; } - nByte = sizeof(*pFKey) + (nCol-1)*sizeof(pFKey->aCol[0]) + pTo->n + 1; + nByte = SZ_FKEY(nCol) + pTo->n + 1; if( pToCol ){ for(i=0; inExpr; i++){ nByte += sqlite3Strlen30(pToCol->a[i].zEName) + 1; @@ -124489,7 +128179,7 @@ static void sqlite3RefillIndex(Parse *pParse, Index *pIndex, int memRootPage){ ** not work for UNIQUE constraint indexes on WITHOUT ROWID tables ** with DESC primary keys, since those indexes have there keys in ** a different order from the main table. - ** See ticket: https://www.sqlite.org/src/info/bba7b69f9849b5bf + ** See ticket: https://sqlite.org/src/info/bba7b69f9849b5bf */ sqlite3VdbeAddOp1(v, OP_SeekEnd, iIdx); } @@ -124513,13 +128203,14 @@ static void sqlite3RefillIndex(Parse *pParse, Index *pIndex, int memRootPage){ */ SQLITE_PRIVATE Index *sqlite3AllocateIndexObject( sqlite3 *db, /* Database connection */ - i16 nCol, /* Total number of columns in the index */ + int nCol, /* Total number of columns in the index */ int nExtra, /* Number of bytes of extra space to alloc */ char **ppExtra /* Pointer to the "extra" space */ ){ Index *p; /* Allocated index object */ - int nByte; /* Bytes of space for Index object + arrays */ + i64 nByte; /* Bytes of space for Index object + arrays */ + assert( nCol <= 2*db->aLimit[SQLITE_LIMIT_COLUMN] ); nByte = ROUND8(sizeof(Index)) + /* Index structure */ ROUND8(sizeof(char*)*nCol) + /* Index.azColl */ ROUND8(sizeof(LogEst)*(nCol+1) + /* Index.aiRowLogEst */ @@ -124532,8 +128223,9 @@ SQLITE_PRIVATE Index *sqlite3AllocateIndexObject( p->aiRowLogEst = (LogEst*)pExtra; pExtra += sizeof(LogEst)*(nCol+1); p->aiColumn = (i16*)pExtra; pExtra += sizeof(i16)*nCol; p->aSortOrder = (u8*)pExtra; - p->nColumn = nCol; - p->nKeyCol = nCol - 1; + assert( nCol>0 ); + p->nColumn = (u16)nCol; + p->nKeyCol = (u16)(nCol - 1); *ppExtra = ((char*)p) + nByte; } return p; @@ -124673,9 +128365,6 @@ SQLITE_PRIVATE void sqlite3CreateIndex( if( sqlite3StrNICmp(pTab->zName, "sqlite_", 7)==0 && db->init.busy==0 && pTblName!=0 -#if SQLITE_USER_AUTHENTICATION - && sqlite3UserAuthTable(pTab->zName)==0 -#endif ){ sqlite3ErrorMsg(pParse, "table %s may not be indexed", pTab->zName); goto exit_create_index; @@ -125240,15 +128929,17 @@ SQLITE_PRIVATE void sqlite3DropIndex(Parse *pParse, SrcList *pName, int ifExists } assert( pParse->nErr==0 ); /* Never called with prior non-OOM errors */ assert( pName->nSrc==1 ); + assert( pName->a[0].fg.fixedSchema==0 ); + assert( pName->a[0].fg.isSubquery==0 ); if( SQLITE_OK!=sqlite3ReadSchema(pParse) ){ goto exit_drop_index; } - pIndex = sqlite3FindIndex(db, pName->a[0].zName, pName->a[0].zDatabase); + pIndex = sqlite3FindIndex(db, pName->a[0].zName, pName->a[0].u4.zDatabase); if( pIndex==0 ){ if( !ifExists ){ sqlite3ErrorMsg(pParse, "no such index: %S", pName->a); }else{ - sqlite3CodeVerifyNamedSchema(pParse, pName->a[0].zDatabase); + sqlite3CodeVerifyNamedSchema(pParse, pName->a[0].u4.zDatabase); sqlite3ForceNotReadOnly(pParse); } pParse->checkSchema = 1; @@ -125345,12 +129036,11 @@ SQLITE_PRIVATE IdList *sqlite3IdListAppend(Parse *pParse, IdList *pList, Token * sqlite3 *db = pParse->db; int i; if( pList==0 ){ - pList = sqlite3DbMallocZero(db, sizeof(IdList) ); + pList = sqlite3DbMallocZero(db, SZ_IDLIST(1)); if( pList==0 ) return 0; }else{ IdList *pNew; - pNew = sqlite3DbRealloc(db, pList, - sizeof(IdList) + pList->nId*sizeof(pList->a)); + pNew = sqlite3DbRealloc(db, pList, SZ_IDLIST(pList->nId+1)); if( pNew==0 ){ sqlite3IdListDelete(db, pList); return 0; @@ -125372,7 +129062,6 @@ SQLITE_PRIVATE void sqlite3IdListDelete(sqlite3 *db, IdList *pList){ int i; assert( db!=0 ); if( pList==0 ) return; - assert( pList->eU4!=EU4_EXPR ); /* EU4_EXPR mode is not currently used */ for(i=0; inId; i++){ sqlite3DbFree(db, pList->a[i].zName); } @@ -125450,8 +129139,7 @@ SQLITE_PRIVATE SrcList *sqlite3SrcListEnlarge( return 0; } if( nAlloc>SQLITE_MAX_SRCLIST ) nAlloc = SQLITE_MAX_SRCLIST; - pNew = sqlite3DbRealloc(db, pSrc, - sizeof(*pSrc) + (nAlloc-1)*sizeof(pSrc->a[0]) ); + pNew = sqlite3DbRealloc(db, pSrc, SZ_SRCLIST(nAlloc)); if( pNew==0 ){ assert( db->mallocFailed ); return 0; @@ -125526,7 +129214,7 @@ SQLITE_PRIVATE SrcList *sqlite3SrcListAppend( assert( pParse->db!=0 ); db = pParse->db; if( pList==0 ){ - pList = sqlite3DbMallocRawNN(pParse->db, sizeof(SrcList) ); + pList = sqlite3DbMallocRawNN(pParse->db, SZ_SRCLIST(1)); if( pList==0 ) return 0; pList->nAlloc = 1; pList->nSrc = 1; @@ -125545,12 +129233,14 @@ SQLITE_PRIVATE SrcList *sqlite3SrcListAppend( if( pDatabase && pDatabase->z==0 ){ pDatabase = 0; } + assert( pItem->fg.fixedSchema==0 ); + assert( pItem->fg.isSubquery==0 ); if( pDatabase ){ pItem->zName = sqlite3NameFromToken(db, pDatabase); - pItem->zDatabase = sqlite3NameFromToken(db, pTable); + pItem->u4.zDatabase = sqlite3NameFromToken(db, pTable); }else{ pItem->zName = sqlite3NameFromToken(db, pTable); - pItem->zDatabase = 0; + pItem->u4.zDatabase = 0; } return pList; } @@ -125566,13 +129256,40 @@ SQLITE_PRIVATE void sqlite3SrcListAssignCursors(Parse *pParse, SrcList *pList){ for(i=0, pItem=pList->a; inSrc; i++, pItem++){ if( pItem->iCursor>=0 ) continue; pItem->iCursor = pParse->nTab++; - if( pItem->pSelect ){ - sqlite3SrcListAssignCursors(pParse, pItem->pSelect->pSrc); + if( pItem->fg.isSubquery ){ + assert( pItem->u4.pSubq!=0 ); + assert( pItem->u4.pSubq->pSelect!=0 ); + assert( pItem->u4.pSubq->pSelect->pSrc!=0 ); + sqlite3SrcListAssignCursors(pParse, pItem->u4.pSubq->pSelect->pSrc); } } } } +/* +** Delete a Subquery object and its substructure. +*/ +SQLITE_PRIVATE void sqlite3SubqueryDelete(sqlite3 *db, Subquery *pSubq){ + assert( pSubq!=0 && pSubq->pSelect!=0 ); + sqlite3SelectDelete(db, pSubq->pSelect); + sqlite3DbFree(db, pSubq); +} + +/* +** Remove a Subquery from a SrcItem. Return the associated Select object. +** The returned Select becomes the responsibility of the caller. +*/ +SQLITE_PRIVATE Select *sqlite3SubqueryDetach(sqlite3 *db, SrcItem *pItem){ + Select *pSel; + assert( pItem!=0 ); + assert( pItem->fg.isSubquery ); + pSel = pItem->u4.pSubq->pSelect; + sqlite3DbFree(db, pItem->u4.pSubq); + pItem->u4.pSubq = 0; + pItem->fg.isSubquery = 0; + return pSel; +} + /* ** Delete an entire SrcList including all its substructure. */ @@ -125582,13 +129299,24 @@ SQLITE_PRIVATE void sqlite3SrcListDelete(sqlite3 *db, SrcList *pList){ assert( db!=0 ); if( pList==0 ) return; for(pItem=pList->a, i=0; inSrc; i++, pItem++){ - if( pItem->zDatabase ) sqlite3DbNNFreeNN(db, pItem->zDatabase); + + /* Check invariants on SrcItem */ + assert( !pItem->fg.isIndexedBy || !pItem->fg.isTabFunc ); + assert( !pItem->fg.isCte || !pItem->fg.isIndexedBy ); + assert( !pItem->fg.fixedSchema || !pItem->fg.isSubquery ); + assert( !pItem->fg.isSubquery || (pItem->u4.pSubq!=0 && + pItem->u4.pSubq->pSelect!=0) ); + if( pItem->zName ) sqlite3DbNNFreeNN(db, pItem->zName); if( pItem->zAlias ) sqlite3DbNNFreeNN(db, pItem->zAlias); + if( pItem->fg.isSubquery ){ + sqlite3SubqueryDelete(db, pItem->u4.pSubq); + }else if( pItem->fg.fixedSchema==0 && pItem->u4.zDatabase!=0 ){ + sqlite3DbNNFreeNN(db, pItem->u4.zDatabase); + } if( pItem->fg.isIndexedBy ) sqlite3DbFree(db, pItem->u1.zIndexedBy); if( pItem->fg.isTabFunc ) sqlite3ExprListDelete(db, pItem->u1.pFuncArg); - sqlite3DeleteTable(db, pItem->pTab); - if( pItem->pSelect ) sqlite3SelectDelete(db, pItem->pSelect); + sqlite3DeleteTable(db, pItem->pSTab); if( pItem->fg.isUsing ){ sqlite3IdListDelete(db, pItem->u3.pUsing); }else if( pItem->u3.pOn ){ @@ -125598,6 +129326,54 @@ SQLITE_PRIVATE void sqlite3SrcListDelete(sqlite3 *db, SrcList *pList){ sqlite3DbNNFreeNN(db, pList); } +/* +** Attach a Subquery object to pItem->uv.pSubq. Set the +** pSelect value but leave all the other values initialized +** to zero. +** +** A copy of the Select object is made if dupSelect is true, and the +** SrcItem takes responsibility for deleting the copy. If dupSelect is +** false, ownership of the Select passes to the SrcItem. Either way, +** the SrcItem will take responsibility for deleting the Select. +** +** When dupSelect is zero, that means the Select might get deleted right +** away if there is an OOM error. Beware. +** +** Return non-zero on success. Return zero on an OOM error. +*/ +SQLITE_PRIVATE int sqlite3SrcItemAttachSubquery( + Parse *pParse, /* Parsing context */ + SrcItem *pItem, /* Item to which the subquery is to be attached */ + Select *pSelect, /* The subquery SELECT. Must be non-NULL */ + int dupSelect /* If true, attach a copy of pSelect, not pSelect itself.*/ +){ + Subquery *p; + assert( pSelect!=0 ); + assert( pItem->fg.isSubquery==0 ); + if( pItem->fg.fixedSchema ){ + pItem->u4.pSchema = 0; + pItem->fg.fixedSchema = 0; + }else if( pItem->u4.zDatabase!=0 ){ + sqlite3DbFree(pParse->db, pItem->u4.zDatabase); + pItem->u4.zDatabase = 0; + } + if( dupSelect ){ + pSelect = sqlite3SelectDup(pParse->db, pSelect, 0); + if( pSelect==0 ) return 0; + } + p = pItem->u4.pSubq = sqlite3DbMallocRawNN(pParse->db, sizeof(Subquery)); + if( p==0 ){ + sqlite3SelectDelete(pParse->db, pSelect); + return 0; + } + pItem->fg.isSubquery = 1; + p->pSelect = pSelect; + assert( offsetof(Subquery, pSelect)==0 ); + memset(((char*)p)+sizeof(p->pSelect), 0, sizeof(*p)-sizeof(p->pSelect)); + return 1; +} + + /* ** This routine is called by the parser to add a new term to the ** end of a growing FROM clause. The "p" parameter is the part of @@ -125647,10 +129423,12 @@ SQLITE_PRIVATE SrcList *sqlite3SrcListAppendFromTerm( if( pAlias->n ){ pItem->zAlias = sqlite3NameFromToken(db, pAlias); } + assert( pSubquery==0 || pDatabase==0 ); if( pSubquery ){ - pItem->pSelect = pSubquery; - if( pSubquery->selFlags & SF_NestedFrom ){ - pItem->fg.isNestedFrom = 1; + if( sqlite3SrcItemAttachSubquery(pParse, pItem, pSubquery, 0) ){ + if( pSubquery->selFlags & SF_NestedFrom ){ + pItem->fg.isNestedFrom = 1; + } } } assert( pOnUsing==0 || pOnUsing->pOn==0 || pOnUsing->pUsing==0 ); @@ -125703,16 +129481,22 @@ SQLITE_PRIVATE void sqlite3SrcListIndexedBy(Parse *pParse, SrcList *p, Token *pI ** are deleted by this function. */ SQLITE_PRIVATE SrcList *sqlite3SrcListAppendList(Parse *pParse, SrcList *p1, SrcList *p2){ - assert( p1 && p1->nSrc==1 ); + assert( p1 ); + assert( p2 || pParse->nErr ); + assert( p2==0 || p2->nSrc>=1 ); + testcase( p1->nSrc==0 ); if( p2 ){ - SrcList *pNew = sqlite3SrcListEnlarge(pParse, p1, p2->nSrc, 1); + int nOld = p1->nSrc; + SrcList *pNew = sqlite3SrcListEnlarge(pParse, p1, p2->nSrc, nOld); if( pNew==0 ){ sqlite3SrcListDelete(pParse->db, p2); }else{ p1 = pNew; - memcpy(&p1->a[1], p2->a, p2->nSrc*sizeof(SrcItem)); + memcpy(&p1->a[nOld], p2->a, p2->nSrc*sizeof(SrcItem)); + assert( nOld==1 || (p2->a[0].fg.jointype & JT_LTORJ)==0 ); + assert( p1->nSrc>=1 ); + p1->a[0].fg.jointype |= (JT_LTORJ & p2->a[0].fg.jointype); sqlite3DbFree(pParse->db, p2); - p1->a[0].fg.jointype |= (JT_LTORJ & p1->a[1].fg.jointype); } } return p1; @@ -126223,14 +130007,19 @@ SQLITE_PRIVATE KeyInfo *sqlite3KeyInfoOfIndex(Parse *pParse, Index *pIdx){ } if( pParse->nErr ){ assert( pParse->rc==SQLITE_ERROR_MISSING_COLLSEQ ); - if( pIdx->bNoQuery==0 ){ + if( pIdx->bNoQuery==0 + && sqlite3HashFind(&pIdx->pSchema->idxHash, pIdx->zName) + ){ /* Deactivate the index because it contains an unknown collating ** sequence. The only way to reactive the index is to reload the ** schema. Adding the missing collating sequence later does not ** reactive the index. The application had the chance to register ** the missing index using the collation-needed callback. For ** simplicity, SQLite will not give the application a second chance. - */ + ** + ** Except, do not do this if the index is not in the schema hash + ** table. In this case the index is currently being constructed + ** by a CREATE INDEX statement, and retrying will not help. */ pIdx->bNoQuery = 1; pParse->rc = SQLITE_ERROR_RETRY; } @@ -126322,10 +130111,9 @@ SQLITE_PRIVATE With *sqlite3WithAdd( } if( pWith ){ - sqlite3_int64 nByte = sizeof(*pWith) + (sizeof(pWith->a[1]) * pWith->nCte); - pNew = sqlite3DbRealloc(db, pWith, nByte); + pNew = sqlite3DbRealloc(db, pWith, SZ_WITH(pWith->nCte+1)); }else{ - pNew = sqlite3DbMallocZero(db, sizeof(*pWith)); + pNew = sqlite3DbMallocZero(db, SZ_WITH(1)); } assert( (pNew!=0 && zName!=0) || db->mallocFailed ); @@ -126663,12 +130451,18 @@ static int matchQuality( u8 enc /* Desired text encoding */ ){ int match; - assert( p->nArg>=-1 ); + assert( p->nArg>=(-4) && p->nArg!=(-2) ); + assert( nArg>=(-2) ); /* Wrong number of arguments means "no match" */ if( p->nArg!=nArg ){ - if( nArg==(-2) ) return (p->xSFunc==0) ? 0 : FUNC_PERFECT_MATCH; + if( nArg==(-2) ) return p->xSFunc==0 ? 0 : FUNC_PERFECT_MATCH; if( p->nArg>=0 ) return 0; + /* Special p->nArg values available to built-in functions only: + ** -3 1 or more arguments required + ** -4 2 or more arguments required + */ + if( p->nArg<(-2) && nArg<(-2-p->nArg) ) return 0; } /* Give a better score to a function with a specific number of arguments @@ -126862,6 +130656,7 @@ SQLITE_PRIVATE void sqlite3SchemaClear(void *p){ for(pElem=sqliteHashFirst(&temp2); pElem; pElem=sqliteHashNext(pElem)){ sqlite3DeleteTrigger(&xdb, (Trigger*)sqliteHashData(pElem)); } + sqlite3HashClear(&temp2); sqlite3HashInit(&pSchema->tblHash); for(pElem=sqliteHashFirst(&temp1); pElem; pElem=sqliteHashNext(pElem)){ @@ -126928,8 +130723,8 @@ SQLITE_PRIVATE Schema *sqlite3SchemaGet(sqlite3 *db, Btree *pBt){ ** ** The following fields are initialized appropriate in pSrc: ** -** pSrc->a[0].pTab Pointer to the Table object -** pSrc->a[0].pIndex Pointer to the INDEXED BY index, if there is one +** pSrc->a[0].spTab Pointer to the Table object +** pSrc->a[0].u2.pIBIndex Pointer to the INDEXED BY index, if there is one ** */ SQLITE_PRIVATE Table *sqlite3SrcListLookup(Parse *pParse, SrcList *pSrc){ @@ -126937,8 +130732,8 @@ SQLITE_PRIVATE Table *sqlite3SrcListLookup(Parse *pParse, SrcList *pSrc){ Table *pTab; assert( pItem && pSrc->nSrc>=1 ); pTab = sqlite3LocateTableItem(pParse, 0, pItem); - if( pItem->pTab ) sqlite3DeleteTable(pParse->db, pItem->pTab); - pItem->pTab = pTab; + if( pItem->pSTab ) sqlite3DeleteTable(pParse->db, pItem->pSTab); + pItem->pSTab = pTab; pItem->fg.notCte = 1; if( pTab ){ pTab->nTabRef++; @@ -126979,6 +130774,7 @@ SQLITE_PRIVATE void sqlite3CodeChangeCount(Vdbe *v, int regCounter, const char * ** is for a top-level SQL statement. */ static int vtabIsReadOnly(Parse *pParse, Table *pTab){ + assert( IsVirtual(pTab) ); if( sqlite3GetVTable(pParse->db, pTab)->pMod->pModule->xUpdate==0 ){ return 1; } @@ -127060,7 +130856,8 @@ SQLITE_PRIVATE void sqlite3MaterializeView( if( pFrom ){ assert( pFrom->nSrc==1 ); pFrom->a[0].zName = sqlite3DbStrDup(db, pView->zName); - pFrom->a[0].zDatabase = sqlite3DbStrDup(db, db->aDb[iDb].zDbSName); + assert( pFrom->a[0].fg.fixedSchema==0 && pFrom->a[0].fg.isSubquery==0 ); + pFrom->a[0].u4.zDatabase = sqlite3DbStrDup(db, db->aDb[iDb].zDbSName); assert( pFrom->a[0].fg.isUsing==0 ); assert( pFrom->a[0].u3.pOn==0 ); } @@ -127122,7 +130919,7 @@ SQLITE_PRIVATE Expr *sqlite3LimitWhere( ** ); */ - pTab = pSrc->a[0].pTab; + pTab = pSrc->a[0].pSTab; if( HasRowid(pTab) ){ pLhs = sqlite3PExpr(pParse, TK_ROW, 0, 0); pEList = sqlite3ExprListAppend( @@ -127155,9 +130952,9 @@ SQLITE_PRIVATE Expr *sqlite3LimitWhere( /* duplicate the FROM clause as it is needed by both the DELETE/UPDATE tree ** and the SELECT subtree. */ - pSrc->a[0].pTab = 0; + pSrc->a[0].pSTab = 0; pSelectSrc = sqlite3SrcListDup(db, pSrc, 0); - pSrc->a[0].pTab = pTab; + pSrc->a[0].pSTab = pTab; if( pSrc->a[0].fg.isIndexedBy ){ assert( pSrc->a[0].fg.isCte==0 ); pSrc->a[0].u2.pIBIndex = 0; @@ -128289,16 +132086,10 @@ static void substrFunc( int len; int p0type; i64 p1, p2; - int negP2 = 0; assert( argc==3 || argc==2 ); - if( sqlite3_value_type(argv[1])==SQLITE_NULL - || (argc==3 && sqlite3_value_type(argv[2])==SQLITE_NULL) - ){ - return; - } p0type = sqlite3_value_type(argv[0]); - p1 = sqlite3_value_int(argv[1]); + p1 = sqlite3_value_int64(argv[1]); if( p0type==SQLITE_BLOB ){ len = sqlite3_value_bytes(argv[0]); z = sqlite3_value_blob(argv[0]); @@ -128314,28 +132105,31 @@ static void substrFunc( } } } -#ifdef SQLITE_SUBSTR_COMPATIBILITY - /* If SUBSTR_COMPATIBILITY is defined then substr(X,0,N) work the same as - ** as substr(X,1,N) - it returns the first N characters of X. This - ** is essentially a back-out of the bug-fix in check-in [5fc125d362df4b8] - ** from 2009-02-02 for compatibility of applications that exploited the - ** old buggy behavior. */ - if( p1==0 ) p1 = 1; /* */ -#endif if( argc==3 ){ - p2 = sqlite3_value_int(argv[2]); - if( p2<0 ){ - p2 = -p2; - negP2 = 1; - } + p2 = sqlite3_value_int64(argv[2]); + if( p2==0 && sqlite3_value_type(argv[2])==SQLITE_NULL ) return; }else{ p2 = sqlite3_context_db_handle(context)->aLimit[SQLITE_LIMIT_LENGTH]; } + if( p1==0 ){ +#ifdef SQLITE_SUBSTR_COMPATIBILITY + /* If SUBSTR_COMPATIBILITY is defined then substr(X,0,N) work the same as + ** as substr(X,1,N) - it returns the first N characters of X. This + ** is essentially a back-out of the bug-fix in check-in [5fc125d362df4b8] + ** from 2009-02-02 for compatibility of applications that exploited the + ** old buggy behavior. */ + p1 = 1; /* */ +#endif + if( sqlite3_value_type(argv[1])==SQLITE_NULL ) return; + } if( p1<0 ){ p1 += len; if( p1<0 ){ - p2 += p1; - if( p2<0 ) p2 = 0; + if( p2<0 ){ + p2 = 0; + }else{ + p2 += p1; + } p1 = 0; } }else if( p1>0 ){ @@ -128343,12 +132137,13 @@ static void substrFunc( }else if( p2>0 ){ p2--; } - if( negP2 ){ - p1 -= p2; - if( p1<0 ){ - p2 += p1; - p1 = 0; + if( p2<0 ){ + if( p2<-p1 ){ + p2 = p1; + }else{ + p2 = -p2; } + p1 -= p2; } assert( p1>=0 && p2>=0 ); if( p0type!=SQLITE_BLOB ){ @@ -128362,9 +132157,11 @@ static void substrFunc( sqlite3_result_text64(context, (char*)z, z2-z, SQLITE_TRANSIENT, SQLITE_UTF8); }else{ - if( p1+p2>len ){ + if( p1>=len ){ + p1 = p2 = 0; + }else if( p2>len-p1 ){ p2 = len-p1; - if( p2<0 ) p2 = 0; + assert( p2>0 ); } sqlite3_result_blob64(context, (char*)&z[p1], (u64)p2, SQLITE_TRANSIENT); } @@ -128375,13 +132172,13 @@ static void substrFunc( */ #ifndef SQLITE_OMIT_FLOATING_POINT static void roundFunc(sqlite3_context *context, int argc, sqlite3_value **argv){ - int n = 0; + i64 n = 0; double r; char *zBuf; assert( argc==1 || argc==2 ); if( argc==2 ){ if( SQLITE_NULL==sqlite3_value_type(argv[1]) ) return; - n = sqlite3_value_int(argv[1]); + n = sqlite3_value_int64(argv[1]); if( n>30 ) n = 30; if( n<0 ) n = 0; } @@ -128396,7 +132193,7 @@ static void roundFunc(sqlite3_context *context, int argc, sqlite3_value **argv){ }else if( n==0 ){ r = (double)((sqlite_int64)(r+(r<0?-0.5:+0.5))); }else{ - zBuf = sqlite3_mprintf("%!.*f",n,r); + zBuf = sqlite3_mprintf("%!.*f",(int)n,r); if( zBuf==0 ){ sqlite3_result_error_nomem(context); return; @@ -128420,7 +132217,7 @@ static void *contextMalloc(sqlite3_context *context, i64 nByte){ sqlite3 *db = sqlite3_context_db_handle(context); assert( nByte>0 ); testcase( nByte==db->aLimit[SQLITE_LIMIT_LENGTH] ); - testcase( nByte==db->aLimit[SQLITE_LIMIT_LENGTH]+1 ); + testcase( nByte==(i64)db->aLimit[SQLITE_LIMIT_LENGTH]+1 ); if( nByte>db->aLimit[SQLITE_LIMIT_LENGTH] ){ sqlite3_result_error_toobig(context); z = 0; @@ -129025,7 +132822,7 @@ static const char hexdigits[] = { ** Append to pStr text that is the SQL literal representation of the ** value contained in pValue. */ -SQLITE_PRIVATE void sqlite3QuoteValue(StrAccum *pStr, sqlite3_value *pValue){ +SQLITE_PRIVATE void sqlite3QuoteValue(StrAccum *pStr, sqlite3_value *pValue, int bEscape){ /* As currently implemented, the string must be initially empty. ** we might relax this requirement in the future, but that will ** require enhancements to the implementation. */ @@ -129073,7 +132870,7 @@ SQLITE_PRIVATE void sqlite3QuoteValue(StrAccum *pStr, sqlite3_value *pValue){ } case SQLITE_TEXT: { const unsigned char *zArg = sqlite3_value_text(pValue); - sqlite3_str_appendf(pStr, "%Q", zArg); + sqlite3_str_appendf(pStr, bEscape ? "%#Q" : "%Q", zArg); break; } default: { @@ -129084,6 +132881,105 @@ SQLITE_PRIVATE void sqlite3QuoteValue(StrAccum *pStr, sqlite3_value *pValue){ } } +/* +** Return true if z[] begins with N hexadecimal digits, and write +** a decoding of those digits into *pVal. Or return false if any +** one of the first N characters in z[] is not a hexadecimal digit. +*/ +static int isNHex(const char *z, int N, u32 *pVal){ + int i; + u32 v = 0; + for(i=0; i0 ){ + memmove(&zOut[j], &zIn[i], n); + j += n; + i += n; + } + if( zIn[i+1]=='\\' ){ + i += 2; + zOut[j++] = '\\'; + }else if( sqlite3Isxdigit(zIn[i+1]) ){ + if( !isNHex(&zIn[i+1], 4, &v) ) goto unistr_error; + i += 5; + j += sqlite3AppendOneUtf8Character(&zOut[j], v); + }else if( zIn[i+1]=='+' ){ + if( !isNHex(&zIn[i+2], 6, &v) ) goto unistr_error; + i += 8; + j += sqlite3AppendOneUtf8Character(&zOut[j], v); + }else if( zIn[i+1]=='u' ){ + if( !isNHex(&zIn[i+2], 4, &v) ) goto unistr_error; + i += 6; + j += sqlite3AppendOneUtf8Character(&zOut[j], v); + }else if( zIn[i+1]=='U' ){ + if( !isNHex(&zIn[i+2], 8, &v) ) goto unistr_error; + i += 10; + j += sqlite3AppendOneUtf8Character(&zOut[j], v); + }else{ + goto unistr_error; + } + } + zOut[j] = 0; + sqlite3_result_text64(context, zOut, j, sqlite3_free, SQLITE_UTF8); + return; + +unistr_error: + sqlite3_free(zOut); + sqlite3_result_error(context, "invalid Unicode escape", -1); + return; +} + + /* ** Implementation of the QUOTE() function. ** @@ -129093,6 +132989,10 @@ SQLITE_PRIVATE void sqlite3QuoteValue(StrAccum *pStr, sqlite3_value *pValue){ ** as needed. BLOBs are encoded as hexadecimal literals. Strings with ** embedded NUL characters cannot be represented as string literals in SQL ** and hence the returned string literal is truncated prior to the first NUL. +** +** If sqlite3_user_data() is non-zero, then the UNISTR_QUOTE() function is +** implemented instead. The difference is that UNISTR_QUOTE() uses the +** UNISTR() function to escape control characters. */ static void quoteFunc(sqlite3_context *context, int argc, sqlite3_value **argv){ sqlite3_str str; @@ -129100,7 +133000,7 @@ static void quoteFunc(sqlite3_context *context, int argc, sqlite3_value **argv){ assert( argc==1 ); UNUSED_PARAMETER(argc); sqlite3StrAccumInit(&str, db, 0, 0, db->aLimit[SQLITE_LIMIT_LENGTH]); - sqlite3QuoteValue(&str,argv[0]); + sqlite3QuoteValue(&str,argv[0],SQLITE_PTR_TO_INT(sqlite3_user_data(context))); sqlite3_result_text(context, sqlite3StrAccumFinish(&str), str.nChar, SQLITE_DYNAMIC); if( str.accError!=SQLITE_OK ){ @@ -129355,7 +133255,7 @@ static void replaceFunc( assert( zRep==sqlite3_value_text(argv[2]) ); nOut = nStr + 1; assert( nOut0 ){ + if( sqlite3_value_type(argv[i])!=SQLITE_NULL ){ + int k = sqlite3_value_bytes(argv[i]); const char *v = (const char*)sqlite3_value_text(argv[i]); if( v!=0 ){ - if( j>0 && nSep>0 ){ + if( bNotNull && nSep>0 ){ memcpy(&z[j], zSep, nSep); j += nSep; } memcpy(&z[j], v, k); j += k; + bNotNull = 1; } } } @@ -129751,7 +133653,7 @@ static void kahanBabuskaNeumaierInit( ** that it returns NULL if it sums over no inputs. TOTAL returns ** 0.0 in that case. In addition, TOTAL always returns a float where ** SUM might return an integer if it never encounters a floating point -** value. TOTAL never fails, but SUM might through an exception if +** value. TOTAL never fails, but SUM might throw an exception if ** it overflows an integer. */ static void sumStep(sqlite3_context *context, int argc, sqlite3_value **argv){ @@ -129803,7 +133705,10 @@ static void sumInverse(sqlite3_context *context, int argc, sqlite3_value**argv){ assert( p->cnt>0 ); p->cnt--; if( !p->approx ){ - p->iSum -= sqlite3_value_int64(argv[0]); + if( sqlite3SubInt64(&p->iSum, sqlite3_value_int64(argv[0])) ){ + p->ovrfl = 1; + p->approx = 1; + } }else if( type==SQLITE_INTEGER ){ i64 iVal = sqlite3_value_int64(argv[0]); if( iVal!=SMALLEST_INT64 ){ @@ -129827,7 +133732,7 @@ static void sumFinalize(sqlite3_context *context){ if( p->approx ){ if( p->ovrfl ){ sqlite3_result_error(context,"integer overflow",-1); - }else if( !sqlite3IsNaN(p->rErr) ){ + }else if( !sqlite3IsOverflow(p->rErr) ){ sqlite3_result_double(context, p->rSum+p->rErr); }else{ sqlite3_result_double(context, p->rSum); @@ -129844,7 +133749,7 @@ static void avgFinalize(sqlite3_context *context){ double r; if( p->approx ){ r = p->rSum; - if( !sqlite3IsNaN(p->rErr) ) r += p->rErr; + if( !sqlite3IsOverflow(p->rErr) ) r += p->rErr; }else{ r = (double)(p->iSum); } @@ -129858,7 +133763,7 @@ static void totalFinalize(sqlite3_context *context){ if( p ){ if( p->approx ){ r = p->rSum; - if( !sqlite3IsNaN(p->rErr) ) r += p->rErr; + if( !sqlite3IsOverflow(p->rErr) ) r += p->rErr; }else{ r = (double)(p->iSum); } @@ -129984,7 +133889,11 @@ static void minMaxFinalize(sqlite3_context *context){ ** group_concat(EXPR, ?SEPARATOR?) ** string_agg(EXPR, SEPARATOR) ** -** The SEPARATOR goes before the EXPR string. This is tragic. The +** Content is accumulated in GroupConcatCtx.str with the SEPARATOR +** coming before the EXPR value, except for the first entry which +** omits the SEPARATOR. +** +** It is tragic that the SEPARATOR goes before the EXPR string. The ** groupConcatInverse() implementation would have been easier if the ** SEPARATOR were appended after EXPR. And the order is undocumented, ** so we could change it, in theory. But the old behavior has been @@ -130088,7 +133997,7 @@ static void groupConcatInverse( /* pGCC is always non-NULL since groupConcatStep() will have always ** run first to initialize it */ if( ALWAYS(pGCC) ){ - int nVS; + int nVS; /* Number of characters to remove */ /* Must call sqlite3_value_text() to convert the argument into text prior ** to invoking sqlite3_value_bytes(), in case the text encoding is UTF16 */ (void)sqlite3_value_text(argv[0]); @@ -130141,6 +134050,8 @@ static void groupConcatValue(sqlite3_context *context){ sqlite3_result_error_toobig(context); }else if( pAccum->accError==SQLITE_NOMEM ){ sqlite3_result_error_nomem(context); + }else if( pGCC->nAccum>0 && pAccum->nChar==0 ){ + sqlite3_result_text(context, "", 1, SQLITE_STATIC); }else{ const char *zText = sqlite3_str_value(pAccum); sqlite3_result_text(context, zText, pAccum->nChar, SQLITE_TRANSIENT); @@ -130459,12 +134370,514 @@ static void signFunc( sqlite3_result_int(context, x<0.0 ? -1 : x>0.0 ? +1 : 0); } +#if defined(SQLITE_ENABLE_PERCENTILE) +/*********************************************************************** +** This section implements the percentile(Y,P) SQL function and similar. +** Requirements: +** +** (1) The percentile(Y,P) function is an aggregate function taking +** exactly two arguments. +** +** (2) If the P argument to percentile(Y,P) is not the same for every +** row in the aggregate then an error is thrown. The word "same" +** in the previous sentence means that the value differ by less +** than 0.001. +** +** (3) If the P argument to percentile(Y,P) evaluates to anything other +** than a number in the range of 0.0 to 100.0 inclusive then an +** error is thrown. +** +** (4) If any Y argument to percentile(Y,P) evaluates to a value that +** is not NULL and is not numeric then an error is thrown. +** +** (5) If any Y argument to percentile(Y,P) evaluates to plus or minus +** infinity then an error is thrown. (SQLite always interprets NaN +** values as NULL.) +** +** (6) Both Y and P in percentile(Y,P) can be arbitrary expressions, +** including CASE WHEN expressions. +** +** (7) The percentile(Y,P) aggregate is able to handle inputs of at least +** one million (1,000,000) rows. +** +** (8) If there are no non-NULL values for Y, then percentile(Y,P) +** returns NULL. +** +** (9) If there is exactly one non-NULL value for Y, the percentile(Y,P) +** returns the one Y value. +** +** (10) If there N non-NULL values of Y where N is two or more and +** the Y values are ordered from least to greatest and a graph is +** drawn from 0 to N-1 such that the height of the graph at J is +** the J-th Y value and such that straight lines are drawn between +** adjacent Y values, then the percentile(Y,P) function returns +** the height of the graph at P*(N-1)/100. +** +** (11) The percentile(Y,P) function always returns either a floating +** point number or NULL. +** +** (12) The percentile(Y,P) is implemented as a single C99 source-code +** file that compiles into a shared-library or DLL that can be loaded +** into SQLite using the sqlite3_load_extension() interface. +** +** (13) A separate median(Y) function is the equivalent percentile(Y,50). +** +** (14) A separate percentile_cont(Y,P) function is equivalent to +** percentile(Y,P/100.0). In other words, the fraction value in +** the second argument is in the range of 0 to 1 instead of 0 to 100. +** +** (15) A separate percentile_disc(Y,P) function is like +** percentile_cont(Y,P) except that instead of returning the weighted +** average of the nearest two input values, it returns the next lower +** value. So the percentile_disc(Y,P) will always return a value +** that was one of the inputs. +** +** (16) All of median(), percentile(Y,P), percentile_cont(Y,P) and +** percentile_disc(Y,P) can be used as window functions. +** +** Differences from standard SQL: +** +** * The percentile_cont(X,P) function is equivalent to the following in +** standard SQL: +** +** (percentile_cont(P) WITHIN GROUP (ORDER BY X)) +** +** The SQLite syntax is much more compact. The standard SQL syntax +** is also supported if SQLite is compiled with the +** -DSQLITE_ENABLE_ORDERED_SET_AGGREGATES option. +** +** * No median(X) function exists in the SQL standard. App developers +** are expected to write "percentile_cont(0.5)WITHIN GROUP(ORDER BY X)". +** +** * No percentile(Y,P) function exists in the SQL standard. Instead of +** percential(Y,P), developers must write this: +** "percentile_cont(P/100.0) WITHIN GROUP (ORDER BY Y)". Note that +** the fraction parameter to percentile() goes from 0 to 100 whereas +** the fraction parameter in SQL standard percentile_cont() goes from +** 0 to 1. +** +** Implementation notes as of 2024-08-31: +** +** * The regular aggregate-function versions of these routines work +** by accumulating all values in an array of doubles, then sorting +** that array using quicksort before computing the answer. Thus +** the runtime is O(NlogN) where N is the number of rows of input. +** +** * For the window-function versions of these routines, the array of +** inputs is sorted as soon as the first value is computed. Thereafter, +** the array is kept in sorted order using an insert-sort. This +** results in O(N*K) performance where K is the size of the window. +** One can imagine alternative implementations that give O(N*logN*logK) +** performance, but they require more complex logic and data structures. +** The developers have elected to keep the asymptotically slower +** algorithm for now, for simplicity, under the theory that window +** functions are seldom used and when they are, the window size K is +** often small. The developers might revisit that decision later, +** should the need arise. +*/ + +/* The following object is the group context for a single percentile() +** aggregate. Remember all input Y values until the very end. +** Those values are accumulated in the Percentile.a[] array. +*/ +typedef struct Percentile Percentile; +struct Percentile { + u64 nAlloc; /* Number of slots allocated for a[] */ + u64 nUsed; /* Number of slots actually used in a[] */ + char bSorted; /* True if a[] is already in sorted order */ + char bKeepSorted; /* True if advantageous to keep a[] sorted */ + char bPctValid; /* True if rPct is valid */ + double rPct; /* Fraction. 0.0 to 1.0 */ + double *a; /* Array of Y values */ +}; + +/* +** Return TRUE if the input floating-point number is an infinity. +*/ +static int percentIsInfinity(double r){ + sqlite3_uint64 u; + assert( sizeof(u)==sizeof(r) ); + memcpy(&u, &r, sizeof(u)); + return ((u>>52)&0x7ff)==0x7ff; +} + +/* +** Return TRUE if two doubles differ by 0.001 or less. +*/ +static int percentSameValue(double a, double b){ + a -= b; + return a>=-0.001 && a<=0.001; +} + +/* +** Search p (which must have p->bSorted) looking for an entry with +** value y. Return the index of that entry. +** +** If bExact is true, return -1 if the entry is not found. +** +** If bExact is false, return the index at which a new entry with +** value y should be insert in order to keep the values in sorted +** order. The smallest return value in this case will be 0, and +** the largest return value will be p->nUsed. +*/ +static i64 percentBinarySearch(Percentile *p, double y, int bExact){ + i64 iFirst = 0; /* First element of search range */ + i64 iLast = (i64)p->nUsed - 1; /* Last element of search range */ + while( iLast>=iFirst ){ + i64 iMid = (iFirst+iLast)/2; + double x = p->a[iMid]; + if( xy ){ + iLast = iMid - 1; + }else{ + return iMid; + } + } + if( bExact ) return -1; + return iFirst; +} + +/* +** Generate an error for a percentile function. +** +** The error format string must have exactly one occurrence of "%%s()" +** (with two '%' characters). That substring will be replaced by the name +** of the function. +*/ +static void percentError(sqlite3_context *pCtx, const char *zFormat, ...){ + char *zMsg1; + char *zMsg2; + va_list ap; + + va_start(ap, zFormat); + zMsg1 = sqlite3_vmprintf(zFormat, ap); + va_end(ap); + zMsg2 = zMsg1 ? sqlite3_mprintf(zMsg1, sqlite3VdbeFuncName(pCtx)) : 0; + sqlite3_result_error(pCtx, zMsg2, -1); + sqlite3_free(zMsg1); + sqlite3_free(zMsg2); +} + +/* +** The "step" function for percentile(Y,P) is called once for each +** input row. +*/ +static void percentStep(sqlite3_context *pCtx, int argc, sqlite3_value **argv){ + Percentile *p; + double rPct; + int eType; + double y; + assert( argc==2 || argc==1 ); + + if( argc==1 ){ + /* Requirement 13: median(Y) is the same as percentile(Y,50). */ + rPct = 0.5; + }else{ + /* P must be a number between 0 and 100 for percentile() or between + ** 0.0 and 1.0 for percentile_cont() and percentile_disc(). + ** + ** The user-data is an integer which is 10 times the upper bound. + */ + double mxFrac = (SQLITE_PTR_TO_INT(sqlite3_user_data(pCtx))&2)? 100.0 : 1.0; + eType = sqlite3_value_numeric_type(argv[1]); + rPct = sqlite3_value_double(argv[1])/mxFrac; + if( (eType!=SQLITE_INTEGER && eType!=SQLITE_FLOAT) + || rPct<0.0 || rPct>1.0 + ){ + percentError(pCtx, "the fraction argument to %%s()" + " is not between 0.0 and %.1f", + (double)mxFrac); + return; + } + } + + /* Allocate the session context. */ + p = (Percentile*)sqlite3_aggregate_context(pCtx, sizeof(*p)); + if( p==0 ) return; + + /* Remember the P value. Throw an error if the P value is different + ** from any prior row, per Requirement (2). */ + if( !p->bPctValid ){ + p->rPct = rPct; + p->bPctValid = 1; + }else if( !percentSameValue(p->rPct,rPct) ){ + percentError(pCtx, "the fraction argument to %%s()" + " is not the same for all input rows"); + return; + } + + /* Ignore rows for which Y is NULL */ + eType = sqlite3_value_type(argv[0]); + if( eType==SQLITE_NULL ) return; + + /* If not NULL, then Y must be numeric. Otherwise throw an error. + ** Requirement 4 */ + if( eType!=SQLITE_INTEGER && eType!=SQLITE_FLOAT ){ + percentError(pCtx, "input to %%s() is not numeric"); + return; + } + + /* Throw an error if the Y value is infinity or NaN */ + y = sqlite3_value_double(argv[0]); + if( percentIsInfinity(y) ){ + percentError(pCtx, "Inf input to %%s()"); + return; + } + + /* Allocate and store the Y */ + if( p->nUsed>=p->nAlloc ){ + u64 n = p->nAlloc*2 + 250; + double *a = sqlite3_realloc64(p->a, sizeof(double)*n); + if( a==0 ){ + sqlite3_free(p->a); + memset(p, 0, sizeof(*p)); + sqlite3_result_error_nomem(pCtx); + return; + } + p->nAlloc = n; + p->a = a; + } + if( p->nUsed==0 ){ + p->a[p->nUsed++] = y; + p->bSorted = 1; + }else if( !p->bSorted || y>=p->a[p->nUsed-1] ){ + p->a[p->nUsed++] = y; + }else if( p->bKeepSorted ){ + i64 i; + i = percentBinarySearch(p, y, 0); + if( i<(int)p->nUsed ){ + memmove(&p->a[i+1], &p->a[i], (p->nUsed-i)*sizeof(p->a[0])); + } + p->a[i] = y; + p->nUsed++; + }else{ + p->a[p->nUsed++] = y; + p->bSorted = 0; + } +} + +/* +** Interchange two doubles. +*/ +#define SWAP_DOUBLE(X,Y) {double ttt=(X);(X)=(Y);(Y)=ttt;} + +/* +** Sort an array of doubles. +** +** Algorithm: quicksort +** +** This is implemented separately rather than using the qsort() routine +** from the standard library because: +** +** (1) To avoid a dependency on qsort() +** (2) To avoid the function call to the comparison routine for each +** comparison. +*/ +static void percentSort(double *a, unsigned int n){ + int iLt; /* Entries before a[iLt] are less than rPivot */ + int iGt; /* Entries at or after a[iGt] are greater than rPivot */ + int i; /* Loop counter */ + double rPivot; /* The pivot value */ + + assert( n>=2 ); + if( a[0]>a[n-1] ){ + SWAP_DOUBLE(a[0],a[n-1]) + } + if( n==2 ) return; + iGt = n-1; + i = n/2; + if( a[0]>a[i] ){ + SWAP_DOUBLE(a[0],a[i]) + }else if( a[i]>a[iGt] ){ + SWAP_DOUBLE(a[i],a[iGt]) + } + if( n==3 ) return; + rPivot = a[i]; + iLt = i = 1; + do{ + if( a[i]iLt ) SWAP_DOUBLE(a[i],a[iLt]) + iLt++; + i++; + }else if( a[i]>rPivot ){ + do{ + iGt--; + }while( iGt>i && a[iGt]>rPivot ); + SWAP_DOUBLE(a[i],a[iGt]) + }else{ + i++; + } + }while( i=2 ) percentSort(a, iLt); + if( n-iGt>=2 ) percentSort(a+iGt, n-iGt); + +/* Uncomment for testing */ +#if 0 + for(i=0; ibSorted==0 ){ + assert( p->nUsed>1 ); + percentSort(p->a, p->nUsed); + p->bSorted = 1; + } + p->bKeepSorted = 1; + + /* Find and remove the row */ + i = percentBinarySearch(p, y, 1); + if( i>=0 ){ + p->nUsed--; + if( i<(int)p->nUsed ){ + memmove(&p->a[i], &p->a[i+1], (p->nUsed - i)*sizeof(p->a[0])); + } + } +} + +/* +** Compute the final output of percentile(). Clean up all allocated +** memory if and only if bIsFinal is true. +*/ +static void percentCompute(sqlite3_context *pCtx, int bIsFinal){ + Percentile *p; + int settings = SQLITE_PTR_TO_INT(sqlite3_user_data(pCtx))&1; /* Discrete? */ + unsigned i1, i2; + double v1, v2; + double ix, vx; + p = (Percentile*)sqlite3_aggregate_context(pCtx, 0); + if( p==0 ) return; + if( p->a==0 ) return; + if( p->nUsed ){ + if( p->bSorted==0 ){ + assert( p->nUsed>1 ); + percentSort(p->a, p->nUsed); + p->bSorted = 1; + } + ix = p->rPct*(p->nUsed-1); + i1 = (unsigned)ix; + if( settings & 1 ){ + vx = p->a[i1]; + }else{ + i2 = ix==(double)i1 || i1==p->nUsed-1 ? i1 : i1+1; + v1 = p->a[i1]; + v2 = p->a[i2]; + vx = v1 + (v2-v1)*(ix-i1); + } + sqlite3_result_double(pCtx, vx); + } + if( bIsFinal ){ + sqlite3_free(p->a); + memset(p, 0, sizeof(*p)); + }else{ + p->bKeepSorted = 1; + } +} +static void percentFinal(sqlite3_context *pCtx){ + percentCompute(pCtx, 1); +} +static void percentValue(sqlite3_context *pCtx){ + percentCompute(pCtx, 0); +} +/****** End of percentile family of functions ******/ +#endif /* SQLITE_ENABLE_PERCENTILE */ + +#if defined(SQLITE_DEBUG) || defined(SQLITE_ENABLE_FILESTAT) +/* +** Implementation of sqlite_filestat(SCHEMA). +** +** Return JSON text that describes low-level debug/diagnostic information +** about the sqlite3_file object associated with SCHEMA. +*/ +static void filestatFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + sqlite3 *db = sqlite3_context_db_handle(context); + const char *zDbName; + sqlite3_str *pStr; + Btree *pBtree; + + zDbName = (const char*)sqlite3_value_text(argv[0]); + pBtree = sqlite3DbNameToBtree(db, zDbName); + if( pBtree ){ + Pager *pPager; + sqlite3_file *fd; + int rc; + sqlite3BtreeEnter(pBtree); + pPager = sqlite3BtreePager(pBtree); + assert( pPager!=0 ); + fd = sqlite3PagerFile(pPager); + pStr = sqlite3_str_new(db); + if( pStr==0 ){ + sqlite3_result_error_nomem(context); + }else{ + sqlite3_str_append(pStr, "{\"db\":", 6); + rc = sqlite3OsFileControl(fd, SQLITE_FCNTL_FILESTAT, pStr); + if( rc ) sqlite3_str_append(pStr, "null", 4); + fd = sqlite3PagerJrnlFile(pPager); + if( fd && fd->pMethods!=0 ){ + sqlite3_str_appendall(pStr, ",\"journal\":"); + rc = sqlite3OsFileControl(fd, SQLITE_FCNTL_FILESTAT, pStr); + if( rc ) sqlite3_str_append(pStr, "null", 4); + } + sqlite3_str_append(pStr, "}", 1); + sqlite3_result_text(context, sqlite3_str_finish(pStr), -1, + sqlite3_free); + } + sqlite3BtreeLeave(pBtree); + }else{ + sqlite3_result_text(context, "{}", 2, SQLITE_STATIC); + } +} +#endif /* SQLITE_DEBUG || SQLITE_ENABLE_FILESTAT */ + #ifdef SQLITE_DEBUG /* ** Implementation of fpdecode(x,y,z) function. ** ** x is a real number that is to be decoded. y is the precision. -** z is the maximum real precision. +** z is the maximum real precision. Return a string that shows the +** results of the sqlite3FpDecode() function. +** +** Used for testing and debugging only, specifically testing and debugging +** of the sqlite3FpDecode() function. This SQL function does not appear +** in production builds. This function is not an API and is subject to +** modification or removal in future versions of SQLite. */ static void fpdecodeFunc( sqlite3_context *context, @@ -130480,6 +134893,7 @@ static void fpdecodeFunc( x = sqlite3_value_double(argv[0]); y = sqlite3_value_int(argv[1]); z = sqlite3_value_int(argv[2]); + if( z<=0 ) z = 1; sqlite3FpDecode(&s, x, y, z); if( s.isSpecial==2 ){ sqlite3_snprintf(sizeof(zBuf), zBuf, "NaN"); @@ -130490,6 +134904,82 @@ static void fpdecodeFunc( } #endif /* SQLITE_DEBUG */ +#ifdef SQLITE_DEBUG +/* +** Implementation of parseuri(uri,flags) function. +** +** Required Arguments: +** "uri" The URI to parse. +** "flags" Bitmask of flags, as if to sqlite3_open_v2(). +** +** Additional arguments beyond the first two make calls to +** sqlite3_uri_key() for integers and sqlite3_uri_parameter for +** anything else. +** +** The result is a string showing the results of calling sqlite3ParseUri(). +** +** Used for testing and debugging only, specifically testing and debugging +** of the sqlite3ParseUri() function. This SQL function does not appear +** in production builds. This function is not an API and is subject to +** modification or removal in future versions of SQLite. +*/ +static void parseuriFunc( + sqlite3_context *ctx, + int argc, + sqlite3_value **argv +){ + sqlite3_str *pResult; + const char *zVfs; + const char *zUri; + unsigned int flgs; + int rc; + sqlite3_vfs *pVfs = 0; + char *zFile = 0; + char *zErr = 0; + + if( argc<2 ) return; + pVfs = sqlite3_vfs_find(0); + assert( pVfs ); + zVfs = pVfs->zName; + zUri = (const char*)sqlite3_value_text(argv[0]); + if( zUri==0 ) return; + flgs = (unsigned int)sqlite3_value_int(argv[1]); + rc = sqlite3ParseUri(zVfs, zUri, &flgs, &pVfs, &zFile, &zErr); + pResult = sqlite3_str_new(0); + if( pResult ){ + int i; + sqlite3_str_appendf(pResult, "rc=%d", rc); + sqlite3_str_appendf(pResult, ", flags=0x%x", flgs); + sqlite3_str_appendf(pResult, ", vfs=%Q", pVfs ? pVfs->zName: 0); + sqlite3_str_appendf(pResult, ", err=%Q", zErr); + sqlite3_str_appendf(pResult, ", file=%Q", zFile); + if( zFile ){ + const char *z = zFile; + z += sqlite3Strlen30(z)+1; + while( z[0] ){ + sqlite3_str_appendf(pResult, ", %Q", z); + z += sqlite3Strlen30(z)+1; + } + for(i=2; ia; - pItem->pTab = pFKey->pFrom; + pItem->pSTab = pFKey->pFrom; pItem->zName = pFKey->pFrom->zName; - pItem->pTab->nTabRef++; + pItem->pSTab->nTabRef++; pItem->iCursor = pParse->nTab++; if( regNew!=0 ){ @@ -132021,7 +136524,8 @@ static Trigger *fkActionTrigger( SrcList *pSrc; Expr *pRaise; - pRaise = sqlite3Expr(db, TK_RAISE, "FOREIGN KEY constraint failed"); + pRaise = sqlite3Expr(db, TK_STRING, "FOREIGN KEY constraint failed"), + pRaise = sqlite3PExpr(pParse, TK_RAISE, pRaise, 0); if( pRaise ){ pRaise->affExpr = OE_Abort; } @@ -132029,7 +136533,8 @@ static Trigger *fkActionTrigger( if( pSrc ){ assert( pSrc->nSrc==1 ); pSrc->a[0].zName = sqlite3DbStrDup(db, zFrom); - pSrc->a[0].zDatabase = sqlite3DbStrDup(db, db->aDb[iDb].zDbSName); + assert( pSrc->a[0].fg.fixedSchema==0 && pSrc->a[0].fg.isSubquery==0 ); + pSrc->a[0].u4.zDatabase = sqlite3DbStrDup(db, db->aDb[iDb].zDbSName); } pSelect = sqlite3SelectNew(pParse, sqlite3ExprListAppend(pParse, 0, pRaise), @@ -132363,12 +136868,15 @@ SQLITE_PRIVATE void sqlite3TableAffinity(Vdbe *v, Table *pTab, int iReg){ ** by one slot and insert a new OP_TypeCheck where the current ** OP_MakeRecord is found */ VdbeOp *pPrev; + int p3; sqlite3VdbeAppendP4(v, pTab, P4_TABLE); pPrev = sqlite3VdbeGetLastOp(v); assert( pPrev!=0 ); assert( pPrev->opcode==OP_MakeRecord || sqlite3VdbeDb(v)->mallocFailed ); pPrev->opcode = OP_TypeCheck; - sqlite3VdbeAddOp3(v, OP_MakeRecord, pPrev->p1, pPrev->p2, pPrev->p3); + p3 = pPrev->p3; + pPrev->p3 = 0; + sqlite3VdbeAddOp3(v, OP_MakeRecord, pPrev->p1, pPrev->p2, p3); }else{ /* Insert an isolated OP_Typecheck */ sqlite3VdbeAddOp2(v, OP_TypeCheck, iReg, pTab->nNVCol); @@ -132755,6 +137263,210 @@ SQLITE_PRIVATE void sqlite3AutoincrementEnd(Parse *pParse){ # define autoIncStep(A,B,C) #endif /* SQLITE_OMIT_AUTOINCREMENT */ +/* +** If argument pVal is a Select object returned by an sqlite3MultiValues() +** that was able to use the co-routine optimization, finish coding the +** co-routine. +*/ +SQLITE_PRIVATE void sqlite3MultiValuesEnd(Parse *pParse, Select *pVal){ + if( ALWAYS(pVal) && pVal->pSrc->nSrc>0 ){ + SrcItem *pItem = &pVal->pSrc->a[0]; + assert( (pItem->fg.isSubquery && pItem->u4.pSubq!=0) || pParse->nErr ); + if( pItem->fg.isSubquery ){ + sqlite3VdbeEndCoroutine(pParse->pVdbe, pItem->u4.pSubq->regReturn); + sqlite3VdbeJumpHere(pParse->pVdbe, pItem->u4.pSubq->addrFillSub - 1); + } + } +} + +/* +** Return true if all expressions in the expression-list passed as the +** only argument are constant. +*/ +static int exprListIsConstant(Parse *pParse, ExprList *pRow){ + int ii; + for(ii=0; iinExpr; ii++){ + if( 0==sqlite3ExprIsConstant(pParse, pRow->a[ii].pExpr) ) return 0; + } + return 1; +} + +/* +** Return true if all expressions in the expression-list passed as the +** only argument are both constant and have no affinity. +*/ +static int exprListIsNoAffinity(Parse *pParse, ExprList *pRow){ + int ii; + if( exprListIsConstant(pParse,pRow)==0 ) return 0; + for(ii=0; iinExpr; ii++){ + Expr *pExpr = pRow->a[ii].pExpr; + assert( pExpr->op!=TK_RAISE ); + assert( pExpr->affExpr==0 ); + if( 0!=sqlite3ExprAffinity(pExpr) ) return 0; + } + return 1; + +} + +/* +** This function is called by the parser for the second and subsequent +** rows of a multi-row VALUES clause. Argument pLeft is the part of +** the VALUES clause already parsed, argument pRow is the vector of values +** for the new row. The Select object returned represents the complete +** VALUES clause, including the new row. +** +** There are two ways in which this may be achieved - by incremental +** coding of a co-routine (the "co-routine" method) or by returning a +** Select object equivalent to the following (the "UNION ALL" method): +** +** "pLeft UNION ALL SELECT pRow" +** +** If the VALUES clause contains a lot of rows, this compound Select +** object may consume a lot of memory. +** +** When the co-routine method is used, each row that will be returned +** by the VALUES clause is coded into part of a co-routine as it is +** passed to this function. The returned Select object is equivalent to: +** +** SELECT * FROM ( +** Select object to read co-routine +** ) +** +** The co-routine method is used in most cases. Exceptions are: +** +** a) If the current statement has a WITH clause. This is to avoid +** statements like: +** +** WITH cte AS ( VALUES('x'), ('y') ... ) +** SELECT * FROM cte AS a, cte AS b; +** +** This will not work, as the co-routine uses a hard-coded register +** for its OP_Yield instructions, and so it is not possible for two +** cursors to iterate through it concurrently. +** +** b) The schema is currently being parsed (i.e. the VALUES clause is part +** of a schema item like a VIEW or TRIGGER). In this case there is no VM +** being generated when parsing is taking place, and so generating +** a co-routine is not possible. +** +** c) There are non-constant expressions in the VALUES clause (e.g. +** the VALUES clause is part of a correlated sub-query). +** +** d) One or more of the values in the first row of the VALUES clause +** has an affinity (i.e. is a CAST expression). This causes problems +** because the complex rules SQLite uses (see function +** sqlite3SubqueryColumnTypes() in select.c) to determine the effective +** affinity of such a column for all rows require access to all values in +** the column simultaneously. +*/ +SQLITE_PRIVATE Select *sqlite3MultiValues(Parse *pParse, Select *pLeft, ExprList *pRow){ + + if( pParse->bHasWith /* condition (a) above */ + || pParse->db->init.busy /* condition (b) above */ + || exprListIsConstant(pParse,pRow)==0 /* condition (c) above */ + || (pLeft->pSrc->nSrc==0 && + exprListIsNoAffinity(pParse,pLeft->pEList)==0) /* condition (d) above */ + || IN_SPECIAL_PARSE + ){ + /* The co-routine method cannot be used. Fall back to UNION ALL. */ + Select *pSelect = 0; + int f = SF_Values | SF_MultiValue; + if( pLeft->pSrc->nSrc ){ + sqlite3MultiValuesEnd(pParse, pLeft); + f = SF_Values; + }else if( pLeft->pPrior ){ + /* In this case set the SF_MultiValue flag only if it was set on pLeft */ + f = (f & pLeft->selFlags); + } + pSelect = sqlite3SelectNew(pParse, pRow, 0, 0, 0, 0, 0, f, 0); + pLeft->selFlags &= ~(u32)SF_MultiValue; + if( pSelect ){ + pSelect->op = TK_ALL; + pSelect->pPrior = pLeft; + pLeft = pSelect; + } + }else{ + SrcItem *p = 0; /* SrcItem that reads from co-routine */ + + if( pLeft->pSrc->nSrc==0 ){ + /* Co-routine has not yet been started and the special Select object + ** that accesses the co-routine has not yet been created. This block + ** does both those things. */ + Vdbe *v = sqlite3GetVdbe(pParse); + Select *pRet = sqlite3SelectNew(pParse, 0, 0, 0, 0, 0, 0, 0, 0); + + /* Ensure the database schema has been read. This is to ensure we have + ** the correct text encoding. */ + if( (pParse->db->mDbFlags & DBFLAG_SchemaKnownOk)==0 ){ + sqlite3ReadSchema(pParse); + } + + if( pRet ){ + SelectDest dest; + Subquery *pSubq; + pRet->pSrc->nSrc = 1; + pRet->pPrior = pLeft->pPrior; + pRet->op = pLeft->op; + if( pRet->pPrior ) pRet->selFlags |= SF_Values; + pLeft->pPrior = 0; + pLeft->op = TK_SELECT; + assert( pLeft->pNext==0 ); + assert( pRet->pNext==0 ); + p = &pRet->pSrc->a[0]; + p->fg.viaCoroutine = 1; + p->iCursor = -1; + assert( !p->fg.isIndexedBy && !p->fg.isTabFunc ); + p->u1.nRow = 2; + if( sqlite3SrcItemAttachSubquery(pParse, p, pLeft, 0) ){ + pSubq = p->u4.pSubq; + pSubq->addrFillSub = sqlite3VdbeCurrentAddr(v) + 1; + pSubq->regReturn = ++pParse->nMem; + sqlite3VdbeAddOp3(v, OP_InitCoroutine, + pSubq->regReturn, 0, pSubq->addrFillSub); + sqlite3SelectDestInit(&dest, SRT_Coroutine, pSubq->regReturn); + + /* Allocate registers for the output of the co-routine. Do so so + ** that there are two unused registers immediately before those + ** used by the co-routine. This allows the code in sqlite3Insert() + ** to use these registers directly, instead of copying the output + ** of the co-routine to a separate array for processing. */ + dest.iSdst = pParse->nMem + 3; + dest.nSdst = pLeft->pEList->nExpr; + pParse->nMem += 2 + dest.nSdst; + + pLeft->selFlags |= SF_MultiValue; + sqlite3Select(pParse, pLeft, &dest); + pSubq->regResult = dest.iSdst; + assert( pParse->nErr || dest.iSdst>0 ); + } + pLeft = pRet; + } + }else{ + p = &pLeft->pSrc->a[0]; + assert( !p->fg.isTabFunc && !p->fg.isIndexedBy ); + p->u1.nRow++; + } + + if( pParse->nErr==0 ){ + Subquery *pSubq; + assert( p!=0 ); + assert( p->fg.isSubquery ); + pSubq = p->u4.pSubq; + assert( pSubq!=0 ); + assert( pSubq->pSelect!=0 ); + assert( pSubq->pSelect->pEList!=0 ); + if( pSubq->pSelect->pEList->nExpr!=pRow->nExpr ){ + sqlite3SelectWrongNumTermsError(pParse, pSubq->pSelect); + }else{ + sqlite3ExprCodeExprList(pParse, pRow, pSubq->regResult, 0, 0); + sqlite3VdbeAddOp1(pParse->pVdbe, OP_Yield, pSubq->regReturn); + } + } + sqlite3ExprListDelete(pParse->db, pRow); + } + + return pLeft; +} /* Forward declaration */ static int xferOptimization( @@ -132901,6 +137613,7 @@ SQLITE_PRIVATE void sqlite3Insert( int regRowid; /* registers holding insert rowid */ int regData; /* register holding first column to insert */ int *aRegIdx = 0; /* One register allocated to each index */ + int *aTabColMap = 0; /* Mapping from pTab columns to pCol entries */ #ifndef SQLITE_OMIT_TRIGGER int isView; /* True if attempting to insert into a view */ @@ -133045,31 +137758,25 @@ SQLITE_PRIVATE void sqlite3Insert( */ bIdListInOrder = (pTab->tabFlags & (TF_OOOHidden|TF_HasStored))==0; if( pColumn ){ - assert( pColumn->eU4!=EU4_EXPR ); - pColumn->eU4 = EU4_IDX; - for(i=0; inId; i++){ - pColumn->a[i].u4.idx = -1; - } + aTabColMap = sqlite3DbMallocZero(db, pTab->nCol*sizeof(int)); + if( aTabColMap==0 ) goto insert_cleanup; for(i=0; inId; i++){ - for(j=0; jnCol; j++){ - if( sqlite3StrICmp(pColumn->a[i].zName, pTab->aCol[j].zCnName)==0 ){ - pColumn->a[i].u4.idx = j; - if( i!=j ) bIdListInOrder = 0; - if( j==pTab->iPKey ){ - ipkColumn = i; assert( !withoutRowid ); - } + j = sqlite3ColumnIndex(pTab, pColumn->a[i].zName); + if( j>=0 ){ + if( aTabColMap[j]==0 ) aTabColMap[j] = i+1; + if( i!=j ) bIdListInOrder = 0; + if( j==pTab->iPKey ){ + ipkColumn = i; assert( !withoutRowid ); + } #ifndef SQLITE_OMIT_GENERATED_COLUMNS - if( pTab->aCol[j].colFlags & (COLFLAG_STORED|COLFLAG_VIRTUAL) ){ - sqlite3ErrorMsg(pParse, - "cannot INSERT into generated column \"%s\"", - pTab->aCol[j].zCnName); - goto insert_cleanup; - } -#endif - break; + if( pTab->aCol[j].colFlags & (COLFLAG_STORED|COLFLAG_VIRTUAL) ){ + sqlite3ErrorMsg(pParse, + "cannot INSERT into generated column \"%s\"", + pTab->aCol[j].zCnName); + goto insert_cleanup; } - } - if( j>=pTab->nCol ){ +#endif + }else{ if( sqlite3IsRowid(pColumn->a[i].zName) && !withoutRowid ){ ipkColumn = i; bIdListInOrder = 0; @@ -133091,25 +137798,45 @@ SQLITE_PRIVATE void sqlite3Insert( if( pSelect ){ /* Data is coming from a SELECT or from a multi-row VALUES clause. ** Generate a co-routine to run the SELECT. */ - int regYield; /* Register holding co-routine entry-point */ - int addrTop; /* Top of the co-routine */ int rc; /* Result code */ - regYield = ++pParse->nMem; - addrTop = sqlite3VdbeCurrentAddr(v) + 1; - sqlite3VdbeAddOp3(v, OP_InitCoroutine, regYield, 0, addrTop); - sqlite3SelectDestInit(&dest, SRT_Coroutine, regYield); - dest.iSdst = bIdListInOrder ? regData : 0; - dest.nSdst = pTab->nCol; - rc = sqlite3Select(pParse, pSelect, &dest); - regFromSelect = dest.iSdst; - assert( db->pParse==pParse ); - if( rc || pParse->nErr ) goto insert_cleanup; - assert( db->mallocFailed==0 ); - sqlite3VdbeEndCoroutine(v, regYield); - sqlite3VdbeJumpHere(v, addrTop - 1); /* label B: */ - assert( pSelect->pEList ); - nColumn = pSelect->pEList->nExpr; + if( pSelect->pSrc->nSrc==1 + && pSelect->pSrc->a[0].fg.viaCoroutine + && pSelect->pPrior==0 + ){ + SrcItem *pItem = &pSelect->pSrc->a[0]; + Subquery *pSubq; + assert( pItem->fg.isSubquery ); + pSubq = pItem->u4.pSubq; + dest.iSDParm = pSubq->regReturn; + regFromSelect = pSubq->regResult; + assert( pSubq->pSelect!=0 ); + assert( pSubq->pSelect->pEList!=0 ); + nColumn = pSubq->pSelect->pEList->nExpr; + ExplainQueryPlan((pParse, 0, "SCAN %S", pItem)); + if( bIdListInOrder && nColumn==pTab->nCol ){ + regData = regFromSelect; + regRowid = regData - 1; + regIns = regRowid - (IsVirtual(pTab) ? 1 : 0); + } + }else{ + int addrTop; /* Top of the co-routine */ + int regYield = ++pParse->nMem; + addrTop = sqlite3VdbeCurrentAddr(v) + 1; + sqlite3VdbeAddOp3(v, OP_InitCoroutine, regYield, 0, addrTop); + sqlite3SelectDestInit(&dest, SRT_Coroutine, regYield); + dest.iSdst = bIdListInOrder ? regData : 0; + dest.nSdst = pTab->nCol; + rc = sqlite3Select(pParse, pSelect, &dest); + regFromSelect = dest.iSdst; + assert( db->pParse==pParse ); + if( rc || pParse->nErr ) goto insert_cleanup; + assert( db->mallocFailed==0 ); + sqlite3VdbeEndCoroutine(v, regYield); + sqlite3VdbeJumpHere(v, addrTop - 1); /* label B: */ + assert( pSelect->pEList ); + nColumn = pSelect->pEList->nExpr; + } /* Set useTempTable to TRUE if the result of the SELECT statement ** should be written into a temporary table (template 4). Set to @@ -133347,7 +138074,7 @@ SQLITE_PRIVATE void sqlite3Insert( continue; }else if( pColumn==0 ){ /* Hidden columns that are not explicitly named in the INSERT - ** get there default value */ + ** get their default value */ sqlite3ExprCodeFactorable(pParse, sqlite3ColumnExpr(pTab, &pTab->aCol[i]), iRegStore); @@ -133355,9 +138082,9 @@ SQLITE_PRIVATE void sqlite3Insert( } } if( pColumn ){ - assert( pColumn->eU4==EU4_IDX ); - for(j=0; jnId && pColumn->a[j].u4.idx!=i; j++){} - if( j>=pColumn->nId ){ + j = aTabColMap[i]; + assert( j>=0 && j<=pColumn->nId ); + if( j==0 ){ /* A column not named in the insert column list gets its ** default value */ sqlite3ExprCodeFactorable(pParse, @@ -133365,7 +138092,7 @@ SQLITE_PRIVATE void sqlite3Insert( iRegStore); continue; } - k = j; + k = j - 1; }else if( nColumn==0 ){ /* This is INSERT INTO ... DEFAULT VALUES. Load the default value. */ sqlite3ExprCodeFactorable(pParse, @@ -133610,7 +138337,10 @@ SQLITE_PRIVATE void sqlite3Insert( sqlite3ExprListDelete(db, pList); sqlite3UpsertDelete(db, pUpsert); sqlite3SelectDelete(db, pSelect); - sqlite3IdListDelete(db, pColumn); + if( pColumn ){ + sqlite3IdListDelete(db, pColumn); + sqlite3DbFree(db, aTabColMap); + } if( aRegIdx ) sqlite3DbNNFreeNN(db, aRegIdx); } @@ -134069,7 +138799,7 @@ SQLITE_PRIVATE void sqlite3GenerateConstraintChecks( ** could happen in any order, but they are grouped up front for ** convenience. ** - ** 2018-08-14: Ticket https://www.sqlite.org/src/info/908f001483982c43 + ** 2018-08-14: Ticket https://sqlite.org/src/info/908f001483982c43 ** The order of constraints used to have OE_Update as (2) and OE_Abort ** and so forth as (1). But apparently PostgreSQL checks the OE_Update ** constraint before any others, so it had to be moved. @@ -135005,7 +139735,7 @@ static int xferOptimization( if( pSelect->pSrc->nSrc!=1 ){ return 0; /* FROM clause must have exactly one term */ } - if( pSelect->pSrc->a[0].pSelect ){ + if( pSelect->pSrc->a[0].fg.isSubquery ){ return 0; /* FROM clause cannot contain a subquery */ } if( pSelect->pWhere ){ @@ -135156,7 +139886,10 @@ static int xferOptimization( } } #ifndef SQLITE_OMIT_CHECK - if( pDest->pCheck && sqlite3ExprListCompare(pSrc->pCheck,pDest->pCheck,-1) ){ + if( pDest->pCheck + && (db->mDbFlags & DBFLAG_Vacuum)==0 + && sqlite3ExprListCompare(pSrc->pCheck,pDest->pCheck,-1) + ){ return 0; /* Tables have different CHECK constraints. Ticket #2252 */ } #endif @@ -135876,6 +140609,12 @@ struct sqlite3_api_routines { /* Version 3.44.0 and later */ void *(*get_clientdata)(sqlite3*,const char*); int (*set_clientdata)(sqlite3*, const char*, void*, void(*)(void*)); + /* Version 3.50.0 and later */ + int (*setlk_timeout)(sqlite3*,int,int); + /* Version 3.51.0 and later */ + int (*set_errmsg)(sqlite3*,int,const char*); + int (*db_status64)(sqlite3*,int,sqlite3_int64*,sqlite3_int64*,int); + }; /* @@ -136209,6 +140948,11 @@ typedef int (*sqlite3_loadext_entry)( /* Version 3.44.0 and later */ #define sqlite3_get_clientdata sqlite3_api->get_clientdata #define sqlite3_set_clientdata sqlite3_api->set_clientdata +/* Version 3.50.0 and later */ +#define sqlite3_setlk_timeout sqlite3_api->setlk_timeout +/* Version 3.51.0 and later */ +#define sqlite3_set_errmsg sqlite3_api->set_errmsg +#define sqlite3_db_status64 sqlite3_api->db_status64 #endif /* !defined(SQLITE_CORE) && !defined(SQLITE_OMIT_LOAD_EXTENSION) */ #if !defined(SQLITE_CORE) && !defined(SQLITE_OMIT_LOAD_EXTENSION) @@ -136730,7 +141474,12 @@ static const sqlite3_api_routines sqlite3Apis = { sqlite3_stmt_explain, /* Version 3.44.0 and later */ sqlite3_get_clientdata, - sqlite3_set_clientdata + sqlite3_set_clientdata, + /* Version 3.50.0 and later */ + sqlite3_setlk_timeout, + /* Version 3.51.0 and later */ + sqlite3_set_errmsg, + sqlite3_db_status64 }; /* True if x is the directory separator character @@ -137252,48 +142001,48 @@ static const char *const pragCName[] = { /* 13 */ "pk", /* 14 */ "hidden", /* table_info reuses 8 */ - /* 15 */ "schema", /* Used by: table_list */ - /* 16 */ "name", + /* 15 */ "name", /* Used by: function_list */ + /* 16 */ "builtin", /* 17 */ "type", - /* 18 */ "ncol", - /* 19 */ "wr", - /* 20 */ "strict", - /* 21 */ "seqno", /* Used by: index_xinfo */ - /* 22 */ "cid", - /* 23 */ "name", - /* 24 */ "desc", - /* 25 */ "coll", - /* 26 */ "key", - /* 27 */ "name", /* Used by: function_list */ - /* 28 */ "builtin", - /* 29 */ "type", - /* 30 */ "enc", - /* 31 */ "narg", - /* 32 */ "flags", - /* 33 */ "tbl", /* Used by: stats */ - /* 34 */ "idx", - /* 35 */ "wdth", - /* 36 */ "hght", - /* 37 */ "flgs", - /* 38 */ "seq", /* Used by: index_list */ - /* 39 */ "name", - /* 40 */ "unique", - /* 41 */ "origin", - /* 42 */ "partial", + /* 18 */ "enc", + /* 19 */ "narg", + /* 20 */ "flags", + /* 21 */ "schema", /* Used by: table_list */ + /* 22 */ "name", + /* 23 */ "type", + /* 24 */ "ncol", + /* 25 */ "wr", + /* 26 */ "strict", + /* 27 */ "seqno", /* Used by: index_xinfo */ + /* 28 */ "cid", + /* 29 */ "name", + /* 30 */ "desc", + /* 31 */ "coll", + /* 32 */ "key", + /* 33 */ "seq", /* Used by: index_list */ + /* 34 */ "name", + /* 35 */ "unique", + /* 36 */ "origin", + /* 37 */ "partial", + /* 38 */ "tbl", /* Used by: stats */ + /* 39 */ "idx", + /* 40 */ "wdth", + /* 41 */ "hght", + /* 42 */ "flgs", /* 43 */ "table", /* Used by: foreign_key_check */ /* 44 */ "rowid", /* 45 */ "parent", /* 46 */ "fkid", - /* index_info reuses 21 */ - /* 47 */ "seq", /* Used by: database_list */ - /* 48 */ "name", - /* 49 */ "file", - /* 50 */ "busy", /* Used by: wal_checkpoint */ - /* 51 */ "log", - /* 52 */ "checkpointed", - /* collation_list reuses 38 */ + /* 47 */ "busy", /* Used by: wal_checkpoint */ + /* 48 */ "log", + /* 49 */ "checkpointed", + /* 50 */ "seq", /* Used by: database_list */ + /* 51 */ "name", + /* 52 */ "file", + /* index_info reuses 27 */ /* 53 */ "database", /* Used by: lock_status */ /* 54 */ "status", + /* collation_list reuses 33 */ /* 55 */ "cache_size", /* Used by: default_cache_size */ /* module_list pragma_list reuses 9 */ /* 56 */ "timeout", /* Used by: busy_timeout */ @@ -137386,7 +142135,7 @@ static const PragmaName aPragmaName[] = { {/* zName: */ "collation_list", /* ePragTyp: */ PragTyp_COLLATION_LIST, /* ePragFlg: */ PragFlg_Result0, - /* ColNames: */ 38, 2, + /* ColNames: */ 33, 2, /* iArg: */ 0 }, #endif #if !defined(SQLITE_OMIT_COMPILEOPTION_DIAGS) @@ -137421,7 +142170,7 @@ static const PragmaName aPragmaName[] = { {/* zName: */ "database_list", /* ePragTyp: */ PragTyp_DATABASE_LIST, /* ePragFlg: */ PragFlg_Result0, - /* ColNames: */ 47, 3, + /* ColNames: */ 50, 3, /* iArg: */ 0 }, #endif #if !defined(SQLITE_OMIT_PAGER_PRAGMAS) && !defined(SQLITE_OMIT_DEPRECATED) @@ -137501,7 +142250,7 @@ static const PragmaName aPragmaName[] = { {/* zName: */ "function_list", /* ePragTyp: */ PragTyp_FUNCTION_LIST, /* ePragFlg: */ PragFlg_Result0, - /* ColNames: */ 27, 6, + /* ColNames: */ 15, 6, /* iArg: */ 0 }, #endif #endif @@ -137530,17 +142279,17 @@ static const PragmaName aPragmaName[] = { {/* zName: */ "index_info", /* ePragTyp: */ PragTyp_INDEX_INFO, /* ePragFlg: */ PragFlg_NeedSchema|PragFlg_Result1|PragFlg_SchemaOpt, - /* ColNames: */ 21, 3, + /* ColNames: */ 27, 3, /* iArg: */ 0 }, {/* zName: */ "index_list", /* ePragTyp: */ PragTyp_INDEX_LIST, /* ePragFlg: */ PragFlg_NeedSchema|PragFlg_Result1|PragFlg_SchemaOpt, - /* ColNames: */ 38, 5, + /* ColNames: */ 33, 5, /* iArg: */ 0 }, {/* zName: */ "index_xinfo", /* ePragTyp: */ PragTyp_INDEX_INFO, /* ePragFlg: */ PragFlg_NeedSchema|PragFlg_Result1|PragFlg_SchemaOpt, - /* ColNames: */ 21, 6, + /* ColNames: */ 27, 6, /* iArg: */ 1 }, #endif #if !defined(SQLITE_OMIT_INTEGRITY_CHECK) @@ -137719,7 +142468,7 @@ static const PragmaName aPragmaName[] = { {/* zName: */ "stats", /* ePragTyp: */ PragTyp_STATS, /* ePragFlg: */ PragFlg_NeedSchema|PragFlg_Result0|PragFlg_SchemaReq, - /* ColNames: */ 33, 5, + /* ColNames: */ 38, 5, /* iArg: */ 0 }, #endif #if !defined(SQLITE_OMIT_PAGER_PRAGMAS) @@ -137738,7 +142487,7 @@ static const PragmaName aPragmaName[] = { {/* zName: */ "table_list", /* ePragTyp: */ PragTyp_TABLE_LIST, /* ePragFlg: */ PragFlg_NeedSchema|PragFlg_Result1, - /* ColNames: */ 15, 6, + /* ColNames: */ 21, 6, /* iArg: */ 0 }, {/* zName: */ "table_xinfo", /* ePragTyp: */ PragTyp_TABLE_INFO, @@ -137815,7 +142564,7 @@ static const PragmaName aPragmaName[] = { {/* zName: */ "wal_checkpoint", /* ePragTyp: */ PragTyp_WAL_CHECKPOINT, /* ePragFlg: */ PragFlg_NeedSchema, - /* ColNames: */ 50, 3, + /* ColNames: */ 47, 3, /* iArg: */ 0 }, #endif #if !defined(SQLITE_OMIT_FLAG_PRAGMAS) @@ -137831,6 +142580,34 @@ static const PragmaName aPragmaName[] = { /************** End of pragma.h **********************************************/ /************** Continuing where we left off in pragma.c *********************/ +/* +** When the 0x10 bit of PRAGMA optimize is set, any ANALYZE commands +** will be run with an analysis_limit set to the lessor of the value of +** the following macro or to the actual analysis_limit if it is non-zero, +** in order to prevent PRAGMA optimize from running for too long. +** +** The value of 2000 is chosen empirically so that the worst-case run-time +** for PRAGMA optimize does not exceed 100 milliseconds against a variety +** of test databases on a RaspberryPI-4 compiled using -Os and without +** -DSQLITE_DEBUG. Of course, your mileage may vary. For the purpose of +** this paragraph, "worst-case" means that ANALYZE ends up being +** run on every table in the database. The worst case typically only +** happens if PRAGMA optimize is run on a database file for which ANALYZE +** has not been previously run and the 0x10000 flag is included so that +** all tables are analyzed. The usual case for PRAGMA optimize is that +** no ANALYZE commands will be run at all, or if any ANALYZE happens it +** will be against a single table, so that expected timing for PRAGMA +** optimize on a PI-4 is more like 1 millisecond or less with the 0x10000 +** flag or less than 100 microseconds without the 0x10000 flag. +** +** An analysis limit of 2000 is almost always sufficient for the query +** planner to fully characterize an index. The additional accuracy from +** a larger analysis is not usually helpful. +*/ +#ifndef SQLITE_DEFAULT_OPTIMIZE_LIMIT +# define SQLITE_DEFAULT_OPTIMIZE_LIMIT 2000 +#endif + /* ** Interpret the given string as a safety level. Return 0 for OFF, ** 1 for ON or NORMAL, 2 for FULL, and 3 for EXTRA. Return 1 for an empty or @@ -138164,6 +142941,22 @@ static int integrityCheckResultRow(Vdbe *v){ return addr; } +/* +** Should table pTab be skipped when doing an integrity_check? +** Return true or false. +** +** If pObjTab is not null, the return true if pTab matches pObjTab. +** +** If pObjTab is null, then return true only if pTab is an imposter table. +*/ +static int tableSkipIntegrityCheck(const Table *pTab, const Table *pObjTab){ + if( pObjTab ){ + return pTab!=pObjTab; + }else{ + return (pTab->tabFlags & TF_Imposter)!=0; + } +} + /* ** Process a pragma statement. ** @@ -138917,12 +143710,6 @@ SQLITE_PRIVATE void sqlite3Pragma( ** in auto-commit mode. */ mask &= ~(SQLITE_ForeignKeys); } -#if SQLITE_USER_AUTHENTICATION - if( db->auth.authLevel==UAUTH_User ){ - /* Do not allow non-admin users to modify the schema arbitrarily */ - mask &= ~(SQLITE_WriteSchema); - } -#endif if( sqlite3GetBoolean(zRight, 0) ){ if( (mask & SQLITE_WriteSchema)==0 @@ -138932,7 +143719,10 @@ SQLITE_PRIVATE void sqlite3Pragma( } }else{ db->flags &= ~mask; - if( mask==SQLITE_DeferFKs ) db->nDeferredImmCons = 0; + if( mask==SQLITE_DeferFKs ){ + db->nDeferredImmCons = 0; + db->nDeferredCons = 0; + } if( (mask & SQLITE_WriteSchema)!=0 && sqlite3_stricmp(zRight, "reset")==0 ){ @@ -139058,7 +143848,8 @@ SQLITE_PRIVATE void sqlite3Pragma( char *zSql = sqlite3MPrintf(db, "SELECT*FROM\"%w\"", pTab->zName); if( zSql ){ sqlite3_stmt *pDummy = 0; - (void)sqlite3_prepare(db, zSql, -1, &pDummy, 0); + (void)sqlite3_prepare_v3(db, zSql, -1, SQLITE_PREPARE_DONT_LOG, + &pDummy, 0); (void)sqlite3_finalize(pDummy); sqlite3DbFree(db, zSql); } @@ -139476,7 +144267,7 @@ SQLITE_PRIVATE void sqlite3Pragma( /* Set the maximum error count */ mxErr = SQLITE_INTEGRITY_CHECK_ERROR_MAX; if( zRight ){ - if( sqlite3GetInt32(zRight, &mxErr) ){ + if( sqlite3GetInt32(pValue->z, &mxErr) ){ if( mxErr<=0 ){ mxErr = SQLITE_INTEGRITY_CHECK_ERROR_MAX; } @@ -139493,7 +144284,6 @@ SQLITE_PRIVATE void sqlite3Pragma( Hash *pTbls; /* Set of all tables in the schema */ int *aRoot; /* Array of root page numbers of all btrees */ int cnt = 0; /* Number of entries in aRoot[] */ - int mxIdx = 0; /* Maximum number of indexes for any table */ if( OMIT_TEMPDB && i==1 ) continue; if( iDb>=0 && i!=iDb ) continue; @@ -139512,10 +144302,9 @@ SQLITE_PRIVATE void sqlite3Pragma( Table *pTab = sqliteHashData(x); /* Current table */ Index *pIdx; /* An index on pTab */ int nIdx; /* Number of indexes on pTab */ - if( pObjTab && pObjTab!=pTab ) continue; + if( tableSkipIntegrityCheck(pTab,pObjTab) ) continue; if( HasRowid(pTab) ) cnt++; for(nIdx=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, nIdx++){ cnt++; } - if( nIdx>mxIdx ) mxIdx = nIdx; } if( cnt==0 ) continue; if( pObjTab ) cnt++; @@ -139526,7 +144315,7 @@ SQLITE_PRIVATE void sqlite3Pragma( for(x=sqliteHashFirst(pTbls); x; x=sqliteHashNext(x)){ Table *pTab = sqliteHashData(x); Index *pIdx; - if( pObjTab && pObjTab!=pTab ) continue; + if( tableSkipIntegrityCheck(pTab,pObjTab) ) continue; if( HasRowid(pTab) ) aRoot[++cnt] = pTab->tnum; for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ aRoot[++cnt] = pIdx->tnum; @@ -139535,12 +144324,13 @@ SQLITE_PRIVATE void sqlite3Pragma( aRoot[0] = cnt; /* Make sure sufficient number of registers have been allocated */ - sqlite3TouchRegister(pParse, 8+mxIdx); + sqlite3TouchRegister(pParse, 8+cnt); + sqlite3VdbeAddOp3(v, OP_Null, 0, 8, 8+cnt); sqlite3ClearTempRegCache(pParse); /* Do the b-tree integrity checks */ - sqlite3VdbeAddOp4(v, OP_IntegrityCk, 2, cnt, 1, (char*)aRoot,P4_INTARRAY); - sqlite3VdbeChangeP5(v, (u8)i); + sqlite3VdbeAddOp4(v, OP_IntegrityCk, 1, cnt, 8, (char*)aRoot,P4_INTARRAY); + sqlite3VdbeChangeP5(v, (u16)i); addr = sqlite3VdbeAddOp1(v, OP_IsNull, 2); VdbeCoverage(v); sqlite3VdbeAddOp4(v, OP_String8, 0, 3, 0, sqlite3MPrintf(db, "*** in database %s ***\n", db->aDb[i].zDbSName), @@ -139549,6 +144339,36 @@ SQLITE_PRIVATE void sqlite3Pragma( integrityCheckResultRow(v); sqlite3VdbeJumpHere(v, addr); + /* Check that the indexes all have the right number of rows */ + cnt = pObjTab ? 1 : 0; + sqlite3VdbeLoadString(v, 2, "wrong # of entries in index "); + for(x=sqliteHashFirst(pTbls); x; x=sqliteHashNext(x)){ + int iTab = 0; + Table *pTab = sqliteHashData(x); + Index *pIdx; + if( tableSkipIntegrityCheck(pTab,pObjTab) ) continue; + if( HasRowid(pTab) ){ + iTab = cnt++; + }else{ + iTab = cnt; + for(pIdx=pTab->pIndex; ALWAYS(pIdx); pIdx=pIdx->pNext){ + if( IsPrimaryKeyIndex(pIdx) ) break; + iTab++; + } + } + for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ + if( pIdx->pPartIdxWhere==0 ){ + addr = sqlite3VdbeAddOp3(v, OP_Eq, 8+cnt, 0, 8+iTab); + VdbeCoverageNeverNull(v); + sqlite3VdbeLoadString(v, 4, pIdx->zName); + sqlite3VdbeAddOp3(v, OP_Concat, 4, 2, 3); + integrityCheckResultRow(v); + sqlite3VdbeJumpHere(v, addr); + } + cnt++; + } + } + /* Make sure all the indices are constructed correctly. */ for(x=sqliteHashFirst(pTbls); x; x=sqliteHashNext(x)){ @@ -139562,7 +144382,7 @@ SQLITE_PRIVATE void sqlite3Pragma( int r2; /* Previous key for WITHOUT ROWID tables */ int mxCol; /* Maximum non-virtual column number */ - if( pObjTab && pObjTab!=pTab ) continue; + if( tableSkipIntegrityCheck(pTab,pObjTab) ) continue; if( !IsOrdinaryTable(pTab) ) continue; if( isQuick || HasRowid(pTab) ){ pPk = 0; @@ -139872,21 +144692,9 @@ SQLITE_PRIVATE void sqlite3Pragma( } sqlite3VdbeAddOp2(v, OP_Next, iDataCur, loopTop); VdbeCoverage(v); sqlite3VdbeJumpHere(v, loopTop-1); - if( !isQuick ){ - sqlite3VdbeLoadString(v, 2, "wrong # of entries in index "); - for(j=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, j++){ - if( pPk==pIdx ) continue; - sqlite3VdbeAddOp2(v, OP_Count, iIdxCur+j, 3); - addr = sqlite3VdbeAddOp3(v, OP_Eq, 8+j, 0, 3); VdbeCoverage(v); - sqlite3VdbeChangeP5(v, SQLITE_NOTNULL); - sqlite3VdbeLoadString(v, 4, pIdx->zName); - sqlite3VdbeAddOp3(v, OP_Concat, 4, 2, 3); - integrityCheckResultRow(v); - sqlite3VdbeJumpHere(v, addr); - } - if( pPk ){ - sqlite3ReleaseTempRange(pParse, r2, pPk->nKeyCol); - } + if( pPk ){ + assert( !isQuick ); + sqlite3ReleaseTempRange(pParse, r2, pPk->nKeyCol); } } @@ -139898,7 +144706,7 @@ SQLITE_PRIVATE void sqlite3Pragma( Table *pTab = sqliteHashData(x); sqlite3_vtab *pVTab; int a1; - if( pObjTab && pObjTab!=pTab ) continue; + if( tableSkipIntegrityCheck(pTab,pObjTab) ) continue; if( IsOrdinaryTable(pTab) ) continue; if( !IsVirtual(pTab) ) continue; if( pTab->nCol<=0 ){ @@ -140130,6 +144938,8 @@ SQLITE_PRIVATE void sqlite3Pragma( eMode = SQLITE_CHECKPOINT_RESTART; }else if( sqlite3StrICmp(zRight, "truncate")==0 ){ eMode = SQLITE_CHECKPOINT_TRUNCATE; + }else if( sqlite3StrICmp(zRight, "noop")==0 ){ + eMode = SQLITE_CHECKPOINT_NOOP; } } pParse->nMem = 3; @@ -140184,44 +144994,63 @@ SQLITE_PRIVATE void sqlite3Pragma( ** ** The optional argument is a bitmask of optimizations to perform: ** - ** 0x0001 Debugging mode. Do not actually perform any optimizations - ** but instead return one line of text for each optimization - ** that would have been done. Off by default. + ** 0x00001 Debugging mode. Do not actually perform any optimizations + ** but instead return one line of text for each optimization + ** that would have been done. Off by default. ** - ** 0x0002 Run ANALYZE on tables that might benefit. On by default. - ** See below for additional information. + ** 0x00002 Run ANALYZE on tables that might benefit. On by default. + ** See below for additional information. ** - ** 0x0004 (Not yet implemented) Record usage and performance - ** information from the current session in the - ** database file so that it will be available to "optimize" - ** pragmas run by future database connections. + ** 0x00010 Run all ANALYZE operations using an analysis_limit that + ** is the lessor of the current analysis_limit and the + ** SQLITE_DEFAULT_OPTIMIZE_LIMIT compile-time option. + ** The default value of SQLITE_DEFAULT_OPTIMIZE_LIMIT is + ** currently (2024-02-19) set to 2000, which is such that + ** the worst case run-time for PRAGMA optimize on a 100MB + ** database will usually be less than 100 milliseconds on + ** a RaspberryPI-4 class machine. On by default. ** - ** 0x0008 (Not yet implemented) Create indexes that might have - ** been helpful to recent queries + ** 0x10000 Look at tables to see if they need to be reanalyzed + ** due to growth or shrinkage even if they have not been + ** queried during the current connection. Off by default. ** - ** The default MASK is and always shall be 0xfffe. 0xfffe means perform all - ** of the optimizations listed above except Debug Mode, including new - ** optimizations that have not yet been invented. If new optimizations are - ** ever added that should be off by default, those off-by-default - ** optimizations will have bitmasks of 0x10000 or larger. + ** The default MASK is and always shall be 0x0fffe. In the current + ** implementation, the default mask only covers the 0x00002 optimization, + ** though additional optimizations that are covered by 0x0fffe might be + ** added in the future. Optimizations that are off by default and must + ** be explicitly requested have masks of 0x10000 or greater. ** ** DETERMINATION OF WHEN TO RUN ANALYZE ** ** In the current implementation, a table is analyzed if only if all of ** the following are true: ** - ** (1) MASK bit 0x02 is set. + ** (1) MASK bit 0x00002 is set. + ** + ** (2) The table is an ordinary table, not a virtual table or view. ** - ** (2) The query planner used sqlite_stat1-style statistics for one or - ** more indexes of the table at some point during the lifetime of - ** the current connection. + ** (3) The table name does not begin with "sqlite_". ** - ** (3) One or more indexes of the table are currently unanalyzed OR - ** the number of rows in the table has increased by 25 times or more - ** since the last time ANALYZE was run. + ** (4) One or more of the following is true: + ** (4a) The 0x10000 MASK bit is set. + ** (4b) One or more indexes on the table lacks an entry + ** in the sqlite_stat1 table. + ** (4c) The query planner used sqlite_stat1-style statistics for one + ** or more indexes of the table at some point during the lifetime + ** of the current connection. + ** + ** (5) One or more of the following is true: + ** (5a) One or more indexes on the table lacks an entry + ** in the sqlite_stat1 table. (Same as 4a) + ** (5b) The number of rows in the table has increased or decreased by + ** 10-fold. In other words, the current size of the table is + ** 10 times larger than the size in sqlite_stat1 or else the + ** current size is less than 1/10th the size in sqlite_stat1. ** ** The rules for when tables are analyzed are likely to change in - ** future releases. + ** future releases. Future versions of SQLite might accept a string + ** literal argument to this pragma that contains a mnemonic description + ** of the options rather than a bitmap. */ case PragTyp_OPTIMIZE: { int iDbLast; /* Loop termination point for the schema loop */ @@ -140233,6 +145062,10 @@ SQLITE_PRIVATE void sqlite3Pragma( LogEst szThreshold; /* Size threshold above which reanalysis needed */ char *zSubSql; /* SQL statement for the OP_SqlExec opcode */ u32 opMask; /* Mask of operations to perform */ + int nLimit; /* Analysis limit to use */ + int nCheck = 0; /* Number of tables to be optimized */ + int nBtree = 0; /* Number of btrees to scan */ + int nIndex; /* Number of indexes on the current table */ if( zRight ){ opMask = (u32)sqlite3Atoi(zRight); @@ -140240,6 +145073,14 @@ SQLITE_PRIVATE void sqlite3Pragma( }else{ opMask = 0xfffe; } + if( (opMask & 0x10)==0 ){ + nLimit = 0; + }else if( db->nAnalysisLimit>0 + && db->nAnalysisLimitnTab++; for(iDbLast = zDb?iDb:db->nDb-1; iDb<=iDbLast; iDb++){ if( iDb==1 ) continue; @@ -140248,23 +145089,61 @@ SQLITE_PRIVATE void sqlite3Pragma( for(k=sqliteHashFirst(&pSchema->tblHash); k; k=sqliteHashNext(k)){ pTab = (Table*)sqliteHashData(k); - /* If table pTab has not been used in a way that would benefit from - ** having analysis statistics during the current session, then skip it. - ** This also has the effect of skipping virtual tables and views */ - if( (pTab->tabFlags & TF_StatsUsed)==0 ) continue; + /* This only works for ordinary tables */ + if( !IsOrdinaryTable(pTab) ) continue; + + /* Do not scan system tables */ + if( 0==sqlite3StrNICmp(pTab->zName, "sqlite_", 7) ) continue; - /* Reanalyze if the table is 25 times larger than the last analysis */ - szThreshold = pTab->nRowLogEst + 46; assert( sqlite3LogEst(25)==46 ); + /* Find the size of the table as last recorded in sqlite_stat1. + ** If any index is unanalyzed, then the threshold is -1 to + ** indicate a new, unanalyzed index + */ + szThreshold = pTab->nRowLogEst; + nIndex = 0; for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ + nIndex++; if( !pIdx->hasStat1 ){ - szThreshold = 0; /* Always analyze if any index lacks statistics */ - break; + szThreshold = -1; /* Always analyze if any index lacks statistics */ } } - if( szThreshold ){ - sqlite3OpenTable(pParse, iTabCur, iDb, pTab, OP_OpenRead); - sqlite3VdbeAddOp3(v, OP_IfSmaller, iTabCur, - sqlite3VdbeCurrentAddr(v)+2+(opMask&1), szThreshold); + + /* If table pTab has not been used in a way that would benefit from + ** having analysis statistics during the current session, then skip it, + ** unless the 0x10000 MASK bit is set. */ + if( (pTab->tabFlags & TF_MaybeReanalyze)!=0 ){ + /* Check for size change if stat1 has been used for a query */ + }else if( opMask & 0x10000 ){ + /* Check for size change if 0x10000 is set */ + }else if( pTab->pIndex!=0 && szThreshold<0 ){ + /* Do analysis if unanalyzed indexes exists */ + }else{ + /* Otherwise, we can skip this table */ + continue; + } + + nCheck++; + if( nCheck==2 ){ + /* If ANALYZE might be invoked two or more times, hold a write + ** transaction for efficiency */ + sqlite3BeginWriteOperation(pParse, 0, iDb); + } + nBtree += nIndex+1; + + /* Reanalyze if the table is 10 times larger or smaller than + ** the last analysis. Unconditional reanalysis if there are + ** unanalyzed indexes. */ + sqlite3OpenTable(pParse, iTabCur, iDb, pTab, OP_OpenRead); + if( szThreshold>=0 ){ + const LogEst iRange = 33; /* 10x size change */ + sqlite3VdbeAddOp4Int(v, OP_IfSizeBetween, iTabCur, + sqlite3VdbeCurrentAddr(v)+2+(opMask&1), + szThreshold>=iRange ? szThreshold-iRange : -1, + szThreshold+iRange); + VdbeCoverage(v); + }else{ + sqlite3VdbeAddOp2(v, OP_Rewind, iTabCur, + sqlite3VdbeCurrentAddr(v)+2+(opMask&1)); VdbeCoverage(v); } zSubSql = sqlite3MPrintf(db, "ANALYZE \"%w\".\"%w\"", @@ -140274,11 +145153,27 @@ SQLITE_PRIVATE void sqlite3Pragma( sqlite3VdbeAddOp4(v, OP_String8, 0, r1, 0, zSubSql, P4_DYNAMIC); sqlite3VdbeAddOp2(v, OP_ResultRow, r1, 1); }else{ - sqlite3VdbeAddOp4(v, OP_SqlExec, 0, 0, 0, zSubSql, P4_DYNAMIC); + sqlite3VdbeAddOp4(v, OP_SqlExec, nLimit ? 0x02 : 00, nLimit, 0, + zSubSql, P4_DYNAMIC); } } } sqlite3VdbeAddOp0(v, OP_Expire); + + /* In a schema with a large number of tables and indexes, scale back + ** the analysis_limit to avoid excess run-time in the worst case. + */ + if( !db->mallocFailed && nLimit>0 && nBtree>100 ){ + int iAddr, iEnd; + VdbeOp *aOp; + nLimit = 100*nLimit/nBtree; + if( nLimit<100 ) nLimit = 100; + aOp = sqlite3VdbeGetOp(v, 0); + iEnd = sqlite3VdbeCurrentAddr(v); + for(iAddr=0; iAddrnConstraint; i++, pConstraint++){ - if( pConstraint->usable==0 ) continue; - if( pConstraint->op!=SQLITE_INDEX_CONSTRAINT_EQ ) continue; if( pConstraint->iColumn < pTab->iHidden ) continue; + if( pConstraint->op!=SQLITE_INDEX_CONSTRAINT_EQ ) continue; + if( pConstraint->usable==0 ) return SQLITE_CONSTRAINT; j = pConstraint->iColumn - pTab->iHidden; assert( j < 2 ); seen[j] = i+1; @@ -140557,12 +145452,13 @@ static int pragmaVtabBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){ j = seen[0]-1; pIdxInfo->aConstraintUsage[j].argvIndex = 1; pIdxInfo->aConstraintUsage[j].omit = 1; - if( seen[1]==0 ) return SQLITE_OK; pIdxInfo->estimatedCost = (double)20; pIdxInfo->estimatedRows = 20; - j = seen[1]-1; - pIdxInfo->aConstraintUsage[j].argvIndex = 2; - pIdxInfo->aConstraintUsage[j].omit = 1; + if( seen[1] ){ + j = seen[1]-1; + pIdxInfo->aConstraintUsage[j].argvIndex = 2; + pIdxInfo->aConstraintUsage[j].omit = 1; + } return SQLITE_OK; } @@ -140582,6 +145478,7 @@ static void pragmaVtabCursorClear(PragmaVtabCursor *pCsr){ int i; sqlite3_finalize(pCsr->pPragma); pCsr->pPragma = 0; + pCsr->iRowid = 0; for(i=0; iazArg); i++){ sqlite3_free(pCsr->azArg[i]); pCsr->azArg[i] = 0; @@ -141055,14 +145952,7 @@ SQLITE_PRIVATE int sqlite3InitOne(sqlite3 *db, int iDb, char **pzErrMsg, u32 mFl #else encoding = SQLITE_UTF8; #endif - if( db->nVdbeActive>0 && encoding!=ENC(db) - && (db->mDbFlags & DBFLAG_Vacuum)==0 - ){ - rc = SQLITE_LOCKED; - goto initone_error_out; - }else{ - sqlite3SetTextEncoding(db, encoding); - } + sqlite3SetTextEncoding(db, encoding); }else{ /* If opening an attached database, the encoding much match ENC(db) */ if( (meta[BTREE_TEXT_ENCODING-1] & 3)!=ENC(db) ){ @@ -141382,7 +146272,13 @@ SQLITE_PRIVATE void *sqlite3ParserAddCleanup( void (*xCleanup)(sqlite3*,void*), /* The cleanup routine */ void *pPtr /* Pointer to object to be cleaned up */ ){ - ParseCleanup *pCleanup = sqlite3DbMallocRaw(pParse->db, sizeof(*pCleanup)); + ParseCleanup *pCleanup; + if( sqlite3FaultSim(300) ){ + pCleanup = 0; + sqlite3OomFault(pParse->db); + }else{ + pCleanup = sqlite3DbMallocRaw(pParse->db, sizeof(*pCleanup)); + } if( pCleanup ){ pCleanup->pNext = pParse->pCleanup; pParse->pCleanup = pCleanup; @@ -141610,9 +146506,11 @@ static int sqlite3LockAndPrepare( rc = sqlite3Prepare(db, zSql, nBytes, prepFlags, pOld, ppStmt, pzTail); assert( rc==SQLITE_OK || *ppStmt==0 ); if( rc==SQLITE_OK || db->mallocFailed ) break; - }while( (rc==SQLITE_ERROR_RETRY && (cnt++)errMask)==rc ); db->busyHandler.nBusy = 0; @@ -141750,12 +146648,24 @@ static int sqlite3Prepare16( if( !sqlite3SafetyCheckOk(db)||zSql==0 ){ return SQLITE_MISUSE_BKPT; } + + /* Make sure nBytes is non-negative and correct. It should be the + ** number of bytes until the end of the input buffer or until the first + ** U+0000 character. If the input nBytes is odd, convert it into + ** an even number. If the input nBytes is negative, then the input + ** must be terminated by at least one U+0000 character */ if( nBytes>=0 ){ int sz; const char *z = (const char*)zSql; for(sz=0; szmutex); zSql8 = sqlite3Utf16to8(db, zSql, nBytes, SQLITE_UTF16NATIVE); if( zSql8 ){ @@ -141769,7 +146679,7 @@ static int sqlite3Prepare16( ** the same number of characters into the UTF-16 string. */ int chars_parsed = sqlite3Utf8CharLen(zSql8, (int)(zTail8-zSql8)); - *pzTail = (u8 *)zSql + sqlite3Utf16ByteLen(zSql, chars_parsed); + *pzTail = (u8 *)zSql + sqlite3Utf16ByteLen(zSql, nBytes, chars_parsed); } sqlite3DbFree(db, zSql8); rc = sqlite3ApiExit(db, rc); @@ -141985,7 +146895,7 @@ SQLITE_PRIVATE Select *sqlite3SelectNew( pNew->addrOpenEphm[0] = -1; pNew->addrOpenEphm[1] = -1; pNew->nSelectRow = 0; - if( pSrc==0 ) pSrc = sqlite3DbMallocZero(pParse->db, sizeof(*pSrc)); + if( pSrc==0 ) pSrc = sqlite3DbMallocZero(pParse->db, SZ_SRCLIST_1); pNew->pSrc = pSrc; pNew->pWhere = pWhere; pNew->pGroupBy = pGroupBy; @@ -142150,10 +147060,33 @@ SQLITE_PRIVATE int sqlite3JoinType(Parse *pParse, Token *pA, Token *pB, Token *p */ SQLITE_PRIVATE int sqlite3ColumnIndex(Table *pTab, const char *zCol){ int i; - u8 h = sqlite3StrIHash(zCol); - Column *pCol; - for(pCol=pTab->aCol, i=0; inCol; pCol++, i++){ - if( pCol->hName==h && sqlite3StrICmp(pCol->zCnName, zCol)==0 ) return i; + u8 h; + const Column *aCol; + int nCol; + + h = sqlite3StrIHash(zCol); + aCol = pTab->aCol; + nCol = pTab->nCol; + + /* See if the aHx gives us a lucky match */ + i = pTab->aHx[h % sizeof(pTab->aHx)]; + assert( i=nCol ) break; } return -1; } @@ -142163,11 +147096,13 @@ SQLITE_PRIVATE int sqlite3ColumnIndex(Table *pTab, const char *zCol){ */ SQLITE_PRIVATE void sqlite3SrcItemColumnUsed(SrcItem *pItem, int iCol){ assert( pItem!=0 ); - assert( (int)pItem->fg.isNestedFrom == IsNestedFrom(pItem->pSelect) ); + assert( (int)pItem->fg.isNestedFrom == IsNestedFrom(pItem) ); if( pItem->fg.isNestedFrom ){ ExprList *pResults; - assert( pItem->pSelect!=0 ); - pResults = pItem->pSelect->pEList; + assert( pItem->fg.isSubquery ); + assert( pItem->u4.pSubq!=0 ); + assert( pItem->u4.pSubq->pSelect!=0 ); + pResults = pItem->u4.pSubq->pSelect->pEList; assert( pResults!=0 ); assert( iCol>=0 && iColnExpr ); pResults->a[iCol].fg.bUsed = 1; @@ -142190,7 +147125,7 @@ static int tableAndColumnIndex( int iEnd, /* Last member of pSrc->a[] to check */ const char *zCol, /* Name of the column we are looking for */ int *piTab, /* Write index of pSrc->a[] here */ - int *piCol, /* Write index of pSrc->a[*piTab].pTab->aCol[] here */ + int *piCol, /* Write index of pSrc->a[*piTab].pSTab->aCol[] here */ int bIgnoreHidden /* Ignore hidden columns */ ){ int i; /* For looping over tables in pSrc */ @@ -142201,9 +147136,9 @@ static int tableAndColumnIndex( assert( (piTab==0)==(piCol==0) ); /* Both or neither are NULL */ for(i=iStart; i<=iEnd; i++){ - iCol = sqlite3ColumnIndex(pSrc->a[i].pTab, zCol); + iCol = sqlite3ColumnIndex(pSrc->a[i].pSTab, zCol); if( iCol>=0 - && (bIgnoreHidden==0 || IsHiddenColumn(&pSrc->a[i].pTab->aCol[iCol])==0) + && (bIgnoreHidden==0 || IsHiddenColumn(&pSrc->a[i].pSTab->aCol[iCol])==0) ){ if( piTab ){ sqlite3SrcItemColumnUsed(&pSrc->a[i], iCol); @@ -142249,8 +147184,7 @@ SQLITE_PRIVATE void sqlite3SetJoinExpr(Expr *p, int iTable, u32 joinFlag){ assert( !ExprHasProperty(p, EP_TokenOnly|EP_Reduced) ); ExprSetVVAProperty(p, EP_NoReduce); p->w.iJoin = iTable; - if( p->op==TK_FUNCTION ){ - assert( ExprUseXList(p) ); + if( ExprUseXList(p) ){ if( p->x.pList ){ int i; for(i=0; ix.pList->nExpr; i++){ @@ -142332,10 +147266,10 @@ static int sqlite3ProcessJoin(Parse *pParse, Select *p){ pLeft = &pSrc->a[0]; pRight = &pLeft[1]; for(i=0; inSrc-1; i++, pRight++, pLeft++){ - Table *pRightTab = pRight->pTab; + Table *pRightTab = pRight->pSTab; u32 joinType; - if( NEVER(pLeft->pTab==0 || pRightTab==0) ) continue; + if( NEVER(pLeft->pSTab==0 || pRightTab==0) ) continue; joinType = (pRight->fg.jointype & JT_OUTER)!=0 ? EP_OuterON : EP_InnerON; /* If this is a NATURAL join, synthesize an appropriate USING clause @@ -142402,7 +147336,7 @@ static int sqlite3ProcessJoin(Parse *pParse, Select *p){ } pE1 = sqlite3CreateColumnExpr(db, pSrc, iLeft, iLeftCol); sqlite3SrcItemColumnUsed(&pSrc->a[iLeft], iLeftCol); - if( (pSrc->a[0].fg.jointype & JT_LTORJ)!=0 ){ + if( (pSrc->a[0].fg.jointype & JT_LTORJ)!=0 && pParse->nErr==0 ){ /* This branch runs if the query contains one or more RIGHT or FULL ** JOINs. If only a single table on the left side of this join ** contains the zName column, then this branch is a no-op. @@ -142418,6 +147352,8 @@ static int sqlite3ProcessJoin(Parse *pParse, Select *p){ */ ExprList *pFuncArgs = 0; /* Arguments to the coalesce() */ static const Token tkCoalesce = { "coalesce", 8 }; + assert( pE1!=0 ); + ExprSetProperty(pE1, EP_CanBeNull); while( tableAndColumnIndex(pSrc, iLeft+1, i, zName, &iLeft, &iLeftCol, pRight->fg.isSynthUsing)!=0 ){ if( pSrc->a[iLeft].fg.isUsing==0 @@ -142434,7 +147370,13 @@ static int sqlite3ProcessJoin(Parse *pParse, Select *p){ if( pFuncArgs ){ pFuncArgs = sqlite3ExprListAppend(pParse, pFuncArgs, pE1); pE1 = sqlite3ExprFunction(pParse, pFuncArgs, &tkCoalesce, 0); + if( pE1 ){ + pE1->affExpr = SQLITE_AFF_DEFER; + } } + }else if( (pSrc->a[i+1].fg.jointype & JT_LEFT)!=0 && pParse->nErr==0 ){ + assert( pE1!=0 ); + ExprSetProperty(pE1, EP_CanBeNull); } pE2 = sqlite3CreateColumnExpr(db, pSrc, i+1, iRightCol); sqlite3SrcItemColumnUsed(pRight, iRightCol); @@ -142458,6 +147400,7 @@ static int sqlite3ProcessJoin(Parse *pParse, Select *p){ p->pWhere = sqlite3ExprAnd(pParse, p->pWhere, pRight->u3.pOn); pRight->u3.pOn = 0; pRight->fg.isOn = 1; + p->selFlags |= SF_OnToWhere; } } return 0; @@ -143208,12 +148151,18 @@ static void selectInnerLoop( ** case the order does matter */ pushOntoSorter( pParse, pSort, p, regResult, regOrig, nResultCol, nPrefixReg); + pDest->iSDParm2 = 0; /* Signal that any Bloom filter is unpopulated */ }else{ int r1 = sqlite3GetTempReg(pParse); assert( sqlite3Strlen30(pDest->zAffSdst)==nResultCol ); sqlite3VdbeAddOp4(v, OP_MakeRecord, regResult, nResultCol, r1, pDest->zAffSdst, nResultCol); sqlite3VdbeAddOp4Int(v, OP_IdxInsert, iParm, r1, regResult, nResultCol); + if( pDest->iSDParm2 ){ + sqlite3VdbeAddOp4Int(v, OP_FilterAdd, pDest->iSDParm2, 0, + regResult, nResultCol); + ExplainQueryPlan((pParse, 0, "CREATE BLOOM FILTER")); + } sqlite3ReleaseTempReg(pParse, r1); } break; @@ -143337,8 +148286,11 @@ static void selectInnerLoop( ** X extra columns. */ SQLITE_PRIVATE KeyInfo *sqlite3KeyInfoAlloc(sqlite3 *db, int N, int X){ - int nExtra = (N+X)*(sizeof(CollSeq*)+1) - sizeof(CollSeq*); - KeyInfo *p = sqlite3DbMallocRawNN(db, sizeof(KeyInfo) + nExtra); + int nExtra = (N+X)*(sizeof(CollSeq*)+1); + KeyInfo *p; + assert( X>=0 ); + if( NEVER(N+X>0xffff) ) return (KeyInfo*)sqlite3OomFault(db); + p = sqlite3DbMallocRawNN(db, SZ_KEYINFO(0) + nExtra); if( p ){ p->aSortFlags = (u8*)&p->aColl[N+X]; p->nKeyField = (u16)N; @@ -143346,7 +148298,7 @@ SQLITE_PRIVATE KeyInfo *sqlite3KeyInfoAlloc(sqlite3 *db, int N, int X){ p->enc = ENC(db); p->db = db; p->nRef = 1; - memset(&p[1], 0, nExtra); + memset(p->aColl, 0, nExtra); }else{ return (KeyInfo*)sqlite3OomFault(db); } @@ -143504,9 +148456,16 @@ static void generateSortTail( int addrExplain; /* Address of OP_Explain instruction */ #endif - ExplainQueryPlan2(addrExplain, (pParse, 0, - "USE TEMP B-TREE FOR %sORDER BY", pSort->nOBSat>0?"RIGHT PART OF ":"") - ); + nKey = pOrderBy->nExpr - pSort->nOBSat; + if( pSort->nOBSat==0 || nKey==1 ){ + ExplainQueryPlan2(addrExplain, (pParse, 0, + "USE TEMP B-TREE FOR %sORDER BY", pSort->nOBSat?"LAST TERM OF ":"" + )); + }else{ + ExplainQueryPlan2(addrExplain, (pParse, 0, + "USE TEMP B-TREE FOR LAST %d TERMS OF ORDER BY", nKey + )); + } sqlite3VdbeScanStatusRange(v, addrExplain,pSort->addrPush,pSort->addrPushEnd); sqlite3VdbeScanStatusCounters(v, addrExplain, addrExplain, pSort->addrPush); @@ -143544,7 +148503,6 @@ static void generateSortTail( regRow = sqlite3GetTempRange(pParse, nColumn); } } - nKey = pOrderBy->nExpr - pSort->nOBSat; if( pSort->sortFlags & SORTFLAG_UseSorter ){ int regSortOut = ++pParse->nMem; iSortTab = pParse->nTab++; @@ -143749,8 +148707,12 @@ static const char *columnTypeImpl( SrcList *pTabList = pNC->pSrcList; for(j=0;jnSrc && pTabList->a[j].iCursor!=pExpr->iTable;j++); if( jnSrc ){ - pTab = pTabList->a[j].pTab; - pS = pTabList->a[j].pSelect; + pTab = pTabList->a[j].pSTab; + if( pTabList->a[j].fg.isSubquery ){ + pS = pTabList->a[j].u4.pSubq->pSelect; + }else{ + pS = 0; + } }else{ pNC = pNC->pNext; } @@ -143784,11 +148746,7 @@ static const char *columnTypeImpl( ** data for the result-set column of the sub-select. */ if( iColpEList->nExpr -#ifdef SQLITE_ALLOW_ROWID_IN_VIEW - && iCol>=0 -#else - && ALWAYS(iCol>=0) -#endif + && (!ViewCanHaveRowid || iCol>=0) ){ /* If iCol is less than zero, then the expression requests the ** rowid of the sub-select or view. This expression is legal (see @@ -143899,6 +148857,10 @@ static void generateColumnTypes( #endif sqlite3VdbeSetColName(v, i, COLNAME_DECLTYPE, zType, SQLITE_TRANSIENT); } +#else + UNUSED_PARAMETER(pParse); + UNUSED_PARAMETER(pTabList); + UNUSED_PARAMETER(pEList); #endif /* !defined(SQLITE_OMIT_DECLTYPE) */ } @@ -144153,8 +149115,7 @@ SQLITE_PRIVATE void sqlite3SubqueryColumnTypes( NameContext sNC; assert( pSelect!=0 ); - testcase( (pSelect->selFlags & SF_Resolved)==0 ); - assert( (pSelect->selFlags & SF_Resolved)!=0 || IN_RENAME_OBJECT ); + assert( (pSelect->selFlags & SF_Resolved)!=0 ); assert( pTab->nCol==pSelect->pEList->nExpr || pParse->nErr>0 ); assert( aff==SQLITE_AFF_NONE || aff==SQLITE_AFF_BLOB ); if( db->mallocFailed || IN_RENAME_OBJECT ) return; @@ -144165,17 +149126,22 @@ SQLITE_PRIVATE void sqlite3SubqueryColumnTypes( for(i=0, pCol=pTab->aCol; inCol; i++, pCol++){ const char *zType; i64 n; + int m = 0; + Select *pS2 = pSelect; pTab->tabFlags |= (pCol->colFlags & COLFLAG_NOINSERT); p = a[i].pExpr; /* pCol->szEst = ... // Column size est for SELECT tables never used */ pCol->affinity = sqlite3ExprAffinity(p); + while( pCol->affinity<=SQLITE_AFF_NONE && pS2->pNext!=0 ){ + m |= sqlite3ExprDataType(pS2->pEList->a[i].pExpr); + pS2 = pS2->pNext; + pCol->affinity = sqlite3ExprAffinity(pS2->pEList->a[i].pExpr); + } if( pCol->affinity<=SQLITE_AFF_NONE ){ pCol->affinity = aff; } - if( pCol->affinity>=SQLITE_AFF_TEXT && pSelect->pNext ){ - int m = 0; - Select *pS2; - for(m=0, pS2=pSelect->pNext; pS2; pS2=pS2->pNext){ + if( pCol->affinity>=SQLITE_AFF_TEXT && (pS2->pNext || pS2!=pSelect) ){ + for(pS2=pS2->pNext; pS2; pS2=pS2->pNext){ m |= sqlite3ExprDataType(pS2->pEList->a[i].pExpr); } if( pCol->affinity==SQLITE_AFF_TEXT && (m&0x01)!=0 ){ @@ -144205,12 +149171,12 @@ SQLITE_PRIVATE void sqlite3SubqueryColumnTypes( } } if( zType ){ - i64 m = sqlite3Strlen30(zType); + const i64 k = sqlite3Strlen30(zType); n = sqlite3Strlen30(pCol->zCnName); - pCol->zCnName = sqlite3DbReallocOrFree(db, pCol->zCnName, n+m+2); + pCol->zCnName = sqlite3DbReallocOrFree(db, pCol->zCnName, n+k+2); pCol->colFlags &= ~(COLFLAG_HASTYPE|COLFLAG_HASCOLL); if( pCol->zCnName ){ - memcpy(&pCol->zCnName[n+1], zType, m+1); + memcpy(&pCol->zCnName[n+1], zType, k+1); pCol->colFlags |= COLFLAG_HASTYPE; } } @@ -144317,7 +149283,7 @@ static void computeLimitRegisters(Parse *pParse, Select *p, int iBreak){ p->iLimit = iLimit = ++pParse->nMem; v = sqlite3GetVdbe(pParse); assert( v!=0 ); - if( sqlite3ExprIsInteger(pLimit->pLeft, &n) ){ + if( sqlite3ExprIsInteger(pLimit->pLeft, &n, pParse) ){ sqlite3VdbeAddOp2(v, OP_Integer, n, iLimit); VdbeComment((v, "LIMIT counter")); if( n==0 ){ @@ -144797,7 +149763,7 @@ static int multiSelect( p->pPrior = pPrior; p->nSelectRow = sqlite3LogEstAdd(p->nSelectRow, pPrior->nSelectRow); if( p->pLimit - && sqlite3ExprIsInteger(p->pLimit->pLeft, &nLimit) + && sqlite3ExprIsInteger(p->pLimit->pLeft, &nLimit, pParse) && nLimit>0 && p->nSelectRow > sqlite3LogEst((u64)nLimit) ){ p->nSelectRow = sqlite3LogEst((u64)nLimit); @@ -144814,8 +149780,10 @@ static int multiSelect( int priorOp; /* The SRT_ operation to apply to prior selects */ Expr *pLimit; /* Saved values of p->nLimit */ int addr; + int emptyBypass = 0; /* IfEmpty opcode to bypass RHS */ SelectDest uniondest; + testcase( p->op==TK_EXCEPT ); testcase( p->op==TK_UNION ); priorOp = SRT_Union; @@ -144853,6 +149821,8 @@ static int multiSelect( */ if( p->op==TK_EXCEPT ){ op = SRT_Except; + emptyBypass = sqlite3VdbeAddOp1(v, OP_IfEmpty, unionTab); + VdbeCoverage(v); }else{ assert( p->op==TK_UNION ); op = SRT_Union; @@ -144873,6 +149843,7 @@ static int multiSelect( if( p->op==TK_UNION ){ p->nSelectRow = sqlite3LogEstAdd(p->nSelectRow, pPrior->nSelectRow); } + if( emptyBypass ) sqlite3VdbeJumpHere(v, emptyBypass); sqlite3ExprDelete(db, p->pLimit); p->pLimit = pLimit; p->iLimit = 0; @@ -144903,9 +149874,10 @@ static int multiSelect( int tab1, tab2; int iCont, iBreak, iStart; Expr *pLimit; - int addr; + int addr, iLimit, iOffset; SelectDest intersectdest; int r1; + int emptyBypass; /* INTERSECT is different from the others since it requires ** two temporary tables. Hence it has its own case. Begin @@ -144930,14 +149902,28 @@ static int multiSelect( goto multi_select_end; } + /* Initialize LIMIT counters before checking to see if the LHS + ** is empty, in case the jump is taken */ + iBreak = sqlite3VdbeMakeLabel(pParse); + computeLimitRegisters(pParse, p, iBreak); + emptyBypass = sqlite3VdbeAddOp1(v, OP_IfEmpty, tab1); VdbeCoverage(v); + /* Code the current SELECT into temporary table "tab2" */ addr = sqlite3VdbeAddOp2(v, OP_OpenEphemeral, tab2, 0); assert( p->addrOpenEphm[1] == -1 ); p->addrOpenEphm[1] = addr; - p->pPrior = 0; + + /* Disable prior SELECTs and the LIMIT counters during the computation + ** of the RHS select */ pLimit = p->pLimit; + iLimit = p->iLimit; + iOffset = p->iOffset; + p->pPrior = 0; p->pLimit = 0; + p->iLimit = 0; + p->iOffset = 0; + intersectdest.iSDParm = tab2; ExplainQueryPlan((pParse, 1, "%s USING TEMP B-TREE", sqlite3SelectOpName(p->op))); @@ -144950,19 +149936,21 @@ static int multiSelect( p->nSelectRow = pPrior->nSelectRow; } sqlite3ExprDelete(db, p->pLimit); + + /* Reinstate the LIMIT counters prior to running the final intersect */ p->pLimit = pLimit; + p->iLimit = iLimit; + p->iOffset = iOffset; /* Generate code to take the intersection of the two temporary ** tables. */ if( rc ) break; assert( p->pEList ); - iBreak = sqlite3VdbeMakeLabel(pParse); - iCont = sqlite3VdbeMakeLabel(pParse); - computeLimitRegisters(pParse, p, iBreak); - sqlite3VdbeAddOp2(v, OP_Rewind, tab1, iBreak); VdbeCoverage(v); + sqlite3VdbeAddOp1(v, OP_Rewind, tab1); r1 = sqlite3GetTempReg(pParse); iStart = sqlite3VdbeAddOp2(v, OP_RowData, tab1, r1); + iCont = sqlite3VdbeMakeLabel(pParse); sqlite3VdbeAddOp4Int(v, OP_NotFound, tab2, iCont, r1, 0); VdbeCoverage(v); sqlite3ReleaseTempReg(pParse, r1); @@ -144972,6 +149960,7 @@ static int multiSelect( sqlite3VdbeAddOp2(v, OP_Next, tab1, iStart); VdbeCoverage(v); sqlite3VdbeResolveLabel(v, iBreak); sqlite3VdbeAddOp2(v, OP_Close, tab2, 0); + sqlite3VdbeJumpHere(v, emptyBypass); sqlite3VdbeAddOp2(v, OP_Close, tab1, 0); break; } @@ -145037,6 +150026,7 @@ static int multiSelect( multi_select_end: pDest->iSdst = dest.iSdst; pDest->nSdst = dest.nSdst; + pDest->iSDParm2 = dest.iSDParm2; if( pDelete ){ sqlite3ParserAddCleanup(pParse, sqlite3SelectDeleteGeneric, pDelete); } @@ -145141,6 +150131,11 @@ static int generateOutputSubroutine( r1, pDest->zAffSdst, pIn->nSdst); sqlite3VdbeAddOp4Int(v, OP_IdxInsert, pDest->iSDParm, r1, pIn->iSdst, pIn->nSdst); + if( pDest->iSDParm2>0 ){ + sqlite3VdbeAddOp4Int(v, OP_FilterAdd, pDest->iSDParm2, 0, + pIn->iSdst, pIn->nSdst); + ExplainQueryPlan((pParse, 0, "CREATE BLOOM FILTER")); + } sqlite3ReleaseTempReg(pParse, r1); break; } @@ -145614,7 +150609,7 @@ static int multiSelectOrderBy( ** ## About "isOuterJoin": ** ** The isOuterJoin column indicates that the replacement will occur into a -** position in the parent that NULL-able due to an OUTER JOIN. Either the +** position in the parent that is NULL-able due to an OUTER JOIN. Either the ** target slot in the parent is the right operand of a LEFT JOIN, or one of ** the left operands of a RIGHT JOIN. In either case, we need to potentially ** bypass the substituted expression with OP_IfNullRow. @@ -145644,6 +150639,7 @@ typedef struct SubstContext { int iTable; /* Replace references to this table */ int iNewTable; /* New table number */ int isOuterJoin; /* Add TK_IF_NULL_ROW opcodes on each replacement */ + int nSelDepth; /* Depth of sub-query recursion. Top==1 */ ExprList *pEList; /* Replacement expressions */ ExprList *pCList; /* Collation sequences for replacement expr */ } SubstContext; @@ -145719,38 +150715,41 @@ static Expr *substExpr( if( pSubst->isOuterJoin ){ ExprSetProperty(pNew, EP_CanBeNull); } - if( ExprHasProperty(pExpr,EP_OuterON|EP_InnerON) ){ - sqlite3SetJoinExpr(pNew, pExpr->w.iJoin, - pExpr->flags & (EP_OuterON|EP_InnerON)); - } - sqlite3ExprDelete(db, pExpr); - pExpr = pNew; - if( pExpr->op==TK_TRUEFALSE ){ - pExpr->u.iValue = sqlite3ExprTruthValue(pExpr); - pExpr->op = TK_INTEGER; - ExprSetProperty(pExpr, EP_IntValue); + if( pNew->op==TK_TRUEFALSE ){ + pNew->u.iValue = sqlite3ExprTruthValue(pNew); + pNew->op = TK_INTEGER; + ExprSetProperty(pNew, EP_IntValue); } /* Ensure that the expression now has an implicit collation sequence, ** just as it did when it was a column of a view or sub-query. */ { - CollSeq *pNat = sqlite3ExprCollSeq(pSubst->pParse, pExpr); + CollSeq *pNat = sqlite3ExprCollSeq(pSubst->pParse, pNew); CollSeq *pColl = sqlite3ExprCollSeq(pSubst->pParse, pSubst->pCList->a[iColumn].pExpr ); - if( pNat!=pColl || (pExpr->op!=TK_COLUMN && pExpr->op!=TK_COLLATE) ){ - pExpr = sqlite3ExprAddCollateString(pSubst->pParse, pExpr, + if( pNat!=pColl || (pNew->op!=TK_COLUMN && pNew->op!=TK_COLLATE) ){ + pNew = sqlite3ExprAddCollateString(pSubst->pParse, pNew, (pColl ? pColl->zName : "BINARY") ); } } - ExprClearProperty(pExpr, EP_Collate); + ExprClearProperty(pNew, EP_Collate); + if( ExprHasProperty(pExpr,EP_OuterON|EP_InnerON) ){ + sqlite3SetJoinExpr(pNew, pExpr->w.iJoin, + pExpr->flags & (EP_OuterON|EP_InnerON)); + } + sqlite3ExprDelete(db, pExpr); + pExpr = pNew; } } }else{ if( pExpr->op==TK_IF_NULL_ROW && pExpr->iTable==pSubst->iTable ){ pExpr->iTable = pSubst->iNewTable; } + if( pExpr->op==TK_AGG_FUNCTION && pExpr->op2>=pSubst->nSelDepth ){ + pExpr->op2--; + } pExpr->pLeft = substExpr(pSubst, pExpr->pLeft); pExpr->pRight = substExpr(pSubst, pExpr->pRight); if( ExprUseXSelect(pExpr) ){ @@ -145788,6 +150787,7 @@ static void substSelect( SrcItem *pItem; int i; if( !p ) return; + pSubst->nSelDepth++; do{ substExprList(pSubst, p->pEList); substExprList(pSubst, p->pGroupBy); @@ -145797,12 +150797,15 @@ static void substSelect( pSrc = p->pSrc; assert( pSrc!=0 ); for(i=pSrc->nSrc, pItem=pSrc->a; i>0; i--, pItem++){ - substSelect(pSubst, pItem->pSelect, 1); + if( pItem->fg.isSubquery ){ + substSelect(pSubst, pItem->u4.pSubq->pSelect, 1); + } if( pItem->fg.isTabFunc ){ substExprList(pSubst, pItem->u1.pFuncArg); } } }while( doPrior && (p = p->pPrior)!=0 ); + pSubst->nSelDepth--; } #endif /* !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) */ @@ -145828,7 +150831,7 @@ static void recomputeColumnsUsed( SrcItem *pSrcItem /* Which FROM clause item to recompute */ ){ Walker w; - if( NEVER(pSrcItem->pTab==0) ) return; + if( NEVER(pSrcItem->pSTab==0) ) return; memset(&w, 0, sizeof(w)); w.xExprCallback = recomputeColumnsUsedExpr; w.xSelectCallback = sqlite3SelectWalkNoop; @@ -145868,8 +150871,10 @@ static void srclistRenumberCursors( aCsrMap[pItem->iCursor+1] = pParse->nTab++; } pItem->iCursor = aCsrMap[pItem->iCursor+1]; - for(p=pItem->pSelect; p; p=p->pPrior){ - srclistRenumberCursors(pParse, aCsrMap, p->pSrc, -1); + if( pItem->fg.isSubquery ){ + for(p=pItem->u4.pSubq->pSelect; p; p=p->pPrior){ + srclistRenumberCursors(pParse, aCsrMap, p->pSrc, -1); + } } } } @@ -146016,9 +151021,9 @@ static int compoundHasDifferentAffinities(Select *p){ ** from 2015-02-09.) ** ** (3) If the subquery is the right operand of a LEFT JOIN then -** (3a) the subquery may not be a join and -** (3b) the FROM clause of the subquery may not contain a virtual -** table and +** (3a) the subquery may not be a join +** (**) Was (3b): "the FROM clause of the subquery may not contain +** a virtual table" ** (**) Was: "The outer query may not have a GROUP BY." This case ** is now managed correctly ** (3d) the outer query may not be DISTINCT. @@ -146180,7 +151185,8 @@ static int flattenSubquery( assert( pSrc && iFrom>=0 && iFromnSrc ); pSubitem = &pSrc->a[iFrom]; iParent = pSubitem->iCursor; - pSub = pSubitem->pSelect; + assert( pSubitem->fg.isSubquery ); + pSub = pSubitem->u4.pSubq->pSelect; assert( pSub!=0 ); #ifndef SQLITE_OMIT_WINDOWFUNC @@ -146233,7 +151239,7 @@ static int flattenSubquery( */ if( (pSubitem->fg.jointype & (JT_OUTER|JT_LTORJ))!=0 ){ if( pSubSrc->nSrc>1 /* (3a) */ - || IsVirtual(pSubSrc->a[0].pTab) /* (3b) */ + /**** || IsVirtual(pSubSrc->a[0].pSTab) (3b)-omitted */ || (p->selFlags & SF_Distinct)!=0 /* (3d) */ || (pSubitem->fg.jointype & JT_RIGHT)!=0 /* (26) */ ){ @@ -146319,14 +151325,18 @@ static int flattenSubquery( pParse->zAuthContext = zSavedAuthContext; /* Delete the transient structures associated with the subquery */ - pSub1 = pSubitem->pSelect; - sqlite3DbFree(db, pSubitem->zDatabase); + + if( ALWAYS(pSubitem->fg.isSubquery) ){ + pSub1 = sqlite3SubqueryDetach(db, pSubitem); + }else{ + pSub1 = 0; + } + assert( pSubitem->fg.isSubquery==0 ); + assert( pSubitem->fg.fixedSchema==0 ); sqlite3DbFree(db, pSubitem->zName); sqlite3DbFree(db, pSubitem->zAlias); - pSubitem->zDatabase = 0; pSubitem->zName = 0; pSubitem->zAlias = 0; - pSubitem->pSelect = 0; assert( pSubitem->fg.isUsing!=0 || pSubitem->u3.pOn==0 ); /* If the sub-query is a compound SELECT statement, then (by restrictions @@ -146367,8 +151377,8 @@ static int flattenSubquery( ExprList *pOrderBy = p->pOrderBy; Expr *pLimit = p->pLimit; Select *pPrior = p->pPrior; - Table *pItemTab = pSubitem->pTab; - pSubitem->pTab = 0; + Table *pItemTab = pSubitem->pSTab; + pSubitem->pSTab = 0; p->pOrderBy = 0; p->pPrior = 0; p->pLimit = 0; @@ -146376,7 +151386,7 @@ static int flattenSubquery( p->pLimit = pLimit; p->pOrderBy = pOrderBy; p->op = TK_ALL; - pSubitem->pTab = pItemTab; + pSubitem->pSTab = pItemTab; if( pNew==0 ){ p->pPrior = pPrior; }else{ @@ -146391,11 +151401,14 @@ static int flattenSubquery( TREETRACE(0x4,pParse,p,("compound-subquery flattener" " creates %u as peer\n",pNew->selId)); } - assert( pSubitem->pSelect==0 ); + assert( pSubitem->fg.isSubquery==0 ); } sqlite3DbFree(db, aCsrMap); if( db->mallocFailed ){ - pSubitem->pSelect = pSub1; + assert( pSubitem->fg.fixedSchema==0 ); + assert( pSubitem->fg.isSubquery==0 ); + assert( pSubitem->u4.zDatabase==0 ); + sqlite3SrcItemAttachSubquery(pParse, pSubitem, pSub1, 0); return 1; } @@ -146404,10 +151417,10 @@ static int flattenSubquery( ** complete, since there may still exist Expr.pTab entries that ** refer to the subquery even after flattening. Ticket #3346. ** - ** pSubitem->pTab is always non-NULL by test restrictions and tests above. + ** pSubitem->pSTab is always non-NULL by test restrictions and tests above. */ - if( ALWAYS(pSubitem->pTab!=0) ){ - Table *pTabToDel = pSubitem->pTab; + if( ALWAYS(pSubitem->pSTab!=0) ){ + Table *pTabToDel = pSubitem->pSTab; if( pTabToDel->nTabRef==1 ){ Parse *pToplevel = sqlite3ParseToplevel(pParse); sqlite3ParserAddCleanup(pToplevel, sqlite3DeleteTableGeneric, pTabToDel); @@ -146415,7 +151428,7 @@ static int flattenSubquery( }else{ pTabToDel->nTabRef--; } - pSubitem->pTab = 0; + pSubitem->pSTab = 0; } /* The following loop runs once for each term in a compound-subquery @@ -146434,17 +151447,12 @@ static int flattenSubquery( pSub = pSub1; for(pParent=p; pParent; pParent=pParent->pPrior, pSub=pSub->pPrior){ int nSubSrc; - u8 jointype = 0; - u8 ltorj = pSrc->a[iFrom].fg.jointype & JT_LTORJ; + u8 jointype = pSubitem->fg.jointype; assert( pSub!=0 ); pSubSrc = pSub->pSrc; /* FROM clause of subquery */ nSubSrc = pSubSrc->nSrc; /* Number of terms in subquery FROM clause */ pSrc = pParent->pSrc; /* FROM clause of the outer query */ - if( pParent==p ){ - jointype = pSubitem->fg.jointype; /* First time through the loop */ - } - /* The subquery uses a single slot of the FROM clause of the outer ** query. If the subquery has more than one element in its FROM clause, ** then expand the outer query to make space for it to hold all elements @@ -146464,22 +151472,25 @@ static int flattenSubquery( pSrc = sqlite3SrcListEnlarge(pParse, pSrc, nSubSrc-1,iFrom+1); if( pSrc==0 ) break; pParent->pSrc = pSrc; + pSubitem = &pSrc->a[iFrom]; } /* Transfer the FROM clause terms from the subquery into the ** outer query. */ + iNewParent = pSubSrc->a[0].iCursor; for(i=0; ia[i+iFrom]; - if( pItem->fg.isUsing ) sqlite3IdListDelete(db, pItem->u3.pUsing); assert( pItem->fg.isTabFunc==0 ); + assert( pItem->fg.isSubquery + || pItem->fg.fixedSchema + || pItem->u4.zDatabase==0 ); + if( pItem->fg.isUsing ) sqlite3IdListDelete(db, pItem->u3.pUsing); *pItem = pSubSrc->a[i]; - pItem->fg.jointype |= ltorj; - iNewParent = pSubSrc->a[i].iCursor; + pItem->fg.jointype |= (jointype & JT_LTORJ); memset(&pSubSrc->a[i], 0, sizeof(pSubSrc->a[i])); } - pSrc->a[iFrom].fg.jointype &= JT_LTORJ; - pSrc->a[iFrom].fg.jointype |= jointype | ltorj; + pSubitem->fg.jointype |= jointype; /* Now begin substituting subquery result set expressions for ** references to the iParent in the outer query. @@ -146515,6 +151526,7 @@ static int flattenSubquery( pWhere = pSub->pWhere; pSub->pWhere = 0; if( isOuterJoin>0 ){ + assert( pSubSrc->nSrc==1 ); sqlite3SetJoinExpr(pWhere, iNewParent, EP_OuterON); } if( pWhere ){ @@ -146530,6 +151542,7 @@ static int flattenSubquery( x.iTable = iParent; x.iNewTable = iNewParent; x.isOuterJoin = isOuterJoin; + x.nSelDepth = 0; x.pEList = pSub->pEList; x.pCList = findLeftmostExprlist(pSub); substSelect(&x, pParent, 0); @@ -146607,7 +151620,7 @@ static void constInsert( ){ int i; assert( pColumn->op==TK_COLUMN ); - assert( sqlite3ExprIsConstant(pValue) ); + assert( sqlite3ExprIsConstant(pConst->pParse, pValue) ); if( ExprHasProperty(pColumn, EP_FixedCol) ) return; if( sqlite3ExprAffinity(pValue)!=0 ) return; @@ -146626,7 +151639,8 @@ static void constInsert( return; /* Already present. Return without doing anything. */ } } - if( sqlite3ExprAffinity(pColumn)==SQLITE_AFF_BLOB ){ + assert( SQLITE_AFF_NONEbHasAffBlob = 1; } @@ -146665,10 +151679,10 @@ static void findConstInWhere(WhereConst *pConst, Expr *pExpr){ pLeft = pExpr->pLeft; assert( pRight!=0 ); assert( pLeft!=0 ); - if( pRight->op==TK_COLUMN && sqlite3ExprIsConstant(pLeft) ){ + if( pRight->op==TK_COLUMN && sqlite3ExprIsConstant(pConst->pParse, pLeft) ){ constInsert(pConst,pRight,pLeft,pExpr); } - if( pLeft->op==TK_COLUMN && sqlite3ExprIsConstant(pRight) ){ + if( pLeft->op==TK_COLUMN && sqlite3ExprIsConstant(pConst->pParse, pRight) ){ constInsert(pConst,pLeft,pRight,pExpr); } } @@ -146701,7 +151715,8 @@ static int propagateConstantExprRewriteOne( if( pColumn==pExpr ) continue; if( pColumn->iTable!=pExpr->iTable ) continue; if( pColumn->iColumn!=pExpr->iColumn ) continue; - if( bIgnoreAffBlob && sqlite3ExprAffinity(pColumn)==SQLITE_AFF_BLOB ){ + assert( SQLITE_AFF_NONEop==TK_ISNULL || pWhere->op==TK_NOTNULL) ){ + Expr *pLeft = pWhere->pLeft; + if( ALWAYS(pLeft) + && pLeft->op==TK_COLUMN + && pLeft->iColumn < 0 + ){ + return 0; /* Restriction (12) */ + } + } +#endif + + if( sqlite3ExprIsSingleTableConstraint(pWhere, pSrcList, iSrc, 1) ){ nChng++; pSubq->selFlags |= SF_PushDown; while( pSubq ){ @@ -147084,6 +152128,7 @@ static int pushDownWhereTerms( x.iTable = pSrc->iCursor; x.iNewTable = pSrc->iCursor; x.isOuterJoin = 0; + x.nSelDepth = 0; x.pEList = pSubq->pEList; x.pCList = findLeftmostExprlist(pSubq); pNew = substExpr(&x, pNew); @@ -147127,10 +152172,10 @@ static int disableUnusedSubqueryResultColumns(SrcItem *pItem){ if( pItem->fg.isCorrelated || pItem->fg.isCte ){ return 0; } - assert( pItem->pTab!=0 ); - pTab = pItem->pTab; - assert( pItem->pSelect!=0 ); - pSub = pItem->pSelect; + assert( pItem->pSTab!=0 ); + pTab = pItem->pSTab; + assert( pItem->fg.isSubquery ); + pSub = pItem->u4.pSubq->pSelect; assert( pSub->pEList->nExpr==pTab->nCol ); for(pX=pSub; pX; pX=pX->pPrior){ if( (pX->selFlags & (SF_Distinct|SF_Aggregate))!=0 ){ @@ -147259,13 +152304,13 @@ static Table *isSimpleCount(Select *p, AggInfo *pAggInfo){ if( p->pWhere || p->pEList->nExpr!=1 || p->pSrc->nSrc!=1 - || p->pSrc->a[0].pSelect + || p->pSrc->a[0].fg.isSubquery || pAggInfo->nFunc!=1 || p->pHaving ){ return 0; } - pTab = p->pSrc->a[0].pTab; + pTab = p->pSrc->a[0].pSTab; assert( pTab!=0 ); assert( !IsView(pTab) ); if( !IsOrdinaryTable(pTab) ) return 0; @@ -147290,7 +152335,7 @@ static Table *isSimpleCount(Select *p, AggInfo *pAggInfo){ ** pFrom->pIndex and return SQLITE_OK. */ SQLITE_PRIVATE int sqlite3IndexedByLookup(Parse *pParse, SrcItem *pFrom){ - Table *pTab = pFrom->pTab; + Table *pTab = pFrom->pSTab; char *zIndexedBy = pFrom->u1.zIndexedBy; Index *pIdx; assert( pTab!=0 ); @@ -147325,7 +152370,7 @@ SQLITE_PRIVATE int sqlite3IndexedByLookup(Parse *pParse, SrcItem *pFrom){ ** above that generates the code for a compound SELECT with an ORDER BY clause ** uses a merge algorithm that requires the same collating sequence on the ** result columns as on the ORDER BY clause. See ticket -** http://www.sqlite.org/src/info/6709574d2a +** http://sqlite.org/src/info/6709574d2a ** ** This transformation is only needed for EXCEPT, INTERSECT, and UNION. ** The UNION ALL operator works fine with multiSelectOrderBy() even when @@ -147367,7 +152412,11 @@ static int convertCompoundSelectToSubquery(Walker *pWalker, Select *p){ if( pNew==0 ) return WRC_Abort; memset(&dummy, 0, sizeof(dummy)); pNewSrc = sqlite3SrcListAppendFromTerm(pParse,0,0,0,&dummy,pNew,0); - if( pNewSrc==0 ) return WRC_Abort; + assert( pNewSrc!=0 || pParse->nErr ); + if( pParse->nErr ){ + sqlite3SrcListDelete(db, pNewSrc); + return WRC_Abort; + } *pNew = *p; p->pSrc = pNewSrc; p->pEList = sqlite3ExprListAppend(pParse, 0, sqlite3Expr(db, TK_ASTERISK, 0)); @@ -147382,7 +152431,7 @@ static int convertCompoundSelectToSubquery(Walker *pWalker, Select *p){ #ifndef SQLITE_OMIT_WINDOWFUNC p->pWinDefn = 0; #endif - p->selFlags &= ~SF_Compound; + p->selFlags &= ~(u32)SF_Compound; assert( (p->selFlags & SF_Converted)==0 ); p->selFlags |= SF_Converted; assert( pNew->pPrior!=0 ); @@ -147422,7 +152471,7 @@ static struct Cte *searchWith( ){ const char *zName = pItem->zName; With *p; - assert( pItem->zDatabase==0 ); + assert( pItem->fg.fixedSchema || pItem->u4.zDatabase==0 ); assert( zName!=0 ); for(p=pWith; p; p=p->pOuter){ int i; @@ -147477,7 +152526,7 @@ SQLITE_PRIVATE With *sqlite3WithPush(Parse *pParse, With *pWith, u8 bFree){ ** CTE expression, through routine checks to see if the reference is ** a recursive reference to the CTE. ** -** If pFrom matches a CTE according to either of these two above, pFrom->pTab +** If pFrom matches a CTE according to either of these two above, pFrom->pSTab ** and other fields are populated accordingly. ** ** Return 0 if no match is found. @@ -147492,7 +152541,7 @@ static int resolveFromTermToCte( Cte *pCte; /* Matched CTE (or NULL if no match) */ With *pWith; /* The matching WITH */ - assert( pFrom->pTab==0 ); + assert( pFrom->pSTab==0 ); if( pParse->pWith==0 ){ /* There are no WITH clauses in the stack. No match is possible */ return 0; @@ -147502,7 +152551,8 @@ static int resolveFromTermToCte( ** go no further. */ return 0; } - if( pFrom->zDatabase!=0 ){ + assert( pFrom->fg.hadSchema==0 || pFrom->fg.notCte!=0 ); + if( pFrom->fg.fixedSchema==0 && pFrom->u4.zDatabase!=0 ){ /* The FROM term contains a schema qualifier (ex: main.t1) and so ** it cannot possibly be a CTE reference. */ return 0; @@ -147538,7 +152588,7 @@ static int resolveFromTermToCte( } if( cannotBeFunction(pParse, pFrom) ) return 2; - assert( pFrom->pTab==0 ); + assert( pFrom->pSTab==0 ); pTab = sqlite3DbMallocZero(db, sizeof(Table)); if( pTab==0 ) return 2; pCteUse = pCte->pUse; @@ -147552,26 +152602,29 @@ static int resolveFromTermToCte( } pCteUse->eM10d = pCte->eM10d; } - pFrom->pTab = pTab; + pFrom->pSTab = pTab; pTab->nTabRef = 1; pTab->zName = sqlite3DbStrDup(db, pCte->zName); pTab->iPKey = -1; pTab->nRowLogEst = 200; assert( 200==sqlite3LogEst(1048576) ); pTab->tabFlags |= TF_Ephemeral | TF_NoVisibleRowid; - pFrom->pSelect = sqlite3SelectDup(db, pCte->pSelect, 0); + sqlite3SrcItemAttachSubquery(pParse, pFrom, pCte->pSelect, 1); if( db->mallocFailed ) return 2; - pFrom->pSelect->selFlags |= SF_CopyCte; - assert( pFrom->pSelect ); + assert( pFrom->fg.isSubquery && pFrom->u4.pSubq ); + pSel = pFrom->u4.pSubq->pSelect; + assert( pSel!=0 ); + pSel->selFlags |= SF_CopyCte; if( pFrom->fg.isIndexedBy ){ sqlite3ErrorMsg(pParse, "no such index: \"%s\"", pFrom->u1.zIndexedBy); return 2; } + assert( !pFrom->fg.isIndexedBy ); pFrom->fg.isCte = 1; pFrom->u2.pCteUse = pCteUse; pCteUse->nUse++; /* Check if this is a recursive CTE. */ - pRecTerm = pSel = pFrom->pSelect; + pRecTerm = pSel; bMayRecursive = ( pSel->op==TK_ALL || pSel->op==TK_UNION ); while( bMayRecursive && pRecTerm->op==pSel->op ){ int i; @@ -147579,11 +152632,13 @@ static int resolveFromTermToCte( assert( pRecTerm->pPrior!=0 ); for(i=0; inSrc; i++){ SrcItem *pItem = &pSrc->a[i]; - if( pItem->zDatabase==0 - && pItem->zName!=0 + if( pItem->zName!=0 + && !pItem->fg.hadSchema + && ALWAYS( !pItem->fg.isSubquery ) + && (pItem->fg.fixedSchema || pItem->u4.zDatabase==0) && 0==sqlite3StrICmp(pItem->zName, pCte->zName) ){ - pItem->pTab = pTab; + pItem->pSTab = pTab; pTab->nTabRef++; pItem->fg.isRecursive = 1; if( pRecTerm->selFlags & SF_Recursive ){ @@ -147685,11 +152740,14 @@ SQLITE_PRIVATE void sqlite3SelectPopWith(Walker *pWalker, Select *p){ ** SQLITE_NOMEM. */ SQLITE_PRIVATE int sqlite3ExpandSubquery(Parse *pParse, SrcItem *pFrom){ - Select *pSel = pFrom->pSelect; + Select *pSel; Table *pTab; + assert( pFrom->fg.isSubquery ); + assert( pFrom->u4.pSubq!=0 ); + pSel = pFrom->u4.pSubq->pSelect; assert( pSel ); - pFrom->pTab = pTab = sqlite3DbMallocZero(pParse->db, sizeof(Table)); + pFrom->pSTab = pTab = sqlite3DbMallocZero(pParse->db, sizeof(Table)); if( pTab==0 ) return SQLITE_NOMEM; pTab->nTabRef = 1; if( pFrom->zAlias ){ @@ -147700,12 +152758,14 @@ SQLITE_PRIVATE int sqlite3ExpandSubquery(Parse *pParse, SrcItem *pFrom){ while( pSel->pPrior ){ pSel = pSel->pPrior; } sqlite3ColumnsFromExprList(pParse, pSel->pEList,&pTab->nCol,&pTab->aCol); pTab->iPKey = -1; + pTab->eTabType = TABTYP_VIEW; pTab->nRowLogEst = 200; assert( 200==sqlite3LogEst(1048576) ); #ifndef SQLITE_ALLOW_ROWID_IN_VIEW /* The usual case - do not allow ROWID on a subquery */ pTab->tabFlags |= TF_Ephemeral | TF_NoVisibleRowid; #else - pTab->tabFlags |= TF_Ephemeral; /* Legacy compatibility mode */ + /* Legacy compatibility mode */ + pTab->tabFlags |= TF_Ephemeral | sqlite3Config.mNoVisibleRowid; #endif return pParse->nErr ? SQLITE_ERROR : SQLITE_OK; } @@ -147787,7 +152847,7 @@ static int selectExpander(Walker *pWalker, Select *p){ pEList = p->pEList; if( pParse->pWith && (p->selFlags & SF_View) ){ if( p->pWith==0 ){ - p->pWith = (With*)sqlite3DbMallocZero(db, sizeof(With)); + p->pWith = (With*)sqlite3DbMallocZero(db, SZ_WITH(1) ); if( p->pWith==0 ){ return WRC_Abort; } @@ -147807,33 +152867,35 @@ static int selectExpander(Walker *pWalker, Select *p){ */ for(i=0, pFrom=pTabList->a; inSrc; i++, pFrom++){ Table *pTab; - assert( pFrom->fg.isRecursive==0 || pFrom->pTab!=0 ); - if( pFrom->pTab ) continue; + assert( pFrom->fg.isRecursive==0 || pFrom->pSTab!=0 ); + if( pFrom->pSTab ) continue; assert( pFrom->fg.isRecursive==0 ); if( pFrom->zName==0 ){ #ifndef SQLITE_OMIT_SUBQUERY - Select *pSel = pFrom->pSelect; + Select *pSel; + assert( pFrom->fg.isSubquery && pFrom->u4.pSubq!=0 ); + pSel = pFrom->u4.pSubq->pSelect; /* A sub-query in the FROM clause of a SELECT */ assert( pSel!=0 ); - assert( pFrom->pTab==0 ); + assert( pFrom->pSTab==0 ); if( sqlite3WalkSelect(pWalker, pSel) ) return WRC_Abort; if( sqlite3ExpandSubquery(pParse, pFrom) ) return WRC_Abort; #endif #ifndef SQLITE_OMIT_CTE }else if( (rc = resolveFromTermToCte(pParse, pWalker, pFrom))!=0 ){ if( rc>1 ) return WRC_Abort; - pTab = pFrom->pTab; + pTab = pFrom->pSTab; assert( pTab!=0 ); #endif }else{ /* An ordinary table or view name in the FROM clause */ - assert( pFrom->pTab==0 ); - pFrom->pTab = pTab = sqlite3LocateTableItem(pParse, 0, pFrom); + assert( pFrom->pSTab==0 ); + pFrom->pSTab = pTab = sqlite3LocateTableItem(pParse, 0, pFrom); if( pTab==0 ) return WRC_Abort; if( pTab->nTabRef>=0xffff ){ sqlite3ErrorMsg(pParse, "too many references to \"%s\": max 65535", pTab->zName); - pFrom->pTab = 0; + pFrom->pSTab = 0; return WRC_Abort; } pTab->nTabRef++; @@ -147845,7 +152907,7 @@ static int selectExpander(Walker *pWalker, Select *p){ i16 nCol; u8 eCodeOrig = pWalker->eCode; if( sqlite3ViewGetColumnNames(pParse, pTab) ) return WRC_Abort; - assert( pFrom->pSelect==0 ); + assert( pFrom->fg.isSubquery==0 ); if( IsView(pTab) ){ if( (db->flags & SQLITE_EnableView)==0 && pTab->pSchema!=db->aDb[1].pSchema @@ -147853,7 +152915,7 @@ static int selectExpander(Walker *pWalker, Select *p){ sqlite3ErrorMsg(pParse, "access to view \"%s\" prohibited", pTab->zName); } - pFrom->pSelect = sqlite3SelectDup(db, pTab->u.view.pSelect, 0); + sqlite3SrcItemAttachSubquery(pParse, pFrom, pTab->u.view.pSelect, 1); } #ifndef SQLITE_OMIT_VIRTUALTABLE else if( ALWAYS(IsVirtual(pTab)) @@ -147869,7 +152931,9 @@ static int selectExpander(Walker *pWalker, Select *p){ nCol = pTab->nCol; pTab->nCol = -1; pWalker->eCode = 1; /* Turn on Select.selId renumbering */ - sqlite3WalkSelect(pWalker, pFrom->pSelect); + if( pFrom->fg.isSubquery ){ + sqlite3WalkSelect(pWalker, pFrom->u4.pSubq->pSelect); + } pWalker->eCode = eCodeOrig; pTab->nCol = nCol; } @@ -147956,7 +153020,7 @@ static int selectExpander(Walker *pWalker, Select *p){ } for(i=0, pFrom=pTabList->a; inSrc; i++, pFrom++){ int nAdd; /* Number of cols including rowid */ - Table *pTab = pFrom->pTab; /* Table for this data source */ + Table *pTab = pFrom->pSTab; /* Table for this data source */ ExprList *pNestedFrom; /* Result-set of a nested FROM clause */ char *zTabName; /* AS name for this data source */ const char *zSchemaName = 0; /* Schema name for this data source */ @@ -147967,13 +153031,14 @@ static int selectExpander(Walker *pWalker, Select *p){ zTabName = pTab->zName; } if( db->mallocFailed ) break; - assert( (int)pFrom->fg.isNestedFrom == IsNestedFrom(pFrom->pSelect) ); + assert( (int)pFrom->fg.isNestedFrom == IsNestedFrom(pFrom) ); if( pFrom->fg.isNestedFrom ){ - assert( pFrom->pSelect!=0 ); - pNestedFrom = pFrom->pSelect->pEList; + assert( pFrom->fg.isSubquery && pFrom->u4.pSubq ); + assert( pFrom->u4.pSubq->pSelect!=0 ); + pNestedFrom = pFrom->u4.pSubq->pSelect->pEList; assert( pNestedFrom!=0 ); assert( pNestedFrom->nExpr==pTab->nCol ); - assert( VisibleRowid(pTab)==0 ); + assert( VisibleRowid(pTab)==0 || ViewCanHaveRowid ); }else{ if( zTName && sqlite3StrICmp(zTName, zTabName)!=0 ){ continue; @@ -148005,7 +153070,8 @@ static int selectExpander(Walker *pWalker, Select *p){ pUsing = 0; } - nAdd = pTab->nCol + (VisibleRowid(pTab) && (selFlags&SF_NestedFrom)); + nAdd = pTab->nCol; + if( VisibleRowid(pTab) && (selFlags & SF_NestedFrom)!=0 ) nAdd++; for(j=0; ja[pNew->nExpr-1]; assert( pX->zEName==0 ); if( (selFlags & SF_NestedFrom)!=0 && !IN_RENAME_OBJECT ){ - if( pNestedFrom ){ + if( pNestedFrom && (!ViewCanHaveRowid || jnExpr) ){ + assert( jnExpr ); pX->zEName = sqlite3DbStrDup(db, pNestedFrom->a[j].zEName); testcase( pX->zEName==0 ); }else{ @@ -148204,18 +153271,15 @@ static void selectAddSubqueryTypeInfo(Walker *pWalker, Select *p){ if( p->selFlags & SF_HasTypeInfo ) return; p->selFlags |= SF_HasTypeInfo; pParse = pWalker->pParse; - testcase( (p->selFlags & SF_Resolved)==0 ); - assert( (p->selFlags & SF_Resolved) || IN_RENAME_OBJECT ); + assert( (p->selFlags & SF_Resolved) ); pTabList = p->pSrc; for(i=0, pFrom=pTabList->a; inSrc; i++, pFrom++){ - Table *pTab = pFrom->pTab; + Table *pTab = pFrom->pSTab; assert( pTab!=0 ); - if( (pTab->tabFlags & TF_Ephemeral)!=0 ){ + if( (pTab->tabFlags & TF_Ephemeral)!=0 && pFrom->fg.isSubquery ){ /* A sub-query in the FROM clause of a SELECT */ - Select *pSel = pFrom->pSelect; - if( pSel ){ - sqlite3SubqueryColumnTypes(pParse, pTab, pSel, SQLITE_AFF_NONE); - } + Select *pSel = pFrom->u4.pSubq->pSelect; + sqlite3SubqueryColumnTypes(pParse, pTab, pSel, SQLITE_AFF_NONE); } } } @@ -148275,6 +153339,8 @@ SQLITE_PRIVATE void sqlite3SelectPrep( */ static void printAggInfo(AggInfo *pAggInfo){ int ii; + sqlite3DebugPrintf("AggInfo %d/%p:\n", + pAggInfo->selId, pAggInfo); for(ii=0; iinColumn; ii++){ struct AggInfo_col *pCol = &pAggInfo->aCol[ii]; sqlite3DebugPrintf( @@ -148498,6 +153564,7 @@ static void resetAccumulator(Parse *pParse, AggInfo *pAggInfo){ if( pFunc->bOBPayload ){ /* extra columns for the function arguments */ assert( ExprUseXList(pFunc->pFExpr) ); + assert( pFunc->pFExpr->x.pList!=0 ); nExtra += pFunc->pFExpr->x.pList->nExpr; } if( pFunc->bUseSubtype ){ @@ -148527,6 +153594,7 @@ static void finalizeAggFunctions(Parse *pParse, AggInfo *pAggInfo){ for(i=0, pF=pAggInfo->aFunc; inFunc; i++, pF++){ ExprList *pList; assert( ExprUseXList(pF->pFExpr) ); + if( pParse->nErr ) return; pList = pF->pFExpr->x.pList; if( pF->iOBTab>=0 ){ /* For an ORDER BY aggregate, calls to OP_AggStep were deferred. Inputs @@ -148567,7 +153635,7 @@ static void finalizeAggFunctions(Parse *pParse, AggInfo *pAggInfo){ } sqlite3VdbeAddOp3(v, OP_AggStep, 0, regAgg, AggInfoFuncReg(pAggInfo,i)); sqlite3VdbeAppendP4(v, pF->pFunc, P4_FUNCDEF); - sqlite3VdbeChangeP5(v, (u8)nArg); + sqlite3VdbeChangeP5(v, (u16)nArg); sqlite3VdbeAddOp2(v, OP_Next, pF->iOBTab, iTop+1); VdbeCoverage(v); sqlite3VdbeJumpHere(v, iTop); sqlite3ReleaseTempRange(pParse, regAgg, nArg); @@ -148730,12 +153798,13 @@ static void updateAccumulator( } sqlite3VdbeAddOp3(v, OP_AggStep, 0, regAgg, AggInfoFuncReg(pAggInfo,i)); sqlite3VdbeAppendP4(v, pF->pFunc, P4_FUNCDEF); - sqlite3VdbeChangeP5(v, (u8)nArg); + sqlite3VdbeChangeP5(v, (u16)nArg); sqlite3ReleaseTempRange(pParse, regAgg, nArg); } if( addrNext ){ sqlite3VdbeResolveLabel(v, addrNext); } + if( pParse->nErr ) return; } if( regHit==0 && pAggInfo->nAccumulator ){ regHit = regAcc; @@ -148745,6 +153814,7 @@ static void updateAccumulator( } for(i=0, pC=pAggInfo->aCol; inAccumulator; i++, pC++){ sqlite3ExprCode(pParse, pC->pCExpr, AggInfoColumnReg(pAggInfo,i)); + if( pParse->nErr ) return; } pAggInfo->directMode = 0; @@ -148860,25 +153930,28 @@ static SrcItem *isSelfJoinView( int iFirst, int iEnd /* Range of FROM-clause entries to search. */ ){ SrcItem *pItem; - assert( pThis->pSelect!=0 ); - if( pThis->pSelect->selFlags & SF_PushDown ) return 0; + Select *pSel; + assert( pThis->fg.isSubquery ); + pSel = pThis->u4.pSubq->pSelect; + assert( pSel!=0 ); + if( pSel->selFlags & SF_PushDown ) return 0; while( iFirsta[iFirst++]; - if( pItem->pSelect==0 ) continue; + if( !pItem->fg.isSubquery ) continue; if( pItem->fg.viaCoroutine ) continue; if( pItem->zName==0 ) continue; - assert( pItem->pTab!=0 ); - assert( pThis->pTab!=0 ); - if( pItem->pTab->pSchema!=pThis->pTab->pSchema ) continue; + assert( pItem->pSTab!=0 ); + assert( pThis->pSTab!=0 ); + if( pItem->pSTab->pSchema!=pThis->pSTab->pSchema ) continue; if( sqlite3_stricmp(pItem->zName, pThis->zName)!=0 ) continue; - pS1 = pItem->pSelect; - if( pItem->pTab->pSchema==0 && pThis->pSelect->selId!=pS1->selId ){ + pS1 = pItem->u4.pSubq->pSelect; + if( pItem->pSTab->pSchema==0 && pSel->selId!=pS1->selId ){ /* The query flattener left two different CTE tables with identical ** names in the same FROM clause. */ continue; } - if( pItem->pSelect->selFlags & SF_PushDown ){ + if( pS1->selFlags & SF_PushDown ){ /* The view was modified by some other optimization such as ** pushDownWhereTerms() */ continue; @@ -148914,6 +153987,7 @@ static void agginfoFree(sqlite3 *db, void *pArg){ ** * There is no WHERE or GROUP BY or HAVING clauses on the subqueries ** * The outer query is a simple count(*) with no WHERE clause or other ** extraneous syntax. +** * None of the subqueries are DISTINCT (forumpost/a860f5fb2e 2025-03-10) ** ** Return TRUE if the optimization is undertaken. */ @@ -148922,6 +153996,7 @@ static int countOfViewOptimization(Parse *pParse, Select *p){ Expr *pExpr; Expr *pCount; sqlite3 *db; + SrcItem *pFrom; if( (p->selFlags & SF_Aggregate)==0 ) return 0; /* This is an aggregate */ if( p->pEList->nExpr!=1 ) return 0; /* Single result column */ if( p->pWhere ) return 0; @@ -148936,17 +154011,22 @@ static int countOfViewOptimization(Parse *pParse, Select *p){ if( pExpr->x.pList!=0 ) return 0; /* Must be count(*) */ if( p->pSrc->nSrc!=1 ) return 0; /* One table in FROM */ if( ExprHasProperty(pExpr, EP_WinFunc) ) return 0;/* Not a window function */ - pSub = p->pSrc->a[0].pSelect; - if( pSub==0 ) return 0; /* The FROM is a subquery */ + pFrom = p->pSrc->a; + if( pFrom->fg.isSubquery==0 ) return 0; /* FROM is a subquery */ + pSub = pFrom->u4.pSubq->pSelect; if( pSub->pPrior==0 ) return 0; /* Must be a compound */ if( pSub->selFlags & SF_CopyCte ) return 0; /* Not a CTE */ do{ if( pSub->op!=TK_ALL && pSub->pPrior ) return 0; /* Must be UNION ALL */ if( pSub->pWhere ) return 0; /* No WHERE clause */ if( pSub->pLimit ) return 0; /* No LIMIT clause */ - if( pSub->selFlags & SF_Aggregate ) return 0; /* Not an aggregate */ + if( pSub->selFlags & (SF_Aggregate|SF_Distinct) ){ + testcase( pSub->selFlags & SF_Aggregate ); + testcase( pSub->selFlags & SF_Distinct ); + return 0; /* Not an aggregate nor DISTINCT */ + } assert( pSub->pHaving==0 ); /* Due to the previous */ - pSub = pSub->pPrior; /* Repeat over compound */ + pSub = pSub->pPrior; /* Repeat over compound */ }while( pSub ); /* If we reach this point then it is OK to perform the transformation */ @@ -148954,17 +154034,16 @@ static int countOfViewOptimization(Parse *pParse, Select *p){ db = pParse->db; pCount = pExpr; pExpr = 0; - pSub = p->pSrc->a[0].pSelect; - p->pSrc->a[0].pSelect = 0; + pSub = sqlite3SubqueryDetach(db, pFrom); sqlite3SrcListDelete(db, p->pSrc); - p->pSrc = sqlite3DbMallocZero(pParse->db, sizeof(*p->pSrc)); + p->pSrc = sqlite3DbMallocZero(pParse->db, SZ_SRCLIST_1); while( pSub ){ Expr *pTerm; pPrior = pSub->pPrior; pSub->pPrior = 0; pSub->pNext = 0; pSub->selFlags |= SF_Aggregate; - pSub->selFlags &= ~SF_Compound; + pSub->selFlags &= ~(u32)SF_Compound; pSub->nSelectRow = 0; sqlite3ParserAddCleanup(pParse, sqlite3ExprListDeleteGeneric, pSub->pEList); pTerm = pPrior ? sqlite3ExprDup(db, pCount, 0) : pCount; @@ -148979,7 +154058,7 @@ static int countOfViewOptimization(Parse *pParse, Select *p){ pSub = pPrior; } p->pEList->a[0].pExpr = pExpr; - p->selFlags &= ~SF_Aggregate; + p->selFlags &= ~(u32)SF_Aggregate; #if TREETRACE_ENABLED if( sqlite3TreeTrace & 0x200 ){ @@ -149000,12 +154079,12 @@ static int sameSrcAlias(SrcItem *p0, SrcList *pSrc){ for(i=0; inSrc; i++){ SrcItem *p1 = &pSrc->a[i]; if( p1==p0 ) continue; - if( p0->pTab==p1->pTab && 0==sqlite3_stricmp(p0->zAlias, p1->zAlias) ){ + if( p0->pSTab==p1->pSTab && 0==sqlite3_stricmp(p0->zAlias, p1->zAlias) ){ return 1; } - if( p1->pSelect - && (p1->pSelect->selFlags & SF_NestedFrom)!=0 - && sameSrcAlias(p0, p1->pSelect->pSrc) + if( p1->fg.isSubquery + && (p1->u4.pSubq->pSelect->selFlags & SF_NestedFrom)!=0 + && sameSrcAlias(p0, p1->u4.pSubq->pSelect->pSrc) ){ return 1; } @@ -149070,13 +154149,200 @@ static int fromClauseTermCanBeCoroutine( if( i==0 ) break; i--; pItem--; - if( pItem->pSelect!=0 ) return 0; /* (1c-i) */ + if( pItem->fg.isSubquery ) return 0; /* (1c-i) */ } return 1; } /* -** Generate code for the SELECT statement given in the p argument. +** Argument pWhere is the WHERE clause belonging to SELECT statement p. This +** function attempts to transform expressions of the form: +** +** EXISTS (SELECT ...) +** +** into joins. For example, given +** +** CREATE TABLE sailors(sid INTEGER PRIMARY KEY, name TEXT); +** CREATE TABLE reserves(sid INT, day DATE, PRIMARY KEY(sid, day)); +** +** SELECT name FROM sailors AS S WHERE EXISTS ( +** SELECT * FROM reserves AS R WHERE S.sid = R.sid AND R.day = '2022-10-25' +** ); +** +** the SELECT statement may be transformed as follows: +** +** SELECT name FROM sailors AS S, reserves AS R +** WHERE S.sid = R.sid AND R.day = '2022-10-25'; +** +** **Approximately**. Really, we have to ensure that the FROM-clause term +** that was formerly inside the EXISTS is only executed once. This is handled +** by setting the SrcItem.fg.fromExists flag, which then causes code in +** the where.c file to exit the corresponding loop after the first successful +** match (if any). +*/ +static SQLITE_NOINLINE void existsToJoin( + Parse *pParse, /* Parsing context */ + Select *p, /* The SELECT statement being optimized */ + Expr *pWhere /* part of the WHERE clause currently being examined */ +){ + if( pParse->nErr==0 + && pWhere!=0 + && !ExprHasProperty(pWhere, EP_OuterON|EP_InnerON) + && ALWAYS(p->pSrc!=0) + && p->pSrc->nSrcop==TK_AND ){ + Expr *pRight = pWhere->pRight; + existsToJoin(pParse, p, pWhere->pLeft); + existsToJoin(pParse, p, pRight); + } + else if( pWhere->op==TK_EXISTS ){ + Select *pSub = pWhere->x.pSelect; + Expr *pSubWhere = pSub->pWhere; + if( pSub->pSrc->nSrc==1 + && (pSub->selFlags & SF_Aggregate)==0 + && !pSub->pSrc->a[0].fg.isSubquery + && pSub->pLimit==0 + ){ + memset(pWhere, 0, sizeof(*pWhere)); + pWhere->op = TK_INTEGER; + pWhere->u.iValue = 1; + ExprSetProperty(pWhere, EP_IntValue); + + assert( p->pWhere!=0 ); + pSub->pSrc->a[0].fg.fromExists = 1; + pSub->pSrc->a[0].fg.jointype |= JT_CROSS; + p->pSrc = sqlite3SrcListAppendList(pParse, p->pSrc, pSub->pSrc); + if( pSubWhere ){ + p->pWhere = sqlite3PExpr(pParse, TK_AND, p->pWhere, pSubWhere); + pSub->pWhere = 0; + } + pSub->pSrc = 0; + sqlite3ParserAddCleanup(pParse, sqlite3SelectDeleteGeneric, pSub); +#if TREETRACE_ENABLED + if( sqlite3TreeTrace & 0x100000 ){ + TREETRACE(0x100000,pParse,p, + ("After EXISTS-to-JOIN optimization:\n")); + sqlite3TreeViewSelect(0, p, 0); + } +#endif + existsToJoin(pParse, p, pSubWhere); + } + } + } +} + +/* +** Type used for Walker callbacks by selectCheckOnClauses(). +*/ +typedef struct CheckOnCtx CheckOnCtx; +struct CheckOnCtx { + SrcList *pSrc; /* SrcList for this context */ + int iJoin; /* Cursor numbers must be =< than this */ + CheckOnCtx *pParent; /* Parent context */ +}; + +/* +** True if the SrcList passed as the only argument contains at least +** one RIGHT or FULL JOIN. False otherwise. +*/ +#define hasRightJoin(pSrc) (((pSrc)->a[0].fg.jointype & JT_LTORJ)!=0) + +/* +** The xExpr callback for the search of invalid ON clause terms. +*/ +static int selectCheckOnClausesExpr(Walker *pWalker, Expr *pExpr){ + CheckOnCtx *pCtx = pWalker->u.pCheckOnCtx; + + /* Check if pExpr is root or near-root of an ON clause constraint that needs + ** to be checked to ensure that it does not refer to tables in its FROM + ** clause to the right of itself. i.e. it is either: + ** + ** + an ON clause on an OUTER join, or + ** + an ON clause on an INNER join within a FROM that features at + ** least one RIGHT or FULL join. + */ + if( (ExprHasProperty(pExpr, EP_OuterON)) + || (ExprHasProperty(pExpr, EP_InnerON) && hasRightJoin(pCtx->pSrc)) + ){ + /* If CheckOnCtx.iJoin is already set, then fall through and process + ** this expression node as normal. Or, if CheckOnCtx.iJoin is still 0, + ** set it to the cursor number of the RHS of the join to which this + ** ON expression was attached and then iterate through the entire + ** expression. */ + assert( pCtx->iJoin==0 || pCtx->iJoin==pExpr->w.iJoin ); + if( pCtx->iJoin==0 ){ + pCtx->iJoin = pExpr->w.iJoin; + sqlite3WalkExprNN(pWalker, pExpr); + pCtx->iJoin = 0; + return WRC_Prune; + } + } + + if( pExpr->op==TK_COLUMN ){ + /* A column expression. Find the SrcList (if any) to which it refers. + ** Then, if CheckOnCtx.iJoin indicates that this expression is part of an + ** ON clause from that SrcList (i.e. if iJoin is non-zero), check that it + ** does not refer to a table to the right of CheckOnCtx.iJoin. */ + do { + SrcList *pSrc = pCtx->pSrc; + int iTab = pExpr->iTable; + if( iTab>=pSrc->a[0].iCursor && iTab<=pSrc->a[pSrc->nSrc-1].iCursor ){ + if( pCtx->iJoin && iTab>pCtx->iJoin ){ + sqlite3ErrorMsg(pWalker->pParse, + "ON clause references tables to its right"); + return WRC_Abort; + } + break; + } + pCtx = pCtx->pParent; + }while( pCtx ); + } + return WRC_Continue; +} + +/* +** The xSelect callback for the search of invalid ON clause terms. +*/ +static int selectCheckOnClausesSelect(Walker *pWalker, Select *pSelect){ + CheckOnCtx *pCtx = pWalker->u.pCheckOnCtx; + if( pSelect->pSrc==pCtx->pSrc || pSelect->pSrc->nSrc==0 ){ + return WRC_Continue; + }else{ + CheckOnCtx sCtx; + memset(&sCtx, 0, sizeof(sCtx)); + sCtx.pSrc = pSelect->pSrc; + sCtx.pParent = pCtx; + pWalker->u.pCheckOnCtx = &sCtx; + sqlite3WalkSelect(pWalker, pSelect); + pWalker->u.pCheckOnCtx = pCtx; + pSelect->selFlags &= ~SF_OnToWhere; + return WRC_Prune; + } +} + +/* +** Check all ON clauses in pSelect to verify that they do not reference +** columns to the right. +*/ +static void selectCheckOnClauses(Parse *pParse, Select *pSelect){ + Walker w; + CheckOnCtx sCtx; + assert( pSelect->selFlags & SF_OnToWhere ); + assert( pSelect->pSrc!=0 && pSelect->pSrc->nSrc>=2 ); + memset(&w, 0, sizeof(w)); + w.pParse = pParse; + w.xExprCallback = selectCheckOnClausesExpr; + w.xSelectCallback = selectCheckOnClausesSelect; + w.u.pCheckOnCtx = &sCtx; + memset(&sCtx, 0, sizeof(sCtx)); + sCtx.pSrc = pSelect->pSrc; + sqlite3WalkExprNN(&w, pSelect->pWhere); + pSelect->selFlags &= ~SF_OnToWhere; +} + +/* +** Generate byte-code for the SELECT statement given in the p argument. ** ** The results are returned according to the SelectDest structure. ** See comments in sqliteInt.h for further information. @@ -149087,6 +154353,40 @@ static int fromClauseTermCanBeCoroutine( ** ** This routine does NOT free the Select structure passed in. The ** calling function needs to do that. +** +** This is a long function. The following is an outline of the processing +** steps, with tags referencing various milestones: +** +** * Resolve names and similar preparation tag-select-0100 +** * Scan of the FROM clause tag-select-0200 +** + OUTER JOIN strength reduction tag-select-0220 +** + Sub-query ORDER BY removal tag-select-0230 +** + Query flattening tag-select-0240 +** * Separate subroutine for compound-SELECT tag-select-0300 +** * WHERE-clause constant propagation tag-select-0330 +** * Count()-of-VIEW optimization tag-select-0350 +** * Scan of the FROM clause again tag-select-0400 +** + Authorize unreferenced tables tag-select-0410 +** + Predicate push-down optimization tag-select-0420 +** + Omit unused subquery columns optimization tag-select-0440 +** + Generate code to implement subqueries tag-select-0480 +** - Co-routines tag-select-0482 +** - Reuse previously computed CTE tag-select-0484 +** - REuse previously computed VIEW tag-select-0486 +** - Materialize a VIEW or CTE tag-select-0488 +** * DISTINCT ORDER BY -> GROUP BY optimization tag-select-0500 +** * Set up for ORDER BY tag-select-0600 +** * Create output table tag-select-0630 +** * Prepare registers for LIMIT tag-select-0650 +** * Setup for DISTINCT tag-select-0680 +** * Generate code for non-aggregate and non-GROUP BY tag-select-0700 +** * Generate code for aggregate and/or GROUP BY tag-select-0800 +** + GROUP BY queries tag-select-0810 +** + non-GROUP BY queries tag-select-0820 +** - Special case of count() w/o GROUP BY tag-select-0821 +** - General case of non-GROUP BY aggregates tag-select-0822 +** * Sort results, as needed tag-select-0900 +** * Internal self-checks tag-select-1000 */ SQLITE_PRIVATE int sqlite3Select( Parse *pParse, /* The parser context */ @@ -149130,6 +154430,7 @@ SQLITE_PRIVATE int sqlite3Select( } #endif + /* tag-select-0100 */ assert( p->pOrderBy==0 || pDest->eDest!=SRT_DistFifo ); assert( p->pOrderBy==0 || pDest->eDest!=SRT_Fifo ); assert( p->pOrderBy==0 || pDest->eDest!=SRT_DistQueue ); @@ -149151,7 +154452,7 @@ SQLITE_PRIVATE int sqlite3Select( testcase( pParse->earlyCleanup ); p->pOrderBy = 0; } - p->selFlags &= ~SF_Distinct; + p->selFlags &= ~(u32)SF_Distinct; p->selFlags |= SF_NoopOrderBy; } sqlite3SelectPrep(pParse, p, 0); @@ -149167,6 +154468,18 @@ SQLITE_PRIVATE int sqlite3Select( } #endif + /* If the SELECT statement contains ON clauses that were moved into + ** the WHERE clause, go through and verify that none of the terms + ** in the ON clauses reference tables to the right of the ON clause. + ** Do this now, after name resolution, but before query flattening + */ + if( p->selFlags & SF_OnToWhere ){ + selectCheckOnClauses(pParse, p); + if( pParse->nErr ){ + goto select_end; + } + } + /* If the SF_UFSrcCheck flag is set, then this function is being called ** as part of populating the temp table for an UPDATE...FROM statement. ** In this case, it is an error if the target object (pSrc->a[0]) name @@ -149181,7 +154494,7 @@ SQLITE_PRIVATE int sqlite3Select( if( sameSrcAlias(p0, p->pSrc) ){ sqlite3ErrorMsg(pParse, "target object/alias may not appear in FROM clause: %s", - p0->zAlias ? p0->zAlias : p0->pTab->zName + p0->zAlias ? p0->zAlias : p0->pSTab->zName ); goto select_end; } @@ -149190,7 +154503,7 @@ SQLITE_PRIVATE int sqlite3Select( ** and leaving this flag set can cause errors if a compound sub-query ** in p->pSrc is flattened into this query and this function called ** again as part of compound SELECT processing. */ - p->selFlags &= ~SF_UFSrcCheck; + p->selFlags &= ~(u32)SF_UFSrcCheck; } if( pDest->eDest==SRT_Output ){ @@ -149216,12 +154529,13 @@ SQLITE_PRIVATE int sqlite3Select( /* Try to do various optimizations (flattening subqueries, and strength ** reduction of join operators) in the FROM clause up into the main query + ** tag-select-0200 */ #if !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) for(i=0; !p->pPrior && inSrc; i++){ SrcItem *pItem = &pTabList->a[i]; - Select *pSub = pItem->pSelect; - Table *pTab = pItem->pTab; + Select *pSub = pItem->fg.isSubquery ? pItem->u4.pSubq->pSelect : 0; + Table *pTab = pItem->pSTab; /* The expander should have already created transient Table objects ** even for FROM clause elements such as subqueries that do not correspond @@ -149238,6 +154552,7 @@ SQLITE_PRIVATE int sqlite3Select( ** way that the i-th table cannot be the NULL row of a join, then ** perform the appropriate simplification. This is called ** "OUTER JOIN strength reduction" in the SQLite documentation. + ** tag-select-0220 */ if( (pItem->fg.jointype & (JT_LEFT|JT_LTORJ))!=0 && sqlite3ExprImpliesNonNullRow(p->pWhere, pItem->iCursor, @@ -149308,7 +154623,8 @@ SQLITE_PRIVATE int sqlite3Select( if( (pSub->selFlags & SF_Aggregate)!=0 ) continue; assert( pSub->pGroupBy==0 ); - /* If a FROM-clause subquery has an ORDER BY clause that is not + /* tag-select-0230: + ** If a FROM-clause subquery has an ORDER BY clause that is not ** really doing anything, then delete it now so that it does not ** interfere with query flattening. See the discussion at ** https://sqlite.org/forum/forumpost/2d76f2bcf65d256a @@ -149327,13 +154643,16 @@ SQLITE_PRIVATE int sqlite3Select( ** (a) The outer query has a different ORDER BY clause ** (b) The subquery is part of a join ** See forum post 062d576715d277c8 + ** (6) The subquery is not a recursive CTE. ORDER BY has a different + ** meaning for recursive CTEs and this optimization does not + ** apply. ** ** Also retain the ORDER BY if the OmitOrderBy optimization is disabled. */ if( pSub->pOrderBy!=0 && (p->pOrderBy!=0 || pTabList->nSrc>1) /* Condition (5) */ && pSub->pLimit==0 /* Condition (1) */ - && (pSub->selFlags & SF_OrderByReqd)==0 /* Condition (2) */ + && (pSub->selFlags & (SF_OrderByReqd|SF_Recursive))==0 /* (2) and (6) */ && (p->selFlags & SF_OrderByReqd)==0 /* Condition (3) and (4) */ && OptimizationEnabled(db, SQLITE_OmitOrderBy) ){ @@ -149371,6 +154690,7 @@ SQLITE_PRIVATE int sqlite3Select( continue; } + /* tag-select-0240 */ if( flattenSubquery(pParse, p, i, isAgg) ){ if( pParse->nErr ) goto select_end; /* This subquery can be absorbed into its parent. */ @@ -149386,7 +154706,7 @@ SQLITE_PRIVATE int sqlite3Select( #ifndef SQLITE_OMIT_COMPOUND_SELECT /* Handle compound SELECT statements using the separate multiSelect() - ** procedure. + ** procedure. tag-select-0300 */ if( p->pPrior ){ rc = multiSelect(pParse, p, pDest); @@ -149401,10 +154721,17 @@ SQLITE_PRIVATE int sqlite3Select( } #endif + /* If there may be an "EXISTS (SELECT ...)" in the WHERE clause, attempt + ** to change it into a join. */ + if( pParse->bHasExists && OptimizationEnabled(db,SQLITE_ExistsToJoin) ){ + existsToJoin(pParse, p, p->pWhere); + pTabList = p->pSrc; + } + /* Do the WHERE-clause constant propagation optimization if this is - ** a join. No need to speed time on this operation for non-join queries + ** a join. No need to spend time on this operation for non-join queries ** as the equivalent optimization will be handled by query planner in - ** sqlite3WhereBegin(). + ** sqlite3WhereBegin(). tag-select-0330 */ if( p->pWhere!=0 && p->pWhere->op==TK_AND @@ -149421,6 +154748,7 @@ SQLITE_PRIVATE int sqlite3Select( TREETRACE(0x2000,pParse,p,("Constant propagation not helpful\n")); } + /* tag-select-0350 */ if( OptimizationEnabled(db, SQLITE_QueryFlattener|SQLITE_CountOfView) && countOfViewOptimization(pParse, p) ){ @@ -149428,20 +154756,26 @@ SQLITE_PRIVATE int sqlite3Select( pTabList = p->pSrc; } - /* For each term in the FROM clause, do two things: - ** (1) Authorized unreferenced tables - ** (2) Generate code for all sub-queries + /* Loop over all terms in the FROM clause and do two things for each term: + ** + ** (1) Authorize unreferenced tables + ** (2) Generate code for all sub-queries + ** + ** tag-select-0400 */ for(i=0; inSrc; i++){ SrcItem *pItem = &pTabList->a[i]; SrcItem *pPrior; SelectDest dest; + Subquery *pSubq; Select *pSub; #if !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) const char *zSavedAuthContext; #endif - /* Issue SQLITE_READ authorizations with a fake column name for any + /* Authorized unreferenced tables. tag-select-0410 + ** + ** Issue SQLITE_READ authorizations with a fake column name for any ** tables that are referenced but from which no values are extracted. ** Examples of where these kinds of null SQLITE_READ authorizations ** would occur: @@ -149458,17 +154792,28 @@ SQLITE_PRIVATE int sqlite3Select( ** string for the fake column name seems safer. */ if( pItem->colUsed==0 && pItem->zName!=0 ){ - sqlite3AuthCheck(pParse, SQLITE_READ, pItem->zName, "", pItem->zDatabase); + const char *zDb; + if( pItem->fg.fixedSchema ){ + int iDb = sqlite3SchemaToIndex(pParse->db, pItem->u4.pSchema); + zDb = db->aDb[iDb].zDbSName; + }else if( pItem->fg.isSubquery ){ + zDb = 0; + }else{ + zDb = pItem->u4.zDatabase; + } + sqlite3AuthCheck(pParse, SQLITE_READ, pItem->zName, "", zDb); } #if !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) /* Generate code for all sub-queries in the FROM clause */ - pSub = pItem->pSelect; - if( pSub==0 ) continue; + if( pItem->fg.isSubquery==0 ) continue; + pSubq = pItem->u4.pSubq; + assert( pSubq!=0 ); + pSub = pSubq->pSelect; /* The code for a subquery should only be generated once. */ - assert( pItem->addrFillSub==0 ); + if( pSubq->addrFillSub!=0 ) continue; /* Increment Parse.nHeight by the height of the largest expression ** tree referred to by this, the parent select. The child select @@ -149481,6 +154826,7 @@ SQLITE_PRIVATE int sqlite3Select( /* Make copies of constant WHERE-clause terms in the outer query down ** inside the subquery. This can help the subquery to run more efficiently. + ** This is the "predicate push-down optimization". tag-select-0420 */ if( OptimizationEnabled(db, SQLITE_PushDown) && (pItem->fg.isCte==0 @@ -149494,13 +154840,14 @@ SQLITE_PRIVATE int sqlite3Select( sqlite3TreeViewSelect(0, p, 0); } #endif - assert( pItem->pSelect && (pItem->pSelect->selFlags & SF_PushDown)!=0 ); + assert( pSubq->pSelect && (pSub->selFlags & SF_PushDown)!=0 ); }else{ - TREETRACE(0x4000,pParse,p,("Push-down not possible\n")); + TREETRACE(0x4000,pParse,p,("WHERE-clause push-down not possible\n")); } /* Convert unused result columns of the subquery into simple NULL ** expressions, to avoid unneeded searching and computation. + ** tag-select-0440 */ if( OptimizationEnabled(db, SQLITE_NullUnusedCols) && disableUnusedSubqueryResultColumns(pItem) @@ -149518,32 +154865,33 @@ SQLITE_PRIVATE int sqlite3Select( zSavedAuthContext = pParse->zAuthContext; pParse->zAuthContext = pItem->zName; - /* Generate code to implement the subquery + /* Generate byte-code to implement the subquery tag-select-0480 */ if( fromClauseTermCanBeCoroutine(pParse, pTabList, i, p->selFlags) ){ /* Implement a co-routine that will return a single row of the result - ** set on each invocation. + ** set on each invocation. tag-select-0482 */ int addrTop = sqlite3VdbeCurrentAddr(v)+1; - pItem->regReturn = ++pParse->nMem; - sqlite3VdbeAddOp3(v, OP_InitCoroutine, pItem->regReturn, 0, addrTop); + pSubq->regReturn = ++pParse->nMem; + sqlite3VdbeAddOp3(v, OP_InitCoroutine, pSubq->regReturn, 0, addrTop); VdbeComment((v, "%!S", pItem)); - pItem->addrFillSub = addrTop; - sqlite3SelectDestInit(&dest, SRT_Coroutine, pItem->regReturn); + pSubq->addrFillSub = addrTop; + sqlite3SelectDestInit(&dest, SRT_Coroutine, pSubq->regReturn); ExplainQueryPlan((pParse, 1, "CO-ROUTINE %!S", pItem)); sqlite3Select(pParse, pSub, &dest); - pItem->pTab->nRowLogEst = pSub->nSelectRow; + pItem->pSTab->nRowLogEst = pSub->nSelectRow; pItem->fg.viaCoroutine = 1; - pItem->regResult = dest.iSdst; - sqlite3VdbeEndCoroutine(v, pItem->regReturn); + pSubq->regResult = dest.iSdst; + sqlite3VdbeEndCoroutine(v, pSubq->regReturn); + VdbeComment((v, "end %!S", pItem)); sqlite3VdbeJumpHere(v, addrTop-1); sqlite3ClearTempRegCache(pParse); }else if( pItem->fg.isCte && pItem->u2.pCteUse->addrM9e>0 ){ /* This is a CTE for which materialization code has already been ** generated. Invoke the subroutine to compute the materialization, - ** the make the pItem->iCursor be a copy of the ephemeral table that - ** holds the result of the materialization. */ + ** then make the pItem->iCursor be a copy of the ephemeral table that + ** holds the result of the materialization. tag-select-0484 */ CteUse *pCteUse = pItem->u2.pCteUse; sqlite3VdbeAddOp2(v, OP_Gosub, pCteUse->regRtn, pCteUse->addrM9e); if( pItem->iCursor!=pCteUse->iCur ){ @@ -149553,25 +154901,30 @@ SQLITE_PRIVATE int sqlite3Select( pSub->nSelectRow = pCteUse->nRowEst; }else if( (pPrior = isSelfJoinView(pTabList, pItem, 0, i))!=0 ){ /* This view has already been materialized by a prior entry in - ** this same FROM clause. Reuse it. */ - if( pPrior->addrFillSub ){ - sqlite3VdbeAddOp2(v, OP_Gosub, pPrior->regReturn, pPrior->addrFillSub); + ** this same FROM clause. Reuse it. tag-select-0486 */ + Subquery *pPriorSubq; + assert( pPrior->fg.isSubquery ); + pPriorSubq = pPrior->u4.pSubq; + assert( pPriorSubq!=0 ); + if( pPriorSubq->addrFillSub ){ + sqlite3VdbeAddOp2(v, OP_Gosub, pPriorSubq->regReturn, + pPriorSubq->addrFillSub); } sqlite3VdbeAddOp2(v, OP_OpenDup, pItem->iCursor, pPrior->iCursor); - pSub->nSelectRow = pPrior->pSelect->nSelectRow; + pSub->nSelectRow = pPriorSubq->pSelect->nSelectRow; }else{ /* Materialize the view. If the view is not correlated, generate a ** subroutine to do the materialization so that subsequent uses of - ** the same view can reuse the materialization. */ + ** the same view can reuse the materialization. tag-select-0488 */ int topAddr; int onceAddr = 0; #ifdef SQLITE_ENABLE_STMT_SCANSTATUS int addrExplain; #endif - pItem->regReturn = ++pParse->nMem; + pSubq->regReturn = ++pParse->nMem; topAddr = sqlite3VdbeAddOp0(v, OP_Goto); - pItem->addrFillSub = topAddr+1; + pSubq->addrFillSub = topAddr+1; pItem->fg.isMaterialized = 1; if( pItem->fg.isCorrelated==0 ){ /* If the subquery is not correlated and if we are not inside of @@ -149586,17 +154939,17 @@ SQLITE_PRIVATE int sqlite3Select( ExplainQueryPlan2(addrExplain, (pParse, 1, "MATERIALIZE %!S", pItem)); sqlite3Select(pParse, pSub, &dest); - pItem->pTab->nRowLogEst = pSub->nSelectRow; + pItem->pSTab->nRowLogEst = pSub->nSelectRow; if( onceAddr ) sqlite3VdbeJumpHere(v, onceAddr); - sqlite3VdbeAddOp2(v, OP_Return, pItem->regReturn, topAddr+1); + sqlite3VdbeAddOp2(v, OP_Return, pSubq->regReturn, topAddr+1); VdbeComment((v, "end %!S", pItem)); sqlite3VdbeScanStatusRange(v, addrExplain, addrExplain, -1); sqlite3VdbeJumpHere(v, topAddr); sqlite3ClearTempRegCache(pParse); if( pItem->fg.isCte && pItem->fg.isCorrelated==0 ){ CteUse *pCteUse = pItem->u2.pCteUse; - pCteUse->addrM9e = pItem->addrFillSub; - pCteUse->regRtn = pItem->regReturn; + pCteUse->addrM9e = pSubq->addrFillSub; + pCteUse->regRtn = pSubq->regReturn; pCteUse->iCur = pItem->iCursor; pCteUse->nRowEst = pSub->nSelectRow; } @@ -149622,7 +154975,9 @@ SQLITE_PRIVATE int sqlite3Select( } #endif - /* If the query is DISTINCT with an ORDER BY but is not an aggregate, and + /* tag-select-0500 + ** + ** If the query is DISTINCT with an ORDER BY but is not an aggregate, and ** if the select-list is the same as the ORDER BY list, then this query ** can be rewritten as a GROUP BY. In other words, this: ** @@ -149639,12 +154994,18 @@ SQLITE_PRIVATE int sqlite3Select( */ if( (p->selFlags & (SF_Distinct|SF_Aggregate))==SF_Distinct && sqlite3ExprListCompare(sSort.pOrderBy, pEList, -1)==0 + && OptimizationEnabled(db, SQLITE_GroupByOrder) #ifndef SQLITE_OMIT_WINDOWFUNC && p->pWin==0 #endif ){ - p->selFlags &= ~SF_Distinct; + p->selFlags &= ~(u32)SF_Distinct; pGroupBy = p->pGroupBy = sqlite3ExprListDup(db, pEList, 0); + if( pGroupBy ){ + for(i=0; inExpr; i++){ + pGroupBy->a[i].u.x.iOrderByCol = i+1; + } + } p->selFlags |= SF_Aggregate; /* Notice that even thought SF_Distinct has been cleared from p->selFlags, ** the sDistinct.isTnct is still set. Hence, isTnct represents the @@ -149666,7 +155027,7 @@ SQLITE_PRIVATE int sqlite3Select( ** If that is the case, then the OP_OpenEphemeral instruction will be ** changed to an OP_Noop once we figure out that the sorting index is ** not needed. The sSort.addrSortIndex variable is used to facilitate - ** that change. + ** that change. tag-select-0600 */ if( sSort.pOrderBy ){ KeyInfo *pKeyInfo; @@ -149683,6 +155044,7 @@ SQLITE_PRIVATE int sqlite3Select( } /* If the output is destined for a temporary table, open that table. + ** tag-select-0630 */ if( pDest->eDest==SRT_EphemTab ){ sqlite3VdbeAddOp2(v, OP_OpenEphemeral, pDest->iSDParm, pEList->nExpr); @@ -149700,7 +155062,7 @@ SQLITE_PRIVATE int sqlite3Select( } } - /* Set the limiter. + /* Set the limiter. tag-select-0650 */ iEnd = sqlite3VdbeMakeLabel(pParse); if( (p->selFlags & SF_FixedLimit)==0 ){ @@ -149712,7 +155074,7 @@ SQLITE_PRIVATE int sqlite3Select( sSort.sortFlags |= SORTFLAG_UseSorter; } - /* Open an ephemeral index to use for the distinct set. + /* Open an ephemeral index to use for the distinct set. tag-select-0680 */ if( p->selFlags & SF_Distinct ){ sDistinct.tabTnct = pParse->nTab++; @@ -149727,7 +155089,7 @@ SQLITE_PRIVATE int sqlite3Select( } if( !isAgg && pGroupBy==0 ){ - /* No aggregate functions and no GROUP BY clause */ + /* No aggregate functions and no GROUP BY clause. tag-select-0700 */ u16 wctrlFlags = (sDistinct.isTnct ? WHERE_WANT_DISTINCT : 0) | (p->selFlags & SF_FixedLimit); #ifndef SQLITE_OMIT_WINDOWFUNC @@ -149746,6 +155108,12 @@ SQLITE_PRIVATE int sqlite3Select( if( pWInfo==0 ) goto select_end; if( sqlite3WhereOutputRowCount(pWInfo) < p->nSelectRow ){ p->nSelectRow = sqlite3WhereOutputRowCount(pWInfo); + if( pDest->eDest<=SRT_DistQueue && pDest->eDest>=SRT_DistFifo ){ + /* TUNING: For a UNION CTE, because UNION is implies DISTINCT, + ** reduce the estimated output row count by 8 (LogEst 30). + ** Search for tag-20250414a to see other cases */ + p->nSelectRow -= 30; + } } if( sDistinct.isTnct && sqlite3WhereIsDistinct(pWInfo) ){ sDistinct.eTnctType = sqlite3WhereIsDistinct(pWInfo); @@ -149800,8 +155168,8 @@ SQLITE_PRIVATE int sqlite3Select( sqlite3WhereEnd(pWInfo); } }else{ - /* This case when there exist aggregate functions or a GROUP BY clause - ** or both */ + /* This case is for when there exist aggregate functions or a GROUP BY + ** clause or both. tag-select-0800 */ NameContext sNC; /* Name context for processing aggregate information */ int iAMem; /* First Mem address for storing current GROUP BY */ int iBMem; /* First Mem address for previous GROUP BY */ @@ -149920,7 +155288,7 @@ SQLITE_PRIVATE int sqlite3Select( /* Processing for aggregates with GROUP BY is very different and - ** much more complex than aggregates without a GROUP BY. + ** much more complex than aggregates without a GROUP BY. tag-select-0810 */ if( pGroupBy ){ KeyInfo *pKeyInfo; /* Keying information for the group by clause */ @@ -149976,6 +155344,7 @@ SQLITE_PRIVATE int sqlite3Select( sqlite3VdbeAddOp2(v, OP_Integer, 0, iAbortFlag); VdbeComment((v, "clear abort flag")); sqlite3VdbeAddOp3(v, OP_Null, 0, iAMem, iAMem+pGroupBy->nExpr-1); + sqlite3ExprNullRegisterRange(pParse, iAMem, pGroupBy->nExpr); /* Begin a loop that will extract all source rows in GROUP BY order. ** This might involve two separate loops with an OP_Sort in between, or @@ -150107,12 +155476,29 @@ SQLITE_PRIVATE int sqlite3Select( sortOut, sortPTab); } for(j=0; jnExpr; j++){ + int iOrderByCol = pGroupBy->a[j].u.x.iOrderByCol; + if( groupBySort ){ sqlite3VdbeAddOp3(v, OP_Column, sortPTab, j, iBMem+j); }else{ pAggInfo->directMode = 1; sqlite3ExprCode(pParse, pGroupBy->a[j].pExpr, iBMem+j); } + + if( iOrderByCol ){ + Expr *pX = p->pEList->a[iOrderByCol-1].pExpr; + Expr *pBase = sqlite3ExprSkipCollateAndLikely(pX); + while( ALWAYS(pBase!=0) && pBase->op==TK_IF_NULL_ROW ){ + pX = pBase->pLeft; + pBase = sqlite3ExprSkipCollateAndLikely(pX); + } + if( ALWAYS(pBase!=0) + && pBase->op!=TK_AGG_COLUMN + && pBase->op!=TK_REGISTER + ){ + sqlite3ExprToRegister(pX, iAMem+j); + } + } } sqlite3VdbeAddOp4(v, OP_Compare, iAMem, iBMem, pGroupBy->nExpr, (char*)sqlite3KeyInfoRef(pKeyInfo), P4_KEYINFO); @@ -150128,13 +155514,13 @@ SQLITE_PRIVATE int sqlite3Select( ** and resets the aggregate accumulator registers in preparation ** for the next GROUP BY batch. */ - sqlite3ExprCodeMove(pParse, iBMem, iAMem, pGroupBy->nExpr); sqlite3VdbeAddOp2(v, OP_Gosub, regOutputRow, addrOutputRow); - VdbeComment((v, "output one row")); + VdbeComment((v, "output one row of %d", p->selId)); + sqlite3ExprCodeMove(pParse, iBMem, iAMem, pGroupBy->nExpr); sqlite3VdbeAddOp2(v, OP_IfPos, iAbortFlag, addrEnd); VdbeCoverage(v); VdbeComment((v, "check abort flag")); sqlite3VdbeAddOp2(v, OP_Gosub, regReset, addrReset); - VdbeComment((v, "reset accumulator")); + VdbeComment((v, "reset accumulator %d", p->selId)); /* Update the aggregate accumulators based on the content of ** the current row @@ -150142,7 +155528,7 @@ SQLITE_PRIVATE int sqlite3Select( sqlite3VdbeJumpHere(v, addr1); updateAccumulator(pParse, iUseFlag, pAggInfo, eDist); sqlite3VdbeAddOp2(v, OP_Integer, 1, iUseFlag); - VdbeComment((v, "indicate data in accumulator")); + VdbeComment((v, "indicate data in accumulator %d", p->selId)); /* End of the loop */ @@ -150159,7 +155545,7 @@ SQLITE_PRIVATE int sqlite3Select( /* Output the final row of result */ sqlite3VdbeAddOp2(v, OP_Gosub, regOutputRow, addrOutputRow); - VdbeComment((v, "output final row")); + VdbeComment((v, "output final row of %d", p->selId)); /* Jump over the subroutines */ @@ -150180,7 +155566,7 @@ SQLITE_PRIVATE int sqlite3Select( addrOutputRow = sqlite3VdbeCurrentAddr(v); sqlite3VdbeAddOp2(v, OP_IfPos, iUseFlag, addrOutputRow+2); VdbeCoverage(v); - VdbeComment((v, "Groupby result generator entry point")); + VdbeComment((v, "Groupby result generator entry point %d", p->selId)); sqlite3VdbeAddOp1(v, OP_Return, regOutputRow); finalizeAggFunctions(pParse, pAggInfo); sqlite3ExprIfFalse(pParse, pHaving, addrOutputRow+1, SQLITE_JUMPIFNULL); @@ -150188,14 +155574,14 @@ SQLITE_PRIVATE int sqlite3Select( &sDistinct, pDest, addrOutputRow+1, addrSetAbort); sqlite3VdbeAddOp1(v, OP_Return, regOutputRow); - VdbeComment((v, "end groupby result generator")); + VdbeComment((v, "end groupby result generator %d", p->selId)); /* Generate a subroutine that will reset the group-by accumulator */ sqlite3VdbeResolveLabel(v, addrReset); resetAccumulator(pParse, pAggInfo); sqlite3VdbeAddOp2(v, OP_Integer, 0, iUseFlag); - VdbeComment((v, "indicate accumulator empty")); + VdbeComment((v, "indicate accumulator %d empty", p->selId)); sqlite3VdbeAddOp1(v, OP_Return, regReset); if( distFlag!=0 && eDist!=WHERE_DISTINCT_NOOP ){ @@ -150204,9 +155590,12 @@ SQLITE_PRIVATE int sqlite3Select( } } /* endif pGroupBy. Begin aggregate queries without GROUP BY: */ else { + /* Aggregate functions without GROUP BY. tag-select-0820 */ Table *pTab; if( (pTab = isSimpleCount(p, pAggInfo))!=0 ){ - /* If isSimpleCount() returns a pointer to a Table structure, then + /* tag-select-0821 + ** + ** If isSimpleCount() returns a pointer to a Table structure, then ** the SQL statement is of the form: ** ** SELECT count(*) FROM @@ -150265,6 +155654,8 @@ SQLITE_PRIVATE int sqlite3Select( sqlite3VdbeAddOp1(v, OP_Close, iCsr); explainSimpleCount(pParse, pTab, pBest); }else{ + /* The general case of an aggregate query without GROUP BY + ** tag-select-0822 */ int regAcc = 0; /* "populate accumulators" flag */ ExprList *pDistinct = 0; u16 distFlag = 0; @@ -150353,7 +155744,7 @@ SQLITE_PRIVATE int sqlite3Select( } /* If there is an ORDER BY clause, then we need to sort the results - ** and send them to the callback one by one. + ** and send them to the callback one by one. tag-select-0900 */ if( sSort.pOrderBy ){ assert( p->pEList==pEList ); @@ -150376,7 +155767,14 @@ SQLITE_PRIVATE int sqlite3Select( assert( db->mallocFailed==0 || pParse->nErr!=0 ); sqlite3ExprListDelete(db, pMinMaxOrderBy); #ifdef SQLITE_DEBUG + /* Internal self-checks. tag-select-1000 */ if( pAggInfo && !db->mallocFailed ){ +#if TREETRACE_ENABLED + if( sqlite3TreeTrace & 0x20 ){ + TREETRACE(0x20,pParse,p,("Finished with AggInfo\n")); + printAggInfo(pAggInfo); + } +#endif for(i=0; inColumn; i++){ Expr *pExpr = pAggInfo->aCol[i].pCExpr; if( pExpr==0 ) continue; @@ -150677,7 +156075,8 @@ SQLITE_PRIVATE Trigger *sqlite3TriggerList(Parse *pParse, Table *pTab){ assert( pParse->db->pVtabCtx==0 ); #endif assert( pParse->bReturning ); - assert( &(pParse->u1.pReturning->retTrig) == pTrig ); + assert( !pParse->isCreate ); + assert( &(pParse->u1.d.pReturning->retTrig) == pTrig ); pTrig->table = pTab->zName; pTrig->pTabSchema = pTab->pSchema; pTrig->pNext = pList; @@ -150759,8 +156158,10 @@ SQLITE_PRIVATE void sqlite3BeginTrigger( ** name on pTableName if we are reparsing out of the schema table */ if( db->init.busy && iDb!=1 ){ - sqlite3DbFree(db, pTableName->a[0].zDatabase); - pTableName->a[0].zDatabase = 0; + assert( pTableName->a[0].fg.fixedSchema==0 ); + assert( pTableName->a[0].fg.isSubquery==0 ); + sqlite3DbFree(db, pTableName->a[0].u4.zDatabase); + pTableName->a[0].u4.zDatabase = 0; } /* If the trigger name was unqualified, and the table is a temp table, @@ -151238,7 +156639,8 @@ SQLITE_PRIVATE void sqlite3DropTrigger(Parse *pParse, SrcList *pName, int noErr) } assert( pName->nSrc==1 ); - zDb = pName->a[0].zDatabase; + assert( pName->a[0].fg.fixedSchema==0 && pName->a[0].fg.isSubquery==0 ); + zDb = pName->a[0].u4.zDatabase; zName = pName->a[0].zName; assert( zDb!=0 || sqlite3BtreeHoldsAllMutexes(db) ); for(i=OMIT_TEMPDB; inDb; i++){ @@ -151475,7 +156877,9 @@ SQLITE_PRIVATE SrcList *sqlite3TriggerStepSrc( Schema *pSchema = pStep->pTrig->pSchema; pSrc->a[0].zName = zName; if( pSchema!=db->aDb[1].pSchema ){ - pSrc->a[0].pSchema = pSchema; + assert( pSrc->a[0].fg.fixedSchema || pSrc->a[0].u4.zDatabase==0 ); + pSrc->a[0].u4.pSchema = pSchema; + pSrc->a[0].fg.fixedSchema = 1; } if( pStep->pFrom ){ SrcList *pDup = sqlite3SrcListDup(db, pStep->pFrom, 0); @@ -151558,6 +156962,72 @@ static ExprList *sqlite3ExpandReturning( return pNew; } +/* If the Expr node is a subquery or an EXISTS operator or an IN operator that +** uses a subquery, and if the subquery is SF_Correlated, then mark the +** expression as EP_VarSelect. +*/ +static int sqlite3ReturningSubqueryVarSelect(Walker *NotUsed, Expr *pExpr){ + UNUSED_PARAMETER(NotUsed); + if( ExprUseXSelect(pExpr) + && (pExpr->x.pSelect->selFlags & SF_Correlated)!=0 + ){ + testcase( ExprHasProperty(pExpr, EP_VarSelect) ); + ExprSetProperty(pExpr, EP_VarSelect); + } + return WRC_Continue; +} + + +/* +** If the SELECT references the table pWalker->u.pTab, then do two things: +** +** (1) Mark the SELECT as as SF_Correlated. +** (2) Set pWalker->eCode to non-zero so that the caller will know +** that (1) has happened. +*/ +static int sqlite3ReturningSubqueryCorrelated(Walker *pWalker, Select *pSelect){ + int i; + SrcList *pSrc; + assert( pSelect!=0 ); + pSrc = pSelect->pSrc; + assert( pSrc!=0 ); + for(i=0; inSrc; i++){ + if( pSrc->a[i].pSTab==pWalker->u.pTab ){ + testcase( pSelect->selFlags & SF_Correlated ); + pSelect->selFlags |= SF_Correlated; + pWalker->eCode = 1; + break; + } + } + return WRC_Continue; +} + +/* +** Scan the expression list that is the argument to RETURNING looking +** for subqueries that depend on the table which is being modified in the +** statement that is hosting the RETURNING clause (pTab). Mark all such +** subqueries as SF_Correlated. If the subqueries are part of an +** expression, mark the expression as EP_VarSelect. +** +** https://sqlite.org/forum/forumpost/2c83569ce8945d39 +*/ +static void sqlite3ProcessReturningSubqueries( + ExprList *pEList, + Table *pTab +){ + Walker w; + memset(&w, 0, sizeof(w)); + w.xExprCallback = sqlite3ExprWalkNoop; + w.xSelectCallback = sqlite3ReturningSubqueryCorrelated; + w.u.pTab = pTab; + sqlite3WalkExprList(&w, pEList); + if( w.eCode ){ + w.xExprCallback = sqlite3ReturningSubqueryVarSelect; + w.xSelectCallback = sqlite3SelectWalkNoop; + sqlite3WalkExprList(&w, pEList); + } +} + /* ** Generate code for the RETURNING trigger. Unlike other triggers ** that invoke a subprogram in the bytecode, the code for RETURNING @@ -151574,7 +157044,11 @@ static void codeReturningTrigger( ExprList *pNew; Returning *pReturning; Select sSelect; - SrcList sFrom; + SrcList *pFrom; + union { + SrcList sSrc; + u8 fromSpace[SZ_SRCLIST_1]; + } uSrc; assert( v!=0 ); if( !pParse->bReturning ){ @@ -151583,18 +157057,21 @@ static void codeReturningTrigger( return; } assert( db->pParse==pParse ); - pReturning = pParse->u1.pReturning; + assert( !pParse->isCreate ); + pReturning = pParse->u1.d.pReturning; if( pTrigger != &(pReturning->retTrig) ){ /* This RETURNING trigger is for a different statement */ return; } memset(&sSelect, 0, sizeof(sSelect)); - memset(&sFrom, 0, sizeof(sFrom)); + memset(&uSrc, 0, sizeof(uSrc)); + pFrom = &uSrc.sSrc; sSelect.pEList = sqlite3ExprListDup(db, pReturning->pReturnEL, 0); - sSelect.pSrc = &sFrom; - sFrom.nSrc = 1; - sFrom.a[0].pTab = pTab; - sFrom.a[0].iCursor = -1; + sSelect.pSrc = pFrom; + pFrom->nSrc = 1; + pFrom->a[0].pSTab = pTab; + pFrom->a[0].zName = pTab->zName; /* tag-20240424-1 */ + pFrom->a[0].iCursor = -1; sqlite3SelectPrep(pParse, &sSelect, 0); if( pParse->nErr==0 ){ assert( db->mallocFailed==0 ); @@ -151620,6 +157097,7 @@ static void codeReturningTrigger( int i; int nCol = pNew->nExpr; int reg = pParse->nMem+1; + sqlite3ProcessReturningSubqueries(pNew, pTab); pParse->nMem += nCol+2; pReturning->iRetReg = reg; for(i=0; iop; sSubParse.nQueryLoop = pParse->nQueryLoop; sSubParse.prepFlags = pParse->prepFlags; + sSubParse.oldmask = 0; + sSubParse.newmask = 0; v = sqlite3GetVdbe(&sSubParse); if( v ){ @@ -151943,7 +157423,7 @@ SQLITE_PRIVATE void sqlite3CodeRowTriggerDirect( ** invocation is disallowed if (a) the sub-program is really a trigger, ** not a foreign key action, and (b) the flag to enable recursive triggers ** is clear. */ - sqlite3VdbeChangeP5(v, (u8)bRecursive); + sqlite3VdbeChangeP5(v, (u16)bRecursive); } } @@ -152302,7 +157782,7 @@ static void updateFromSelect( Expr *pLimit2 = 0; ExprList *pOrderBy2 = 0; sqlite3 *db = pParse->db; - Table *pTab = pTabList->a[0].pTab; + Table *pTab = pTabList->a[0].pSTab; SrcList *pSrc; Expr *pWhere2; int eDest; @@ -152326,8 +157806,8 @@ static void updateFromSelect( if( pSrc ){ assert( pSrc->a[0].fg.notCte ); pSrc->a[0].iCursor = -1; - pSrc->a[0].pTab->nTabRef--; - pSrc->a[0].pTab = 0; + pSrc->a[0].pSTab->nTabRef--; + pSrc->a[0].pSTab = 0; } if( pPk ){ for(i=0; inKeyCol; i++){ @@ -152565,38 +158045,32 @@ SQLITE_PRIVATE void sqlite3Update( */ chngRowid = chngPk = 0; for(i=0; inExpr; i++){ - u8 hCol = sqlite3StrIHash(pChanges->a[i].zEName); /* If this is an UPDATE with a FROM clause, do not resolve expressions ** here. The call to sqlite3Select() below will do that. */ if( nChangeFrom==0 && sqlite3ResolveExprNames(&sNC, pChanges->a[i].pExpr) ){ goto update_cleanup; } - for(j=0; jnCol; j++){ - if( pTab->aCol[j].hName==hCol - && sqlite3StrICmp(pTab->aCol[j].zCnName, pChanges->a[i].zEName)==0 - ){ - if( j==pTab->iPKey ){ - chngRowid = 1; - pRowidExpr = pChanges->a[i].pExpr; - iRowidExpr = i; - }else if( pPk && (pTab->aCol[j].colFlags & COLFLAG_PRIMKEY)!=0 ){ - chngPk = 1; - } + j = sqlite3ColumnIndex(pTab, pChanges->a[i].zEName); + if( j>=0 ){ + if( j==pTab->iPKey ){ + chngRowid = 1; + pRowidExpr = pChanges->a[i].pExpr; + iRowidExpr = i; + }else if( pPk && (pTab->aCol[j].colFlags & COLFLAG_PRIMKEY)!=0 ){ + chngPk = 1; + } #ifndef SQLITE_OMIT_GENERATED_COLUMNS - else if( pTab->aCol[j].colFlags & COLFLAG_GENERATED ){ - testcase( pTab->aCol[j].colFlags & COLFLAG_VIRTUAL ); - testcase( pTab->aCol[j].colFlags & COLFLAG_STORED ); - sqlite3ErrorMsg(pParse, - "cannot UPDATE generated column \"%s\"", - pTab->aCol[j].zCnName); - goto update_cleanup; - } -#endif - aXRef[j] = i; - break; + else if( pTab->aCol[j].colFlags & COLFLAG_GENERATED ){ + testcase( pTab->aCol[j].colFlags & COLFLAG_VIRTUAL ); + testcase( pTab->aCol[j].colFlags & COLFLAG_STORED ); + sqlite3ErrorMsg(pParse, + "cannot UPDATE generated column \"%s\"", + pTab->aCol[j].zCnName); + goto update_cleanup; } - } - if( j>=pTab->nCol ){ +#endif + aXRef[j] = i; + }else{ if( pPk==0 && sqlite3IsRowid(pChanges->a[i].zEName) ){ j = -1; chngRowid = 1; @@ -153021,6 +158495,9 @@ SQLITE_PRIVATE void sqlite3Update( } } if( chngRowid==0 && pPk==0 ){ +#ifdef SQLITE_ALLOW_ROWID_IN_VIEW + if( isView ) sqlite3VdbeAddOp2(v, OP_Null, 0, regOldRowid); +#endif sqlite3VdbeAddOp2(v, OP_Copy, regOldRowid, regNewRowid); } } @@ -153572,7 +159049,7 @@ SQLITE_PRIVATE int sqlite3UpsertAnalyzeTarget( int nClause = 0; /* Counter of ON CONFLICT clauses */ assert( pTabList->nSrc==1 ); - assert( pTabList->a[0].pTab!=0 ); + assert( pTabList->a[0].pSTab!=0 ); assert( pUpsert!=0 ); assert( pUpsert->pUpsertTarget!=0 ); @@ -153591,7 +159068,7 @@ SQLITE_PRIVATE int sqlite3UpsertAnalyzeTarget( if( rc ) return rc; /* Check to see if the conflict target matches the rowid. */ - pTab = pTabList->a[0].pTab; + pTab = pTabList->a[0].pSTab; pTarget = pUpsert->pUpsertTarget; iCursor = pTabList->a[0].iCursor; if( HasRowid(pTab) @@ -153916,7 +159393,7 @@ SQLITE_PRIVATE void sqlite3Vacuum(Parse *pParse, Token *pNm, Expr *pInto){ #else /* When SQLITE_BUG_COMPATIBLE_20160819 is defined, unrecognized arguments ** to VACUUM are silently ignored. This is a back-out of a bug fix that - ** occurred on 2016-08-19 (https://www.sqlite.org/src/info/083f9e6270). + ** occurred on 2016-08-19 (https://sqlite.org/src/info/083f9e6270). ** The buggy behavior is required for binary compatibility with some ** legacy applications. */ iDb = sqlite3FindDb(pParse->db, pNm); @@ -153962,6 +159439,9 @@ SQLITE_PRIVATE SQLITE_NOINLINE int sqlite3RunVacuum( const char *zDbMain; /* Schema name of database to vacuum */ const char *zOut; /* Name of output file */ u32 pgflags = PAGER_SYNCHRONOUS_OFF; /* sync flags for output db */ + u64 iRandom; /* Random value used for zDbVacuum[] */ + char zDbVacuum[42]; /* Name of the ATTACH-ed database used for vacuum */ + if( !db->autoCommit ){ sqlite3SetString(pzErrMsg, db, "cannot VACUUM from within a transaction"); @@ -153992,7 +159472,8 @@ SQLITE_PRIVATE SQLITE_NOINLINE int sqlite3RunVacuum( saved_nChange = db->nChange; saved_nTotalChange = db->nTotalChange; saved_mTrace = db->mTrace; - db->flags |= SQLITE_WriteSchema | SQLITE_IgnoreChecks; + db->flags |= SQLITE_WriteSchema | SQLITE_IgnoreChecks | SQLITE_Comments + | SQLITE_AttachCreate | SQLITE_AttachWrite; db->mDbFlags |= DBFLAG_PreferBuiltin | DBFLAG_Vacuum; db->flags &= ~(u64)(SQLITE_ForeignKeys | SQLITE_ReverseOrder | SQLITE_Defensive | SQLITE_CountRows); @@ -154002,27 +159483,29 @@ SQLITE_PRIVATE SQLITE_NOINLINE int sqlite3RunVacuum( pMain = db->aDb[iDb].pBt; isMemDb = sqlite3PagerIsMemdb(sqlite3BtreePager(pMain)); - /* Attach the temporary database as 'vacuum_db'. The synchronous pragma + /* Attach the temporary database as 'vacuum_XXXXXX'. The synchronous pragma ** can be set to 'off' for this file, as it is not recovered if a crash ** occurs anyway. The integrity of the database is maintained by a ** (possibly synchronous) transaction opened on the main database before ** sqlite3BtreeCopyFile() is called. ** ** An optimization would be to use a non-journaled pager. - ** (Later:) I tried setting "PRAGMA vacuum_db.journal_mode=OFF" but + ** (Later:) I tried setting "PRAGMA vacuum_XXXXXX.journal_mode=OFF" but ** that actually made the VACUUM run slower. Very little journalling ** actually occurs when doing a vacuum since the vacuum_db is initially ** empty. Only the journal header is written. Apparently it takes more ** time to parse and run the PRAGMA to turn journalling off than it does ** to write the journal header file. */ + sqlite3_randomness(sizeof(iRandom),&iRandom); + sqlite3_snprintf(sizeof(zDbVacuum), zDbVacuum, "vacuum_%016llx", iRandom); nDb = db->nDb; - rc = execSqlF(db, pzErrMsg, "ATTACH %Q AS vacuum_db", zOut); + rc = execSqlF(db, pzErrMsg, "ATTACH %Q AS %s", zOut, zDbVacuum); db->openFlags = saved_openFlags; if( rc!=SQLITE_OK ) goto end_of_vacuum; assert( (db->nDb-1)==nDb ); pDb = &db->aDb[nDb]; - assert( strcmp(pDb->zDbSName,"vacuum_db")==0 ); + assert( strcmp(pDb->zDbSName,zDbVacuum)==0 ); pTemp = pDb->pBt; if( pOut ){ sqlite3_file *id = sqlite3PagerFile(sqlite3BtreePager(pTemp)); @@ -154099,11 +159582,11 @@ SQLITE_PRIVATE SQLITE_NOINLINE int sqlite3RunVacuum( ** the contents to the temporary database. */ rc = execSqlF(db, pzErrMsg, - "SELECT'INSERT INTO vacuum_db.'||quote(name)" + "SELECT'INSERT INTO %s.'||quote(name)" "||' SELECT*FROM\"%w\".'||quote(name)" - "FROM vacuum_db.sqlite_schema " + "FROM %s.sqlite_schema " "WHERE type='table'AND coalesce(rootpage,1)>0", - zDbMain + zDbVacuum, zDbMain, zDbVacuum ); assert( (db->mDbFlags & DBFLAG_Vacuum)!=0 ); db->mDbFlags &= ~DBFLAG_Vacuum; @@ -154115,11 +159598,11 @@ SQLITE_PRIVATE SQLITE_NOINLINE int sqlite3RunVacuum( ** from the schema table. */ rc = execSqlF(db, pzErrMsg, - "INSERT INTO vacuum_db.sqlite_schema" + "INSERT INTO %s.sqlite_schema" " SELECT*FROM \"%w\".sqlite_schema" " WHERE type IN('view','trigger')" " OR(type='table'AND rootpage=0)", - zDbMain + zDbVacuum, zDbMain ); if( rc ) goto end_of_vacuum; @@ -154695,11 +160178,12 @@ SQLITE_PRIVATE void sqlite3VtabFinishParse(Parse *pParse, Token *pEnd){ ** schema table. We just need to update that slot with all ** the information we've collected. ** - ** The VM register number pParse->regRowid holds the rowid of an + ** The VM register number pParse->u1.cr.regRowid holds the rowid of an ** entry in the sqlite_schema table that was created for this vtab ** by sqlite3StartTable(). */ iDb = sqlite3SchemaToIndex(db, pTab->pSchema); + assert( pParse->isCreate ); sqlite3NestedParse(pParse, "UPDATE %Q." LEGACY_SCHEMA_TABLE " " "SET type='table', name=%Q, tbl_name=%Q, rootpage=0, sql=%Q " @@ -154708,7 +160192,7 @@ SQLITE_PRIVATE void sqlite3VtabFinishParse(Parse *pParse, Token *pEnd){ pTab->zName, pTab->zName, zStmt, - pParse->regRowid + pParse->u1.cr.regRowid ); v = sqlite3GetVdbe(pParse); sqlite3ChangeCookie(pParse, iDb); @@ -154827,6 +160311,8 @@ static int vtabCallConstructor( db->pVtabCtx = &sCtx; pTab->nTabRef++; rc = xConstruct(db, pMod->pAux, nArg, azArg, &pVTable->pVtab, &zErr); + assert( pTab!=0 ); + assert( pTab->nTabRef>1 || rc!=SQLITE_OK ); sqlite3DeleteTable(db, pTab); db->pVtabCtx = sCtx.pPrior; if( rc==SQLITE_NOMEM ) sqlite3OomFault(db); @@ -154849,7 +160335,7 @@ static int vtabCallConstructor( pVTable->nRef = 1; if( sCtx.bDeclared==0 ){ const char *zFormat = "vtable constructor did not declare schema: %s"; - *pzErr = sqlite3MPrintf(db, zFormat, pTab->zName); + *pzErr = sqlite3MPrintf(db, zFormat, zModuleName); sqlite3VtabUnlock(pVTable); rc = SQLITE_ERROR; }else{ @@ -155027,12 +160513,32 @@ SQLITE_API int sqlite3_declare_vtab(sqlite3 *db, const char *zCreateTable){ Table *pTab; Parse sParse; int initBusy; + int i; + const unsigned char *z; + static const u8 aKeyword[] = { TK_CREATE, TK_TABLE, 0 }; #ifdef SQLITE_ENABLE_API_ARMOR if( !sqlite3SafetyCheckOk(db) || zCreateTable==0 ){ return SQLITE_MISUSE_BKPT; } #endif + + /* Verify that the first two keywords in the CREATE TABLE statement + ** really are "CREATE" and "TABLE". If this is not the case, then + ** sqlite3_declare_vtab() is being misused. + */ + z = (const unsigned char*)zCreateTable; + for(i=0; aKeyword[i]; i++){ + int tokenType = 0; + do{ + z += sqlite3GetToken(z, &tokenType); + }while( tokenType==TK_SPACE || tokenType==TK_COMMENT ); + if( tokenType!=aKeyword[i] ){ + sqlite3ErrorWithMsg(db, SQLITE_ERROR, "syntax error"); + return SQLITE_ERROR; + } + } + sqlite3_mutex_enter(db->mutex); pCtx = db->pVtabCtx; if( !pCtx || pCtx->bDeclared ){ @@ -155040,6 +160546,7 @@ SQLITE_API int sqlite3_declare_vtab(sqlite3 *db, const char *zCreateTable){ sqlite3_mutex_leave(db->mutex); return SQLITE_MISUSE_BKPT; } + pTab = pCtx->pTab; assert( IsVirtual(pTab) ); @@ -155053,16 +160560,16 @@ SQLITE_API int sqlite3_declare_vtab(sqlite3 *db, const char *zCreateTable){ initBusy = db->init.busy; db->init.busy = 0; sParse.nQueryLoop = 1; - if( SQLITE_OK==sqlite3RunParser(&sParse, zCreateTable) - && ALWAYS(sParse.pNewTable!=0) - && ALWAYS(!db->mallocFailed) - && IsOrdinaryTable(sParse.pNewTable) - ){ + if( SQLITE_OK==sqlite3RunParser(&sParse, zCreateTable) ){ + assert( sParse.pNewTable!=0 ); + assert( !db->mallocFailed ); + assert( IsOrdinaryTable(sParse.pNewTable) ); assert( sParse.zErrMsg==0 ); if( !pTab->aCol ){ Table *pNew = sParse.pNewTable; Index *pIdx; pTab->aCol = pNew->aCol; + assert( IsOrdinaryTable(pNew) ); sqlite3ExprListDelete(db, pNew->u.tab.pDfltList); pTab->nNVCol = pTab->nCol = pNew->nCol; pTab->tabFlags |= pNew->tabFlags & (TF_WithoutRowid|TF_NoVisibleRowid); @@ -155471,9 +160978,12 @@ SQLITE_PRIVATE int sqlite3VtabEponymousTableInit(Parse *pParse, Module *pMod){ addModuleArgument(pParse, pTab, sqlite3DbStrDup(db, pTab->zName)); addModuleArgument(pParse, pTab, 0); addModuleArgument(pParse, pTab, sqlite3DbStrDup(db, pTab->zName)); + db->nSchemaLock++; rc = vtabCallConstructor(db, pTab, pMod, pModule->xConnect, &zErr); + db->nSchemaLock--; if( rc ){ sqlite3ErrorMsg(pParse, "%s", zErr); + pParse->rc = rc; sqlite3DbFree(db, zErr); sqlite3VtabEponymousTableClear(db, pMod); } @@ -155669,6 +161179,7 @@ struct WhereLevel { int iTabCur; /* The VDBE cursor used to access the table */ int iIdxCur; /* The VDBE cursor used to access pIdx */ int addrBrk; /* Jump here to break out of the loop */ + int addrHalt; /* Abort the query due to empty table or similar */ int addrNxt; /* Jump here to start the next IN combination */ int addrSkip; /* Jump here for next iteration of skip-scan */ int addrCont; /* Jump here to continue with the next loop cycle */ @@ -155737,11 +161248,13 @@ struct WhereLoop { u16 nTop; /* Size of TOP vector */ u16 nDistinctCol; /* Index columns used to sort for DISTINCT */ Index *pIndex; /* Index used, or NULL */ + ExprList *pOrderBy; /* ORDER BY clause if this is really a subquery */ } btree; struct { /* Information for virtual tables */ int idxNum; /* Index number */ u32 needFree : 1; /* True if sqlite3_free(idxStr) is needed */ u32 bOmitOffset : 1; /* True to let virtual table handle offset */ + u32 bIdxNumHex : 1; /* Show idxNum as hex in EXPLAIN QUERY PLAN */ i8 isOrdered; /* True if satisfies ORDER BY */ u16 omitMask; /* Terms that may be omitted */ char *idxStr; /* Index identifier string */ @@ -155754,6 +161267,10 @@ struct WhereLoop { /**** whereLoopXfer() copies fields above ***********************/ # define WHERE_LOOP_XFER_SZ offsetof(WhereLoop,nLSlot) u16 nLSlot; /* Number of slots allocated for aLTerm[] */ +#ifdef WHERETRACE_ENABLED + LogEst rStarDelta; /* Cost delta due to star-schema heuristic. Not + ** initialized unless pWInfo->bStarUsed */ +#endif WhereTerm **aLTerm; /* WhereTerms used */ WhereLoop *pNextLoop; /* Next WhereLoop object in the WhereClause */ WhereTerm *aLTermSpace[3]; /* Initial aLTerm[] space */ @@ -155802,7 +161319,7 @@ struct WherePath { Bitmask revLoop; /* aLoop[]s that should be reversed for ORDER BY */ LogEst nRow; /* Estimated number of rows generated by this path */ LogEst rCost; /* Total cost of this path */ - LogEst rUnsorted; /* Total cost of this path ignoring sorting costs */ + LogEst rUnsort; /* Total cost of this path ignoring sorting costs */ i8 isOrdered; /* No. of ORDER BY terms satisfied. -1 for unknown */ WhereLoop **aLoop; /* Array of WhereLoop objects implementing this path */ }; @@ -155868,6 +161385,9 @@ struct WhereTerm { u8 eMatchOp; /* Op for vtab MATCH/LIKE/GLOB/REGEXP terms */ int iParent; /* Disable pWC->a[iParent] when this term disabled */ int leftCursor; /* Cursor number of X in "X " */ +#ifdef SQLITE_DEBUG + int iTerm; /* Which WhereTerm is this, for debug purposes */ +#endif union { struct { int leftColumn; /* Column number of X in "X " */ @@ -156075,8 +161595,13 @@ struct WhereInfo { unsigned bDeferredSeek :1; /* Uses OP_DeferredSeek */ unsigned untestedTerms :1; /* Not all WHERE terms resolved by outer loop */ unsigned bOrderedInnerLoop:1;/* True if only the inner-most loop is ordered */ - unsigned sorted :1; /* True if really sorted (not just grouped) */ + unsigned sorted :1; /* True if really sorted (not just grouped) */ + unsigned bStarDone :1; /* True if check for star-query is complete */ + unsigned bStarUsed :1; /* True if star-query heuristic is used */ LogEst nRowOut; /* Estimated number of output rows */ +#ifdef WHERETRACE_ENABLED + LogEst rTotalCost; /* Total cost of the solution */ +#endif int iTop; /* The very beginning of the WHERE loop */ int iEndWhere; /* End of the WHERE clause itself */ WhereLoop *pLoops; /* List of all WhereLoop objects */ @@ -156084,9 +161609,14 @@ struct WhereInfo { Bitmask revMask; /* Mask of ORDER BY terms that need reversing */ WhereClause sWC; /* Decomposition of the WHERE clause */ WhereMaskSet sMaskSet; /* Map cursor numbers to bitmasks */ - WhereLevel a[1]; /* Information about each nest loop in WHERE */ + WhereLevel a[FLEXARRAY]; /* Information about each nest loop in WHERE */ }; +/* +** The size (in bytes) of a WhereInfo object that holds N WhereLevels. +*/ +#define SZ_WHEREINFO(N) ROUND8(offsetof(WhereInfo,a)+(N)*sizeof(WhereLevel)) + /* ** Private interfaces - callable only by other where.c routines. ** @@ -156122,9 +161652,17 @@ SQLITE_PRIVATE int sqlite3WhereExplainBloomFilter( const WhereInfo *pWInfo, /* WHERE clause */ const WhereLevel *pLevel /* Bloom filter on this level */ ); +SQLITE_PRIVATE void sqlite3WhereAddExplainText( + Parse *pParse, /* Parse context */ + int addr, + SrcList *pTabList, /* Table list this loop refers to */ + WhereLevel *pLevel, /* Scan to write OP_Explain opcode for */ + u16 wctrlFlags /* Flags passed to sqlite3WhereBegin() */ +); #else # define sqlite3WhereExplainOneScan(u,v,w,x) 0 # define sqlite3WhereExplainBloomFilter(u,v,w) 0 +# define sqlite3WhereAddExplainText(u,v,w,x,y) #endif /* SQLITE_OMIT_EXPLAIN */ #ifdef SQLITE_ENABLE_STMT_SCANSTATUS SQLITE_PRIVATE void sqlite3WhereAddScanStatus( @@ -156227,7 +161765,8 @@ SQLITE_PRIVATE void sqlite3WhereTabFuncArgs(Parse*, SrcItem*, WhereClause*); #define WHERE_BLOOMFILTER 0x00400000 /* Consider using a Bloom-filter */ #define WHERE_SELFCULL 0x00800000 /* nOut reduced by extra WHERE terms */ #define WHERE_OMIT_OFFSET 0x01000000 /* Set offset counter to zero */ - /* 0x02000000 -- available for reuse */ +#define WHERE_COROUTINE 0x02000000 /* Implemented by co-routine. + ** NB: False-negatives are possible */ #define WHERE_EXPRIDX 0x04000000 /* Uses an index-on-expressions */ #endif /* !defined(SQLITE_WHEREINT_H) */ @@ -156325,38 +161864,37 @@ static void explainIndexRange(StrAccum *pStr, WhereLoop *pLoop){ } /* -** This function is a no-op unless currently processing an EXPLAIN QUERY PLAN -** command, or if stmt_scanstatus_v2() stats are enabled, or if SQLITE_DEBUG -** was defined at compile-time. If it is not a no-op, a single OP_Explain -** opcode is added to the output to describe the table scan strategy in pLevel. -** -** If an OP_Explain opcode is added to the VM, its address is returned. -** Otherwise, if no OP_Explain is coded, zero is returned. +** This function sets the P4 value of an existing OP_Explain opcode to +** text describing the loop in pLevel. If the OP_Explain opcode already has +** a P4 value, it is freed before it is overwritten. */ -SQLITE_PRIVATE int sqlite3WhereExplainOneScan( +SQLITE_PRIVATE void sqlite3WhereAddExplainText( Parse *pParse, /* Parse context */ + int addr, /* Address of OP_Explain opcode */ SrcList *pTabList, /* Table list this loop refers to */ WhereLevel *pLevel, /* Scan to write OP_Explain opcode for */ u16 wctrlFlags /* Flags passed to sqlite3WhereBegin() */ ){ - int ret = 0; #if !defined(SQLITE_DEBUG) if( sqlite3ParseToplevel(pParse)->explain==2 || IS_STMT_SCANSTATUS(pParse->db) ) #endif { + VdbeOp *pOp = sqlite3VdbeGetOp(pParse->pVdbe, addr); SrcItem *pItem = &pTabList->a[pLevel->iFrom]; - Vdbe *v = pParse->pVdbe; /* VM being constructed */ sqlite3 *db = pParse->db; /* Database handle */ int isSearch; /* True for a SEARCH. False for SCAN. */ WhereLoop *pLoop; /* The controlling WhereLoop object */ u32 flags; /* Flags that describe this loop */ +#if defined(SQLITE_DEBUG) && !defined(SQLITE_OMIT_EXPLAIN) char *zMsg; /* Text to add to EQP output */ +#endif StrAccum str; /* EQP output string */ char zBuf[100]; /* Initial space for EQP output string */ + if( db->mallocFailed ) return; + pLoop = pLevel->pWLoop; flags = pLoop->wsFlags; - if( (flags&WHERE_MULTI_OR) || (wctrlFlags&WHERE_OR_SUBCLAUSE) ) return 0; isSearch = (flags&(WHERE_BTM_LIMIT|WHERE_TOP_LIMIT))!=0 || ((flags&WHERE_VIRTUALTABLE)==0 && (pLoop->u.btree.nEq>0)) @@ -156364,7 +161902,10 @@ SQLITE_PRIVATE int sqlite3WhereExplainOneScan( sqlite3StrAccumInit(&str, db, zBuf, sizeof(zBuf), SQLITE_MAX_LENGTH); str.printfFlags = SQLITE_PRINTF_INTERNAL; - sqlite3_str_appendf(&str, "%s %S", isSearch ? "SEARCH" : "SCAN", pItem); + sqlite3_str_appendf(&str, "%s %S%s", + isSearch ? "SEARCH" : "SCAN", + pItem, + pItem->fg.fromExists ? " EXISTS" : ""); if( (flags & (WHERE_IPK|WHERE_VIRTUALTABLE))==0 ){ const char *zFmt = 0; Index *pIdx; @@ -156372,7 +161913,7 @@ SQLITE_PRIVATE int sqlite3WhereExplainOneScan( assert( pLoop->u.btree.pIndex!=0 ); pIdx = pLoop->u.btree.pIndex; assert( !(flags&WHERE_AUTO_INDEX) || (flags&WHERE_IDX_ONLY) ); - if( !HasRowid(pItem->pTab) && IsPrimaryKeyIndex(pIdx) ){ + if( !HasRowid(pItem->pSTab) && IsPrimaryKeyIndex(pIdx) ){ if( isSearch ){ zFmt = "PRIMARY KEY"; } @@ -156380,7 +161921,7 @@ SQLITE_PRIVATE int sqlite3WhereExplainOneScan( zFmt = "AUTOMATIC PARTIAL COVERING INDEX"; }else if( flags & WHERE_AUTO_INDEX ){ zFmt = "AUTOMATIC COVERING INDEX"; - }else if( flags & WHERE_IDX_ONLY ){ + }else if( flags & (WHERE_IDX_ONLY|WHERE_EXPRIDX) ){ zFmt = "COVERING INDEX %s"; }else{ zFmt = "INDEX %s"; @@ -156415,7 +161956,9 @@ SQLITE_PRIVATE int sqlite3WhereExplainOneScan( } #ifndef SQLITE_OMIT_VIRTUALTABLE else if( (flags & WHERE_VIRTUALTABLE)!=0 ){ - sqlite3_str_appendf(&str, " VIRTUAL TABLE INDEX %d:%s", + sqlite3_str_appendall(&str, " VIRTUAL TABLE INDEX "); + sqlite3_str_appendf(&str, + pLoop->u.vtab.bIdxNumHex ? "0x%x:%s" : "%d:%s", pLoop->u.vtab.idxNum, pLoop->u.vtab.idxStr); } #endif @@ -156430,10 +161973,50 @@ SQLITE_PRIVATE int sqlite3WhereExplainOneScan( sqlite3_str_append(&str, " (~1 row)", 9); } #endif +#if defined(SQLITE_DEBUG) && !defined(SQLITE_OMIT_EXPLAIN) zMsg = sqlite3StrAccumFinish(&str); sqlite3ExplainBreakpoint("",zMsg); - ret = sqlite3VdbeAddOp4(v, OP_Explain, sqlite3VdbeCurrentAddr(v), - pParse->addrExplain, 0, zMsg,P4_DYNAMIC); +#endif + + assert( pOp->opcode==OP_Explain ); + assert( pOp->p4type==P4_DYNAMIC || pOp->p4.z==0 ); + sqlite3DbFree(db, pOp->p4.z); + pOp->p4type = P4_DYNAMIC; + pOp->p4.z = sqlite3StrAccumFinish(&str); + } +} + + +/* +** This function is a no-op unless currently processing an EXPLAIN QUERY PLAN +** command, or if stmt_scanstatus_v2() stats are enabled, or if SQLITE_DEBUG +** was defined at compile-time. If it is not a no-op, a single OP_Explain +** opcode is added to the output to describe the table scan strategy in pLevel. +** +** If an OP_Explain opcode is added to the VM, its address is returned. +** Otherwise, if no OP_Explain is coded, zero is returned. +*/ +SQLITE_PRIVATE int sqlite3WhereExplainOneScan( + Parse *pParse, /* Parse context */ + SrcList *pTabList, /* Table list this loop refers to */ + WhereLevel *pLevel, /* Scan to write OP_Explain opcode for */ + u16 wctrlFlags /* Flags passed to sqlite3WhereBegin() */ +){ + int ret = 0; +#if !defined(SQLITE_DEBUG) + if( sqlite3ParseToplevel(pParse)->explain==2 || IS_STMT_SCANSTATUS(pParse->db) ) +#endif + { + if( (pLevel->pWLoop->wsFlags & WHERE_MULTI_OR)==0 + && (wctrlFlags & WHERE_OR_SUBCLAUSE)==0 + ){ + Vdbe *v = pParse->pVdbe; + int addr = sqlite3VdbeCurrentAddr(v); + ret = sqlite3VdbeAddOp3( + v, OP_Explain, addr, pParse->addrExplain, pLevel->pWLoop->rRun + ); + sqlite3WhereAddExplainText(pParse, addr, pTabList, pLevel, wctrlFlags); + } } return ret; } @@ -156468,7 +162051,7 @@ SQLITE_PRIVATE int sqlite3WhereExplainBloomFilter( sqlite3_str_appendf(&str, "BLOOM FILTER ON %S (", pItem); pLoop = pLevel->pWLoop; if( pLoop->wsFlags & WHERE_IPK ){ - const Table *pTab = pItem->pTab; + const Table *pTab = pItem->pSTab; if( pTab->iPKey>=0 ){ sqlite3_str_appendf(&str, "%s=?", pTab->aCol[pTab->iPKey].zCnName); }else{ @@ -156531,8 +162114,11 @@ SQLITE_PRIVATE void sqlite3WhereAddScanStatus( sqlite3VdbeScanStatusRange(v, addrExplain, -1, pLvl->iIdxCur); } }else{ - int addr = pSrclist->a[pLvl->iFrom].addrFillSub; - VdbeOp *pOp = sqlite3VdbeGetOp(v, addr-1); + int addr; + VdbeOp *pOp; + assert( pSrclist->a[pLvl->iFrom].fg.isSubquery ); + addr = pSrclist->a[pLvl->iFrom].u4.pSubq->addrFillSub; + pOp = sqlite3VdbeGetOp(v, addr-1); assert( sqlite3VdbeDb(v)->mallocFailed || pOp->opcode==OP_InitCoroutine ); assert( sqlite3VdbeDb(v)->mallocFailed || pOp->p2>addr ); sqlite3VdbeScanStatusRange(v, addrExplain, addr, pOp->p2-1); @@ -156675,11 +162261,44 @@ static void updateRangeAffinityStr( } } +/* +** The pOrderBy->a[].u.x.iOrderByCol values might be incorrect because +** columns might have been rearranged in the result set. This routine +** fixes them up. +** +** pEList is the new result set. The pEList->a[].u.x.iOrderByCol values +** contain the *old* locations of each expression. This is a temporary +** use of u.x.iOrderByCol, not its intended use. The caller must reset +** u.x.iOrderByCol back to zero for all entries in pEList before the +** caller returns. +** +** This routine changes pOrderBy->a[].u.x.iOrderByCol values from +** pEList->a[N].u.x.iOrderByCol into N+1. (The "+1" is because of the 1-based +** indexing used by iOrderByCol.) Or if no match, iOrderByCol is set to zero. +*/ +static void adjustOrderByCol(ExprList *pOrderBy, ExprList *pEList){ + int i, j; + if( pOrderBy==0 ) return; + for(i=0; inExpr; i++){ + int t = pOrderBy->a[i].u.x.iOrderByCol; + if( t==0 ) continue; + for(j=0; jnExpr; j++){ + if( pEList->a[j].u.x.iOrderByCol==t ){ + pOrderBy->a[i].u.x.iOrderByCol = j+1; + break; + } + } + if( j>=pEList->nExpr ){ + pOrderBy->a[i].u.x.iOrderByCol = 0; + } + } +} + /* ** pX is an expression of the form: (vector) IN (SELECT ...) ** In other words, it is a vector IN operator with a SELECT clause on the -** LHS. But not all terms in the vector are indexable and the terms might +** RHS. But not all terms in the vector are indexable and the terms might ** not be in the correct order for indexing. ** ** This routine makes a copy of the input pX expression and then adjusts @@ -156735,9 +162354,12 @@ static Expr *removeUnindexableInClauseTerms( int iField; assert( (pLoop->aLTerm[i]->eOperator & (WO_OR|WO_AND))==0 ); iField = pLoop->aLTerm[i]->u.x.iField - 1; - if( pOrigRhs->a[iField].pExpr==0 ) continue; /* Duplicate PK column */ + if( NEVER(pOrigRhs->a[iField].pExpr==0) ){ + continue; /* Duplicate PK column */ + } pRhs = sqlite3ExprListAppend(pParse, pRhs, pOrigRhs->a[iField].pExpr); pOrigRhs->a[iField].pExpr = 0; + if( pRhs ) pRhs->a[pRhs->nExpr-1].u.x.iOrderByCol = iField+1; if( pOrigLhs ){ assert( pOrigLhs->a[iField].pExpr!=0 ); pLhs = sqlite3ExprListAppend(pParse,pLhs,pOrigLhs->a[iField].pExpr); @@ -156751,6 +162373,7 @@ static Expr *removeUnindexableInClauseTerms( pNew->pLeft->x.pList = pLhs; } pSelect->pEList = pRhs; + pSelect->selId = ++pParse->nSelect; /* Req'd for SubrtnSig validity */ if( pLhs && pLhs->nExpr==1 ){ /* Take care here not to generate a TK_VECTOR containing only a ** single value. Since the parser never creates such a vector, some @@ -156760,18 +162383,16 @@ static Expr *removeUnindexableInClauseTerms( sqlite3ExprDelete(db, pNew->pLeft); pNew->pLeft = p; } - if( pSelect->pOrderBy ){ - /* If the SELECT statement has an ORDER BY clause, zero the - ** iOrderByCol variables. These are set to non-zero when an - ** ORDER BY term exactly matches one of the terms of the - ** result-set. Since the result-set of the SELECT statement may - ** have been modified or reordered, these variables are no longer - ** set correctly. Since setting them is just an optimization, - ** it's easiest just to zero them here. */ - ExprList *pOrderBy = pSelect->pOrderBy; - for(i=0; inExpr; i++){ - pOrderBy->a[i].u.x.iOrderByCol = 0; - } + + /* If either the ORDER BY clause or the GROUP BY clause contains + ** references to result-set columns, those references might now be + ** obsolete. So fix them up. + */ + assert( pRhs!=0 || db->mallocFailed ); + if( pRhs ){ + adjustOrderByCol(pSelect->pOrderBy, pRhs); + adjustOrderByCol(pSelect->pGroupBy, pRhs); + for(i=0; inExpr; i++) pRhs->a[i].u.x.iOrderByCol = 0; } #if 0 @@ -156786,6 +162407,138 @@ static Expr *removeUnindexableInClauseTerms( } +#ifndef SQLITE_OMIT_SUBQUERY +/* +** Generate code for a single X IN (....) term of the WHERE clause. +** +** This is a special-case of codeEqualityTerm() that works for IN operators +** only. It is broken out into a subroutine because this case is +** uncommon and by splitting it off into a subroutine, the common case +** runs faster. +** +** The current value for the constraint is left in register iTarget. +** This routine sets up a loop that will iterate over all values of X. +*/ +static SQLITE_NOINLINE void codeINTerm( + Parse *pParse, /* The parsing context */ + WhereTerm *pTerm, /* The term of the WHERE clause to be coded */ + WhereLevel *pLevel, /* The level of the FROM clause we are working on */ + int iEq, /* Index of the equality term within this level */ + int bRev, /* True for reverse-order IN operations */ + int iTarget /* Attempt to leave results in this register */ +){ + Expr *pX = pTerm->pExpr; + int eType = IN_INDEX_NOOP; + int iTab; + struct InLoop *pIn; + WhereLoop *pLoop = pLevel->pWLoop; + Vdbe *v = pParse->pVdbe; + int i; + int nEq = 0; + int *aiMap = 0; + + if( (pLoop->wsFlags & WHERE_VIRTUALTABLE)==0 + && pLoop->u.btree.pIndex!=0 + && pLoop->u.btree.pIndex->aSortOrder[iEq] + ){ + testcase( iEq==0 ); + testcase( bRev ); + bRev = !bRev; + } + assert( pX->op==TK_IN ); + + for(i=0; iaLTerm[i] && pLoop->aLTerm[i]->pExpr==pX ){ + disableTerm(pLevel, pTerm); + return; + } + } + for(i=iEq; inLTerm; i++){ + assert( pLoop->aLTerm[i]!=0 ); + if( pLoop->aLTerm[i]->pExpr==pX ) nEq++; + } + + iTab = 0; + if( !ExprUseXSelect(pX) || pX->x.pSelect->pEList->nExpr==1 ){ + eType = sqlite3FindInIndex(pParse, pX, IN_INDEX_LOOP, 0, 0, &iTab); + }else{ + sqlite3 *db = pParse->db; + Expr *pXMod = removeUnindexableInClauseTerms(pParse, iEq, pLoop, pX); + if( !db->mallocFailed ){ + aiMap = (int*)sqlite3DbMallocZero(db, sizeof(int)*nEq); + eType = sqlite3FindInIndex(pParse, pXMod, IN_INDEX_LOOP, 0, aiMap, &iTab); + } + sqlite3ExprDelete(db, pXMod); + } + + if( eType==IN_INDEX_INDEX_DESC ){ + testcase( bRev ); + bRev = !bRev; + } + sqlite3VdbeAddOp2(v, bRev ? OP_Last : OP_Rewind, iTab, 0); + VdbeCoverageIf(v, bRev); + VdbeCoverageIf(v, !bRev); + + assert( (pLoop->wsFlags & WHERE_MULTI_OR)==0 ); + pLoop->wsFlags |= WHERE_IN_ABLE; + if( pLevel->u.in.nIn==0 ){ + pLevel->addrNxt = sqlite3VdbeMakeLabel(pParse); + } + if( iEq>0 && (pLoop->wsFlags & WHERE_IN_SEEKSCAN)==0 ){ + pLoop->wsFlags |= WHERE_IN_EARLYOUT; + } + + i = pLevel->u.in.nIn; + pLevel->u.in.nIn += nEq; + pLevel->u.in.aInLoop = + sqlite3WhereRealloc(pTerm->pWC->pWInfo, + pLevel->u.in.aInLoop, + sizeof(pLevel->u.in.aInLoop[0])*pLevel->u.in.nIn); + pIn = pLevel->u.in.aInLoop; + if( pIn ){ + int iMap = 0; /* Index in aiMap[] */ + pIn += i; + for(i=iEq; inLTerm; i++){ + if( pLoop->aLTerm[i]->pExpr==pX ){ + int iOut = iTarget + i - iEq; + if( eType==IN_INDEX_ROWID ){ + pIn->addrInTop = sqlite3VdbeAddOp2(v, OP_Rowid, iTab, iOut); + }else{ + int iCol = aiMap ? aiMap[iMap++] : 0; + pIn->addrInTop = sqlite3VdbeAddOp3(v,OP_Column,iTab, iCol, iOut); + } + sqlite3VdbeAddOp1(v, OP_IsNull, iOut); VdbeCoverage(v); + if( i==iEq ){ + pIn->iCur = iTab; + pIn->eEndLoopOp = bRev ? OP_Prev : OP_Next; + if( iEq>0 ){ + pIn->iBase = iTarget - i; + pIn->nPrefix = i; + }else{ + pIn->nPrefix = 0; + } + }else{ + pIn->eEndLoopOp = OP_Noop; + } + pIn++; + } + } + testcase( iEq>0 + && (pLoop->wsFlags & WHERE_IN_SEEKSCAN)==0 + && (pLoop->wsFlags & WHERE_VIRTUALTABLE)!=0 ); + if( iEq>0 + && (pLoop->wsFlags & (WHERE_IN_SEEKSCAN|WHERE_VIRTUALTABLE))==0 + ){ + sqlite3VdbeAddOp3(v, OP_SeekHit, pLevel->iIdxCur, 0, iEq); + } + }else{ + pLevel->u.in.nIn = 0; + } + sqlite3DbFree(pParse->db, aiMap); +} +#endif + + /* ** Generate code for a single equality term of the WHERE clause. An equality ** term can be either X=expr or X IN (...). pTerm is the term to be @@ -156810,7 +162563,6 @@ static int codeEqualityTerm( int iTarget /* Attempt to leave results in this register */ ){ Expr *pX = pTerm->pExpr; - Vdbe *v = pParse->pVdbe; int iReg; /* Register holding results */ assert( pLevel->pWLoop->aLTerm[iEq]==pTerm ); @@ -156819,125 +162571,12 @@ static int codeEqualityTerm( iReg = sqlite3ExprCodeTarget(pParse, pX->pRight, iTarget); }else if( pX->op==TK_ISNULL ){ iReg = iTarget; - sqlite3VdbeAddOp2(v, OP_Null, 0, iReg); + sqlite3VdbeAddOp2(pParse->pVdbe, OP_Null, 0, iReg); #ifndef SQLITE_OMIT_SUBQUERY }else{ - int eType = IN_INDEX_NOOP; - int iTab; - struct InLoop *pIn; - WhereLoop *pLoop = pLevel->pWLoop; - int i; - int nEq = 0; - int *aiMap = 0; - - if( (pLoop->wsFlags & WHERE_VIRTUALTABLE)==0 - && pLoop->u.btree.pIndex!=0 - && pLoop->u.btree.pIndex->aSortOrder[iEq] - ){ - testcase( iEq==0 ); - testcase( bRev ); - bRev = !bRev; - } assert( pX->op==TK_IN ); iReg = iTarget; - - for(i=0; iaLTerm[i] && pLoop->aLTerm[i]->pExpr==pX ){ - disableTerm(pLevel, pTerm); - return iTarget; - } - } - for(i=iEq;inLTerm; i++){ - assert( pLoop->aLTerm[i]!=0 ); - if( pLoop->aLTerm[i]->pExpr==pX ) nEq++; - } - - iTab = 0; - if( !ExprUseXSelect(pX) || pX->x.pSelect->pEList->nExpr==1 ){ - eType = sqlite3FindInIndex(pParse, pX, IN_INDEX_LOOP, 0, 0, &iTab); - }else{ - Expr *pExpr = pTerm->pExpr; - if( pExpr->iTable==0 || !ExprHasProperty(pExpr, EP_Subrtn) ){ - sqlite3 *db = pParse->db; - pX = removeUnindexableInClauseTerms(pParse, iEq, pLoop, pX); - if( !db->mallocFailed ){ - aiMap = (int*)sqlite3DbMallocZero(pParse->db, sizeof(int)*nEq); - eType = sqlite3FindInIndex(pParse, pX, IN_INDEX_LOOP, 0, aiMap,&iTab); - pExpr->iTable = iTab; - } - sqlite3ExprDelete(db, pX); - }else{ - int n = sqlite3ExprVectorSize(pX->pLeft); - aiMap = (int*)sqlite3DbMallocZero(pParse->db, sizeof(int)*MAX(nEq,n)); - eType = sqlite3FindInIndex(pParse, pX, IN_INDEX_LOOP, 0, aiMap, &iTab); - } - pX = pExpr; - } - - if( eType==IN_INDEX_INDEX_DESC ){ - testcase( bRev ); - bRev = !bRev; - } - sqlite3VdbeAddOp2(v, bRev ? OP_Last : OP_Rewind, iTab, 0); - VdbeCoverageIf(v, bRev); - VdbeCoverageIf(v, !bRev); - - assert( (pLoop->wsFlags & WHERE_MULTI_OR)==0 ); - pLoop->wsFlags |= WHERE_IN_ABLE; - if( pLevel->u.in.nIn==0 ){ - pLevel->addrNxt = sqlite3VdbeMakeLabel(pParse); - } - if( iEq>0 && (pLoop->wsFlags & WHERE_IN_SEEKSCAN)==0 ){ - pLoop->wsFlags |= WHERE_IN_EARLYOUT; - } - - i = pLevel->u.in.nIn; - pLevel->u.in.nIn += nEq; - pLevel->u.in.aInLoop = - sqlite3WhereRealloc(pTerm->pWC->pWInfo, - pLevel->u.in.aInLoop, - sizeof(pLevel->u.in.aInLoop[0])*pLevel->u.in.nIn); - pIn = pLevel->u.in.aInLoop; - if( pIn ){ - int iMap = 0; /* Index in aiMap[] */ - pIn += i; - for(i=iEq;inLTerm; i++){ - if( pLoop->aLTerm[i]->pExpr==pX ){ - int iOut = iReg + i - iEq; - if( eType==IN_INDEX_ROWID ){ - pIn->addrInTop = sqlite3VdbeAddOp2(v, OP_Rowid, iTab, iOut); - }else{ - int iCol = aiMap ? aiMap[iMap++] : 0; - pIn->addrInTop = sqlite3VdbeAddOp3(v,OP_Column,iTab, iCol, iOut); - } - sqlite3VdbeAddOp1(v, OP_IsNull, iOut); VdbeCoverage(v); - if( i==iEq ){ - pIn->iCur = iTab; - pIn->eEndLoopOp = bRev ? OP_Prev : OP_Next; - if( iEq>0 ){ - pIn->iBase = iReg - i; - pIn->nPrefix = i; - }else{ - pIn->nPrefix = 0; - } - }else{ - pIn->eEndLoopOp = OP_Noop; - } - pIn++; - } - } - testcase( iEq>0 - && (pLoop->wsFlags & WHERE_IN_SEEKSCAN)==0 - && (pLoop->wsFlags & WHERE_VIRTUALTABLE)!=0 ); - if( iEq>0 - && (pLoop->wsFlags & (WHERE_IN_SEEKSCAN|WHERE_VIRTUALTABLE))==0 - ){ - sqlite3VdbeAddOp3(v, OP_SeekHit, pLevel->iIdxCur, 0, iEq); - } - }else{ - pLevel->u.in.nIn = 0; - } - sqlite3DbFree(pParse->db, aiMap); + codeINTerm(pParse, pTerm, pLevel, iEq, bRev, iTarget); #endif } @@ -157510,6 +163149,7 @@ static SQLITE_NOINLINE void filterPullDown( int addrNxt, /* Jump here to bypass inner loops */ Bitmask notReady /* Loops that are not ready */ ){ + int saved_addrBrk; while( ++iLevel < pWInfo->nLevel ){ WhereLevel *pLevel = &pWInfo->a[iLevel]; WhereLoop *pLoop = pLevel->pWLoop; @@ -157518,7 +163158,7 @@ static SQLITE_NOINLINE void filterPullDown( /* ,--- Because sqlite3ConstructBloomFilter() has will not have set ** vvvvv--' pLevel->regFilter if this were true. */ if( NEVER(pLoop->prereq & notReady) ) continue; - assert( pLevel->addrBrk==0 ); + saved_addrBrk = pLevel->addrBrk; pLevel->addrBrk = addrNxt; if( pLoop->wsFlags & WHERE_IPK ){ WhereTerm *pTerm = pLoop->aLTerm[0]; @@ -157548,10 +163188,31 @@ static SQLITE_NOINLINE void filterPullDown( VdbeCoverage(pParse->pVdbe); } pLevel->regFilter = 0; - pLevel->addrBrk = 0; + pLevel->addrBrk = saved_addrBrk; } } +/* +** Loop pLoop is a WHERE_INDEXED level that uses at least one IN(...) +** operator. Return true if level pLoop is guaranteed to visit only one +** row for each key generated for the index. +*/ +static int whereLoopIsOneRow(WhereLoop *pLoop){ + if( pLoop->u.btree.pIndex->onError + && pLoop->nSkip==0 + && pLoop->u.btree.nEq==pLoop->u.btree.pIndex->nKeyCol + ){ + int ii; + for(ii=0; iiu.btree.nEq; ii++){ + if( pLoop->aLTerm[ii]->eOperator & (WO_IS|WO_ISNULL) ){ + return 0; + } + } + return 1; + } + return 0; +} + /* ** Generate code for the start of the iLevel-th loop in the WHERE clause ** implementation described by pWInfo. @@ -157574,7 +163235,6 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( sqlite3 *db; /* Database connection */ SrcItem *pTabItem; /* FROM clause term being coded */ int addrBrk; /* Jump here to break out of the loop */ - int addrHalt; /* addrBrk for the outermost loop */ int addrCont; /* Jump here to continue with next cycle */ int iRowidReg = 0; /* Rowid is stored in this register, if not zero */ int iReleaseReg = 0; /* Temp register to free before returning */ @@ -157588,7 +163248,8 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( iCur = pTabItem->iCursor; pLevel->notReady = notReady & ~sqlite3WhereGetMask(&pWInfo->sMaskSet, iCur); bRev = (pWInfo->revMask>>iLevel)&1; - VdbeModuleComment((v, "Begin WHERE-loop%d: %s",iLevel,pTabItem->pTab->zName)); + VdbeModuleComment((v, "Begin WHERE-loop%d: %s", + iLevel, pTabItem->pSTab->zName)); #if WHERETRACE_ENABLED /* 0x4001 */ if( sqlite3WhereTrace & 0x1 ){ sqlite3DebugPrintf("Coding level %d of %d: notReady=%llx iFrom=%d\n", @@ -157617,7 +163278,7 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( ** there are no IN operators in the constraints, the "addrNxt" label ** is the same as "addrBrk". */ - addrBrk = pLevel->addrBrk = pLevel->addrNxt = sqlite3VdbeMakeLabel(pParse); + addrBrk = pLevel->addrNxt = pLevel->addrBrk; addrCont = pLevel->addrCont = sqlite3VdbeMakeLabel(pParse); /* If this is the right table of a LEFT OUTER JOIN, allocate and @@ -157630,24 +163291,20 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( if( pLevel->iFrom>0 && (pTabItem[0].fg.jointype & JT_LEFT)!=0 ){ pLevel->iLeftJoin = ++pParse->nMem; sqlite3VdbeAddOp2(v, OP_Integer, 0, pLevel->iLeftJoin); - VdbeComment((v, "init LEFT JOIN no-match flag")); + VdbeComment((v, "init LEFT JOIN match flag")); } - /* Compute a safe address to jump to if we discover that the table for - ** this loop is empty and can never contribute content. */ - for(j=iLevel; j>0; j--){ - if( pWInfo->a[j].iLeftJoin ) break; - if( pWInfo->a[j].pRJ ) break; - } - addrHalt = pWInfo->a[j].addrBrk; - /* Special case of a FROM clause subquery implemented as a co-routine */ if( pTabItem->fg.viaCoroutine ){ - int regYield = pTabItem->regReturn; - sqlite3VdbeAddOp3(v, OP_InitCoroutine, regYield, 0, pTabItem->addrFillSub); + int regYield; + Subquery *pSubq; + assert( pTabItem->fg.isSubquery && pTabItem->u4.pSubq!=0 ); + pSubq = pTabItem->u4.pSubq; + regYield = pSubq->regReturn; + sqlite3VdbeAddOp3(v, OP_InitCoroutine, regYield, 0, pSubq->addrFillSub); pLevel->p2 = sqlite3VdbeAddOp2(v, OP_Yield, regYield, addrBrk); VdbeCoverage(v); - VdbeComment((v, "next row of %s", pTabItem->pTab->zName)); + VdbeComment((v, "next row of %s", pTabItem->pSTab->zName)); pLevel->op = OP_Goto; }else @@ -157692,6 +163349,9 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( } sqlite3VdbeAddOp2(v, OP_Integer, pLoop->u.vtab.idxNum, iReg); sqlite3VdbeAddOp2(v, OP_Integer, nConstraint, iReg+1); + /* The instruction immediately prior to OP_VFilter must be an OP_Integer + ** that sets the "argc" value for xVFilter. This is necessary for + ** resolveP2() to work correctly. See tag-20250207a. */ sqlite3VdbeAddOp4(v, OP_VFilter, iCur, addrNotFound, iReg, pLoop->u.vtab.idxStr, pLoop->u.vtab.needFree ? P4_DYNAMIC : P4_STATIC); @@ -157872,7 +163532,7 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( VdbeCoverageIf(v, pX->op==TK_GE); sqlite3ReleaseTempReg(pParse, rTemp); }else{ - sqlite3VdbeAddOp2(v, bRev ? OP_Last : OP_Rewind, iCur, addrHalt); + sqlite3VdbeAddOp2(v, bRev ? OP_Last : OP_Rewind, iCur, pLevel->addrHalt); VdbeCoverageIf(v, bRev==0); VdbeCoverageIf(v, bRev!=0); } @@ -157912,36 +163572,36 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( sqlite3VdbeChangeP5(v, SQLITE_AFF_NUMERIC | SQLITE_JUMPIFNULL); } }else if( pLoop->wsFlags & WHERE_INDEXED ){ - /* Case 4: A scan using an index. + /* Case 4: Search using an index. ** - ** The WHERE clause may contain zero or more equality - ** terms ("==" or "IN" operators) that refer to the N - ** left-most columns of the index. It may also contain - ** inequality constraints (>, <, >= or <=) on the indexed - ** column that immediately follows the N equalities. Only - ** the right-most column can be an inequality - the rest must - ** use the "==" and "IN" operators. For example, if the - ** index is on (x,y,z), then the following clauses are all - ** optimized: + ** The WHERE clause may contain zero or more equality + ** terms ("==" or "IN" or "IS" operators) that refer to the N + ** left-most columns of the index. It may also contain + ** inequality constraints (>, <, >= or <=) on the indexed + ** column that immediately follows the N equalities. Only + ** the right-most column can be an inequality - the rest must + ** use the "==", "IN", or "IS" operators. For example, if the + ** index is on (x,y,z), then the following clauses are all + ** optimized: ** - ** x=5 - ** x=5 AND y=10 - ** x=5 AND y<10 - ** x=5 AND y>5 AND y<10 - ** x=5 AND y=5 AND z<=10 + ** x=5 + ** x=5 AND y=10 + ** x=5 AND y<10 + ** x=5 AND y>5 AND y<10 + ** x=5 AND y=5 AND z<=10 ** - ** The z<10 term of the following cannot be used, only - ** the x=5 term: + ** The z<10 term of the following cannot be used, only + ** the x=5 term: ** - ** x=5 AND z<10 + ** x=5 AND z<10 ** - ** N may be zero if there are inequality constraints. - ** If there are no inequality constraints, then N is at - ** least one. + ** N may be zero if there are inequality constraints. + ** If there are no inequality constraints, then N is at + ** least one. ** - ** This case is also used when there are no WHERE clause - ** constraints but an index is selected anyway, in order - ** to force the output order to conform to an ORDER BY. + ** This case is also used when there are no WHERE clause + ** constraints but an index is selected anyway, in order + ** to force the output order to conform to an ORDER BY. */ static const u8 aStartOp[] = { 0, @@ -158282,12 +163942,13 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( if( pLevel->iLeftJoin==0 ){ /* If a partial index is driving the loop, try to eliminate WHERE clause ** terms from the query that must be true due to the WHERE clause of - ** the partial index. + ** the partial index. This optimization does not work on an outer join, + ** as shown by: ** - ** 2019-11-02 ticket 623eff57e76d45f6: This optimization does not work - ** for a LEFT JOIN. + ** 2019-11-02 ticket 623eff57e76d45f6 (LEFT JOIN) + ** 2025-05-29 forum post 7dee41d32506c4ae (RIGHT JOIN) */ - if( pIdx->pPartIdxWhere ){ + if( pIdx->pPartIdxWhere && pLevel->pRJ==0 ){ whereApplyPartialIndexConstraints(pIdx->pPartIdxWhere, iCur, pWC); } }else{ @@ -158299,7 +163960,9 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( } /* Record the instruction used to terminate the loop. */ - if( pLoop->wsFlags & WHERE_ONEROW ){ + if( (pLoop->wsFlags & WHERE_ONEROW) + || (pLevel->u.in.nIn && regBignull==0 && whereLoopIsOneRow(pLoop)) + ){ pLevel->op = OP_Noop; }else if( bRev ){ pLevel->op = OP_Prev; @@ -158374,7 +164037,7 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( int untestedTerms = 0; /* Some terms not completely tested */ int ii; /* Loop counter */ Expr *pAndExpr = 0; /* An ".. AND (...)" expression */ - Table *pTab = pTabItem->pTab; + Table *pTab = pTabItem->pSTab; pTerm = pLoop->aLTerm[0]; assert( pTerm!=0 ); @@ -158392,8 +164055,7 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( int nNotReady; /* The number of notReady tables */ SrcItem *origSrc; /* Original list of tables */ nNotReady = pWInfo->nLevel - iLevel - 1; - pOrTab = sqlite3DbMallocRawNN(db, - sizeof(*pOrTab)+ nNotReady*sizeof(pOrTab->a[0])); + pOrTab = sqlite3DbMallocRawNN(db, SZ_SRCLIST(nNotReady+1)); if( pOrTab==0 ) return notReady; pOrTab->nAlloc = (u8)(nNotReady + 1); pOrTab->nSrc = pOrTab->nAlloc; @@ -158444,7 +164106,7 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( ** ** This optimization also only applies if the (x1 OR x2 OR ...) term ** is not contained in the ON clause of a LEFT JOIN. - ** See ticket http://www.sqlite.org/src/info/f2369304e4 + ** See ticket http://sqlite.org/src/info/f2369304e4 ** ** 2022-02-04: Do not push down slices of a row-value comparison. ** In other words, "w" or "y" may not be a slice of a vector. Otherwise, @@ -158665,7 +164327,7 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( codeCursorHint(pTabItem, pWInfo, pLevel, 0); pLevel->op = aStep[bRev]; pLevel->p1 = iCur; - pLevel->p2 = 1 + sqlite3VdbeAddOp2(v, aStart[bRev], iCur, addrHalt); + pLevel->p2 = 1 + sqlite3VdbeAddOp2(v, aStart[bRev],iCur,pLevel->addrHalt); VdbeCoverageIf(v, bRev==0); VdbeCoverageIf(v, bRev!=0); pLevel->p5 = SQLITE_STMTSTATUS_FULLSCAN_STEP; @@ -158689,6 +164351,12 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( ** iLoop==3: Code all remaining expressions. ** ** An effort is made to skip unnecessary iterations of the loop. + ** + ** This optimization of causing simple query restrictions to occur before + ** more complex one is call the "push-down" optimization in MySQL. Here + ** in SQLite, the name is "MySQL push-down", since there is also another + ** totally unrelated optimization called "WHERE-clause push-down". + ** Sometimes the qualifier is omitted, resulting in an ambiguity, so beware. */ iLoop = (pIdx ? 1 : 2); do{ @@ -158827,7 +164495,7 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( ** least once. This is accomplished by storing the PK for the row in ** both the iMatch index and the regBloom Bloom filter. */ - pTab = pWInfo->pTabList->a[pLevel->iFrom].pTab; + pTab = pWInfo->pTabList->a[pLevel->iFrom].pSTab; if( HasRowid(pTab) ){ r = sqlite3GetTempRange(pParse, 2); sqlite3ExprCodeGetColumnOfTable(v, pTab, pLevel->iTabCur, -1, r+1); @@ -158930,16 +164598,33 @@ SQLITE_PRIVATE SQLITE_NOINLINE void sqlite3WhereRightJoinLoop( WhereInfo *pSubWInfo; WhereLoop *pLoop = pLevel->pWLoop; SrcItem *pTabItem = &pWInfo->pTabList->a[pLevel->iFrom]; - SrcList sFrom; + SrcList *pFrom; + union { + SrcList sSrc; + u8 fromSpace[SZ_SRCLIST_1]; + } uSrc; Bitmask mAll = 0; int k; - ExplainQueryPlan((pParse, 1, "RIGHT-JOIN %s", pTabItem->pTab->zName)); + ExplainQueryPlan((pParse, 1, "RIGHT-JOIN %s", pTabItem->pSTab->zName)); sqlite3VdbeNoJumpsOutsideSubrtn(v, pRJ->addrSubrtn, pRJ->endSubrtn, pRJ->regReturn); for(k=0; ka[k].pWLoop->iTab == pWInfo->a[k].iFrom ); + pRight = &pWInfo->pTabList->a[pWInfo->a[k].iFrom]; mAll |= pWInfo->a[k].pWLoop->maskSelf; + if( pRight->fg.viaCoroutine ){ + Subquery *pSubq; + assert( pRight->fg.isSubquery && pRight->u4.pSubq!=0 ); + pSubq = pRight->u4.pSubq; + assert( pSubq->pSelect!=0 && pSubq->pSelect->pEList!=0 ); + sqlite3VdbeAddOp3( + v, OP_Null, 0, pSubq->regResult, + pSubq->regResult + pSubq->pSelect->pEList->nExpr-1 + ); + } sqlite3VdbeAddOp1(v, OP_NullRow, pWInfo->a[k].iTabCur); iIdxCur = pWInfo->a[k].iIdxCur; if( iIdxCur ){ @@ -158961,13 +164646,14 @@ SQLITE_PRIVATE SQLITE_NOINLINE void sqlite3WhereRightJoinLoop( sqlite3ExprDup(pParse->db, pTerm->pExpr, 0)); } } - sFrom.nSrc = 1; - sFrom.nAlloc = 1; - memcpy(&sFrom.a[0], pTabItem, sizeof(SrcItem)); - sFrom.a[0].fg.jointype = 0; + pFrom = &uSrc.sSrc; + pFrom->nSrc = 1; + pFrom->nAlloc = 1; + memcpy(&pFrom->a[0], pTabItem, sizeof(SrcItem)); + pFrom->a[0].fg.jointype = 0; assert( pParse->withinRJSubrtn < 100 ); pParse->withinRJSubrtn++; - pSubWInfo = sqlite3WhereBegin(pParse, &sFrom, pSubWhere, 0, 0, 0, + pSubWInfo = sqlite3WhereBegin(pParse, pFrom, pSubWhere, 0, 0, 0, WHERE_RIGHT_JOIN, 0); if( pSubWInfo ){ int iCur = pLevel->iTabCur; @@ -158975,7 +164661,7 @@ SQLITE_PRIVATE SQLITE_NOINLINE void sqlite3WhereRightJoinLoop( int nPk; int jmp; int addrCont = sqlite3WhereContinueLabel(pSubWInfo); - Table *pTab = pTabItem->pTab; + Table *pTab = pTabItem->pSTab; if( HasRowid(pTab) ){ sqlite3ExprCodeGetColumnOfTable(v, pTab, iCur, -1, r); nPk = 1; @@ -159108,7 +164794,12 @@ static int allowedOp(int op){ assert( TK_LT>TK_EQ && TK_LTTK_EQ && TK_LE=TK_EQ && op<=TK_GE) || op==TK_ISNULL || op==TK_IS; + assert( TK_INTK_GE ) return 0; + if( op>=TK_EQ ) return 1; + return op==TK_IN || op==TK_ISNULL || op==TK_IS; } /* @@ -159141,15 +164832,16 @@ static u16 exprCommute(Parse *pParse, Expr *pExpr){ static u16 operatorMask(int op){ u16 c; assert( allowedOp(op) ); - if( op==TK_IN ){ + if( op>=TK_EQ ){ + assert( (WO_EQ<<(op-TK_EQ)) < 0x7fff ); + c = (u16)(WO_EQ<<(op-TK_EQ)); + }else if( op==TK_IN ){ c = WO_IN; }else if( op==TK_ISNULL ){ c = WO_ISNULL; - }else if( op==TK_IS ){ - c = WO_IS; }else{ - assert( (WO_EQ<<(op-TK_EQ)) < 0x7fff ); - c = (u16)(WO_EQ<<(op-TK_EQ)); + assert( op==TK_IS ); + c = WO_IS; } assert( op!=TK_ISNULL || c==WO_ISNULL ); assert( op!=TK_IN || c==WO_IN ); @@ -159220,12 +164912,28 @@ static int isLikeOrGlob( z = (u8*)pRight->u.zToken; } if( z ){ - - /* Count the number of prefix characters prior to the first wildcard */ + /* Count the number of prefix bytes prior to the first wildcard, + ** U+fffd character, or malformed utf-8. If the underlying database + ** has a UTF16LE encoding, then only consider ASCII characters. Note that + ** the encoding of z[] is UTF8 - we are dealing with only UTF8 here in this + ** code, but the database engine itself might be processing content using a + ** different encoding. */ cnt = 0; while( (c=z[cnt])!=0 && c!=wc[0] && c!=wc[1] && c!=wc[2] ){ cnt++; - if( c==wc[3] && z[cnt]!=0 ) cnt++; + if( c==wc[3] && z[cnt]>0 && z[cnt]<0x80 ){ + cnt++; + }else if( c>=0x80 ){ + const u8 *z2 = z+cnt-1; + if( c==0xff || sqlite3Utf8Read(&z2)==0xfffd /* bad utf-8 */ + || ENC(db)==SQLITE_UTF16LE + ){ + cnt--; + break; + }else{ + cnt = (int)(z2-z); + } + } } /* The optimization is possible only if (1) the pattern does not begin @@ -159236,11 +164944,11 @@ static int isLikeOrGlob( ** range search. The third is because the caller assumes that the pattern ** consists of at least one character after all escapes have been ** removed. */ - if( (cnt>1 || (cnt>0 && z[0]!=wc[3])) && 255!=(u8)z[cnt-1] ){ + if( (cnt>1 || (cnt>0 && z[0]!=wc[3])) && ALWAYS(255!=(u8)z[cnt-1]) ){ Expr *pPrefix; /* A "complete" match if the pattern ends with "*" or "%" */ - *pisComplete = c==wc[0] && z[cnt+1]==0; + *pisComplete = c==wc[0] && z[cnt+1]==0 && ENC(db)!=SQLITE_UTF16LE; /* Get the pattern prefix. Remove all escapes from the prefix. */ pPrefix = sqlite3Expr(db, TK_STRING, (char*)z); @@ -159436,6 +165144,13 @@ static int isAuxiliaryVtabOperator( } } } + }else if( pExpr->op>=TK_EQ ){ + /* Comparison operators are a common case. Save a few comparisons for + ** that common case by terminating early. */ + assert( TK_NE < TK_EQ ); + assert( TK_ISNOT < TK_EQ ); + assert( TK_NOTNULL < TK_EQ ); + return 0; }else if( pExpr->op==TK_NE || pExpr->op==TK_ISNOT || pExpr->op==TK_NOTNULL ){ int res = 0; Expr *pLeft = pExpr->pLeft; @@ -159909,30 +165624,42 @@ static void exprAnalyzeOrTerm( ** 1. The SQLITE_Transitive optimization must be enabled ** 2. Must be either an == or an IS operator ** 3. Not originating in the ON clause of an OUTER JOIN -** 4. The affinities of A and B must be compatible -** 5a. Both operands use the same collating sequence OR -** 5b. The overall collating sequence is BINARY +** 4. The operator is not IS or else the query does not contain RIGHT JOIN +** 5. The affinities of A and B must be compatible +** 6a. Both operands use the same collating sequence OR +** 6b. The overall collating sequence is BINARY ** If this routine returns TRUE, that means that the RHS can be substituted ** for the LHS anyplace else in the WHERE clause where the LHS column occurs. ** This is an optimization. No harm comes from returning 0. But if 1 is ** returned when it should not be, then incorrect answers might result. */ -static int termIsEquivalence(Parse *pParse, Expr *pExpr){ +static int termIsEquivalence(Parse *pParse, Expr *pExpr, SrcList *pSrc){ char aff1, aff2; CollSeq *pColl; - if( !OptimizationEnabled(pParse->db, SQLITE_Transitive) ) return 0; - if( pExpr->op!=TK_EQ && pExpr->op!=TK_IS ) return 0; - if( ExprHasProperty(pExpr, EP_OuterON) ) return 0; + if( !OptimizationEnabled(pParse->db, SQLITE_Transitive) ) return 0; /* (1) */ + if( pExpr->op!=TK_EQ && pExpr->op!=TK_IS ) return 0; /* (2) */ + if( ExprHasProperty(pExpr, EP_OuterON) ) return 0; /* (3) */ + assert( pSrc!=0 ); + if( pExpr->op==TK_IS + && pSrc->nSrc>=2 + && (pSrc->a[0].fg.jointype & JT_LTORJ)!=0 + ){ + return 0; /* (4) */ + } aff1 = sqlite3ExprAffinity(pExpr->pLeft); aff2 = sqlite3ExprAffinity(pExpr->pRight); if( aff1!=aff2 && (!sqlite3IsNumericAffinity(aff1) || !sqlite3IsNumericAffinity(aff2)) ){ - return 0; + return 0; /* (5) */ } pColl = sqlite3ExprCompareCollSeq(pParse, pExpr); - if( sqlite3IsBinary(pColl) ) return 1; - return sqlite3ExprCollSeqMatch(pParse, pExpr->pLeft, pExpr->pRight); + if( !sqlite3IsBinary(pColl) + && !sqlite3ExprCollSeqMatch(pParse, pExpr->pLeft, pExpr->pRight) + ){ + return 0; /* (6) */ + } + return 1; } /* @@ -159952,7 +165679,9 @@ static Bitmask exprSelectUsage(WhereMaskSet *pMaskSet, Select *pS){ if( ALWAYS(pSrc!=0) ){ int i; for(i=0; inSrc; i++){ - mask |= exprSelectUsage(pMaskSet, pSrc->a[i].pSelect); + if( pSrc->a[i].fg.isSubquery ){ + mask |= exprSelectUsage(pMaskSet, pSrc->a[i].u4.pSubq->pSelect); + } if( pSrc->a[i].fg.isUsing==0 ){ mask |= sqlite3WhereExprUsage(pMaskSet, pSrc->a[i].u3.pOn); } @@ -159990,13 +165719,13 @@ static SQLITE_NOINLINE int exprMightBeIndexed2( int iCur; do{ iCur = pFrom->a[j].iCursor; - for(pIdx=pFrom->a[j].pTab->pIndex; pIdx; pIdx=pIdx->pNext){ + for(pIdx=pFrom->a[j].pSTab->pIndex; pIdx; pIdx=pIdx->pNext){ if( pIdx->aColExpr==0 ) continue; for(i=0; inKeyCol; i++){ if( pIdx->aiColumn[i]!=XN_EXPR ) continue; assert( pIdx->bHasExpr ); if( sqlite3ExprCompareSkip(pExpr,pIdx->aColExpr->a[i].pExpr,iCur)==0 - && pExpr->op!=TK_STRING + && !sqlite3ExprIsConstant(0,pIdx->aColExpr->a[i].pExpr) ){ aiCurCol[0] = iCur; aiCurCol[1] = XN_EXPR; @@ -160034,7 +165763,7 @@ static int exprMightBeIndexed( for(i=0; inSrc; i++){ Index *pIdx; - for(pIdx=pFrom->a[i].pTab->pIndex; pIdx; pIdx=pIdx->pNext){ + for(pIdx=pFrom->a[i].pSTab->pIndex; pIdx; pIdx=pIdx->pNext){ if( pIdx->aColExpr ){ return exprMightBeIndexed2(pFrom,aiCurCol,pExpr,i); } @@ -160088,6 +165817,9 @@ static void exprAnalyze( } assert( pWC->nTerm > idxTerm ); pTerm = &pWC->a[idxTerm]; +#ifdef SQLITE_DEBUG + pTerm->iTerm = idxTerm; +#endif pMaskSet = &pWInfo->sMaskSet; pExpr = pTerm->pExpr; assert( pExpr!=0 ); /* Because malloc() has not failed */ @@ -160131,21 +165863,7 @@ static void exprAnalyze( prereqAll |= x; extraRight = x-1; /* ON clause terms may not be used with an index ** on left table of a LEFT JOIN. Ticket #3015 */ - if( (prereqAll>>1)>=x ){ - sqlite3ErrorMsg(pParse, "ON clause references tables to its right"); - return; - } }else if( (prereqAll>>1)>=x ){ - /* The ON clause of an INNER JOIN references a table to its right. - ** Most other SQL database engines raise an error. But SQLite versions - ** 3.0 through 3.38 just put the ON clause constraint into the WHERE - ** clause and carried on. Beginning with 3.39, raise an error only - ** if there is a RIGHT or FULL JOIN in the query. This makes SQLite - ** more like other systems, and also preserves legacy. */ - if( ALWAYS(pSrc->nSrc>0) && (pSrc->a[0].fg.jointype & JT_LTORJ)!=0 ){ - sqlite3ErrorMsg(pParse, "ON clause references tables to its right"); - return; - } ExprClearProperty(pExpr, EP_InnerON); } } @@ -160195,8 +165913,8 @@ static void exprAnalyze( if( op==TK_IS ) pNew->wtFlags |= TERM_IS; pTerm = &pWC->a[idxTerm]; pTerm->wtFlags |= TERM_COPIED; - - if( termIsEquivalence(pParse, pDup) ){ + assert( pWInfo->pTabList!=0 ); + if( termIsEquivalence(pParse, pDup, pWInfo->pTabList) ){ pTerm->eOperator |= WO_EQUIV; eExtraOp = WO_EQUIV; } @@ -160362,9 +166080,8 @@ static void exprAnalyze( } if( !db->mallocFailed ){ - u8 c, *pC; /* Last character before the first wildcard */ + u8 *pC; /* Last character before the first wildcard */ pC = (u8*)&pStr2->u.zToken[sqlite3Strlen30(pStr2->u.zToken)-1]; - c = *pC; if( noCase ){ /* The point is to increment the last character before the first ** wildcard. But if we increment '@', that will push it into the @@ -160372,10 +166089,17 @@ static void exprAnalyze( ** inequality. To avoid this, make sure to also run the full ** LIKE on all candidate expressions by clearing the isComplete flag */ - if( c=='A'-1 ) isComplete = 0; - c = sqlite3UpperToLower[c]; + if( *pC=='A'-1 ) isComplete = 0; + *pC = sqlite3UpperToLower[*pC]; } - *pC = c + 1; + + /* Increment the value of the last utf8 character in the prefix. */ + while( *pC==0xBF && pC>(u8*)pStr2->u.zToken ){ + *pC = 0x80; + pC--; + } + assert( *pC!=0xFF ); /* isLikeOrGlob() guarantees this */ + (*pC)++; } zCollSeqName = noCase ? "NOCASE" : sqlite3StrBINARY; pNewExpr1 = sqlite3ExprDup(db, pLeft, 0); @@ -160496,7 +166220,7 @@ static void exprAnalyze( idxNew = whereClauseInsert(pWC, pNewExpr, TERM_VIRTUAL|TERM_DYNAMIC); testcase( idxNew==0 ); pNewTerm = &pWC->a[idxNew]; - pNewTerm->prereqRight = prereqExpr; + pNewTerm->prereqRight = prereqExpr | extraRight; pNewTerm->leftCursor = pLeft->iTable; pNewTerm->u.x.leftColumn = pLeft->iColumn; pNewTerm->eOperator = WO_AUX; @@ -160577,7 +166301,7 @@ static void whereAddLimitExpr( Expr *pNew; int iVal = 0; - if( sqlite3ExprIsInteger(pExpr, &iVal) && iVal>=0 ){ + if( sqlite3ExprIsInteger(pExpr, &iVal, pParse) && iVal>=0 ){ Expr *pVal = sqlite3Expr(db, TK_INTEGER, 0); if( pVal==0 ) return; ExprSetProperty(pVal, EP_IntValue); @@ -160607,7 +166331,7 @@ static void whereAddLimitExpr( ** ** 1. The SELECT statement has a LIMIT clause, and ** 2. The SELECT statement is not an aggregate or DISTINCT query, and -** 3. The SELECT statement has exactly one object in its from clause, and +** 3. The SELECT statement has exactly one object in its FROM clause, and ** that object is a virtual table, and ** 4. There are no terms in the WHERE clause that will not be passed ** to the virtual table xBestIndex method. @@ -160622,7 +166346,7 @@ SQLITE_PRIVATE void SQLITE_NOINLINE sqlite3WhereAddLimit(WhereClause *pWC, Selec assert( p!=0 && p->pLimit!=0 ); /* 1 -- checked by caller */ if( p->pGroupBy==0 && (p->selFlags & (SF_Distinct|SF_Aggregate))==0 /* 2 */ - && (p->pSrc->nSrc==1 && IsVirtual(p->pSrc->a[0].pTab)) /* 3 */ + && (p->pSrc->nSrc==1 && IsVirtual(p->pSrc->a[0].pSTab)) /* 3 */ ){ ExprList *pOrderBy = p->pOrderBy; int iCsr = p->pSrc->a[0].iCursor; @@ -160644,7 +166368,22 @@ SQLITE_PRIVATE void SQLITE_NOINLINE sqlite3WhereAddLimit(WhereClause *pWC, Selec ** (leftCursor==iCsr) test below. */ continue; } - if( pWC->a[ii].leftCursor!=iCsr ) return; + if( pWC->a[ii].leftCursor==iCsr && pWC->a[ii].prereqRight==0 ) continue; + + /* If this term has a parent with exactly one child, and the parent will + ** be passed through to xBestIndex, then this term can be ignored. */ + if( pWC->a[ii].iParent>=0 ){ + WhereTerm *pParent = &pWC->a[ pWC->a[ii].iParent ]; + if( pParent->leftCursor==iCsr + && pParent->prereqRight==0 + && pParent->nChild==1 + ){ + continue; + } + } + + /* This term will not be passed through. Do not add a LIMIT clause. */ + return; } /* Check condition (5). Return early if it is not met. */ @@ -160659,12 +166398,14 @@ SQLITE_PRIVATE void SQLITE_NOINLINE sqlite3WhereAddLimit(WhereClause *pWC, Selec /* All conditions are met. Add the terms to the where-clause object. */ assert( p->pLimit->op==TK_LIMIT ); - whereAddLimitExpr(pWC, p->iLimit, p->pLimit->pLeft, - iCsr, SQLITE_INDEX_CONSTRAINT_LIMIT); - if( p->iOffset>0 ){ + if( p->iOffset!=0 && (p->selFlags & SF_Compound)==0 ){ whereAddLimitExpr(pWC, p->iOffset, p->pLimit->pRight, iCsr, SQLITE_INDEX_CONSTRAINT_OFFSET); } + if( p->iOffset==0 || (p->selFlags & SF_Compound)==0 ){ + whereAddLimitExpr(pWC, p->iLimit, p->pLimit->pLeft, + iCsr, SQLITE_INDEX_CONSTRAINT_LIMIT); + } } } @@ -160840,7 +166581,7 @@ SQLITE_PRIVATE void sqlite3WhereTabFuncArgs( Expr *pColRef; Expr *pTerm; if( pItem->fg.isTabFunc==0 ) return; - pTab = pItem->pTab; + pTab = pItem->pSTab; assert( pTab!=0 ); pArgs = pItem->u1.pFuncArg; if( pArgs==0 ) return; @@ -160915,11 +166656,16 @@ struct HiddenIndexInfo { int eDistinct; /* Value to return from sqlite3_vtab_distinct() */ u32 mIn; /* Mask of terms that are IN (...) */ u32 mHandleIn; /* Terms that vtab will handle as IN (...) */ - sqlite3_value *aRhs[1]; /* RHS values for constraints. MUST BE LAST - ** because extra space is allocated to hold up - ** to nTerm such values */ + sqlite3_value *aRhs[FLEXARRAY]; /* RHS values for constraints. MUST BE LAST + ** Extra space is allocated to hold up + ** to nTerm such values */ }; +/* Size (in bytes) of a HiddenIndeInfo object sufficient to hold as +** many as N constraints */ +#define SZ_HIDDENINDEXINFO(N) \ + (offsetof(HiddenIndexInfo,aRhs) + (N)*sizeof(sqlite3_value*)) + /* Forward declaration of methods */ static int whereLoopResize(sqlite3*, WhereLoop*, int); @@ -161182,6 +166928,42 @@ static Expr *whereRightSubexprIsColumn(Expr *p){ return 0; } +/* +** Term pTerm is guaranteed to be a WO_IN term. It may be a component term +** of a vector IN expression of the form "(x, y, ...) IN (SELECT ...)". +** This function checks to see if the term is compatible with an index +** column with affinity idxaff (one of the SQLITE_AFF_XYZ values). If so, +** it returns a pointer to the name of the collation sequence (e.g. "BINARY" +** or "NOCASE") used by the comparison in pTerm. If it is not compatible +** with affinity idxaff, NULL is returned. +*/ +static SQLITE_NOINLINE const char *indexInAffinityOk( + Parse *pParse, + WhereTerm *pTerm, + u8 idxaff +){ + Expr *pX = pTerm->pExpr; + Expr inexpr; + + assert( pTerm->eOperator & WO_IN ); + + if( sqlite3ExprIsVector(pX->pLeft) ){ + int iField = pTerm->u.x.iField - 1; + inexpr.flags = 0; + inexpr.op = TK_EQ; + inexpr.pLeft = pX->pLeft->x.pList->a[iField].pExpr; + assert( ExprUseXSelect(pX) ); + inexpr.pRight = pX->x.pSelect->pEList->a[iField].pExpr; + pX = &inexpr; + } + + if( sqlite3IndexAffinityOk(pX, idxaff) ){ + CollSeq *pRet = sqlite3ExprCompareCollSeq(pParse, pX); + return pRet ? pRet->zName : sqlite3StrBINARY; + } + return 0; +} + /* ** Advance to the next WhereTerm that matches according to the criteria ** established when the pScan object was initialized by whereScanInit(). @@ -161232,16 +167014,24 @@ static WhereTerm *whereScanNext(WhereScan *pScan){ if( (pTerm->eOperator & pScan->opMask)!=0 ){ /* Verify the affinity and collating sequence match */ if( pScan->zCollName && (pTerm->eOperator & WO_ISNULL)==0 ){ - CollSeq *pColl; + const char *zCollName; Parse *pParse = pWC->pWInfo->pParse; pX = pTerm->pExpr; - if( !sqlite3IndexAffinityOk(pX, pScan->idxaff) ){ - continue; + + if( (pTerm->eOperator & WO_IN) ){ + zCollName = indexInAffinityOk(pParse, pTerm, pScan->idxaff); + if( !zCollName ) continue; + }else{ + CollSeq *pColl; + if( !sqlite3IndexAffinityOk(pX, pScan->idxaff) ){ + continue; + } + assert(pX->pLeft); + pColl = sqlite3ExprCompareCollSeq(pParse, pX); + zCollName = pColl ? pColl->zName : sqlite3StrBINARY; } - assert(pX->pLeft); - pColl = sqlite3ExprCompareCollSeq(pParse, pX); - if( pColl==0 ) pColl = pParse->db->pDfltColl; - if( sqlite3StrICmp(pColl->zName, pScan->zCollName) ){ + + if( sqlite3StrICmp(zCollName, pScan->zCollName) ){ continue; } } @@ -161257,11 +167047,11 @@ static WhereTerm *whereScanNext(WhereScan *pScan){ pScan->pWC = pWC; pScan->k = k+1; #ifdef WHERETRACE_ENABLED - if( sqlite3WhereTrace & 0x20000 ){ + if( (sqlite3WhereTrace & 0x20000)!=0 && pScan->nEquiv>1 ){ int ii; - sqlite3DebugPrintf("SCAN-TERM %p: nEquiv=%d", - pTerm, pScan->nEquiv); - for(ii=0; iinEquiv; ii++){ + sqlite3DebugPrintf("EQUIVALENT TO {%d:%d} (due to TERM-%d):", + pScan->aiCur[0], pScan->aiColumn[0], pTerm->iTerm); + for(ii=1; iinEquiv; ii++){ sqlite3DebugPrintf(" {%d:%d}", pScan->aiCur[ii], pScan->aiColumn[ii]); } @@ -161480,7 +167270,7 @@ static int isDistinctRedundant( ** clause is redundant. */ if( pTabList->nSrc!=1 ) return 0; iBase = pTabList->a[0].iCursor; - pTab = pTabList->a[0].pTab; + pTab = pTabList->a[0].pSTab; /* If any of the expressions is an IPK column on table iBase, then return ** true. Note: The (p->iTable==iBase) part of this test may be false if the @@ -161555,6 +167345,12 @@ static void translateColumnToCopy( VdbeOp *pOp = sqlite3VdbeGetOp(v, iStart); int iEnd = sqlite3VdbeCurrentAddr(v); if( pParse->db->mallocFailed ) return; +#ifdef SQLITE_DEBUG + if( pParse->db->flags & SQLITE_VdbeAddopTrace ){ + printf("CHECKING for column-to-copy on cursor %d for %d..%d\n", + iTabCur, iStart, iEnd); + } +#endif for(; iStartp1!=iTabCur ) continue; if( pOp->opcode==OP_Column ){ @@ -161593,9 +167389,13 @@ static void translateColumnToCopy( ** are no-ops. */ #if !defined(SQLITE_OMIT_VIRTUALTABLE) && defined(WHERETRACE_ENABLED) -static void whereTraceIndexInfoInputs(sqlite3_index_info *p){ +static void whereTraceIndexInfoInputs( + sqlite3_index_info *p, /* The IndexInfo object */ + Table *pTab /* The TABLE that is the virtual table */ +){ int i; if( (sqlite3WhereTrace & 0x10)==0 ) return; + sqlite3DebugPrintf("sqlite3_index_info inputs for %s:\n", pTab->zName); for(i=0; inConstraint; i++){ sqlite3DebugPrintf( " constraint[%d]: col=%d termid=%d op=%d usabled=%d collseq=%s\n", @@ -161613,9 +167413,13 @@ static void whereTraceIndexInfoInputs(sqlite3_index_info *p){ p->aOrderBy[i].desc); } } -static void whereTraceIndexInfoOutputs(sqlite3_index_info *p){ +static void whereTraceIndexInfoOutputs( + sqlite3_index_info *p, /* The IndexInfo object */ + Table *pTab /* The TABLE that is the virtual table */ +){ int i; if( (sqlite3WhereTrace & 0x10)==0 ) return; + sqlite3DebugPrintf("sqlite3_index_info outputs for %s:\n", pTab->zName); for(i=0; inConstraint; i++){ sqlite3DebugPrintf(" usage[%d]: argvIdx=%d omit=%d\n", i, @@ -161629,8 +167433,8 @@ static void whereTraceIndexInfoOutputs(sqlite3_index_info *p){ sqlite3DebugPrintf(" estimatedRows=%lld\n", p->estimatedRows); } #else -#define whereTraceIndexInfoInputs(A) -#define whereTraceIndexInfoOutputs(A) +#define whereTraceIndexInfoInputs(A,B) +#define whereTraceIndexInfoOutputs(A,B) #endif /* @@ -161661,13 +167465,52 @@ static int constraintCompatibleWithOuterJoin( return 0; } if( (pSrc->fg.jointype & (JT_LEFT|JT_RIGHT))!=0 - && ExprHasProperty(pTerm->pExpr, EP_InnerON) + && NEVER(ExprHasProperty(pTerm->pExpr, EP_InnerON)) ){ return 0; } return 1; } +#ifndef SQLITE_OMIT_AUTOMATIC_INDEX +/* +** Return true if column iCol of table pTab seem like it might be a +** good column to use as part of a query-time index. +** +** Current algorithm (subject to improvement!): +** +** 1. If iCol is already the left-most column of some other index, +** then return false. +** +** 2. If iCol is part of an existing index that has an aiRowLogEst of +** more than 20, then return false. +** +** 3. If no disqualifying conditions above are found, return true. +** +** 2025-01-03: I experimented with a new rule that returns false if the +** the datatype of the column is "BOOLEAN". This did not improve +** performance on any queries at hand, but it did burn CPU cycles, so the +** idea was not committed. +*/ +static SQLITE_NOINLINE int columnIsGoodIndexCandidate( + const Table *pTab, + int iCol +){ + const Index *pIdx; + for(pIdx = pTab->pIndex; pIdx!=0; pIdx=pIdx->pNext){ + int j; + for(j=0; jnKeyCol; j++){ + if( pIdx->aiColumn[j]==iCol ){ + if( j==0 ) return 0; + if( pIdx->hasStat1 && pIdx->aiRowLogEst[j+1]>20 ) return 0; + break; + } + } + } + return 1; +} +#endif /* SQLITE_OMIT_AUTOMATIC_INDEX */ + #ifndef SQLITE_OMIT_AUTOMATIC_INDEX @@ -161682,6 +167525,8 @@ static int termCanDriveIndex( const Bitmask notReady /* Tables in outer loops of the join */ ){ char aff; + int leftCol; + if( pTerm->leftCursor!=pSrc->iCursor ) return 0; if( (pTerm->eOperator & (WO_EQ|WO_IS))==0 ) return 0; assert( (pSrc->fg.jointype & JT_RIGHT)==0 ); @@ -161692,11 +167537,12 @@ static int termCanDriveIndex( } if( (pTerm->prereqRight & notReady)!=0 ) return 0; assert( (pTerm->eOperator & (WO_OR|WO_AND))==0 ); - if( pTerm->u.x.leftColumn<0 ) return 0; - aff = pSrc->pTab->aCol[pTerm->u.x.leftColumn].affinity; + leftCol = pTerm->u.x.leftColumn; + if( leftCol<0 ) return 0; + aff = pSrc->pSTab->aCol[leftCol].affinity; if( !sqlite3IndexAffinityOk(pTerm->pExpr, aff) ) return 0; testcase( pTerm->pExpr->op==TK_IS ); - return 1; + return columnIsGoodIndexCandidate(pSrc->pSTab, leftCol); } #endif @@ -161729,7 +167575,7 @@ static void explainAutomaticIndex( sqlite3_str *pStr = sqlite3_str_new(pParse->db); sqlite3_str_appendf(pStr,"CREATE AUTOMATIC INDEX ON %s(", pTab->zName); assert( pIdx->nColumn>1 ); - assert( pIdx->aiColumn[pIdx->nColumn-1]==XN_ROWID ); + assert( pIdx->aiColumn[pIdx->nColumn-1]==XN_ROWID || !HasRowid(pTab) ); for(ii=0; ii<(pIdx->nColumn-1); ii++){ const char *zName = 0; int iCol = pIdx->aiColumn[ii]; @@ -161804,7 +167650,7 @@ static SQLITE_NOINLINE void constructAutomaticIndex( nKeyCol = 0; pTabList = pWC->pWInfo->pTabList; pSrc = &pTabList->a[pLevel->iFrom]; - pTable = pSrc->pTab; + pTable = pSrc->pSTab; pWCEnd = &pWC->a[pWC->nTerm]; pLoop = pLevel->pWLoop; idxCols = 0; @@ -161814,7 +167660,7 @@ static SQLITE_NOINLINE void constructAutomaticIndex( ** WHERE clause (or the ON clause of a LEFT join) that constrain which ** rows of the target table (pSrc) that can be used. */ if( (pTerm->wtFlags & TERM_VIRTUAL)==0 - && sqlite3ExprIsSingleTableConstraint(pExpr, pTabList, pLevel->iFrom) + && sqlite3ExprIsSingleTableConstraint(pExpr, pTabList, pLevel->iFrom, 0) ){ pPartial = sqlite3ExprAnd(pParse, pPartial, sqlite3ExprDup(pParse->db, pExpr, 0)); @@ -161856,10 +167702,23 @@ static SQLITE_NOINLINE void constructAutomaticIndex( ** if they go out of sync. */ if( IsView(pTable) ){ - extraCols = ALLBITS; + extraCols = ALLBITS & ~idxCols; }else{ extraCols = pSrc->colUsed & (~idxCols | MASKBIT(BMS-1)); } + if( !HasRowid(pTable) ){ + /* For WITHOUT ROWID tables, ensure that all PRIMARY KEY columns are + ** either in the idxCols mask or in the extraCols mask */ + for(i=0; inCol; i++){ + if( (pTable->aCol[i].colFlags & COLFLAG_PRIMKEY)==0 ) continue; + if( i>=BMS-1 ){ + extraCols |= MASKBIT(BMS-1); + break; + } + if( idxCols & MASKBIT(i) ) continue; + extraCols |= MASKBIT(i); + } + } mxBitCol = MIN(BMS-1,pTable->nCol); testcase( pTable->nCol==BMS-1 ); testcase( pTable->nCol==BMS-2 ); @@ -161871,7 +167730,10 @@ static SQLITE_NOINLINE void constructAutomaticIndex( } /* Construct the Index object to describe this index */ - pIdx = sqlite3AllocateIndexObject(pParse->db, nKeyCol+1, 0, &zNotUsed); + assert( nKeyCol <= pTable->nCol + MAX(0, pTable->nCol - BMS + 1) ); + /* ^-- This guarantees that the number of index columns will fit in the u16 */ + pIdx = sqlite3AllocateIndexObject(pParse->db, nKeyCol+HasRowid(pTable), + 0, &zNotUsed); if( pIdx==0 ) goto end_auto_index_create; pLoop->u.btree.pIndex = pIdx; pIdx->zName = "auto-index"; @@ -161927,8 +167789,10 @@ static SQLITE_NOINLINE void constructAutomaticIndex( } } assert( n==nKeyCol ); - pIdx->aiColumn[n] = XN_ROWID; - pIdx->azColl[n] = sqlite3StrBINARY; + if( HasRowid(pTable) ){ + pIdx->aiColumn[n] = XN_ROWID; + pIdx->azColl[n] = sqlite3StrBINARY; + } /* Create the automatic index */ explainAutomaticIndex(pParse, pIdx, pPartial!=0, &addrExp); @@ -161946,14 +167810,21 @@ static SQLITE_NOINLINE void constructAutomaticIndex( /* Fill the automatic index with content */ assert( pSrc == &pWC->pWInfo->pTabList->a[pLevel->iFrom] ); if( pSrc->fg.viaCoroutine ){ - int regYield = pSrc->regReturn; + int regYield; + Subquery *pSubq; + assert( pSrc->fg.isSubquery ); + pSubq = pSrc->u4.pSubq; + assert( pSubq!=0 ); + regYield = pSubq->regReturn; addrCounter = sqlite3VdbeAddOp2(v, OP_Integer, 0, 0); - sqlite3VdbeAddOp3(v, OP_InitCoroutine, regYield, 0, pSrc->addrFillSub); + sqlite3VdbeAddOp3(v, OP_InitCoroutine, regYield, 0, pSubq->addrFillSub); addrTop = sqlite3VdbeAddOp1(v, OP_Yield, regYield); VdbeCoverage(v); - VdbeComment((v, "next row of %s", pSrc->pTab->zName)); + VdbeComment((v, "next row of %s", pSrc->pSTab->zName)); }else{ - addrTop = sqlite3VdbeAddOp1(v, OP_Rewind, pLevel->iTabCur); VdbeCoverage(v); + assert( pLevel->addrHalt ); + addrTop = sqlite3VdbeAddOp2(v, OP_Rewind,pLevel->iTabCur,pLevel->addrHalt); + VdbeCoverage(v); } if( pPartial ){ iContinue = sqlite3VdbeMakeLabel(pParse); @@ -161973,18 +167844,22 @@ static SQLITE_NOINLINE void constructAutomaticIndex( sqlite3VdbeChangeP5(v, OPFLAG_USESEEKRESULT); if( pPartial ) sqlite3VdbeResolveLabel(v, iContinue); if( pSrc->fg.viaCoroutine ){ + assert( pSrc->fg.isSubquery && pSrc->u4.pSubq!=0 ); sqlite3VdbeChangeP2(v, addrCounter, regBase+n); testcase( pParse->db->mallocFailed ); assert( pLevel->iIdxCur>0 ); translateColumnToCopy(pParse, addrTop, pLevel->iTabCur, - pSrc->regResult, pLevel->iIdxCur); + pSrc->u4.pSubq->regResult, pLevel->iIdxCur); sqlite3VdbeGoto(v, addrTop); pSrc->fg.viaCoroutine = 0; + sqlite3VdbeJumpHere(v, addrTop); }else{ sqlite3VdbeAddOp2(v, OP_Next, pLevel->iTabCur, addrTop+1); VdbeCoverage(v); sqlite3VdbeChangeP5(v, SQLITE_STMTSTATUS_AUTOINDEX); + if( (pSrc->fg.jointype & JT_LEFT)!=0 ){ + sqlite3VdbeJumpHere(v, addrTop); + } } - sqlite3VdbeJumpHere(v, addrTop); sqlite3ReleaseTempReg(pParse, regRecord); /* Jump here when skipping the initialization */ @@ -162068,7 +167943,7 @@ static SQLITE_NOINLINE void sqlite3ConstructBloomFilter( iSrc = pLevel->iFrom; pItem = &pTabList->a[iSrc]; assert( pItem!=0 ); - pTab = pItem->pTab; + pTab = pItem->pSTab; assert( pTab!=0 ); sz = sqlite3LogEstToInt(pTab->nRowLogEst); if( sz<10000 ){ @@ -162083,7 +167958,7 @@ static SQLITE_NOINLINE void sqlite3ConstructBloomFilter( for(pTerm=pWInfo->sWC.a; pTermpExpr; if( (pTerm->wtFlags & TERM_VIRTUAL)==0 - && sqlite3ExprIsSingleTableConstraint(pExpr, pTabList, iSrc) + && sqlite3ExprIsSingleTableConstraint(pExpr, pTabList, iSrc, 0) ){ sqlite3ExprIfFalse(pParse, pTerm->pExpr, addrCont, SQLITE_JUMPIFNULL); } @@ -162099,7 +167974,7 @@ static SQLITE_NOINLINE void sqlite3ConstructBloomFilter( int r1 = sqlite3GetTempRange(pParse, n); int jj; for(jj=0; jjpTable==pItem->pTab ); + assert( pIdx->pTable==pItem->pSTab ); sqlite3ExprCodeLoadIndexColumn(pParse, pIdx, iCur, jj, r1+jj); } sqlite3VdbeAddOp4Int(v, OP_FilterAdd, pLevel->regFilter, 0, r1, n); @@ -162137,6 +168012,20 @@ static SQLITE_NOINLINE void sqlite3ConstructBloomFilter( #ifndef SQLITE_OMIT_VIRTUALTABLE +/* +** Return term iTerm of the WhereClause passed as the first argument. Terms +** are numbered from 0 upwards, starting with the terms in pWC->a[], then +** those in pWC->pOuter->a[] (if any), and so on. +*/ +static WhereTerm *termFromWhereClause(WhereClause *pWC, int iTerm){ + WhereClause *p; + for(p=pWC; p; p=p->pOuter){ + if( iTermnTerm ) return &p->a[iTerm]; + iTerm -= p->nTerm; + } + return 0; +} + /* ** Allocate and populate an sqlite3_index_info structure. It is the ** responsibility of the caller to eventually release the structure @@ -162163,9 +168052,10 @@ static sqlite3_index_info *allocateIndexInfo( const Table *pTab; int eDistinct = 0; ExprList *pOrderBy = pWInfo->pOrderBy; + WhereClause *p; assert( pSrc!=0 ); - pTab = pSrc->pTab; + pTab = pSrc->pSTab; assert( pTab!=0 ); assert( IsVirtual(pTab) ); @@ -162173,28 +168063,30 @@ static sqlite3_index_info *allocateIndexInfo( ** Mark each term with the TERM_OK flag. Set nTerm to the number of ** terms found. */ - for(i=nTerm=0, pTerm=pWC->a; inTerm; i++, pTerm++){ - pTerm->wtFlags &= ~TERM_OK; - if( pTerm->leftCursor != pSrc->iCursor ) continue; - if( pTerm->prereqRight & mUnusable ) continue; - assert( IsPowerOfTwo(pTerm->eOperator & ~WO_EQUIV) ); - testcase( pTerm->eOperator & WO_IN ); - testcase( pTerm->eOperator & WO_ISNULL ); - testcase( pTerm->eOperator & WO_IS ); - testcase( pTerm->eOperator & WO_ALL ); - if( (pTerm->eOperator & ~(WO_EQUIV))==0 ) continue; - if( pTerm->wtFlags & TERM_VNULL ) continue; + for(p=pWC, nTerm=0; p; p=p->pOuter){ + for(i=0, pTerm=p->a; inTerm; i++, pTerm++){ + pTerm->wtFlags &= ~TERM_OK; + if( pTerm->leftCursor != pSrc->iCursor ) continue; + if( pTerm->prereqRight & mUnusable ) continue; + assert( IsPowerOfTwo(pTerm->eOperator & ~WO_EQUIV) ); + testcase( pTerm->eOperator & WO_IN ); + testcase( pTerm->eOperator & WO_ISNULL ); + testcase( pTerm->eOperator & WO_IS ); + testcase( pTerm->eOperator & WO_ALL ); + if( (pTerm->eOperator & ~(WO_EQUIV))==0 ) continue; + if( pTerm->wtFlags & TERM_VNULL ) continue; - assert( (pTerm->eOperator & (WO_OR|WO_AND))==0 ); - assert( pTerm->u.x.leftColumn>=XN_ROWID ); - assert( pTerm->u.x.leftColumnnCol ); - if( (pSrc->fg.jointype & (JT_LEFT|JT_LTORJ|JT_RIGHT))!=0 - && !constraintCompatibleWithOuterJoin(pTerm,pSrc) - ){ - continue; + assert( (pTerm->eOperator & (WO_OR|WO_AND))==0 ); + assert( pTerm->u.x.leftColumn>=XN_ROWID ); + assert( pTerm->u.x.leftColumnnCol ); + if( (pSrc->fg.jointype & (JT_LEFT|JT_LTORJ|JT_RIGHT))!=0 + && !constraintCompatibleWithOuterJoin(pTerm,pSrc) + ){ + continue; + } + nTerm++; + pTerm->wtFlags |= TERM_OK; } - nTerm++; - pTerm->wtFlags |= TERM_OK; } /* If the ORDER BY clause contains only columns in the current @@ -162209,7 +168101,7 @@ static sqlite3_index_info *allocateIndexInfo( Expr *pE2; /* Skip over constant terms in the ORDER BY clause */ - if( sqlite3ExprIsConstant(pExpr) ){ + if( sqlite3ExprIsConstant(0, pExpr) ){ continue; } @@ -162244,7 +168136,7 @@ static sqlite3_index_info *allocateIndexInfo( } if( i==n ){ nOrderBy = n; - if( (pWInfo->wctrlFlags & WHERE_DISTINCTBY) ){ + if( (pWInfo->wctrlFlags & WHERE_DISTINCTBY) && !pSrc->fg.rowidUsed ){ eDistinct = 2 + ((pWInfo->wctrlFlags & WHERE_SORTBYGROUP)!=0); }else if( pWInfo->wctrlFlags & WHERE_GROUPBY ){ eDistinct = 1; @@ -162256,8 +168148,8 @@ static sqlite3_index_info *allocateIndexInfo( */ pIdxInfo = sqlite3DbMallocZero(pParse->db, sizeof(*pIdxInfo) + (sizeof(*pIdxCons) + sizeof(*pUsage))*nTerm - + sizeof(*pIdxOrderBy)*nOrderBy + sizeof(*pHidden) - + sizeof(sqlite3_value*)*nTerm ); + + sizeof(*pIdxOrderBy)*nOrderBy + + SZ_HIDDENINDEXINFO(nTerm) ); if( pIdxInfo==0 ){ sqlite3ErrorMsg(pParse, "out of memory"); return 0; @@ -162269,59 +168161,75 @@ static sqlite3_index_info *allocateIndexInfo( pIdxInfo->aConstraint = pIdxCons; pIdxInfo->aOrderBy = pIdxOrderBy; pIdxInfo->aConstraintUsage = pUsage; + pIdxInfo->colUsed = (sqlite3_int64)pSrc->colUsed; + if( HasRowid(pTab)==0 ){ + /* Ensure that all bits associated with PK columns are set. This is to + ** ensure they are available for cases like RIGHT joins or OR loops. */ + Index *pPk = sqlite3PrimaryKeyIndex((Table*)pTab); + assert( pPk!=0 ); + for(i=0; inKeyCol; i++){ + int iCol = pPk->aiColumn[i]; + assert( iCol>=0 ); + if( iCol>=BMS-1 ) iCol = BMS-1; + pIdxInfo->colUsed |= MASKBIT(iCol); + } + } pHidden->pWC = pWC; pHidden->pParse = pParse; pHidden->eDistinct = eDistinct; pHidden->mIn = 0; - for(i=j=0, pTerm=pWC->a; inTerm; i++, pTerm++){ - u16 op; - if( (pTerm->wtFlags & TERM_OK)==0 ) continue; - pIdxCons[j].iColumn = pTerm->u.x.leftColumn; - pIdxCons[j].iTermOffset = i; - op = pTerm->eOperator & WO_ALL; - if( op==WO_IN ){ - if( (pTerm->wtFlags & TERM_SLICE)==0 ){ - pHidden->mIn |= SMASKBIT32(j); - } - op = WO_EQ; - } - if( op==WO_AUX ){ - pIdxCons[j].op = pTerm->eMatchOp; - }else if( op & (WO_ISNULL|WO_IS) ){ - if( op==WO_ISNULL ){ - pIdxCons[j].op = SQLITE_INDEX_CONSTRAINT_ISNULL; - }else{ - pIdxCons[j].op = SQLITE_INDEX_CONSTRAINT_IS; - } - }else{ - pIdxCons[j].op = (u8)op; - /* The direct assignment in the previous line is possible only because - ** the WO_ and SQLITE_INDEX_CONSTRAINT_ codes are identical. The - ** following asserts verify this fact. */ - assert( WO_EQ==SQLITE_INDEX_CONSTRAINT_EQ ); - assert( WO_LT==SQLITE_INDEX_CONSTRAINT_LT ); - assert( WO_LE==SQLITE_INDEX_CONSTRAINT_LE ); - assert( WO_GT==SQLITE_INDEX_CONSTRAINT_GT ); - assert( WO_GE==SQLITE_INDEX_CONSTRAINT_GE ); - assert( pTerm->eOperator&(WO_IN|WO_EQ|WO_LT|WO_LE|WO_GT|WO_GE|WO_AUX) ); - - if( op & (WO_LT|WO_LE|WO_GT|WO_GE) - && sqlite3ExprIsVector(pTerm->pExpr->pRight) - ){ - testcase( j!=i ); - if( j<16 ) mNoOmit |= (1 << j); - if( op==WO_LT ) pIdxCons[j].op = WO_LE; - if( op==WO_GT ) pIdxCons[j].op = WO_GE; + for(p=pWC, i=j=0; p; p=p->pOuter){ + int nLast = i+p->nTerm;; + for(pTerm=p->a; iwtFlags & TERM_OK)==0 ) continue; + pIdxCons[j].iColumn = pTerm->u.x.leftColumn; + pIdxCons[j].iTermOffset = i; + op = pTerm->eOperator & WO_ALL; + if( op==WO_IN ){ + if( (pTerm->wtFlags & TERM_SLICE)==0 ){ + pHidden->mIn |= SMASKBIT32(j); + } + op = WO_EQ; + } + if( op==WO_AUX ){ + pIdxCons[j].op = pTerm->eMatchOp; + }else if( op & (WO_ISNULL|WO_IS) ){ + if( op==WO_ISNULL ){ + pIdxCons[j].op = SQLITE_INDEX_CONSTRAINT_ISNULL; + }else{ + pIdxCons[j].op = SQLITE_INDEX_CONSTRAINT_IS; + } + }else{ + pIdxCons[j].op = (u8)op; + /* The direct assignment in the previous line is possible only because + ** the WO_ and SQLITE_INDEX_CONSTRAINT_ codes are identical. The + ** following asserts verify this fact. */ + assert( WO_EQ==SQLITE_INDEX_CONSTRAINT_EQ ); + assert( WO_LT==SQLITE_INDEX_CONSTRAINT_LT ); + assert( WO_LE==SQLITE_INDEX_CONSTRAINT_LE ); + assert( WO_GT==SQLITE_INDEX_CONSTRAINT_GT ); + assert( WO_GE==SQLITE_INDEX_CONSTRAINT_GE ); + assert( pTerm->eOperator&(WO_IN|WO_EQ|WO_LT|WO_LE|WO_GT|WO_GE|WO_AUX) ); + + if( op & (WO_LT|WO_LE|WO_GT|WO_GE) + && sqlite3ExprIsVector(pTerm->pExpr->pRight) + ){ + testcase( j!=i ); + if( j<16 ) mNoOmit |= (1 << j); + if( op==WO_LT ) pIdxCons[j].op = WO_LE; + if( op==WO_GT ) pIdxCons[j].op = WO_GE; + } } - } - j++; + j++; + } } assert( j==nTerm ); pIdxInfo->nConstraint = j; for(i=j=0; ia[i].pExpr; - if( sqlite3ExprIsConstant(pExpr) ) continue; + if( sqlite3ExprIsConstant(0, pExpr) ) continue; assert( pExpr->op==TK_COLUMN || (pExpr->op==TK_COLLATE && pExpr->pLeft->op==TK_COLUMN && pExpr->iColumn==pExpr->pLeft->iColumn) ); @@ -162335,6 +168243,17 @@ static sqlite3_index_info *allocateIndexInfo( return pIdxInfo; } +/* +** Free and zero the sqlite3_index_info.idxStr value if needed. +*/ +static void freeIdxStr(sqlite3_index_info *pIdxInfo){ + if( pIdxInfo->needToFreeIdxStr ){ + sqlite3_free(pIdxInfo->idxStr); + pIdxInfo->idxStr = 0; + pIdxInfo->needToFreeIdxStr = 0; + } +} + /* ** Free an sqlite3_index_info structure allocated by allocateIndexInfo() ** and possibly modified by xBestIndex methods. @@ -162350,6 +168269,7 @@ static void freeIndexInfo(sqlite3 *db, sqlite3_index_info *pIdxInfo){ sqlite3ValueFree(pHidden->aRhs[i]); /* IMP: R-14553-25174 */ pHidden->aRhs[i] = 0; } + freeIdxStr(pIdxInfo); sqlite3DbFree(db, pIdxInfo); } @@ -162370,14 +168290,16 @@ static void freeIndexInfo(sqlite3 *db, sqlite3_index_info *pIdxInfo){ ** that this is required. */ static int vtabBestIndex(Parse *pParse, Table *pTab, sqlite3_index_info *p){ - sqlite3_vtab *pVtab = sqlite3GetVTable(pParse->db, pTab)->pVtab; int rc; + sqlite3_vtab *pVtab; - whereTraceIndexInfoInputs(p); + assert( IsVirtual(pTab) ); + pVtab = sqlite3GetVTable(pParse->db, pTab)->pVtab; + whereTraceIndexInfoInputs(p, pTab); pParse->db->nSchemaLock++; rc = pVtab->pModule->xBestIndex(pVtab, p); pParse->db->nSchemaLock--; - whereTraceIndexInfoOutputs(p); + whereTraceIndexInfoOutputs(p, pTab); if( rc!=SQLITE_OK && rc!=SQLITE_CONSTRAINT ){ if( rc==SQLITE_NOMEM ){ @@ -163064,7 +168986,7 @@ static int whereInScanEst( #endif /* SQLITE_ENABLE_STAT4 */ -#ifdef WHERETRACE_ENABLED +#if defined(WHERETRACE_ENABLED) || defined(SQLITE_DEBUG) /* ** Print the content of a WhereTerm object */ @@ -163089,6 +169011,7 @@ SQLITE_PRIVATE void sqlite3WhereTermPrint(WhereTerm *pTerm, int iTerm){ }else{ sqlite3_snprintf(sizeof(zLeft),zLeft,"left=%d", pTerm->leftCursor); } + iTerm = pTerm->iTerm = MAX(iTerm,pTerm->iTerm); sqlite3DebugPrintf( "TERM-%-3d %p %s %-12s op=%03x wtFlags=%04x", iTerm, pTerm, zType, zLeft, pTerm->eOperator, pTerm->wtFlags); @@ -163108,6 +169031,9 @@ SQLITE_PRIVATE void sqlite3WhereTermPrint(WhereTerm *pTerm, int iTerm){ sqlite3TreeViewExpr(0, pTerm->pExpr, 0); } } +SQLITE_PRIVATE void sqlite3ShowWhereTerm(WhereTerm *pTerm){ + sqlite3WhereTermPrint(pTerm, 0); +} #endif #ifdef WHERETRACE_ENABLED @@ -163139,17 +169065,19 @@ SQLITE_PRIVATE void sqlite3WhereClausePrint(WhereClause *pWC){ ** 1.002.001 t2.t2xy 2 f 010241 N 2 cost 0,56,31 */ SQLITE_PRIVATE void sqlite3WhereLoopPrint(const WhereLoop *p, const WhereClause *pWC){ + WhereInfo *pWInfo; if( pWC ){ - WhereInfo *pWInfo = pWC->pWInfo; + pWInfo = pWC->pWInfo; int nb = 1+(pWInfo->pTabList->nSrc+3)/4; SrcItem *pItem = pWInfo->pTabList->a + p->iTab; - Table *pTab = pItem->pTab; + Table *pTab = pItem->pSTab; Bitmask mAll = (((Bitmask)1)<<(nb*4)) - 1; sqlite3DebugPrintf("%c%2d.%0*llx.%0*llx", p->cId, p->iTab, nb, p->maskSelf, nb, p->prereq & mAll); sqlite3DebugPrintf(" %12s", pItem->zAlias ? pItem->zAlias : pTab->zName); }else{ + pWInfo = 0; sqlite3DebugPrintf("%c%2d.%03llx.%03llx %c%d", p->cId, p->iTab, p->maskSelf, p->prereq & 0xfff, p->cId, p->iTab); } @@ -163181,7 +169109,12 @@ SQLITE_PRIVATE void sqlite3WhereLoopPrint(const WhereLoop *p, const WhereClause }else{ sqlite3DebugPrintf(" f %06x N %d", p->wsFlags, p->nLTerm); } - sqlite3DebugPrintf(" cost %d,%d,%d\n", p->rSetup, p->rRun, p->nOut); + if( pWInfo && pWInfo->bStarUsed && p->rStarDelta!=0 ){ + sqlite3DebugPrintf(" cost %d,%d,%d delta=%d\n", + p->rSetup, p->rRun, p->nOut, p->rStarDelta); + }else{ + sqlite3DebugPrintf(" cost %d,%d,%d\n", p->rSetup, p->rRun, p->nOut); + } if( p->nLTerm && (sqlite3WhereTrace & 0x4000)!=0 ){ int i; for(i=0; inLTerm; i++){ @@ -163315,7 +169248,7 @@ static void whereInfoFree(sqlite3 *db, WhereInfo *pWInfo){ ** and Y has additional constraints that might speed the search that X lacks ** but the cost of running X is not more than the cost of running Y. ** -** In other words, return true if the cost relationwship between X and Y +** In other words, return true if the cost relationship between X and Y ** is inverted and needs to be adjusted. ** ** Case 1: @@ -163701,7 +169634,7 @@ static void whereLoopOutputAdjust( Expr *pRight = pTerm->pExpr->pRight; int k = 0; testcase( pTerm->pExpr->op==TK_IS ); - if( sqlite3ExprIsInteger(pRight, &k) && k>=(-1) && k<=1 ){ + if( sqlite3ExprIsInteger(pRight, &k, 0) && k>=(-1) && k<=1 ){ k = 10; }else{ k = 20; @@ -163853,9 +169786,8 @@ static int whereLoopAddBtreeIndex( assert( pNew->u.btree.nBtm==0 ); opMask = WO_EQ|WO_IN|WO_GT|WO_GE|WO_LT|WO_LE|WO_ISNULL|WO_IS; } - if( pProbe->bUnordered || pProbe->bLowQual ){ - if( pProbe->bUnordered ) opMask &= ~(WO_GT|WO_GE|WO_LT|WO_LE); - if( pProbe->bLowQual ) opMask &= ~(WO_EQ|WO_IN|WO_IS); + if( pProbe->bUnordered ){ + opMask &= ~(WO_GT|WO_GE|WO_LT|WO_LE); } assert( pNew->u.btree.nEqnColumn ); @@ -163928,6 +169860,7 @@ static int whereLoopAddBtreeIndex( if( ExprUseXSelect(pExpr) ){ /* "x IN (SELECT ...)": TUNING: the SELECT returns 25 rows */ int i; + int bRedundant = 0; nIn = 46; assert( 46==sqlite3LogEst(25) ); /* The expression may actually be of the form (x, y) IN (SELECT...). @@ -163936,7 +169869,20 @@ static int whereLoopAddBtreeIndex( ** for each such term. The following loop checks that pTerm is the ** first such term in use, and sets nIn back to 0 if it is not. */ for(i=0; inLTerm-1; i++){ - if( pNew->aLTerm[i] && pNew->aLTerm[i]->pExpr==pExpr ) nIn = 0; + if( pNew->aLTerm[i] && pNew->aLTerm[i]->pExpr==pExpr ){ + nIn = 0; + if( pNew->aLTerm[i]->u.x.iField == pTerm->u.x.iField ){ + /* Detect when two or more columns of an index match the same + ** column of a vector IN operater, and avoid adding the column + ** to the WhereLoop more than once. See tag-20250707-01 + ** in test/rowvalue.test */ + bRedundant = 1; + } + } + } + if( bRedundant ){ + pNew->nLTerm--; + continue; } }else if( ALWAYS(pExpr->x.pList && pExpr->x.pList->nExpr) ){ /* "x IN (value, value, ...)" */ @@ -163996,7 +169942,7 @@ static int whereLoopAddBtreeIndex( || (iCol>=0 && nInMul==0 && saved_nEq==pProbe->nKeyCol-1) ){ if( iCol==XN_ROWID || pProbe->uniqNotNull - || (pProbe->nKeyCol==1 && pProbe->onError && eOp==WO_EQ) + || (pProbe->nKeyCol==1 && pProbe->onError && (eOp & WO_EQ)) ){ pNew->wsFlags |= WHERE_ONEROW; }else{ @@ -164122,11 +170068,14 @@ static int whereLoopAddBtreeIndex( } } - /* Set rCostIdx to the cost of visiting selected rows in index. Add - ** it to pNew->rRun, which is currently set to the cost of the index - ** seek only. Then, if this is a non-covering index, add the cost of - ** visiting the rows in the main table. */ - assert( pSrc->pTab->szTabRow>0 ); + /* Set rCostIdx to the estimated cost of visiting selected rows in the + ** index. The estimate is the sum of two values: + ** 1. The cost of doing one search-by-key to find the first matching + ** entry + ** 2. Stepping forward in the index pNew->nOut times to find all + ** additional matching entries. + */ + assert( pSrc->pSTab->szTabRow>0 ); if( pProbe->idxType==SQLITE_IDXTYPE_IPK ){ /* The pProbe->szIdxRow is low for an IPK table since the interior ** pages are small. Thus szIdxRow gives a good estimate of seek cost. @@ -164134,9 +170083,17 @@ static int whereLoopAddBtreeIndex( ** under-estimate the scanning cost. */ rCostIdx = pNew->nOut + 16; }else{ - rCostIdx = pNew->nOut + 1 + (15*pProbe->szIdxRow)/pSrc->pTab->szTabRow; + rCostIdx = pNew->nOut + 1 + (15*pProbe->szIdxRow)/pSrc->pSTab->szTabRow; } - pNew->rRun = sqlite3LogEstAdd(rLogSize, rCostIdx); + rCostIdx = sqlite3LogEstAdd(rLogSize, rCostIdx); + + /* Estimate the cost of running the loop. If all data is coming + ** from the index, then this is just the cost of doing the index + ** lookup and scan. But if some data is coming out of the main table, + ** we also have to add in the cost of doing pNew->nOut searches to + ** locate the row in the main table that corresponds to the index entry. + */ + pNew->rRun = rCostIdx; if( (pNew->wsFlags & (WHERE_IDX_ONLY|WHERE_IPK|WHERE_EXPRIDX))==0 ){ pNew->rRun = sqlite3LogEstAdd(pNew->rRun, pNew->nOut + 16); } @@ -164157,7 +170114,7 @@ static int whereLoopAddBtreeIndex( if( (pNew->wsFlags & WHERE_TOP_LIMIT)==0 && pNew->u.btree.nEqnColumn && (pNew->u.btree.nEqnKeyCol || - pProbe->idxType!=SQLITE_IDXTYPE_PRIMARYKEY) + pProbe->idxType!=SQLITE_IDXTYPE_PRIMARYKEY) ){ if( pNew->u.btree.nEq>3 ){ sqlite3ProgressCheck(pParse); @@ -164196,6 +170153,7 @@ static int whereLoopAddBtreeIndex( && pProbe->hasStat1!=0 && OptimizationEnabled(db, SQLITE_SkipScan) && pProbe->aiRowLogEst[saved_nEq+1]>=42 /* TUNING: Minimum for skip-scan */ + && pSrc->fg.fromExists==0 && (rc = whereLoopResize(db, pNew, pNew->nLTerm+1))==SQLITE_OK ){ LogEst nIter; @@ -164242,7 +170200,9 @@ static int indexMightHelpWithOrderBy( for(ii=0; iinExpr; ii++){ Expr *pExpr = sqlite3ExprSkipCollateAndLikely(pOB->a[ii].pExpr); if( NEVER(pExpr==0) ) continue; - if( pExpr->op==TK_COLUMN && pExpr->iTable==iCursor ){ + if( (pExpr->op==TK_COLUMN || pExpr->op==TK_AGG_COLUMN) + && pExpr->iTable==iCursor + ){ if( pExpr->iColumn<0 ) return 1; for(jj=0; jjnKeyCol; jj++){ if( pExpr->iColumn==pIndex->aiColumn[jj] ) return 1; @@ -164278,13 +170238,13 @@ static int whereUsablePartialIndex( if( !whereUsablePartialIndex(iTab,jointype,pWC,pWhere->pLeft) ) return 0; pWhere = pWhere->pRight; } - if( pParse->db->flags & SQLITE_EnableQPSG ) pParse = 0; for(i=0, pTerm=pWC->a; inTerm; i++, pTerm++){ Expr *pExpr; pExpr = pTerm->pExpr; if( (!ExprHasProperty(pExpr, EP_OuterON) || pExpr->w.iJoin==iTab) && ((jointype & JT_OUTER)==0 || ExprHasProperty(pExpr, EP_OuterON)) && sqlite3ExprImpliesExpr(pParse, pExpr, pWhere, iTab) + && !sqlite3ExprImpliesExpr(pParse, pExpr, pWhere, -1) && (pTerm->wtFlags & TERM_VNULL)==0 ){ return 1; @@ -164499,7 +170459,7 @@ static void wherePartIdxExpr( u8 aff; if( pLeft->op!=TK_COLUMN ) return; - if( !sqlite3ExprIsConstant(pRight) ) return; + if( !sqlite3ExprIsConstant(0, pRight) ) return; if( !sqlite3IsBinary(sqlite3ExprCompareCollSeq(pParse, pPart)) ) return; if( pLeft->iColumn<0 ) return; aff = pIdx->pTable->aCol[pLeft->iColumn].affinity; @@ -164589,9 +170549,9 @@ static int whereLoopAddBtree( pWInfo = pBuilder->pWInfo; pTabList = pWInfo->pTabList; pSrc = pTabList->a + pNew->iTab; - pTab = pSrc->pTab; + pTab = pSrc->pSTab; pWC = pBuilder->pWC; - assert( !IsVirtual(pSrc->pTab) ); + assert( !IsVirtual(pSrc->pSTab) ); if( pSrc->fg.isIndexedBy ){ assert( pSrc->fg.isCte==0 ); @@ -164616,7 +170576,7 @@ static int whereLoopAddBtree( sPk.idxType = SQLITE_IDXTYPE_IPK; aiRowEstPk[0] = pTab->nRowLogEst; aiRowEstPk[1] = 0; - pFirst = pSrc->pTab->pIndex; + pFirst = pSrc->pSTab->pIndex; if( pSrc->fg.notIndexed==0 ){ /* The real indices of the table are only considered if the ** NOT INDEXED qualifier is omitted from the FROM clause */ @@ -164633,7 +170593,6 @@ static int whereLoopAddBtree( && (pWInfo->pParse->db->flags & SQLITE_AutoIndex)!=0 && !pSrc->fg.isIndexedBy /* Has no INDEXED BY clause */ && !pSrc->fg.notIndexed /* Has no NOT INDEXED clause */ - && HasRowid(pTab) /* Not WITHOUT ROWID table. (FIXME: Why not?) */ && !pSrc->fg.isCorrelated /* Not a correlated subquery */ && !pSrc->fg.isRecursive /* Not a recursive common table expression. */ && (pSrc->fg.jointype & JT_RIGHT)==0 /* Not the right tab of a RIGHT JOIN */ @@ -164699,6 +170658,7 @@ static int whereLoopAddBtree( pNew->u.btree.nEq = 0; pNew->u.btree.nBtm = 0; pNew->u.btree.nTop = 0; + pNew->u.btree.nDistinctCol = 0; pNew->nSkip = 0; pNew->nLTerm = 0; pNew->iSortIdx = 0; @@ -164706,6 +170666,7 @@ static int whereLoopAddBtree( pNew->prereq = mPrereq; pNew->nOut = rSize; pNew->u.btree.pIndex = pProbe; + pNew->u.btree.pOrderBy = 0; b = indexMightHelpWithOrderBy(pBuilder, pProbe, pSrc->iCursor); /* The ONEPASS_DESIRED flags never occurs together with ORDER BY */ @@ -164735,6 +170696,10 @@ static int whereLoopAddBtree( #endif ApplyCostMultiplier(pNew->rRun, pTab->costMult); whereLoopOutputAdjust(pWC, pNew, rSize); + if( pSrc->fg.isSubquery ){ + if( pSrc->fg.viaCoroutine ) pNew->wsFlags |= WHERE_COROUTINE; + pNew->u.btree.pOrderBy = pSrc->u4.pSubq->pSelect->pOrderBy; + } rc = whereLoopInsert(pBuilder, pNew); pNew->nOut = rSize; if( rc ) break; @@ -164772,9 +170737,11 @@ static int whereLoopAddBtree( " according to whereIsCoveringIndex()\n", pProbe->zName)); } } - }else if( m==0 ){ + }else if( m==0 + && (HasRowid(pTab) || pWInfo->pSelect!=0 || sqlite3FaultSim(700)) + ){ WHERETRACE(0x200, - ("-> %s a covering index according to bitmasks\n", + ("-> %s is a covering index according to bitmasks\n", pProbe->zName, m==0 ? "is" : "is not")); pNew->wsFlags = WHERE_IDX_ONLY | WHERE_INDEXED; } @@ -164848,7 +170815,7 @@ static int whereLoopAddBtree( ** unique index is used (making the index functionally non-unique) ** then the sqlite_stat1 data becomes important for scoring the ** plan */ - pTab->tabFlags |= TF_StatsUsed; + pTab->tabFlags |= TF_MaybeReanalyze; } #ifdef SQLITE_ENABLE_STAT4 sqlite3Stat4ProbeFree(pBuilder->pRec); @@ -164870,6 +170837,21 @@ static int isLimitTerm(WhereTerm *pTerm){ && pTerm->eMatchOp<=SQLITE_INDEX_CONSTRAINT_OFFSET; } +/* +** Return true if the first nCons constraints in the pUsage array are +** marked as in-use (have argvIndex>0). False otherwise. +*/ +static int allConstraintsUsed( + struct sqlite3_index_constraint_usage *aUsage, + int nCons +){ + int ii; + for(ii=0; iipNew->iTab. This @@ -164920,7 +170902,7 @@ static int whereLoopAddVirtualOne( ** arguments mUsable and mExclude. */ pIdxCons = *(struct sqlite3_index_constraint**)&pIdxInfo->aConstraint; for(i=0; ia[pIdxCons->iTermOffset]; + WhereTerm *pTerm = termFromWhereClause(pWC, pIdxCons->iTermOffset); pIdxCons->usable = 0; if( (pTerm->prereqRight & mUsable)==pTerm->prereqRight && (pTerm->eOperator & mExclude)==0 @@ -164939,11 +170921,10 @@ static int whereLoopAddVirtualOne( pIdxInfo->estimatedCost = SQLITE_BIG_DBL / (double)2; pIdxInfo->estimatedRows = 25; pIdxInfo->idxFlags = 0; - pIdxInfo->colUsed = (sqlite3_int64)pSrc->colUsed; pHidden->mHandleIn = 0; /* Invoke the virtual table xBestIndex() method */ - rc = vtabBestIndex(pParse, pSrc->pTab, pIdxInfo); + rc = vtabBestIndex(pParse, pSrc->pSTab, pIdxInfo); if( rc ){ if( rc==SQLITE_CONSTRAINT ){ /* If the xBestIndex method returns SQLITE_CONSTRAINT, that means @@ -164951,6 +170932,7 @@ static int whereLoopAddVirtualOne( ** Make no entries in the loop table. */ WHERETRACE(0xffffffff, (" ^^^^--- non-viable plan rejected!\n")); + freeIdxStr(pIdxInfo); return SQLITE_OK; } return rc; @@ -164968,18 +170950,17 @@ static int whereLoopAddVirtualOne( int j = pIdxCons->iTermOffset; if( iTerm>=nConstraint || j<0 - || j>=pWC->nTerm + || (pTerm = termFromWhereClause(pWC, j))==0 || pNew->aLTerm[iTerm]!=0 || pIdxCons->usable==0 ){ - sqlite3ErrorMsg(pParse,"%s.xBestIndex malfunction",pSrc->pTab->zName); - testcase( pIdxInfo->needToFreeIdxStr ); + sqlite3ErrorMsg(pParse,"%s.xBestIndex malfunction",pSrc->pSTab->zName); + freeIdxStr(pIdxInfo); return SQLITE_ERROR; } testcase( iTerm==nConstraint-1 ); testcase( j==0 ); testcase( j==pWC->nTerm-1 ); - pTerm = &pWC->a[j]; pNew->prereq |= pTerm->prereqRight; assert( iTermnLSlot ); pNew->aLTerm[iTerm] = pTerm; @@ -165010,18 +170991,21 @@ static int whereLoopAddVirtualOne( *pbIn = 1; assert( (mExclude & WO_IN)==0 ); } + /* Unless pbRetryLimit is non-NULL, there should be no LIMIT/OFFSET + ** terms. And if there are any, they should follow all other terms. */ assert( pbRetryLimit || !isLimitTerm(pTerm) ); - if( isLimitTerm(pTerm) && *pbIn ){ + assert( !isLimitTerm(pTerm) || i>=nConstraint-2 ); + assert( !isLimitTerm(pTerm) || i==nConstraint-1 || isLimitTerm(pTerm+1) ); + + if( isLimitTerm(pTerm) && (*pbIn || !allConstraintsUsed(pUsage, i)) ){ /* If there is an IN(...) term handled as an == (separate call to ** xFilter for each value on the RHS of the IN) and a LIMIT or - ** OFFSET term handled as well, the plan is unusable. Set output - ** variable *pbRetryLimit to true to tell the caller to retry with - ** LIMIT and OFFSET disabled. */ - if( pIdxInfo->needToFreeIdxStr ){ - sqlite3_free(pIdxInfo->idxStr); - pIdxInfo->idxStr = 0; - pIdxInfo->needToFreeIdxStr = 0; - } + ** OFFSET term handled as well, the plan is unusable. Similarly, + ** if there is a LIMIT/OFFSET and there are other unused terms, + ** the plan cannot be used. In these cases set variable *pbRetryLimit + ** to true to tell the caller to retry with LIMIT and OFFSET + ** disabled. */ + freeIdxStr(pIdxInfo); *pbRetryLimit = 1; return SQLITE_OK; } @@ -165033,8 +171017,8 @@ static int whereLoopAddVirtualOne( if( pNew->aLTerm[i]==0 ){ /* The non-zero argvIdx values must be contiguous. Raise an ** error if they are not */ - sqlite3ErrorMsg(pParse,"%s.xBestIndex malfunction",pSrc->pTab->zName); - testcase( pIdxInfo->needToFreeIdxStr ); + sqlite3ErrorMsg(pParse,"%s.xBestIndex malfunction",pSrc->pSTab->zName); + freeIdxStr(pIdxInfo); return SQLITE_ERROR; } } @@ -165045,6 +171029,7 @@ static int whereLoopAddVirtualOne( pNew->u.vtab.idxStr = pIdxInfo->idxStr; pNew->u.vtab.isOrdered = (i8)(pIdxInfo->orderByConsumed ? pIdxInfo->nOrderBy : 0); + pNew->u.vtab.bIdxNumHex = (pIdxInfo->idxFlags&SQLITE_INDEX_SCAN_HEX)!=0; pNew->rSetup = 0; pNew->rRun = sqlite3LogEstFromDouble(pIdxInfo->estimatedCost); pNew->nOut = sqlite3LogEst(pIdxInfo->estimatedRows); @@ -165089,7 +171074,7 @@ SQLITE_API const char *sqlite3_vtab_collation(sqlite3_index_info *pIdxInfo, int if( iCons>=0 && iConsnConstraint ){ CollSeq *pC = 0; int iTerm = pIdxInfo->aConstraint[iCons].iTermOffset; - Expr *pX = pHidden->pWC->a[iTerm].pExpr; + Expr *pX = termFromWhereClause(pHidden->pWC, iTerm)->pExpr; if( pX->pLeft ){ pC = sqlite3ExprCompareCollSeq(pHidden->pParse, pX); } @@ -165135,7 +171120,9 @@ SQLITE_API int sqlite3_vtab_rhs_value( rc = SQLITE_MISUSE_BKPT; /* EV: R-30545-25046 */ }else{ if( pH->aRhs[iCons]==0 ){ - WhereTerm *pTerm = &pH->pWC->a[pIdxInfo->aConstraint[iCons].iTermOffset]; + WhereTerm *pTerm = termFromWhereClause( + pH->pWC, pIdxInfo->aConstraint[iCons].iTermOffset + ); rc = sqlite3ValueFromExpr( pH->pParse->db, pTerm->pExpr->pRight, ENC(pH->pParse->db), SQLITE_AFF_BLOB, &pH->aRhs[iCons] @@ -165233,7 +171220,7 @@ static int whereLoopAddVirtual( pWC = pBuilder->pWC; pNew = pBuilder->pNew; pSrc = &pWInfo->pTabList->a[pNew->iTab]; - assert( IsVirtual(pSrc->pTab) ); + assert( IsVirtual(pSrc->pSTab) ); p = allocateIndexInfo(pWInfo, pWC, mUnusable, pSrc, &mNoOmit); if( p==0 ) return SQLITE_NOMEM_BKPT; pNew->rSetup = 0; @@ -165247,7 +171234,7 @@ static int whereLoopAddVirtual( } /* First call xBestIndex() with all constraints usable. */ - WHERETRACE(0x800, ("BEGIN %s.addVirtual()\n", pSrc->pTab->zName)); + WHERETRACE(0x800, ("BEGIN %s.addVirtual()\n", pSrc->pSTab->zName)); WHERETRACE(0x800, (" VirtualOne: all usable\n")); rc = whereLoopAddVirtualOne( pBuilder, mPrereq, ALLBITS, 0, p, mNoOmit, &bIn, &bRetry @@ -165291,9 +171278,8 @@ static int whereLoopAddVirtual( Bitmask mNext = ALLBITS; assert( mNext>0 ); for(i=0; ia[p->aConstraint[i].iTermOffset].prereqRight & ~mPrereq - ); + int iTerm = p->aConstraint[i].iTermOffset; + Bitmask mThis = termFromWhereClause(pWC, iTerm)->prereqRight & ~mPrereq; if( mThis>mPrev && mThisneedToFreeIdxStr ) sqlite3_free(p->idxStr); freeIndexInfo(pParse->db, p); - WHERETRACE(0x800, ("END %s.addVirtual(), rc=%d\n", pSrc->pTab->zName, rc)); + WHERETRACE(0x800, ("END %s.addVirtual(), rc=%d\n", pSrc->pSTab->zName, rc)); return rc; } #endif /* SQLITE_OMIT_VIRTUALTABLE */ @@ -165403,7 +171388,7 @@ static int whereLoopAddOr( } #endif #ifndef SQLITE_OMIT_VIRTUALTABLE - if( IsVirtual(pItem->pTab) ){ + if( IsVirtual(pItem->pSTab) ){ rc = whereLoopAddVirtual(&sSubBuild, mPrereq, mUnusable); }else #endif @@ -165517,7 +171502,7 @@ static int whereLoopAddAll(WhereLoopBuilder *pBuilder){ mPrereq = 0; } #ifndef SQLITE_OMIT_VIRTUALTABLE - if( IsVirtual(pItem->pTab) ){ + if( IsVirtual(pItem->pSTab) ){ SrcItem *p; for(p=&pItem[1]; pfg.jointype & (JT_OUTER|JT_CROSS)) ){ @@ -165549,6 +171534,97 @@ static int whereLoopAddAll(WhereLoopBuilder *pBuilder){ return rc; } +/* Implementation of the order-by-subquery optimization: +** +** WhereLoop pLoop, which the iLoop-th term of the nested loop, is really +** a subquery or CTE that has an ORDER BY clause. See if any of the terms +** in the subquery ORDER BY clause will satisfy pOrderBy from the outer +** query. Mark off all satisfied terms (by setting bits in *pOBSat) and +** return TRUE if they do. If not, return false. +** +** Example: +** +** CREATE TABLE t1(a,b,c, PRIMARY KEY(a,b)); +** CREATE TABLE t2(x,y); +** WITH t3(p,q) AS MATERIALIZED (SELECT x+y, x-y FROM t2 ORDER BY x+y) +** SELECT * FROM t3 JOIN t1 ON a=q ORDER BY p, b; +** +** The CTE named "t3" comes out in the natural order of "p", so the first +** first them of "ORDER BY p,b" is satisfied by a sequential scan of "t3" +** and sorting only needs to occur on the second term "b". +** +** Limitations: +** +** (1) The optimization is not applied if the outer ORDER BY contains +** a COLLATE clause. The optimization might be applied if the +** outer ORDER BY uses NULLS FIRST, NULLS LAST, ASC, and/or DESC as +** long as the subquery ORDER BY does the same. But if the +** outer ORDER BY uses COLLATE, even a redundant COLLATE, the +** optimization is bypassed. +** +** (2) The subquery ORDER BY terms must exactly match subquery result +** columns, including any COLLATE annotations. This routine relies +** on iOrderByCol to do matching between order by terms and result +** columns, and iOrderByCol will not be set if the result column +** and ORDER BY collations differ. +** +** (3) The subquery and outer ORDER BY can be in opposite directions as +** long as the subquery is materialized. If the subquery is +** implemented as a co-routine, the sort orders must be in the same +** direction because there is no way to run a co-routine backwards. +*/ +static SQLITE_NOINLINE int wherePathMatchSubqueryOB( + WhereInfo *pWInfo, /* The WHERE clause */ + WhereLoop *pLoop, /* The nested loop term that is a subquery */ + int iLoop, /* Which level of the nested loop. 0==outermost */ + int iCur, /* Cursor used by the this loop */ + ExprList *pOrderBy, /* The ORDER BY clause on the whole query */ + Bitmask *pRevMask, /* When loops need to go in reverse order */ + Bitmask *pOBSat /* Which terms of pOrderBy are satisfied so far */ +){ + int iOB; /* Index into pOrderBy->a[] */ + int jSub; /* Index into pSubOB->a[] */ + u8 rev = 0; /* True if iOB and jSub sort in opposite directions */ + u8 revIdx = 0; /* Sort direction for jSub */ + Expr *pOBExpr; /* Current term of outer ORDER BY */ + ExprList *pSubOB; /* Complete ORDER BY on the subquery */ + + pSubOB = pLoop->u.btree.pOrderBy; + assert( pSubOB!=0 ); + for(iOB=0; (MASKBIT(iOB) & *pOBSat)!=0; iOB++){} + for(jSub=0; jSubnExpr && iOBnExpr; jSub++, iOB++){ + if( pSubOB->a[jSub].u.x.iOrderByCol==0 ) break; + pOBExpr = pOrderBy->a[iOB].pExpr; + if( pOBExpr->op!=TK_COLUMN && pOBExpr->op!=TK_AGG_COLUMN ) break; + if( pOBExpr->iTable!=iCur ) break; + if( pOBExpr->iColumn!=pSubOB->a[jSub].u.x.iOrderByCol-1 ) break; + if( (pWInfo->wctrlFlags & WHERE_GROUPBY)==0 ){ + u8 sfOB = pOrderBy->a[iOB].fg.sortFlags; /* sortFlags for iOB */ + u8 sfSub = pSubOB->a[jSub].fg.sortFlags; /* sortFlags for jSub */ + if( (sfSub & KEYINFO_ORDER_BIGNULL) != (sfOB & KEYINFO_ORDER_BIGNULL) ){ + break; + } + revIdx = sfSub & KEYINFO_ORDER_DESC; + if( jSub>0 ){ + if( (rev^revIdx)!=(sfOB & KEYINFO_ORDER_DESC) ){ + break; + } + }else{ + rev = revIdx ^ (sfOB & KEYINFO_ORDER_DESC); + if( rev ){ + if( (pLoop->wsFlags & WHERE_COROUTINE)!=0 ){ + /* Cannot run a co-routine in reverse order */ + break; + } + *pRevMask |= MASKBIT(iLoop); + } + } + } + *pOBSat |= MASKBIT(iOB); + } + return jSub>0; +} + /* ** Examine a WherePath (with the addition of the extra WhereLoop of the 6th ** parameters) to see if it outputs rows in the requested ORDER BY @@ -165649,10 +171725,12 @@ static i8 wherePathSatisfiesOrderBy( && ((wctrlFlags&(WHERE_DISTINCTBY|WHERE_SORTBYGROUP))!=WHERE_DISTINCTBY) ){ obSat = obDone; + }else{ + /* No further ORDER BY terms may be matched. So this call should + ** return >=0, not -1. Clear isOrderDistinct to ensure it does so. */ + isOrderDistinct = 0; } break; - }else if( wctrlFlags & WHERE_DISTINCTBY ){ - pLoop->u.btree.nDistinctCol = 0; } iCur = pWInfo->pTabList->a[pLoop->iTab].iCursor; @@ -165694,9 +171772,18 @@ static i8 wherePathSatisfiesOrderBy( if( (pLoop->wsFlags & WHERE_ONEROW)==0 ){ if( pLoop->wsFlags & WHERE_IPK ){ + if( pLoop->u.btree.pOrderBy + && OptimizationEnabled(db, SQLITE_OrderBySubq) + && wherePathMatchSubqueryOB(pWInfo,pLoop,iLoop,iCur, + pOrderBy,pRevMask, &obSat) + ){ + nColumn = 0; + isOrderDistinct = 0; + }else{ + nColumn = 1; + } pIndex = 0; nKeyCol = 0; - nColumn = 1; }else if( (pIndex = pLoop->u.btree.pIndex)==0 || pIndex->bUnordered ){ return 0; }else{ @@ -165706,7 +171793,7 @@ static i8 wherePathSatisfiesOrderBy( assert( pIndex->aiColumn[nColumn-1]==XN_ROWID || !HasRowid(pIndex->pTable)); /* All relevant terms of the index must also be non-NULL in order - ** for isOrderDistinct to be true. So the isOrderDistint value + ** for isOrderDistinct to be true. So the isOrderDistinct value ** computed here might be a false positive. Corrections will be ** made at tag-20210426-1 below */ isOrderDistinct = IsUniqueIndex(pIndex) @@ -165791,7 +171878,7 @@ static i8 wherePathSatisfiesOrderBy( } /* Find the ORDER BY term that corresponds to the j-th column - ** of the index and mark that ORDER BY term off + ** of the index and mark that ORDER BY term having been satisfied. */ isMatch = 0; for(i=0; bOnce && ia[i].pExpr; mTerm = sqlite3WhereExprUsage(&pWInfo->sMaskSet,p); - if( mTerm==0 && !sqlite3ExprIsConstant(p) ) continue; + if( mTerm==0 && !sqlite3ExprIsConstant(0,p) ) continue; if( (mTerm&~orderDistinctMask)==0 ){ obSat |= MASKBIT(i); } @@ -165998,6 +172085,216 @@ static LogEst whereSortingCost( return rSortCost; } +/* +** Compute the maximum number of paths in the solver algorithm, for +** queries that have three or more terms in the FROM clause. Queries with +** two or fewer FROM clause terms are handled by the caller. +** +** Query planning is NP-hard. We must limit the number of paths at +** each step of the solver search algorithm to avoid exponential behavior. +** +** The value returned is a tuning parameter. Currently the value is: +** +** 18 for star queries +** 12 otherwise +** +** For the purposes of this heuristic, a star-query is defined as a query +** with a large central table that is joined using an INNER JOIN, +** not CROSS or OUTER JOINs, against four or more smaller tables. +** The central table is called the "fact" table. The smaller tables +** that get joined are "dimension tables". Also, any table that is +** self-joined cannot be a dimension table; we assume that dimension +** tables may only be joined against fact tables. +** +** SIDE EFFECT: (and really the whole point of this subroutine) +** +** If pWInfo describes a star-query, then the cost for SCANs of dimension +** WhereLoops is increased to be slightly larger than the cost of a SCAN +** in the fact table. Only SCAN costs are increased. SEARCH costs are +** unchanged. This heuristic helps keep fact tables in outer loops. Without +** this heuristic, paths with fact tables in outer loops tend to get pruned +** by the mxChoice limit on the number of paths, resulting in poor query +** plans. See the starschema1.test test module for examples of queries +** that need this heuristic to find good query plans. +** +** This heuristic can be completely disabled, so that no query is +** considered a star-query, using SQLITE_TESTCTRL_OPTIMIZATION to +** disable the SQLITE_StarQuery optimization. In the CLI, the command +** to do that is: ".testctrl opt -starquery". +** +** HISTORICAL NOTES: +** +** This optimization was first added on 2024-05-09 by check-in 38db9b5c83d. +** The original optimization reduced the cost and output size estimate for +** fact tables to help them move to outer loops. But months later (as people +** started upgrading) performance regression reports started caming in, +** including: +** +** forum post b18ef983e68d06d1 (2024-12-21) +** forum post 0025389d0860af82 (2025-01-14) +** forum post d87570a145599033 (2025-01-17) +** +** To address these, the criteria for a star-query was tightened to exclude +** cases where the fact and dimensions are separated by an outer join, and +** the affect of star-schema detection was changed to increase the rRun cost +** on just full table scans of dimension tables, rather than reducing costs +** in the all access methods of the fact table. +*/ +static int computeMxChoice(WhereInfo *pWInfo){ + int nLoop = pWInfo->nLevel; /* Number of terms in the join */ + WhereLoop *pWLoop; /* For looping over WhereLoops */ + +#ifdef SQLITE_DEBUG + /* The star-query detection code below makes use of the following + ** properties of the WhereLoop list, so verify them before + ** continuing: + ** (1) .maskSelf is the bitmask corresponding to .iTab + ** (2) The WhereLoop list is in ascending .iTab order + */ + for(pWLoop=pWInfo->pLoops; pWLoop; pWLoop=pWLoop->pNextLoop){ + assert( pWLoop->maskSelf==MASKBIT(pWLoop->iTab) ); + assert( pWLoop->pNextLoop==0 || pWLoop->iTab<=pWLoop->pNextLoop->iTab ); + } +#endif /* SQLITE_DEBUG */ + + if( nLoop>=5 + && !pWInfo->bStarDone + && OptimizationEnabled(pWInfo->pParse->db, SQLITE_StarQuery) + ){ + SrcItem *aFromTabs; /* All terms of the FROM clause */ + int iFromIdx; /* Term of FROM clause is the candidate fact-table */ + Bitmask m; /* Bitmask for candidate fact-table */ + Bitmask mSelfJoin = 0; /* Tables that cannot be dimension tables */ + WhereLoop *pStart; /* Where to start searching for dimension-tables */ + + pWInfo->bStarDone = 1; /* Only do this computation once */ + + /* Look for fact tables with four or more dimensions where the + ** dimension tables are not separately from the fact tables by an outer + ** or cross join. Adjust cost weights if found. + */ + assert( !pWInfo->bStarUsed ); + aFromTabs = pWInfo->pTabList->a; + pStart = pWInfo->pLoops; + for(iFromIdx=0, m=1; iFromIdxfg.jointype & (JT_OUTER|JT_CROSS))!=0 ){ + /* If the candidate fact-table is the right table of an outer join + ** restrict the search for dimension-tables to be tables to the right + ** of the fact-table. */ + if( iFromIdx+4 > nLoop ) break; /* Impossible to reach nDep>=4 */ + while( pStart && pStart->iTab<=iFromIdx ){ + pStart = pStart->pNextLoop; + } + } + for(pWLoop=pStart; pWLoop; pWLoop=pWLoop->pNextLoop){ + if( (aFromTabs[pWLoop->iTab].fg.jointype & (JT_OUTER|JT_CROSS))!=0 ){ + /* Fact-tables and dimension-tables cannot be separated by an + ** outer join (at least for the definition of fact- and dimension- + ** used by this heuristic). */ + break; + } + if( (pWLoop->prereq & m)!=0 /* pWInfo depends on iFromIdx */ + && (pWLoop->maskSelf & mSeen)==0 /* pWInfo not already a dependency */ + && (pWLoop->maskSelf & mSelfJoin)==0 /* Not a self-join */ + ){ + if( aFromTabs[pWLoop->iTab].pSTab==pFactTab->pSTab ){ + mSelfJoin |= m; + }else{ + nDep++; + mSeen |= pWLoop->maskSelf; + } + } + } + if( nDep<=3 ) continue; + + /* If we reach this point, it means that pFactTab is a fact table + ** with four or more dimensions connected by inner joins. Proceed + ** to make cost adjustments. */ + +#ifdef WHERETRACE_ENABLED + /* Make sure rStarDelta values are initialized */ + if( !pWInfo->bStarUsed ){ + for(pWLoop=pWInfo->pLoops; pWLoop; pWLoop=pWLoop->pNextLoop){ + pWLoop->rStarDelta = 0; + } + } +#endif + pWInfo->bStarUsed = 1; + + /* Compute the maximum cost of any WhereLoop for the + ** fact table plus one epsilon */ + mxRun = LOGEST_MIN; + for(pWLoop=pStart; pWLoop; pWLoop=pWLoop->pNextLoop){ + if( pWLoop->iTabiTab>iFromIdx ) break; + if( pWLoop->rRun>mxRun ) mxRun = pWLoop->rRun; + } + if( ALWAYS(mxRunpNextLoop){ + if( (pWLoop->maskSelf & mSeen)==0 ) continue; + if( pWLoop->nLTerm ) continue; + if( pWLoop->rRuniTab; + sqlite3DebugPrintf( + "Increase SCAN cost of dimension %s(%d) of fact %s(%d) to %d\n", + pDim->zAlias ? pDim->zAlias: pDim->pSTab->zName, pWLoop->iTab, + pFactTab->zAlias ? pFactTab->zAlias : pFactTab->pSTab->zName, + iFromIdx, mxRun + ); + } + pWLoop->rStarDelta = mxRun - pWLoop->rRun; +#endif /* WHERETRACE_ENABLED */ + pWLoop->rRun = mxRun; + } + } + } +#ifdef WHERETRACE_ENABLED /* 0x80000 */ + if( (sqlite3WhereTrace & 0x80000)!=0 && pWInfo->bStarUsed ){ + sqlite3DebugPrintf("WhereLoops changed by star-query heuristic:\n"); + for(pWLoop=pWInfo->pLoops; pWLoop; pWLoop=pWLoop->pNextLoop){ + if( pWLoop->rStarDelta ){ + sqlite3WhereLoopPrint(pWLoop, &pWInfo->sWC); + } + } + } +#endif + } + return pWInfo->bStarUsed ? 18 : 12; +} + +/* +** Two WhereLoop objects, pCandidate and pBaseline, are known to have the +** same cost. Look deep into each to see if pCandidate is even slightly +** better than pBaseline. Return false if it is, if pCandidate is is preferred. +** Return true if pBaseline is preferred or if we cannot tell the difference. +** +** Result Meaning +** -------- ---------------------------------------------------------- +** true We cannot tell the difference in pCandidate and pBaseline +** false pCandidate seems like a better choice than pBaseline +*/ +static SQLITE_NOINLINE int whereLoopIsNoBetter( + const WhereLoop *pCandidate, + const WhereLoop *pBaseline +){ + if( (pCandidate->wsFlags & WHERE_INDEXED)==0 ) return 1; + if( (pBaseline->wsFlags & WHERE_INDEXED)==0 ) return 1; + if( pCandidate->u.btree.pIndex->szIdxRow < + pBaseline->u.btree.pIndex->szIdxRow ) return 0; + return 1; +} + /* ** Given the list of WhereLoop objects at pWInfo->pLoops, this routine ** attempts to find the lowest cost path that visits each WhereLoop @@ -166019,7 +172316,7 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ int mxI = 0; /* Index of next entry to replace */ int nOrderBy; /* Number of ORDER BY clause terms */ LogEst mxCost = 0; /* Maximum cost of a set of paths */ - LogEst mxUnsorted = 0; /* Maximum unsorted cost of a set of path */ + LogEst mxUnsort = 0; /* Maximum unsorted cost of a set of path */ int nTo, nFrom; /* Number of valid entries in aTo[] and aFrom[] */ WherePath *aFrom; /* All nFrom paths at the previous level */ WherePath *aTo; /* The nTo best paths at the current level */ @@ -166033,13 +172330,27 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ pParse = pWInfo->pParse; nLoop = pWInfo->nLevel; - /* TUNING: For simple queries, only the best path is tracked. - ** For 2-way joins, the 5 best paths are followed. - ** For joins of 3 or more tables, track the 10 best paths */ - mxChoice = (nLoop<=1) ? 1 : (nLoop==2 ? 5 : 10); - assert( nLoop<=pWInfo->pTabList->nSrc ); WHERETRACE(0x002, ("---- begin solver. (nRowEst=%d, nQueryLoop=%d)\n", nRowEst, pParse->nQueryLoop)); + /* TUNING: mxChoice is the maximum number of possible paths to preserve + ** at each step. Based on the number of loops in the FROM clause: + ** + ** nLoop mxChoice + ** ----- -------- + ** 1 1 // the most common case + ** 2 5 + ** 3+ 12 or 18 // see computeMxChoice() + */ + if( nLoop<=1 ){ + mxChoice = 1; + }else if( nLoop==2 ){ + mxChoice = 5; + }else if( pParse->nErr ){ + mxChoice = 1; + }else{ + mxChoice = computeMxChoice(pWInfo); + } + assert( nLoop<=pWInfo->pTabList->nSrc ); /* If nRowEst is zero and there is an ORDER BY clause, ignore it. In this ** case the purpose of this call is to estimate the number of rows returned @@ -166104,7 +172415,7 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ for(pWLoop=pWInfo->pLoops; pWLoop; pWLoop=pWLoop->pNextLoop){ LogEst nOut; /* Rows visited by (pFrom+pWLoop) */ LogEst rCost; /* Cost of path (pFrom+pWLoop) */ - LogEst rUnsorted; /* Unsorted cost of (pFrom+pWLoop) */ + LogEst rUnsort; /* Unsorted cost of (pFrom+pWLoop) */ i8 isOrdered; /* isOrdered for (pFrom+pWLoop) */ Bitmask maskNew; /* Mask of src visited by (..) */ Bitmask revMask; /* Mask of rev-order loops for (..) */ @@ -166122,8 +172433,11 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ /* At this point, pWLoop is a candidate to be the next loop. ** Compute its cost */ - rUnsorted = sqlite3LogEstAdd(pWLoop->rSetup,pWLoop->rRun + pFrom->nRow); - rUnsorted = sqlite3LogEstAdd(rUnsorted, pFrom->rUnsorted); + rUnsort = pWLoop->rRun + pFrom->nRow; + if( pWLoop->rSetup ){ + rUnsort = sqlite3LogEstAdd(pWLoop->rSetup, rUnsort); + } + rUnsort = sqlite3LogEstAdd(rUnsort, pFrom->rUnsort); nOut = pFrom->nRow + pWLoop->nOut; maskNew = pFrom->maskLoop | pWLoop->maskSelf; isOrdered = pFrom->isOrdered; @@ -166145,31 +172459,39 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ ** extra encouragement to the query planner to select a plan ** where the rows emerge in the correct order without any sorting ** required. */ - rCost = sqlite3LogEstAdd(rUnsorted, aSortCost[isOrdered]) + 3; + rCost = sqlite3LogEstAdd(rUnsort, aSortCost[isOrdered]) + 3; WHERETRACE(0x002, ("---- sort cost=%-3d (%d/%d) increases cost %3d to %-3d\n", aSortCost[isOrdered], (nOrderBy-isOrdered), nOrderBy, - rUnsorted, rCost)); + rUnsort, rCost)); }else{ - rCost = rUnsorted; - rUnsorted -= 2; /* TUNING: Slight bias in favor of no-sort plans */ + rCost = rUnsort; + rUnsort -= 2; /* TUNING: Slight bias in favor of no-sort plans */ } /* Check to see if pWLoop should be added to the set of ** mxChoice best-so-far paths. ** ** First look for an existing path among best-so-far paths - ** that covers the same set of loops and has the same isOrdered - ** setting as the current path candidate. + ** that: + ** (1) covers the same set of loops, and + ** (2) has a compatible isOrdered value. + ** + ** "Compatible isOrdered value" means either + ** (A) both have isOrdered==-1, or + ** (B) both have isOrder>=0, or + ** (C) ordering does not matter because this is the last round + ** of the solver. ** ** The term "((pTo->isOrdered^isOrdered)&0x80)==0" is equivalent ** to (pTo->isOrdered==(-1))==(isOrdered==(-1))" for the range ** of legal values for isOrdered, -1..64. */ + testcase( nTo==0 ); for(jj=0, pTo=aTo; jjmaskLoop==maskNew - && ((pTo->isOrdered^isOrdered)&0x80)==0 + && ( ((pTo->isOrdered^isOrdered)&0x80)==0 || iLoop==nLoop-1 ) ){ testcase( jj==nTo-1 ); break; @@ -166178,7 +172500,7 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ if( jj>=nTo ){ /* None of the existing best-so-far paths match the candidate. */ if( nTo>=mxChoice - && (rCost>mxCost || (rCost==mxCost && rUnsorted>=mxUnsorted)) + && (rCost>mxCost || (rCost==mxCost && rUnsort>=mxUnsort)) ){ /* The current candidate is no better than any of the mxChoice ** paths currently in the best-so-far buffer. So discard @@ -166186,7 +172508,7 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ #ifdef WHERETRACE_ENABLED /* 0x4 */ if( sqlite3WhereTrace&0x4 ){ sqlite3DebugPrintf("Skip %s cost=%-3d,%3d,%3d order=%c\n", - wherePathName(pFrom, iLoop, pWLoop), rCost, nOut, rUnsorted, + wherePathName(pFrom, iLoop, pWLoop), rCost, nOut, rUnsort, isOrdered>=0 ? isOrdered+'0' : '?'); } #endif @@ -166205,7 +172527,7 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ #ifdef WHERETRACE_ENABLED /* 0x4 */ if( sqlite3WhereTrace&0x4 ){ sqlite3DebugPrintf("New %s cost=%-3d,%3d,%3d order=%c\n", - wherePathName(pFrom, iLoop, pWLoop), rCost, nOut, rUnsorted, + wherePathName(pFrom, iLoop, pWLoop), rCost, nOut, rUnsort, isOrdered>=0 ? isOrdered+'0' : '?'); } #endif @@ -166216,24 +172538,23 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ ** pTo or if the candidate should be skipped. ** ** The conditional is an expanded vector comparison equivalent to: - ** (pTo->rCost,pTo->nRow,pTo->rUnsorted) <= (rCost,nOut,rUnsorted) + ** (pTo->rCost,pTo->nRow,pTo->rUnsort) <= (rCost,nOut,rUnsort) */ - if( pTo->rCostrCost==rCost - && (pTo->nRownRow==nOut && pTo->rUnsorted<=rUnsorted) - ) - ) + if( (pTo->rCostrCost==rCost && pTo->nRowrCost==rCost && pTo->nRow==nOut && pTo->rUnsortrCost==rCost && pTo->nRow==nOut && pTo->rUnsort==rUnsort + && whereLoopIsNoBetter(pWLoop, pTo->aLoop[iLoop]) ) ){ #ifdef WHERETRACE_ENABLED /* 0x4 */ if( sqlite3WhereTrace&0x4 ){ sqlite3DebugPrintf( "Skip %s cost=%-3d,%3d,%3d order=%c", - wherePathName(pFrom, iLoop, pWLoop), rCost, nOut, rUnsorted, + wherePathName(pFrom, iLoop, pWLoop), rCost, nOut, rUnsort, isOrdered>=0 ? isOrdered+'0' : '?'); sqlite3DebugPrintf(" vs %s cost=%-3d,%3d,%3d order=%c\n", wherePathName(pTo, iLoop+1, 0), pTo->rCost, pTo->nRow, - pTo->rUnsorted, pTo->isOrdered>=0 ? pTo->isOrdered+'0' : '?'); + pTo->rUnsort, pTo->isOrdered>=0 ? pTo->isOrdered+'0' : '?'); } #endif /* Discard the candidate path from further consideration */ @@ -166247,11 +172568,11 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ if( sqlite3WhereTrace&0x4 ){ sqlite3DebugPrintf( "Update %s cost=%-3d,%3d,%3d order=%c", - wherePathName(pFrom, iLoop, pWLoop), rCost, nOut, rUnsorted, + wherePathName(pFrom, iLoop, pWLoop), rCost, nOut, rUnsort, isOrdered>=0 ? isOrdered+'0' : '?'); sqlite3DebugPrintf(" was %s cost=%-3d,%3d,%3d order=%c\n", wherePathName(pTo, iLoop+1, 0), pTo->rCost, pTo->nRow, - pTo->rUnsorted, pTo->isOrdered>=0 ? pTo->isOrdered+'0' : '?'); + pTo->rUnsort, pTo->isOrdered>=0 ? pTo->isOrdered+'0' : '?'); } #endif } @@ -166260,20 +172581,20 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ pTo->revLoop = revMask; pTo->nRow = nOut; pTo->rCost = rCost; - pTo->rUnsorted = rUnsorted; + pTo->rUnsort = rUnsort; pTo->isOrdered = isOrdered; memcpy(pTo->aLoop, pFrom->aLoop, sizeof(WhereLoop*)*iLoop); pTo->aLoop[iLoop] = pWLoop; if( nTo>=mxChoice ){ mxI = 0; mxCost = aTo[0].rCost; - mxUnsorted = aTo[0].nRow; + mxUnsort = aTo[0].nRow; for(jj=1, pTo=&aTo[1]; jjrCost>mxCost - || (pTo->rCost==mxCost && pTo->rUnsorted>mxUnsorted) + || (pTo->rCost==mxCost && pTo->rUnsort>mxUnsort) ){ mxCost = pTo->rCost; - mxUnsorted = pTo->rUnsorted; + mxUnsort = pTo->rUnsort; mxI = jj; } } @@ -166283,17 +172604,32 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ #ifdef WHERETRACE_ENABLED /* >=2 */ if( sqlite3WhereTrace & 0x02 ){ + LogEst rMin, rFloor = 0; + int nDone = 0; + int nProgress; sqlite3DebugPrintf("---- after round %d ----\n", iLoop); - for(ii=0, pTo=aTo; iirCost, pTo->nRow, - pTo->isOrdered>=0 ? (pTo->isOrdered+'0') : '?'); - if( pTo->isOrdered>0 ){ - sqlite3DebugPrintf(" rev=0x%llx\n", pTo->revLoop); - }else{ - sqlite3DebugPrintf("\n"); + do{ + nProgress = 0; + rMin = 0x7fff; + for(ii=0, pTo=aTo; iirCost>rFloor && pTo->rCostrCost; + } + for(ii=0, pTo=aTo; iirCost==rMin ){ + sqlite3DebugPrintf(" %s cost=%-3d nrow=%-3d order=%c", + wherePathName(pTo, iLoop+1, 0), pTo->rCost, pTo->nRow, + pTo->isOrdered>=0 ? (pTo->isOrdered+'0') : '?'); + if( pTo->isOrdered>0 ){ + sqlite3DebugPrintf(" rev=0x%llx\n", pTo->revLoop); + }else{ + sqlite3DebugPrintf("\n"); + } + nDone++; + nProgress++; + } } - } + rFloor = rMin; + }while( nDone0 ); } #endif @@ -166310,11 +172646,10 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ return SQLITE_ERROR; } - /* Find the lowest cost path. pFrom will be left pointing to that path */ + /* Only one path is available, which is the best path */ + assert( nFrom==1 ); pFrom = aFrom; - for(ii=1; iirCost>aFrom[ii].rCost ) pFrom = &aFrom[ii]; - } + assert( pWInfo->nLevel==nLoop ); /* Load the lowest cost path into pWInfo */ for(iLoop=0; iLoopisOrdered==pWInfo->pOrderBy->nExpr ){ pWInfo->eDistinct = WHERE_DISTINCT_ORDERED; } - if( pWInfo->pSelect->pOrderBy - && pWInfo->nOBSat > pWInfo->pSelect->pOrderBy->nExpr ){ - pWInfo->nOBSat = pWInfo->pSelect->pOrderBy->nExpr; - } + /* vvv--- See check-in [12ad822d9b827777] on 2023-03-16 ---vvv */ + assert( pWInfo->pSelect->pOrderBy==0 + || pWInfo->nOBSat <= pWInfo->pSelect->pOrderBy->nExpr ); }else{ pWInfo->revMask = pFrom->revLoop; if( pWInfo->nOBSat<=0 ){ @@ -166388,14 +172722,96 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ } } - pWInfo->nRowOut = pFrom->nRow; +#ifdef WHERETRACE_ENABLED + pWInfo->rTotalCost = pFrom->rCost; +#endif /* Free temporary memory and return success */ sqlite3StackFreeNN(pParse->db, pSpace); return SQLITE_OK; } +/* +** This routine implements a heuristic designed to improve query planning. +** This routine is called in between the first and second call to +** wherePathSolver(). Hence the name "Interstage" "Heuristic". +** +** The first call to wherePathSolver() (hereafter just "solver()") computes +** the best path without regard to the order of the outputs. The second call +** to the solver() builds upon the first call to try to find an alternative +** path that satisfies the ORDER BY clause. +** +** This routine looks at the results of the first solver() run, and for +** every FROM clause term in the resulting query plan that uses an equality +** constraint against an index, disable other WhereLoops for that same +** FROM clause term that would try to do a full-table scan. This prevents +** an index search from being converted into a full-table scan in order to +** satisfy an ORDER BY clause, since even though we might get slightly better +** performance using the full-scan without sorting if the output size +** estimates are very precise, we might also get severe performance +** degradation using the full-scan if the output size estimate is too large. +** It is better to err on the side of caution. +** +** Except, if the first solver() call generated a full-table scan in an outer +** loop then stop this analysis at the first full-scan, since the second +** solver() run might try to swap that full-scan for another in order to +** get the output into the correct order. In other words, we allow a +** rewrite like this: +** +** First Solver() Second Solver() +** |-- SCAN t1 |-- SCAN t2 +** |-- SEARCH t2 `-- SEARCH t1 +** `-- SORT USING B-TREE +** +** The purpose of this routine is to disallow rewrites such as: +** +** First Solver() Second Solver() +** |-- SEARCH t1 |-- SCAN t2 <--- bad! +** |-- SEARCH t2 `-- SEARCH t1 +** `-- SORT USING B-TREE +** +** See test cases in test/whereN.test for the real-world query that +** originally provoked this heuristic. +*/ +static SQLITE_NOINLINE void whereInterstageHeuristic(WhereInfo *pWInfo){ + int i; +#ifdef WHERETRACE_ENABLED + int once = 0; +#endif + for(i=0; inLevel; i++){ + WhereLoop *p = pWInfo->a[i].pWLoop; + if( p==0 ) break; + if( (p->wsFlags & WHERE_VIRTUALTABLE)!=0 ){ + /* Treat a vtab scan as similar to a full-table scan */ + break; + } + if( (p->wsFlags & (WHERE_COLUMN_EQ|WHERE_COLUMN_NULL|WHERE_COLUMN_IN))!=0 ){ + u8 iTab = p->iTab; + WhereLoop *pLoop; + for(pLoop=pWInfo->pLoops; pLoop; pLoop=pLoop->pNextLoop){ + if( pLoop->iTab!=iTab ) continue; + if( (pLoop->wsFlags & (WHERE_CONSTRAINT|WHERE_AUTO_INDEX))!=0 ){ + /* Auto-index and index-constrained loops allowed to remain */ + continue; + } +#ifdef WHERETRACE_ENABLED + if( sqlite3WhereTrace & 0x80 ){ + if( once==0 ){ + sqlite3DebugPrintf("Loops disabled by interstage heuristic:\n"); + once = 1; + } + sqlite3WhereLoopPrint(pLoop, &pWInfo->sWC); + } +#endif /* WHERETRACE_ENABLED */ + pLoop->prereq = ALLBITS; /* Prevent 2nd solver() from using this one */ + } + }else{ + break; + } + } +} + /* ** Most queries use only a single table (they are not joins) and have ** simple == constraints against indexed fields. This routine attempts @@ -166423,7 +172839,7 @@ static int whereShortCut(WhereLoopBuilder *pBuilder){ if( pWInfo->wctrlFlags & WHERE_OR_SUBCLAUSE ) return 0; assert( pWInfo->pTabList->nSrc>=1 ); pItem = pWInfo->pTabList->a; - pTab = pItem->pTab; + pTab = pItem->pSTab; if( IsVirtual(pTab) ) return 0; if( pItem->fg.isIndexedBy || pItem->fg.notIndexed ){ testcase( pItem->fg.isIndexedBy ); @@ -166564,6 +172980,10 @@ static void showAllWhereLoops(WhereInfo *pWInfo, WhereClause *pWC){ ** the right-most table of a subquery that was flattened into the ** main query and that subquery was the right-hand operand of an ** inner join that held an ON or USING clause. +** 6) The ORDER BY clause has 63 or fewer terms +** 7) The omit-noop-join optimization is enabled. +** +** Items (1), (6), and (7) are checked by the caller. ** ** For example, given: ** @@ -166609,6 +173029,7 @@ static SQLITE_NOINLINE Bitmask whereOmitNoopJoin( WhereTerm *pTerm, *pEnd; SrcItem *pItem; WhereLoop *pLoop; + Bitmask m1; pLoop = pWInfo->a[i].pWLoop; pItem = &pWInfo->pTabList->a[pLoop->iTab]; if( (pItem->fg.jointype & (JT_LEFT|JT_RIGHT))!=JT_LEFT ) continue; @@ -166629,13 +173050,16 @@ static SQLITE_NOINLINE Bitmask whereOmitNoopJoin( } if( hasRightJoin && ExprHasProperty(pTerm->pExpr, EP_InnerON) - && pTerm->pExpr->w.iJoin==pItem->iCursor + && NEVER(pTerm->pExpr->w.iJoin==pItem->iCursor) ){ break; /* restriction (5) */ } } if( pTerm drop loop %c not used\n", pLoop->cId)); + WHERETRACE(0xffffffff,("-> omit unused FROM-clause term %c\n",pLoop->cId)); + m1 = MASKBIT(i)-1; + testcase( ((pWInfo->revMask>>1) & ~m1)!=0 ); + pWInfo->revMask = (m1 & pWInfo->revMask) | ((pWInfo->revMask>>1) & ~m1); notReady &= ~pLoop->maskSelf; for(pTerm=pWInfo->sWC.a; pTermprereqAll & pLoop->maskSelf)!=0 ){ @@ -166682,9 +173106,9 @@ static SQLITE_NOINLINE void whereCheckIfBloomFilterIsUseful( WhereLoop *pLoop = pWInfo->a[i].pWLoop; const unsigned int reqFlags = (WHERE_SELFCULL|WHERE_COLUMN_EQ); SrcItem *pItem = &pWInfo->pTabList->a[pLoop->iTab]; - Table *pTab = pItem->pTab; + Table *pTab = pItem->pSTab; if( (pTab->tabFlags & TF_HasStat1)==0 ) break; - pTab->tabFlags |= TF_StatsUsed; + pTab->tabFlags |= TF_MaybeReanalyze; if( i>=1 && (pLoop->wsFlags & reqFlags)==reqFlags /* vvvvvv--- Always the case if WHERE_COLUMN_EQ is defined */ @@ -166730,34 +173154,14 @@ static SQLITE_NOINLINE void whereAddIndexedExpr( for(i=0; inColumn; i++){ Expr *pExpr; int j = pIdx->aiColumn[i]; - int bMaybeNullRow; if( j==XN_EXPR ){ pExpr = pIdx->aColExpr->a[i].pExpr; - testcase( pTabItem->fg.jointype & JT_LEFT ); - testcase( pTabItem->fg.jointype & JT_RIGHT ); - testcase( pTabItem->fg.jointype & JT_LTORJ ); - bMaybeNullRow = (pTabItem->fg.jointype & (JT_LEFT|JT_LTORJ|JT_RIGHT))!=0; }else if( j>=0 && (pTab->aCol[j].colFlags & COLFLAG_VIRTUAL)!=0 ){ pExpr = sqlite3ColumnExpr(pTab, &pTab->aCol[j]); - bMaybeNullRow = 0; }else{ continue; } - if( sqlite3ExprIsConstant(pExpr) ) continue; - if( pExpr->op==TK_FUNCTION ){ - /* Functions that might set a subtype should not be replaced by the - ** value taken from an expression index since the index omits the - ** subtype. https://sqlite.org/forum/forumpost/68d284c86b082c3e */ - int n; - FuncDef *pDef; - sqlite3 *db = pParse->db; - assert( ExprUseXList(pExpr) ); - n = pExpr->x.pList ? pExpr->x.pList->nExpr : 0; - pDef = sqlite3FindFunction(db, pExpr->u.zToken, n, ENC(db), 0); - if( pDef==0 || (pDef->funcFlags & SQLITE_RESULT_SUBTYPE)!=0 ){ - continue; - } - } + if( sqlite3ExprIsConstant(0,pExpr) ) continue; p = sqlite3DbMallocRaw(pParse->db, sizeof(IndexedExpr)); if( p==0 ) break; p->pIENext = pParse->pIdxEpr; @@ -166771,7 +173175,7 @@ static SQLITE_NOINLINE void whereAddIndexedExpr( p->iDataCur = pTabItem->iCursor; p->iIdxCur = iIdxCur; p->iIdxCol = i; - p->bMaybeNullRow = bMaybeNullRow; + p->bMaybeNullRow = (pTabItem->fg.jointype & (JT_LEFT|JT_LTORJ|JT_RIGHT))!=0; if( sqlite3IndexAffinityStr(pParse->db, pIdx) ){ p->aff = pIdx->zColAff[i]; } @@ -166800,8 +173204,8 @@ static SQLITE_NOINLINE void whereReverseScanOrder(WhereInfo *pWInfo){ SrcItem *pItem = &pWInfo->pTabList->a[ii]; if( !pItem->fg.isCte || pItem->u2.pCteUse->eM10d!=M10d_Yes - || NEVER(pItem->pSelect==0) - || pItem->pSelect->pOrderBy==0 + || NEVER(pItem->fg.isSubquery==0) + || pItem->u4.pSubq->pSelect->pOrderBy==0 ){ pWInfo->revMask |= MASKBIT(ii); } @@ -166939,6 +173343,7 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( if( pOrderBy && pOrderBy->nExpr>=BMS ){ pOrderBy = 0; wctrlFlags &= ~WHERE_WANT_DISTINCT; + wctrlFlags |= WHERE_KEEP_ALL_JOINS; /* Disable omit-noop-join opt */ } /* The number of tables in the FROM clause is limited by the number of @@ -166964,10 +173369,7 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( ** field (type Bitmask) it must be aligned on an 8-byte boundary on ** some architectures. Hence the ROUND8() below. */ - nByteWInfo = ROUND8P(sizeof(WhereInfo)); - if( nTabList>1 ){ - nByteWInfo = ROUND8P(nByteWInfo + (nTabList-1)*sizeof(WhereLevel)); - } + nByteWInfo = SZ_WHEREINFO(nTabList); pWInfo = sqlite3DbMallocRawNN(db, nByteWInfo + sizeof(WhereLoop)); if( db->mallocFailed ){ sqlite3DbFree(db, pWInfo); @@ -167021,7 +173423,11 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( ){ pWInfo->eDistinct = WHERE_DISTINCT_UNIQUE; } - ExplainQueryPlan((pParse, 0, "SCAN CONSTANT ROW")); + if( ALWAYS(pWInfo->pSelect) + && (pWInfo->pSelect->selFlags & SF_MultiValue)==0 + ){ + ExplainQueryPlan((pParse, 0, "SCAN CONSTANT ROW")); + } }else{ /* Assign a bit from the bitmask to every term in the FROM clause. ** @@ -167174,12 +173580,14 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( wherePathSolver(pWInfo, 0); if( db->mallocFailed ) goto whereBeginError; if( pWInfo->pOrderBy ){ - wherePathSolver(pWInfo, pWInfo->nRowOut+1); + whereInterstageHeuristic(pWInfo); + wherePathSolver(pWInfo, pWInfo->nRowOut<0 ? 1 : pWInfo->nRowOut+1); if( db->mallocFailed ) goto whereBeginError; } /* TUNING: Assume that a DISTINCT clause on a subquery reduces - ** the output size by a factor of 8 (LogEst -30). + ** the output size by a factor of 8 (LogEst -30). Search for + ** tag-20250414a to see other cases. */ if( (pWInfo->wctrlFlags & WHERE_WANT_DISTINCT)!=0 ){ WHERETRACE(0x0080,("nRowOut reduced from %d to %d due to DISTINCT\n", @@ -167198,7 +173606,8 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( assert( db->mallocFailed==0 ); #ifdef WHERETRACE_ENABLED if( sqlite3WhereTrace ){ - sqlite3DebugPrintf("---- Solution nRow=%d", pWInfo->nRowOut); + sqlite3DebugPrintf("---- Solution cost=%d, nRow=%d", + pWInfo->rTotalCost, pWInfo->nRowOut); if( pWInfo->nOBSat>0 ){ sqlite3DebugPrintf(" ORDERBY=%d,0x%llx", pWInfo->nOBSat, pWInfo->revMask); } @@ -167234,10 +173643,10 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( ** in-line sqlite3WhereCodeOneLoopStart() for performance reasons. */ notReady = ~(Bitmask)0; - if( pWInfo->nLevel>=2 - && pResultSet!=0 /* these two combine to guarantee */ - && 0==(wctrlFlags & WHERE_AGG_DISTINCT) /* condition (1) above */ - && OptimizationEnabled(db, SQLITE_OmitNoopJoin) + if( pWInfo->nLevel>=2 /* Must be a join, or this opt8n is pointless */ + && pResultSet!=0 /* Condition (1) */ + && 0==(wctrlFlags & (WHERE_AGG_DISTINCT|WHERE_KEEP_ALL_JOINS)) /* (1),(6) */ + && OptimizationEnabled(db, SQLITE_OmitNoopJoin) /* (7) */ ){ notReady = whereOmitNoopJoin(pWInfo, notReady); nTabList = pWInfo->nLevel; @@ -167285,15 +173694,15 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( if( (wctrlFlags & WHERE_ONEPASS_DESIRED)!=0 ){ int wsFlags = pWInfo->a[0].pWLoop->wsFlags; int bOnerow = (wsFlags & WHERE_ONEROW)!=0; - assert( !(wsFlags & WHERE_VIRTUALTABLE) || IsVirtual(pTabList->a[0].pTab) ); + assert( !(wsFlags&WHERE_VIRTUALTABLE) || IsVirtual(pTabList->a[0].pSTab) ); if( bOnerow || ( 0!=(wctrlFlags & WHERE_ONEPASS_MULTIROW) - && !IsVirtual(pTabList->a[0].pTab) + && !IsVirtual(pTabList->a[0].pSTab) && (0==(wsFlags & WHERE_MULTI_OR) || (wctrlFlags & WHERE_DUPLICATES_OK)) && OptimizationEnabled(db, SQLITE_OnePass) )){ pWInfo->eOnePass = bOnerow ? ONEPASS_SINGLE : ONEPASS_MULTI; - if( HasRowid(pTabList->a[0].pTab) && (wsFlags & WHERE_IDX_ONLY) ){ + if( HasRowid(pTabList->a[0].pSTab) && (wsFlags & WHERE_IDX_ONLY) ){ if( wctrlFlags & WHERE_ONEPASS_MULTIROW ){ bFordelete = OPFLAG_FORDELETE; } @@ -167311,9 +173720,17 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( SrcItem *pTabItem; pTabItem = &pTabList->a[pLevel->iFrom]; - pTab = pTabItem->pTab; + pTab = pTabItem->pSTab; iDb = sqlite3SchemaToIndex(db, pTab->pSchema); pLoop = pLevel->pWLoop; + pLevel->addrBrk = sqlite3VdbeMakeLabel(pParse); + if( ii==0 || (pTabItem[0].fg.jointype & JT_LEFT)!=0 ){ + pLevel->addrHalt = pLevel->addrBrk; + }else if( pWInfo->a[ii-1].pRJ ){ + pLevel->addrHalt = pWInfo->a[ii-1].addrBrk; + }else{ + pLevel->addrHalt = pWInfo->a[ii-1].addrHalt; + } if( (pTab->tabFlags & TF_Ephemeral)!=0 || IsView(pTab) ){ /* Do nothing */ }else @@ -167365,6 +173782,13 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( sqlite3VdbeAddOp4Dup8(v, OP_ColumnsUsed, pTabItem->iCursor, 0, 0, (const u8*)&pTabItem->colUsed, P4_INT64); #endif + if( ii>=2 + && (pTabItem[0].fg.jointype & (JT_LTORJ|JT_LEFT))==0 + && pLevel->addrHalt==pWInfo->a[0].addrHalt + ){ + sqlite3VdbeAddOp2(v, OP_IfEmpty, pTabItem->iCursor, pWInfo->iBreak); + VdbeCoverage(v); + } }else{ sqlite3TableLock(pParse, iDb, pTab->tnum, 0, pTab->zName); } @@ -167382,7 +173806,7 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( iIndexCur = pLevel->iTabCur; op = 0; }else if( pWInfo->eOnePass!=ONEPASS_OFF ){ - Index *pJ = pTabItem->pTab->pIndex; + Index *pJ = pTabItem->pSTab->pIndex; iIndexCur = iAuxArg; assert( wctrlFlags & WHERE_ONEPASS_DESIRED ); while( ALWAYS(pJ) && pJ!=pIx ){ @@ -167449,7 +173873,7 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( sqlite3VdbeAddOp2(v, OP_Blob, 65536, pRJ->regBloom); pRJ->regReturn = ++pParse->nMem; sqlite3VdbeAddOp2(v, OP_Null, 0, pRJ->regReturn); - assert( pTab==pTabItem->pTab ); + assert( pTab==pTabItem->pSTab ); if( HasRowid(pTab) ){ KeyInfo *pInfo; sqlite3VdbeAddOp2(v, OP_OpenEphemeral, pRJ->iMatch, 1); @@ -167488,13 +173912,18 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( wsFlags = pLevel->pWLoop->wsFlags; pSrc = &pTabList->a[pLevel->iFrom]; if( pSrc->fg.isMaterialized ){ - if( pSrc->fg.isCorrelated ){ - sqlite3VdbeAddOp2(v, OP_Gosub, pSrc->regReturn, pSrc->addrFillSub); + Subquery *pSubq; + int iOnce = 0; + assert( pSrc->fg.isSubquery ); + pSubq = pSrc->u4.pSubq; + if( pSrc->fg.isCorrelated==0 ){ + iOnce = sqlite3VdbeAddOp0(v, OP_Once); VdbeCoverage(v); }else{ - int iOnce = sqlite3VdbeAddOp0(v, OP_Once); VdbeCoverage(v); - sqlite3VdbeAddOp2(v, OP_Gosub, pSrc->regReturn, pSrc->addrFillSub); - sqlite3VdbeJumpHere(v, iOnce); + iOnce = 0; } + sqlite3VdbeAddOp2(v, OP_Gosub, pSubq->regReturn, pSubq->addrFillSub); + VdbeComment((v, "materialize %!S", pSrc)); + if( iOnce ) sqlite3VdbeJumpHere(v, iOnce); } assert( pTabList == pWInfo->pTabList ); if( (wsFlags & (WHERE_AUTO_INDEX|WHERE_BLOOMFILTER))!=0 ){ @@ -167554,29 +173983,10 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( ){ if( (db->flags & SQLITE_VdbeAddopTrace)==0 ) return; sqlite3VdbePrintOp(0, pc, pOp); + sqlite3ShowWhereTerm(0); /* So compiler won't complain about unused func */ } #endif -#ifdef SQLITE_DEBUG -/* -** Return true if cursor iCur is opened by instruction k of the -** bytecode. Used inside of assert() only. -*/ -static int cursorIsOpen(Vdbe *v, int iCur, int k){ - while( k>=0 ){ - VdbeOp *pOp = sqlite3VdbeGetOp(v,k--); - if( pOp->p1!=iCur ) continue; - if( pOp->opcode==OP_Close ) return 0; - if( pOp->opcode==OP_OpenRead ) return 1; - if( pOp->opcode==OP_OpenWrite ) return 1; - if( pOp->opcode==OP_OpenDup ) return 1; - if( pOp->opcode==OP_OpenAutoindex ) return 1; - if( pOp->opcode==OP_OpenEphemeral ) return 1; - } - return 0; -} -#endif /* SQLITE_DEBUG */ - /* ** Generate the end of the WHERE loop. See comments on ** sqlite3WhereBegin() for additional information. @@ -167635,6 +174045,23 @@ SQLITE_PRIVATE void sqlite3WhereEnd(WhereInfo *pWInfo){ sqlite3VdbeAddOp2(v, OP_Goto, 1, pLevel->p2); } #endif /* SQLITE_DISABLE_SKIPAHEAD_DISTINCT */ + if( pTabList->a[pLevel->iFrom].fg.fromExists && i==pWInfo->nLevel-1 ){ + /* If the EXISTS-to-JOIN optimization was applied, then the EXISTS + ** loop(s) will be the inner-most loops of the join. There might be + ** multiple EXISTS loops, but they will all be nested, and the join + ** order will not have been changed by the query planner. If the + ** inner-most EXISTS loop sees a single successful row, it should + ** break out of *all* EXISTS loops. But only the inner-most of the + ** nested EXISTS loops should do this breakout. */ + int nOuter = 0; /* Nr of outer EXISTS that this one is nested within */ + while( nOutera[pLevel[-nOuter-1].iFrom].fg.fromExists ) break; + nOuter++; + } + testcase( nOuter>0 ); + sqlite3VdbeAddOp2(v, OP_Goto, 0, pLevel[-nOuter].addrBrk); + VdbeComment((v, "EXISTS break")); + } /* The common case: Advance to the next row */ if( pLevel->addrCont ) sqlite3VdbeResolveLabel(v, pLevel->addrCont); sqlite3VdbeAddOp3(v, pLevel->op, pLevel->p1, pLevel->p2, pLevel->p3); @@ -167723,7 +174150,16 @@ SQLITE_PRIVATE void sqlite3WhereEnd(WhereInfo *pWInfo){ addr = sqlite3VdbeAddOp1(v, OP_IfPos, pLevel->iLeftJoin); VdbeCoverage(v); assert( (ws & WHERE_IDX_ONLY)==0 || (ws & WHERE_INDEXED)!=0 ); if( (ws & WHERE_IDX_ONLY)==0 ){ - assert( pLevel->iTabCur==pTabList->a[pLevel->iFrom].iCursor ); + SrcItem *pSrc = &pTabList->a[pLevel->iFrom]; + assert( pLevel->iTabCur==pSrc->iCursor ); + if( pSrc->fg.viaCoroutine ){ + int m, n; + assert( pSrc->fg.isSubquery ); + n = pSrc->u4.pSubq->regResult; + assert( pSrc->pSTab!=0 ); + m = pSrc->pSTab->nCol; + sqlite3VdbeAddOp3(v, OP_Null, 0, n, n+m-1); + } sqlite3VdbeAddOp1(v, OP_NullRow, pLevel->iTabCur); } if( (ws & WHERE_INDEXED) @@ -167745,7 +174181,7 @@ SQLITE_PRIVATE void sqlite3WhereEnd(WhereInfo *pWInfo){ sqlite3VdbeJumpHere(v, addr); } VdbeModuleComment((v, "End WHERE-loop%d: %s", i, - pWInfo->pTabList->a[pLevel->iFrom].pTab->zName)); + pWInfo->pTabList->a[pLevel->iFrom].pSTab->zName)); } assert( pWInfo->nLevel<=pTabList->nSrc ); @@ -167754,7 +174190,7 @@ SQLITE_PRIVATE void sqlite3WhereEnd(WhereInfo *pWInfo){ VdbeOp *pOp, *pLastOp; Index *pIdx = 0; SrcItem *pTabItem = &pTabList->a[pLevel->iFrom]; - Table *pTab = pTabItem->pTab; + Table *pTab = pTabItem->pSTab; assert( pTab!=0 ); pLoop = pLevel->pWLoop; @@ -167773,8 +174209,10 @@ SQLITE_PRIVATE void sqlite3WhereEnd(WhereInfo *pWInfo){ */ if( pTabItem->fg.viaCoroutine ){ testcase( pParse->db->mallocFailed ); + assert( pTabItem->fg.isSubquery ); + assert( pTabItem->u4.pSubq->regResult>=0 ); translateColumnToCopy(pParse, pLevel->addrBody, pLevel->iTabCur, - pTabItem->regResult, 0); + pTabItem->u4.pSubq->regResult, 0); continue; } @@ -167862,21 +174300,29 @@ SQLITE_PRIVATE void sqlite3WhereEnd(WhereInfo *pWInfo){ pOp->p2 = x; pOp->p1 = pLevel->iIdxCur; OpcodeRewriteTrace(db, k, pOp); - }else{ - /* Unable to translate the table reference into an index - ** reference. Verify that this is harmless - that the - ** table being referenced really is open. - */ -#ifdef SQLITE_ENABLE_OFFSET_SQL_FUNC - assert( (pLoop->wsFlags & WHERE_IDX_ONLY)==0 - || cursorIsOpen(v,pOp->p1,k) - || pOp->opcode==OP_Offset - ); -#else - assert( (pLoop->wsFlags & WHERE_IDX_ONLY)==0 - || cursorIsOpen(v,pOp->p1,k) - ); -#endif + }else if( pLoop->wsFlags & (WHERE_IDX_ONLY|WHERE_EXPRIDX) ){ + if( pLoop->wsFlags & WHERE_IDX_ONLY ){ + /* An error. pLoop is supposed to be a covering index loop, + ** and yet the VM code refers to a column of the table that + ** is not part of the index. */ + sqlite3ErrorMsg(pParse, "internal query planner error"); + pParse->rc = SQLITE_INTERNAL; + }else{ + /* The WHERE_EXPRIDX flag is set by the planner when it is likely + ** that pLoop is a covering index loop, but it is not possible + ** to be 100% sure. In this case, any OP_Explain opcode + ** corresponding to this loop describes the index as a "COVERING + ** INDEX". But, pOp proves that pLoop is not actually a covering + ** index loop. So clear the WHERE_EXPRIDX flag and rewrite the + ** text that accompanies the OP_Explain opcode, if any. */ + pLoop->wsFlags &= ~WHERE_EXPRIDX; + sqlite3WhereAddExplainText(pParse, + pLevel->addrBody-1, + pTabList, + pLevel, + pWInfo->wctrlFlags + ); + } } }else if( pOp->opcode==OP_Rowid ){ pOp->p1 = pLevel->iIdxCur; @@ -168822,7 +175268,7 @@ static ExprList *exprListAppendList( int iDummy; Expr *pSub; pSub = sqlite3ExprSkipCollateAndLikely(pDup); - if( sqlite3ExprIsInteger(pSub, &iDummy) ){ + if( sqlite3ExprIsInteger(pSub, &iDummy, 0) ){ pSub->op = TK_NULL; pSub->flags &= ~(EP_IntValue|EP_IsTrue|EP_IsFalse); pSub->u.zToken = 0; @@ -168908,7 +175354,7 @@ SQLITE_PRIVATE int sqlite3WindowRewrite(Parse *pParse, Select *p){ p->pWhere = 0; p->pGroupBy = 0; p->pHaving = 0; - p->selFlags &= ~SF_Aggregate; + p->selFlags &= ~(u32)SF_Aggregate; p->selFlags |= SF_WinRewrite; /* Create the ORDER BY clause for the sub-select. This is the concatenation @@ -168990,9 +175436,10 @@ SQLITE_PRIVATE int sqlite3WindowRewrite(Parse *pParse, Select *p){ assert( pSub!=0 || p->pSrc==0 ); /* Due to db->mallocFailed test inside ** of sqlite3DbMallocRawNN() called from ** sqlite3SrcListAppend() */ - if( p->pSrc ){ + if( p->pSrc==0 ){ + sqlite3SelectDelete(db, pSub); + }else if( sqlite3SrcItemAttachSubquery(pParse, &p->pSrc->a[0], pSub, 0) ){ Table *pTab2; - p->pSrc->a[0].pSelect = pSub; p->pSrc->a[0].fg.isCorrelated = 1; sqlite3SrcListAssignCursors(pParse, p->pSrc); pSub->selFlags |= SF_Expanded|SF_OrderByReqd; @@ -169006,7 +175453,7 @@ SQLITE_PRIVATE int sqlite3WindowRewrite(Parse *pParse, Select *p){ }else{ memcpy(pTab, pTab2, sizeof(Table)); pTab->tabFlags |= TF_Ephemeral; - p->pSrc->a[0].pTab = pTab; + p->pSrc->a[0].pSTab = pTab; pTab = pTab2; memset(&w, 0, sizeof(w)); w.xExprCallback = sqlite3WindowExtraAggFuncDepth; @@ -169014,8 +175461,6 @@ SQLITE_PRIVATE int sqlite3WindowRewrite(Parse *pParse, Select *p){ w.xSelectCallback2 = sqlite3WalkerDepthDecrease; sqlite3WalkSelect(&w, pSub); } - }else{ - sqlite3SelectDelete(db, pSub); } if( db->mallocFailed ) rc = SQLITE_NOMEM; @@ -169077,7 +175522,7 @@ SQLITE_PRIVATE void sqlite3WindowListDelete(sqlite3 *db, Window *p){ ** variable values in the expression tree. */ static Expr *sqlite3WindowOffsetExpr(Parse *pParse, Expr *pExpr){ - if( 0==sqlite3ExprIsConstant(pExpr) ){ + if( 0==sqlite3ExprIsConstant(0,pExpr) ){ if( IN_RENAME_OBJECT ) sqlite3RenameExprUnmap(pParse, pExpr); sqlite3ExprDelete(pParse->db, pExpr); pExpr = sqlite3ExprAlloc(pParse->db, TK_NULL, 0, 0); @@ -169302,10 +175747,15 @@ SQLITE_PRIVATE int sqlite3WindowCompare( ** and initialize registers and cursors used by sqlite3WindowCodeStep(). */ SQLITE_PRIVATE void sqlite3WindowCodeInit(Parse *pParse, Select *pSelect){ - int nEphExpr = pSelect->pSrc->a[0].pSelect->pEList->nExpr; - Window *pMWin = pSelect->pWin; Window *pWin; - Vdbe *v = sqlite3GetVdbe(pParse); + int nEphExpr; + Window *pMWin; + Vdbe *v; + + assert( pSelect->pSrc->a[0].fg.isSubquery ); + nEphExpr = pSelect->pSrc->a[0].u4.pSubq->pSelect->pEList->nExpr; + pMWin = pSelect->pWin; + v = sqlite3GetVdbe(pParse); sqlite3VdbeAddOp2(v, OP_OpenEphemeral, pMWin->iEphCsr, nEphExpr); sqlite3VdbeAddOp2(v, OP_OpenDup, pMWin->iEphCsr+1, pMWin->iEphCsr); @@ -169579,6 +176029,7 @@ static void windowAggStep( int regArg; int nArg = pWin->bExprArgs ? 0 : windowArgCount(pWin); int i; + int addrIf = 0; assert( bInverse==0 || pWin->eStart!=TK_UNBOUNDED ); @@ -169595,6 +176046,18 @@ static void windowAggStep( } regArg = reg; + if( pWin->pFilter ){ + int regTmp; + assert( ExprUseXList(pWin->pOwner) ); + assert( pWin->bExprArgs || !nArg ||nArg==pWin->pOwner->x.pList->nExpr ); + assert( pWin->bExprArgs || nArg ||pWin->pOwner->x.pList==0 ); + regTmp = sqlite3GetTempReg(pParse); + sqlite3VdbeAddOp3(v, OP_Column, csr, pWin->iArgCol+nArg,regTmp); + addrIf = sqlite3VdbeAddOp3(v, OP_IfNot, regTmp, 0, 1); + VdbeCoverage(v); + sqlite3ReleaseTempReg(pParse, regTmp); + } + if( pMWin->regStartRowid==0 && (pFunc->funcFlags & SQLITE_FUNC_MINMAX) && (pWin->eStart!=TK_UNBOUNDED) @@ -169614,25 +176077,13 @@ static void windowAggStep( } sqlite3VdbeJumpHere(v, addrIsNull); }else if( pWin->regApp ){ + assert( pWin->pFilter==0 ); assert( pFunc->zName==nth_valueName || pFunc->zName==first_valueName ); assert( bInverse==0 || bInverse==1 ); sqlite3VdbeAddOp2(v, OP_AddImm, pWin->regApp+1-bInverse, 1); }else if( pFunc->xSFunc!=noopStepFunc ){ - int addrIf = 0; - if( pWin->pFilter ){ - int regTmp; - assert( ExprUseXList(pWin->pOwner) ); - assert( pWin->bExprArgs || !nArg ||nArg==pWin->pOwner->x.pList->nExpr ); - assert( pWin->bExprArgs || nArg ||pWin->pOwner->x.pList==0 ); - regTmp = sqlite3GetTempReg(pParse); - sqlite3VdbeAddOp3(v, OP_Column, csr, pWin->iArgCol+nArg,regTmp); - addrIf = sqlite3VdbeAddOp3(v, OP_IfNot, regTmp, 0, 1); - VdbeCoverage(v); - sqlite3ReleaseTempReg(pParse, regTmp); - } - if( pWin->bExprArgs ){ int iOp = sqlite3VdbeCurrentAddr(v); int iEnd; @@ -169659,12 +176110,13 @@ static void windowAggStep( sqlite3VdbeAddOp3(v, bInverse? OP_AggInverse : OP_AggStep, bInverse, regArg, pWin->regAccum); sqlite3VdbeAppendP4(v, pFunc, P4_FUNCDEF); - sqlite3VdbeChangeP5(v, (u8)nArg); + sqlite3VdbeChangeP5(v, (u16)nArg); if( pWin->bExprArgs ){ sqlite3ReleaseTempRange(pParse, regArg, nArg); } - if( addrIf ) sqlite3VdbeJumpHere(v, addrIf); } + + if( addrIf ) sqlite3VdbeJumpHere(v, addrIf); } } @@ -170460,7 +176912,7 @@ static int windowExprGtZero(Parse *pParse, Expr *pExpr){ ** ** ROWS BETWEEN FOLLOWING AND FOLLOWING ** -** ... loop started by sqlite3WhereBegin() ... +** ... loop started by sqlite3WhereBegin() ... ** if( new partition ){ ** Gosub flush ** } @@ -170702,7 +177154,7 @@ SQLITE_PRIVATE void sqlite3WindowCodeStep( Vdbe *v = sqlite3GetVdbe(pParse); int csrWrite; /* Cursor used to write to eph. table */ int csrInput = p->pSrc->a[0].iCursor; /* Cursor of sub-select */ - int nInput = p->pSrc->a[0].pTab->nCol; /* Number of cols returned by sub */ + int nInput = p->pSrc->a[0].pSTab->nCol; /* Number of cols returned by sub */ int iInput; /* To iterate through sub cols */ int addrNe; /* Address of OP_Ne */ int addrGosubFlush = 0; /* Address of OP_Gosub to flush: */ @@ -170978,6 +177430,12 @@ SQLITE_PRIVATE void sqlite3WindowCodeStep( addrBreak2 = windowCodeOp(&s, WINDOW_AGGINVERSE, 0, 1); }else{ assert( pMWin->eEnd==TK_FOLLOWING ); + /* assert( regStart>=0 ); + ** regEnd = regEnd - regStart; + ** regStart = 0; */ + sqlite3VdbeAddOp3(v, OP_Subtract, regStart, regEnd, regEnd); + sqlite3VdbeAddOp2(v, OP_Integer, 0, regStart); + addrStart = sqlite3VdbeCurrentAddr(v); addrBreak1 = windowCodeOp(&s, WINDOW_RETURN_ROW, regEnd, 1); addrBreak2 = windowCodeOp(&s, WINDOW_AGGINVERSE, regStart, 1); @@ -171042,6 +177500,11 @@ SQLITE_PRIVATE void sqlite3WindowCodeStep( /* #include "sqliteInt.h" */ +/* +** Verify that the pParse->isCreate field is set +*/ +#define ASSERT_IS_CREATE assert(pParse->isCreate) + /* ** Disable all error recovery processing in the parser push-down ** automaton. @@ -171091,6 +177554,13 @@ struct TrigEvent { int a; IdList * b; }; struct FrameBound { int eType; Expr *pExpr; }; +/* +** Generate a syntax error +*/ +static void parserSyntaxError(Parse *pParse, Token *p){ + sqlite3ErrorMsg(pParse, "near \"%T\": syntax error", p); +} + /* ** Disable lookaside memory allocation for objects that might be ** shared across database connections. @@ -171098,6 +177568,10 @@ struct FrameBound { int eType; Expr *pExpr; }; static void disableLookaside(Parse *pParse){ sqlite3 *db = pParse->db; pParse->disableLookaside++; +#ifdef SQLITE_DEBUG + pParse->isCreate = 1; +#endif + memset(&pParse->u1.cr, 0, sizeof(pParse->u1.cr)); DisableLookaside; } @@ -171147,9 +177621,9 @@ static void updateDeleteLimitError( break; } } - if( (p->selFlags & SF_MultiValue)==0 && - (mxSelect = pParse->db->aLimit[SQLITE_LIMIT_COMPOUND_SELECT])>0 && - cnt>mxSelect + if( (p->selFlags & (SF_MultiValue|SF_Values))==0 + && (mxSelect = pParse->db->aLimit[SQLITE_LIMIT_COMPOUND_SELECT])>0 + && cnt>mxSelect ){ sqlite3ErrorMsg(pParse, "too many terms in compound SELECT"); } @@ -171169,6 +177643,14 @@ static void updateDeleteLimitError( return pSelect; } + /* Memory allocator for parser stack resizing. This is a thin wrapper around + ** sqlite3_realloc() that includes a call to sqlite3FaultSim() to facilitate + ** testing. + */ + static void *parserStackRealloc(void *pOld, sqlite3_uint64 newSize){ + return sqlite3FaultSim(700) ? 0 : sqlite3_realloc(pOld, newSize); + } + /* Construct a new Expr object from a single token */ static Expr *tokenExpr(Parse *pParse, int op, Token t){ @@ -171291,135 +177773,135 @@ static void updateDeleteLimitError( #define TK_OR 43 #define TK_AND 44 #define TK_IS 45 -#define TK_MATCH 46 -#define TK_LIKE_KW 47 -#define TK_BETWEEN 48 -#define TK_IN 49 -#define TK_ISNULL 50 -#define TK_NOTNULL 51 -#define TK_NE 52 -#define TK_EQ 53 -#define TK_GT 54 -#define TK_LE 55 -#define TK_LT 56 -#define TK_GE 57 -#define TK_ESCAPE 58 -#define TK_ID 59 -#define TK_COLUMNKW 60 -#define TK_DO 61 -#define TK_FOR 62 -#define TK_IGNORE 63 -#define TK_INITIALLY 64 -#define TK_INSTEAD 65 -#define TK_NO 66 -#define TK_KEY 67 -#define TK_OF 68 -#define TK_OFFSET 69 -#define TK_PRAGMA 70 -#define TK_RAISE 71 -#define TK_RECURSIVE 72 -#define TK_REPLACE 73 -#define TK_RESTRICT 74 -#define TK_ROW 75 -#define TK_ROWS 76 -#define TK_TRIGGER 77 -#define TK_VACUUM 78 -#define TK_VIEW 79 -#define TK_VIRTUAL 80 -#define TK_WITH 81 -#define TK_NULLS 82 -#define TK_FIRST 83 -#define TK_LAST 84 -#define TK_CURRENT 85 -#define TK_FOLLOWING 86 -#define TK_PARTITION 87 -#define TK_PRECEDING 88 -#define TK_RANGE 89 -#define TK_UNBOUNDED 90 -#define TK_EXCLUDE 91 -#define TK_GROUPS 92 -#define TK_OTHERS 93 -#define TK_TIES 94 -#define TK_GENERATED 95 -#define TK_ALWAYS 96 -#define TK_MATERIALIZED 97 -#define TK_REINDEX 98 -#define TK_RENAME 99 -#define TK_CTIME_KW 100 -#define TK_ANY 101 -#define TK_BITAND 102 -#define TK_BITOR 103 -#define TK_LSHIFT 104 -#define TK_RSHIFT 105 -#define TK_PLUS 106 -#define TK_MINUS 107 -#define TK_STAR 108 -#define TK_SLASH 109 -#define TK_REM 110 -#define TK_CONCAT 111 -#define TK_PTR 112 -#define TK_COLLATE 113 -#define TK_BITNOT 114 -#define TK_ON 115 -#define TK_INDEXED 116 -#define TK_STRING 117 -#define TK_JOIN_KW 118 -#define TK_CONSTRAINT 119 -#define TK_DEFAULT 120 -#define TK_NULL 121 -#define TK_PRIMARY 122 -#define TK_UNIQUE 123 -#define TK_CHECK 124 -#define TK_REFERENCES 125 -#define TK_AUTOINCR 126 -#define TK_INSERT 127 -#define TK_DELETE 128 -#define TK_UPDATE 129 -#define TK_SET 130 -#define TK_DEFERRABLE 131 -#define TK_FOREIGN 132 -#define TK_DROP 133 -#define TK_UNION 134 -#define TK_ALL 135 -#define TK_EXCEPT 136 -#define TK_INTERSECT 137 -#define TK_SELECT 138 -#define TK_VALUES 139 -#define TK_DISTINCT 140 -#define TK_DOT 141 -#define TK_FROM 142 -#define TK_JOIN 143 -#define TK_USING 144 -#define TK_ORDER 145 -#define TK_GROUP 146 -#define TK_HAVING 147 -#define TK_LIMIT 148 -#define TK_WHERE 149 -#define TK_RETURNING 150 -#define TK_INTO 151 -#define TK_NOTHING 152 -#define TK_FLOAT 153 -#define TK_BLOB 154 -#define TK_INTEGER 155 -#define TK_VARIABLE 156 -#define TK_CASE 157 -#define TK_WHEN 158 -#define TK_THEN 159 -#define TK_ELSE 160 -#define TK_INDEX 161 -#define TK_ALTER 162 -#define TK_ADD 163 -#define TK_WINDOW 164 -#define TK_OVER 165 -#define TK_FILTER 166 -#define TK_COLUMN 167 -#define TK_AGG_FUNCTION 168 -#define TK_AGG_COLUMN 169 -#define TK_TRUEFALSE 170 -#define TK_ISNOT 171 +#define TK_ISNOT 46 +#define TK_MATCH 47 +#define TK_LIKE_KW 48 +#define TK_BETWEEN 49 +#define TK_IN 50 +#define TK_ISNULL 51 +#define TK_NOTNULL 52 +#define TK_NE 53 +#define TK_EQ 54 +#define TK_GT 55 +#define TK_LE 56 +#define TK_LT 57 +#define TK_GE 58 +#define TK_ESCAPE 59 +#define TK_ID 60 +#define TK_COLUMNKW 61 +#define TK_DO 62 +#define TK_FOR 63 +#define TK_IGNORE 64 +#define TK_INITIALLY 65 +#define TK_INSTEAD 66 +#define TK_NO 67 +#define TK_KEY 68 +#define TK_OF 69 +#define TK_OFFSET 70 +#define TK_PRAGMA 71 +#define TK_RAISE 72 +#define TK_RECURSIVE 73 +#define TK_REPLACE 74 +#define TK_RESTRICT 75 +#define TK_ROW 76 +#define TK_ROWS 77 +#define TK_TRIGGER 78 +#define TK_VACUUM 79 +#define TK_VIEW 80 +#define TK_VIRTUAL 81 +#define TK_WITH 82 +#define TK_NULLS 83 +#define TK_FIRST 84 +#define TK_LAST 85 +#define TK_CURRENT 86 +#define TK_FOLLOWING 87 +#define TK_PARTITION 88 +#define TK_PRECEDING 89 +#define TK_RANGE 90 +#define TK_UNBOUNDED 91 +#define TK_EXCLUDE 92 +#define TK_GROUPS 93 +#define TK_OTHERS 94 +#define TK_TIES 95 +#define TK_GENERATED 96 +#define TK_ALWAYS 97 +#define TK_MATERIALIZED 98 +#define TK_REINDEX 99 +#define TK_RENAME 100 +#define TK_CTIME_KW 101 +#define TK_ANY 102 +#define TK_BITAND 103 +#define TK_BITOR 104 +#define TK_LSHIFT 105 +#define TK_RSHIFT 106 +#define TK_PLUS 107 +#define TK_MINUS 108 +#define TK_STAR 109 +#define TK_SLASH 110 +#define TK_REM 111 +#define TK_CONCAT 112 +#define TK_PTR 113 +#define TK_COLLATE 114 +#define TK_BITNOT 115 +#define TK_ON 116 +#define TK_INDEXED 117 +#define TK_STRING 118 +#define TK_JOIN_KW 119 +#define TK_CONSTRAINT 120 +#define TK_DEFAULT 121 +#define TK_NULL 122 +#define TK_PRIMARY 123 +#define TK_UNIQUE 124 +#define TK_CHECK 125 +#define TK_REFERENCES 126 +#define TK_AUTOINCR 127 +#define TK_INSERT 128 +#define TK_DELETE 129 +#define TK_UPDATE 130 +#define TK_SET 131 +#define TK_DEFERRABLE 132 +#define TK_FOREIGN 133 +#define TK_DROP 134 +#define TK_UNION 135 +#define TK_ALL 136 +#define TK_EXCEPT 137 +#define TK_INTERSECT 138 +#define TK_SELECT 139 +#define TK_VALUES 140 +#define TK_DISTINCT 141 +#define TK_DOT 142 +#define TK_FROM 143 +#define TK_JOIN 144 +#define TK_USING 145 +#define TK_ORDER 146 +#define TK_GROUP 147 +#define TK_HAVING 148 +#define TK_LIMIT 149 +#define TK_WHERE 150 +#define TK_RETURNING 151 +#define TK_INTO 152 +#define TK_NOTHING 153 +#define TK_FLOAT 154 +#define TK_BLOB 155 +#define TK_INTEGER 156 +#define TK_VARIABLE 157 +#define TK_CASE 158 +#define TK_WHEN 159 +#define TK_THEN 160 +#define TK_ELSE 161 +#define TK_INDEX 162 +#define TK_ALTER 163 +#define TK_ADD 164 +#define TK_WINDOW 165 +#define TK_OVER 166 +#define TK_FILTER 167 +#define TK_COLUMN 168 +#define TK_AGG_FUNCTION 169 +#define TK_AGG_COLUMN 170 +#define TK_TRUEFALSE 171 #define TK_FUNCTION 172 -#define TK_UMINUS 173 -#define TK_UPLUS 174 +#define TK_UPLUS 173 +#define TK_UMINUS 174 #define TK_TRUTH 175 #define TK_REGISTER 176 #define TK_VECTOR 177 @@ -171428,8 +177910,10 @@ static void updateDeleteLimitError( #define TK_ASTERISK 180 #define TK_SPAN 181 #define TK_ERROR 182 -#define TK_SPACE 183 -#define TK_ILLEGAL 184 +#define TK_QNUMBER 183 +#define TK_SPACE 184 +#define TK_COMMENT 185 +#define TK_ILLEGAL 186 #endif /**************** End token definitions ***************************************/ @@ -171470,6 +177954,9 @@ static void updateDeleteLimitError( ** sqlite3ParserARG_STORE Code to store %extra_argument into yypParser ** sqlite3ParserARG_FETCH Code to extract %extra_argument from yypParser ** sqlite3ParserCTX_* As sqlite3ParserARG_ except for %extra_context +** YYREALLOC Name of the realloc() function to use +** YYFREE Name of the free() function to use +** YYDYNSTACK True if stack space should be extended on heap ** YYERRORSYMBOL is the code number of the error symbol. If not ** defined, then do no error processing. ** YYNSTATE the combined number of states. @@ -171483,37 +177970,39 @@ static void updateDeleteLimitError( ** YY_NO_ACTION The yy_action[] code for no-op ** YY_MIN_REDUCE Minimum value for reduce actions ** YY_MAX_REDUCE Maximum value for reduce actions +** YY_MIN_DSTRCTR Minimum symbol value that has a destructor +** YY_MAX_DSTRCTR Maximum symbol value that has a destructor */ #ifndef INTERFACE # define INTERFACE 1 #endif /************* Begin control #defines *****************************************/ #define YYCODETYPE unsigned short int -#define YYNOCODE 319 +#define YYNOCODE 323 #define YYACTIONTYPE unsigned short int -#define YYWILDCARD 101 +#define YYWILDCARD 102 #define sqlite3ParserTOKENTYPE Token typedef union { int yyinit; sqlite3ParserTOKENTYPE yy0; - TriggerStep* yy33; - Window* yy41; - Select* yy47; - SrcList* yy131; - struct TrigEvent yy180; - struct {int value; int mask;} yy231; - IdList* yy254; - u32 yy285; - ExprList* yy322; - Cte* yy385; - int yy394; - Upsert* yy444; - u8 yy516; - With* yy521; - const char* yy522; - Expr* yy528; - OnOrUsing yy561; - struct FrameBound yy595; + u32 yy9; + struct TrigEvent yy28; + With* yy125; + IdList* yy204; + struct FrameBound yy205; + TriggerStep* yy319; + const char* yy342; + Cte* yy361; + ExprList* yy402; + Upsert* yy403; + OnOrUsing yy421; + u8 yy444; + struct {int value; int mask;} yy481; + Window* yy483; + int yy502; + SrcList* yy563; + Expr* yy590; + Select* yy637; } YYMINORTYPE; #ifndef YYSTACKDEPTH #define YYSTACKDEPTH 100 @@ -171523,24 +178012,29 @@ typedef union { #define sqlite3ParserARG_PARAM #define sqlite3ParserARG_FETCH #define sqlite3ParserARG_STORE +#define YYREALLOC parserStackRealloc +#define YYFREE sqlite3_free +#define YYDYNSTACK 1 #define sqlite3ParserCTX_SDECL Parse *pParse; #define sqlite3ParserCTX_PDECL ,Parse *pParse #define sqlite3ParserCTX_PARAM ,pParse #define sqlite3ParserCTX_FETCH Parse *pParse=yypParser->pParse; #define sqlite3ParserCTX_STORE yypParser->pParse=pParse; #define YYFALLBACK 1 -#define YYNSTATE 579 -#define YYNRULE 405 -#define YYNRULE_WITH_ACTION 340 -#define YYNTOKEN 185 -#define YY_MAX_SHIFT 578 -#define YY_MIN_SHIFTREDUCE 838 -#define YY_MAX_SHIFTREDUCE 1242 -#define YY_ERROR_ACTION 1243 -#define YY_ACCEPT_ACTION 1244 -#define YY_NO_ACTION 1245 -#define YY_MIN_REDUCE 1246 -#define YY_MAX_REDUCE 1650 +#define YYNSTATE 583 +#define YYNRULE 409 +#define YYNRULE_WITH_ACTION 344 +#define YYNTOKEN 187 +#define YY_MAX_SHIFT 582 +#define YY_MIN_SHIFTREDUCE 845 +#define YY_MAX_SHIFTREDUCE 1253 +#define YY_ERROR_ACTION 1254 +#define YY_ACCEPT_ACTION 1255 +#define YY_NO_ACTION 1256 +#define YY_MIN_REDUCE 1257 +#define YY_MAX_REDUCE 1665 +#define YY_MIN_DSTRCTR 206 +#define YY_MAX_DSTRCTR 320 /************* End control #defines *******************************************/ #define YY_NLOOKAHEAD ((int)(sizeof(yy_lookahead)/sizeof(yy_lookahead[0]))) @@ -171556,6 +178050,22 @@ typedef union { # define yytestcase(X) #endif +/* Macro to determine if stack space has the ability to grow using +** heap memory. +*/ +#if YYSTACKDEPTH<=0 || YYDYNSTACK +# define YYGROWABLESTACK 1 +#else +# define YYGROWABLESTACK 0 +#endif + +/* Guarantee a minimum number of initial stack slots. +*/ +#if YYSTACKDEPTH<=0 +# undef YYSTACKDEPTH +# define YYSTACKDEPTH 2 /* Need a minimum stack size */ +#endif + /* Next are the tables used to determine what action to take based on the ** current state and lookahead token. These tables are used to implement @@ -171607,619 +178117,643 @@ typedef union { ** yy_default[] Default action for each state. ** *********** Begin parsing tables **********************************************/ -#define YY_ACTTAB_COUNT (2100) +#define YY_ACTTAB_COUNT (2207) static const YYACTIONTYPE yy_action[] = { - /* 0 */ 572, 210, 572, 119, 116, 231, 572, 119, 116, 231, - /* 10 */ 572, 1317, 379, 1296, 410, 566, 566, 566, 572, 411, - /* 20 */ 380, 1317, 1279, 42, 42, 42, 42, 210, 1529, 72, - /* 30 */ 72, 974, 421, 42, 42, 495, 305, 281, 305, 975, - /* 40 */ 399, 72, 72, 126, 127, 81, 1217, 1217, 1054, 1057, - /* 50 */ 1044, 1044, 124, 124, 125, 125, 125, 125, 480, 411, - /* 60 */ 1244, 1, 1, 578, 2, 1248, 554, 119, 116, 231, - /* 70 */ 319, 484, 147, 484, 528, 119, 116, 231, 533, 1330, - /* 80 */ 419, 527, 143, 126, 127, 81, 1217, 1217, 1054, 1057, - /* 90 */ 1044, 1044, 124, 124, 125, 125, 125, 125, 119, 116, - /* 100 */ 231, 329, 123, 123, 123, 123, 122, 122, 121, 121, - /* 110 */ 121, 120, 117, 448, 286, 286, 286, 286, 446, 446, - /* 120 */ 446, 1568, 378, 1570, 1193, 377, 1164, 569, 1164, 569, - /* 130 */ 411, 1568, 541, 261, 228, 448, 102, 146, 453, 318, - /* 140 */ 563, 242, 123, 123, 123, 123, 122, 122, 121, 121, - /* 150 */ 121, 120, 117, 448, 126, 127, 81, 1217, 1217, 1054, - /* 160 */ 1057, 1044, 1044, 124, 124, 125, 125, 125, 125, 143, - /* 170 */ 296, 1193, 341, 452, 121, 121, 121, 120, 117, 448, - /* 180 */ 128, 1193, 1194, 1193, 149, 445, 444, 572, 120, 117, - /* 190 */ 448, 125, 125, 125, 125, 118, 123, 123, 123, 123, - /* 200 */ 122, 122, 121, 121, 121, 120, 117, 448, 458, 114, - /* 210 */ 13, 13, 550, 123, 123, 123, 123, 122, 122, 121, - /* 220 */ 121, 121, 120, 117, 448, 424, 318, 563, 1193, 1194, - /* 230 */ 1193, 150, 1225, 411, 1225, 125, 125, 125, 125, 123, - /* 240 */ 123, 123, 123, 122, 122, 121, 121, 121, 120, 117, - /* 250 */ 448, 469, 344, 1041, 1041, 1055, 1058, 126, 127, 81, - /* 260 */ 1217, 1217, 1054, 1057, 1044, 1044, 124, 124, 125, 125, - /* 270 */ 125, 125, 1282, 526, 224, 1193, 572, 411, 226, 519, - /* 280 */ 177, 83, 84, 123, 123, 123, 123, 122, 122, 121, - /* 290 */ 121, 121, 120, 117, 448, 1010, 16, 16, 1193, 134, - /* 300 */ 134, 126, 127, 81, 1217, 1217, 1054, 1057, 1044, 1044, - /* 310 */ 124, 124, 125, 125, 125, 125, 123, 123, 123, 123, - /* 320 */ 122, 122, 121, 121, 121, 120, 117, 448, 1045, 550, - /* 330 */ 1193, 375, 1193, 1194, 1193, 254, 1438, 401, 508, 505, - /* 340 */ 504, 112, 564, 570, 4, 929, 929, 435, 503, 342, - /* 350 */ 464, 330, 362, 396, 1238, 1193, 1194, 1193, 567, 572, - /* 360 */ 123, 123, 123, 123, 122, 122, 121, 121, 121, 120, - /* 370 */ 117, 448, 286, 286, 371, 1581, 1607, 445, 444, 155, - /* 380 */ 411, 449, 72, 72, 1289, 569, 1222, 1193, 1194, 1193, - /* 390 */ 86, 1224, 273, 561, 547, 520, 520, 572, 99, 1223, - /* 400 */ 6, 1281, 476, 143, 126, 127, 81, 1217, 1217, 1054, - /* 410 */ 1057, 1044, 1044, 124, 124, 125, 125, 125, 125, 554, - /* 420 */ 13, 13, 1031, 511, 1225, 1193, 1225, 553, 110, 110, - /* 430 */ 224, 572, 1239, 177, 572, 429, 111, 199, 449, 573, - /* 440 */ 449, 432, 1555, 1019, 327, 555, 1193, 272, 289, 370, - /* 450 */ 514, 365, 513, 259, 72, 72, 547, 72, 72, 361, - /* 460 */ 318, 563, 1613, 123, 123, 123, 123, 122, 122, 121, - /* 470 */ 121, 121, 120, 117, 448, 1019, 1019, 1021, 1022, 28, - /* 480 */ 286, 286, 1193, 1194, 1193, 1159, 572, 1612, 411, 904, - /* 490 */ 192, 554, 358, 569, 554, 940, 537, 521, 1159, 437, - /* 500 */ 415, 1159, 556, 1193, 1194, 1193, 572, 548, 548, 52, - /* 510 */ 52, 216, 126, 127, 81, 1217, 1217, 1054, 1057, 1044, - /* 520 */ 1044, 124, 124, 125, 125, 125, 125, 1193, 478, 136, - /* 530 */ 136, 411, 286, 286, 1493, 509, 122, 122, 121, 121, - /* 540 */ 121, 120, 117, 448, 1010, 569, 522, 219, 545, 545, - /* 550 */ 318, 563, 143, 6, 536, 126, 127, 81, 1217, 1217, - /* 560 */ 1054, 1057, 1044, 1044, 124, 124, 125, 125, 125, 125, - /* 570 */ 1557, 123, 123, 123, 123, 122, 122, 121, 121, 121, - /* 580 */ 120, 117, 448, 489, 1193, 1194, 1193, 486, 283, 1270, - /* 590 */ 960, 254, 1193, 375, 508, 505, 504, 1193, 342, 574, - /* 600 */ 1193, 574, 411, 294, 503, 960, 879, 193, 484, 318, - /* 610 */ 563, 386, 292, 382, 123, 123, 123, 123, 122, 122, - /* 620 */ 121, 121, 121, 120, 117, 448, 126, 127, 81, 1217, - /* 630 */ 1217, 1054, 1057, 1044, 1044, 124, 124, 125, 125, 125, - /* 640 */ 125, 411, 396, 1139, 1193, 872, 101, 286, 286, 1193, - /* 650 */ 1194, 1193, 375, 1096, 1193, 1194, 1193, 1193, 1194, 1193, - /* 660 */ 569, 459, 33, 375, 235, 126, 127, 81, 1217, 1217, - /* 670 */ 1054, 1057, 1044, 1044, 124, 124, 125, 125, 125, 125, - /* 680 */ 1437, 962, 572, 230, 961, 123, 123, 123, 123, 122, - /* 690 */ 122, 121, 121, 121, 120, 117, 448, 1159, 230, 1193, - /* 700 */ 158, 1193, 1194, 1193, 1556, 13, 13, 303, 960, 1233, - /* 710 */ 1159, 154, 411, 1159, 375, 1584, 1177, 5, 371, 1581, - /* 720 */ 431, 1239, 3, 960, 123, 123, 123, 123, 122, 122, - /* 730 */ 121, 121, 121, 120, 117, 448, 126, 127, 81, 1217, - /* 740 */ 1217, 1054, 1057, 1044, 1044, 124, 124, 125, 125, 125, - /* 750 */ 125, 411, 210, 571, 1193, 1032, 1193, 1194, 1193, 1193, - /* 760 */ 390, 855, 156, 1555, 376, 404, 1101, 1101, 492, 572, - /* 770 */ 469, 344, 1322, 1322, 1555, 126, 127, 81, 1217, 1217, - /* 780 */ 1054, 1057, 1044, 1044, 124, 124, 125, 125, 125, 125, - /* 790 */ 130, 572, 13, 13, 532, 123, 123, 123, 123, 122, - /* 800 */ 122, 121, 121, 121, 120, 117, 448, 304, 572, 457, - /* 810 */ 229, 1193, 1194, 1193, 13, 13, 1193, 1194, 1193, 1300, - /* 820 */ 467, 1270, 411, 1320, 1320, 1555, 1015, 457, 456, 436, - /* 830 */ 301, 72, 72, 1268, 123, 123, 123, 123, 122, 122, - /* 840 */ 121, 121, 121, 120, 117, 448, 126, 127, 81, 1217, - /* 850 */ 1217, 1054, 1057, 1044, 1044, 124, 124, 125, 125, 125, - /* 860 */ 125, 411, 384, 1076, 1159, 286, 286, 421, 314, 280, - /* 870 */ 280, 287, 287, 461, 408, 407, 1539, 1159, 569, 572, - /* 880 */ 1159, 1196, 569, 409, 569, 126, 127, 81, 1217, 1217, - /* 890 */ 1054, 1057, 1044, 1044, 124, 124, 125, 125, 125, 125, - /* 900 */ 457, 1485, 13, 13, 1541, 123, 123, 123, 123, 122, - /* 910 */ 122, 121, 121, 121, 120, 117, 448, 202, 572, 462, - /* 920 */ 1587, 578, 2, 1248, 843, 844, 845, 1563, 319, 409, - /* 930 */ 147, 6, 411, 257, 256, 255, 208, 1330, 9, 1196, - /* 940 */ 264, 72, 72, 1436, 123, 123, 123, 123, 122, 122, - /* 950 */ 121, 121, 121, 120, 117, 448, 126, 127, 81, 1217, - /* 960 */ 1217, 1054, 1057, 1044, 1044, 124, 124, 125, 125, 125, - /* 970 */ 125, 572, 286, 286, 572, 1213, 411, 577, 315, 1248, - /* 980 */ 421, 371, 1581, 356, 319, 569, 147, 495, 529, 1644, - /* 990 */ 397, 935, 495, 1330, 71, 71, 934, 72, 72, 242, - /* 1000 */ 1328, 105, 81, 1217, 1217, 1054, 1057, 1044, 1044, 124, - /* 1010 */ 124, 125, 125, 125, 125, 123, 123, 123, 123, 122, - /* 1020 */ 122, 121, 121, 121, 120, 117, 448, 1117, 286, 286, - /* 1030 */ 1422, 452, 1528, 1213, 443, 286, 286, 1492, 1355, 313, - /* 1040 */ 478, 569, 1118, 454, 351, 495, 354, 1266, 569, 209, - /* 1050 */ 572, 418, 179, 572, 1031, 242, 385, 1119, 523, 123, - /* 1060 */ 123, 123, 123, 122, 122, 121, 121, 121, 120, 117, - /* 1070 */ 448, 1020, 108, 72, 72, 1019, 13, 13, 915, 572, - /* 1080 */ 1498, 572, 286, 286, 98, 530, 1537, 452, 916, 1334, - /* 1090 */ 1329, 203, 411, 286, 286, 569, 152, 211, 1498, 1500, - /* 1100 */ 426, 569, 56, 56, 57, 57, 569, 1019, 1019, 1021, - /* 1110 */ 447, 572, 411, 531, 12, 297, 126, 127, 81, 1217, - /* 1120 */ 1217, 1054, 1057, 1044, 1044, 124, 124, 125, 125, 125, - /* 1130 */ 125, 572, 411, 867, 15, 15, 126, 127, 81, 1217, - /* 1140 */ 1217, 1054, 1057, 1044, 1044, 124, 124, 125, 125, 125, - /* 1150 */ 125, 373, 529, 264, 44, 44, 126, 115, 81, 1217, - /* 1160 */ 1217, 1054, 1057, 1044, 1044, 124, 124, 125, 125, 125, - /* 1170 */ 125, 1498, 478, 1271, 417, 123, 123, 123, 123, 122, - /* 1180 */ 122, 121, 121, 121, 120, 117, 448, 205, 1213, 495, - /* 1190 */ 430, 867, 468, 322, 495, 123, 123, 123, 123, 122, - /* 1200 */ 122, 121, 121, 121, 120, 117, 448, 572, 557, 1140, - /* 1210 */ 1642, 1422, 1642, 543, 572, 123, 123, 123, 123, 122, - /* 1220 */ 122, 121, 121, 121, 120, 117, 448, 572, 1422, 572, - /* 1230 */ 13, 13, 542, 323, 1325, 411, 334, 58, 58, 349, - /* 1240 */ 1422, 1170, 326, 286, 286, 549, 1213, 300, 895, 530, - /* 1250 */ 45, 45, 59, 59, 1140, 1643, 569, 1643, 565, 417, - /* 1260 */ 127, 81, 1217, 1217, 1054, 1057, 1044, 1044, 124, 124, - /* 1270 */ 125, 125, 125, 125, 1367, 373, 500, 290, 1193, 512, - /* 1280 */ 1366, 427, 394, 394, 393, 275, 391, 896, 1138, 852, - /* 1290 */ 478, 258, 1422, 1170, 463, 1159, 12, 331, 428, 333, - /* 1300 */ 1117, 460, 236, 258, 325, 460, 544, 1544, 1159, 1098, - /* 1310 */ 491, 1159, 324, 1098, 440, 1118, 335, 516, 123, 123, - /* 1320 */ 123, 123, 122, 122, 121, 121, 121, 120, 117, 448, - /* 1330 */ 1119, 318, 563, 1138, 572, 1193, 1194, 1193, 112, 564, - /* 1340 */ 201, 4, 238, 433, 935, 490, 285, 228, 1517, 934, - /* 1350 */ 170, 560, 572, 142, 1516, 567, 572, 60, 60, 572, - /* 1360 */ 416, 572, 441, 572, 535, 302, 875, 8, 487, 572, - /* 1370 */ 237, 572, 416, 572, 485, 61, 61, 572, 449, 62, - /* 1380 */ 62, 332, 63, 63, 46, 46, 47, 47, 361, 572, - /* 1390 */ 561, 572, 48, 48, 50, 50, 51, 51, 572, 295, - /* 1400 */ 64, 64, 482, 295, 539, 412, 471, 1031, 572, 538, - /* 1410 */ 318, 563, 65, 65, 66, 66, 409, 475, 572, 1031, - /* 1420 */ 572, 14, 14, 875, 1020, 110, 110, 409, 1019, 572, - /* 1430 */ 474, 67, 67, 111, 455, 449, 573, 449, 98, 317, - /* 1440 */ 1019, 132, 132, 133, 133, 572, 1561, 572, 974, 409, - /* 1450 */ 6, 1562, 68, 68, 1560, 6, 975, 572, 6, 1559, - /* 1460 */ 1019, 1019, 1021, 6, 346, 218, 101, 531, 53, 53, - /* 1470 */ 69, 69, 1019, 1019, 1021, 1022, 28, 1586, 1181, 451, - /* 1480 */ 70, 70, 290, 87, 215, 31, 1363, 394, 394, 393, - /* 1490 */ 275, 391, 350, 109, 852, 107, 572, 112, 564, 483, - /* 1500 */ 4, 1212, 572, 239, 153, 572, 39, 236, 1299, 325, - /* 1510 */ 112, 564, 1298, 4, 567, 572, 32, 324, 572, 54, - /* 1520 */ 54, 572, 1135, 353, 398, 165, 165, 567, 166, 166, - /* 1530 */ 572, 291, 355, 572, 17, 357, 572, 449, 77, 77, - /* 1540 */ 1313, 55, 55, 1297, 73, 73, 572, 238, 470, 561, - /* 1550 */ 449, 472, 364, 135, 135, 170, 74, 74, 142, 163, - /* 1560 */ 163, 374, 561, 539, 572, 321, 572, 886, 540, 137, - /* 1570 */ 137, 339, 1353, 422, 298, 237, 539, 572, 1031, 572, - /* 1580 */ 340, 538, 101, 369, 110, 110, 162, 131, 131, 164, - /* 1590 */ 164, 1031, 111, 368, 449, 573, 449, 110, 110, 1019, - /* 1600 */ 157, 157, 141, 141, 572, 111, 572, 449, 573, 449, - /* 1610 */ 412, 288, 1019, 572, 882, 318, 563, 572, 219, 572, - /* 1620 */ 241, 1012, 477, 263, 263, 894, 893, 140, 140, 138, - /* 1630 */ 138, 1019, 1019, 1021, 1022, 28, 139, 139, 525, 455, - /* 1640 */ 76, 76, 78, 78, 1019, 1019, 1021, 1022, 28, 1181, - /* 1650 */ 451, 572, 1083, 290, 112, 564, 1575, 4, 394, 394, - /* 1660 */ 393, 275, 391, 572, 1023, 852, 572, 479, 345, 263, - /* 1670 */ 101, 567, 882, 1376, 75, 75, 1421, 501, 236, 260, - /* 1680 */ 325, 112, 564, 359, 4, 101, 43, 43, 324, 49, - /* 1690 */ 49, 901, 902, 161, 449, 101, 977, 978, 567, 1079, - /* 1700 */ 1349, 260, 965, 932, 263, 114, 561, 1095, 517, 1095, - /* 1710 */ 1083, 1094, 865, 1094, 151, 933, 1144, 114, 238, 1361, - /* 1720 */ 558, 449, 1023, 559, 1426, 1278, 170, 1269, 1257, 142, - /* 1730 */ 1601, 1256, 1258, 561, 1594, 1031, 496, 278, 213, 1346, - /* 1740 */ 310, 110, 110, 939, 311, 312, 237, 11, 234, 111, - /* 1750 */ 221, 449, 573, 449, 293, 395, 1019, 1408, 337, 1403, - /* 1760 */ 1396, 338, 1031, 299, 343, 1413, 1412, 481, 110, 110, - /* 1770 */ 506, 402, 225, 1296, 206, 367, 111, 1358, 449, 573, - /* 1780 */ 449, 412, 1359, 1019, 1489, 1488, 318, 563, 1019, 1019, - /* 1790 */ 1021, 1022, 28, 562, 207, 220, 80, 564, 389, 4, - /* 1800 */ 1597, 1357, 552, 1356, 1233, 181, 267, 232, 1536, 1534, - /* 1810 */ 455, 1230, 420, 567, 82, 1019, 1019, 1021, 1022, 28, - /* 1820 */ 86, 217, 85, 1494, 190, 175, 183, 465, 185, 466, - /* 1830 */ 36, 1409, 186, 187, 188, 499, 449, 244, 37, 99, - /* 1840 */ 400, 1415, 1414, 488, 1417, 194, 473, 403, 561, 1483, - /* 1850 */ 248, 92, 1505, 494, 198, 279, 112, 564, 250, 4, - /* 1860 */ 348, 497, 405, 352, 1259, 251, 252, 515, 1316, 434, - /* 1870 */ 1315, 1314, 94, 567, 1307, 886, 1306, 1031, 226, 406, - /* 1880 */ 1611, 1610, 438, 110, 110, 1580, 1286, 524, 439, 308, - /* 1890 */ 266, 111, 1285, 449, 573, 449, 449, 309, 1019, 366, - /* 1900 */ 1284, 1609, 265, 1566, 1565, 442, 372, 1381, 561, 129, - /* 1910 */ 550, 1380, 10, 1470, 383, 106, 316, 551, 100, 35, - /* 1920 */ 534, 575, 212, 1339, 381, 387, 1187, 1338, 274, 276, - /* 1930 */ 1019, 1019, 1021, 1022, 28, 277, 413, 1031, 576, 1254, - /* 1940 */ 388, 1521, 1249, 110, 110, 167, 1522, 168, 148, 1520, - /* 1950 */ 1519, 111, 306, 449, 573, 449, 222, 223, 1019, 839, - /* 1960 */ 169, 79, 450, 214, 414, 233, 320, 145, 1093, 1091, - /* 1970 */ 328, 182, 171, 1212, 918, 184, 240, 336, 243, 1107, - /* 1980 */ 189, 172, 173, 423, 425, 88, 180, 191, 89, 90, - /* 1990 */ 1019, 1019, 1021, 1022, 28, 91, 174, 1110, 245, 1106, - /* 2000 */ 246, 159, 18, 247, 347, 1099, 263, 195, 1227, 493, - /* 2010 */ 249, 196, 38, 854, 498, 368, 253, 360, 897, 197, - /* 2020 */ 502, 93, 19, 20, 507, 884, 363, 510, 95, 307, - /* 2030 */ 160, 96, 518, 97, 1175, 1060, 1146, 40, 21, 227, - /* 2040 */ 176, 1145, 282, 284, 969, 200, 963, 114, 262, 1165, - /* 2050 */ 22, 23, 24, 1161, 1169, 25, 1163, 1150, 34, 26, - /* 2060 */ 1168, 546, 27, 204, 101, 103, 104, 1074, 7, 1061, - /* 2070 */ 1059, 1063, 1116, 1064, 1115, 268, 269, 29, 41, 270, - /* 2080 */ 1024, 866, 113, 30, 568, 392, 1183, 144, 178, 1182, - /* 2090 */ 271, 928, 1245, 1245, 1245, 1245, 1245, 1245, 1245, 1602, + /* 0 */ 130, 127, 234, 282, 282, 1328, 576, 1307, 460, 289, + /* 10 */ 289, 576, 1622, 381, 576, 1328, 573, 576, 562, 413, + /* 20 */ 1300, 1542, 573, 481, 562, 524, 460, 459, 558, 82, + /* 30 */ 82, 983, 294, 375, 51, 51, 498, 61, 61, 984, + /* 40 */ 82, 82, 1577, 137, 138, 91, 7, 1228, 1228, 1063, + /* 50 */ 1066, 1053, 1053, 135, 135, 136, 136, 136, 136, 413, + /* 60 */ 288, 288, 182, 288, 288, 481, 536, 288, 288, 130, + /* 70 */ 127, 234, 432, 573, 525, 562, 573, 557, 562, 1290, + /* 80 */ 573, 421, 562, 137, 138, 91, 559, 1228, 1228, 1063, + /* 90 */ 1066, 1053, 1053, 135, 135, 136, 136, 136, 136, 296, + /* 100 */ 460, 398, 1249, 134, 134, 134, 134, 133, 133, 132, + /* 110 */ 132, 132, 131, 128, 451, 451, 1050, 1050, 1064, 1067, + /* 120 */ 1255, 1, 1, 582, 2, 1259, 581, 1174, 1259, 1174, + /* 130 */ 321, 413, 155, 321, 1584, 155, 379, 112, 481, 1341, + /* 140 */ 456, 299, 1341, 134, 134, 134, 134, 133, 133, 132, + /* 150 */ 132, 132, 131, 128, 451, 137, 138, 91, 498, 1228, + /* 160 */ 1228, 1063, 1066, 1053, 1053, 135, 135, 136, 136, 136, + /* 170 */ 136, 1204, 862, 1281, 288, 288, 283, 288, 288, 523, + /* 180 */ 523, 1250, 139, 578, 7, 578, 1345, 573, 1169, 562, + /* 190 */ 573, 1054, 562, 136, 136, 136, 136, 129, 573, 547, + /* 200 */ 562, 1169, 245, 1541, 1169, 245, 133, 133, 132, 132, + /* 210 */ 132, 131, 128, 451, 302, 134, 134, 134, 134, 133, + /* 220 */ 133, 132, 132, 132, 131, 128, 451, 1575, 1204, 1205, + /* 230 */ 1204, 7, 470, 550, 455, 413, 550, 455, 130, 127, + /* 240 */ 234, 134, 134, 134, 134, 133, 133, 132, 132, 132, + /* 250 */ 131, 128, 451, 136, 136, 136, 136, 538, 483, 137, + /* 260 */ 138, 91, 1019, 1228, 1228, 1063, 1066, 1053, 1053, 135, + /* 270 */ 135, 136, 136, 136, 136, 1085, 576, 1204, 132, 132, + /* 280 */ 132, 131, 128, 451, 93, 214, 134, 134, 134, 134, + /* 290 */ 133, 133, 132, 132, 132, 131, 128, 451, 401, 19, + /* 300 */ 19, 134, 134, 134, 134, 133, 133, 132, 132, 132, + /* 310 */ 131, 128, 451, 1498, 426, 267, 344, 467, 332, 134, + /* 320 */ 134, 134, 134, 133, 133, 132, 132, 132, 131, 128, + /* 330 */ 451, 1281, 576, 6, 1204, 1205, 1204, 257, 576, 413, + /* 340 */ 511, 508, 507, 1279, 94, 1019, 464, 1204, 551, 551, + /* 350 */ 506, 1224, 1571, 44, 38, 51, 51, 411, 576, 413, + /* 360 */ 45, 51, 51, 137, 138, 91, 530, 1228, 1228, 1063, + /* 370 */ 1066, 1053, 1053, 135, 135, 136, 136, 136, 136, 398, + /* 380 */ 1148, 82, 82, 137, 138, 91, 39, 1228, 1228, 1063, + /* 390 */ 1066, 1053, 1053, 135, 135, 136, 136, 136, 136, 344, + /* 400 */ 44, 288, 288, 375, 1204, 1205, 1204, 209, 1204, 1224, + /* 410 */ 320, 567, 471, 576, 573, 576, 562, 576, 316, 264, + /* 420 */ 231, 46, 160, 134, 134, 134, 134, 133, 133, 132, + /* 430 */ 132, 132, 131, 128, 451, 303, 82, 82, 82, 82, + /* 440 */ 82, 82, 442, 134, 134, 134, 134, 133, 133, 132, + /* 450 */ 132, 132, 131, 128, 451, 1582, 544, 320, 567, 1250, + /* 460 */ 874, 1582, 380, 382, 413, 1204, 1205, 1204, 360, 182, + /* 470 */ 288, 288, 1576, 557, 1339, 557, 7, 557, 1277, 472, + /* 480 */ 346, 526, 531, 573, 556, 562, 439, 1511, 137, 138, + /* 490 */ 91, 219, 1228, 1228, 1063, 1066, 1053, 1053, 135, 135, + /* 500 */ 136, 136, 136, 136, 465, 1511, 1513, 532, 413, 288, + /* 510 */ 288, 423, 512, 288, 288, 411, 288, 288, 874, 130, + /* 520 */ 127, 234, 573, 1107, 562, 1204, 573, 1107, 562, 573, + /* 530 */ 560, 562, 137, 138, 91, 1293, 1228, 1228, 1063, 1066, + /* 540 */ 1053, 1053, 135, 135, 136, 136, 136, 136, 134, 134, + /* 550 */ 134, 134, 133, 133, 132, 132, 132, 131, 128, 451, + /* 560 */ 493, 503, 1292, 1204, 257, 288, 288, 511, 508, 507, + /* 570 */ 1204, 1628, 1169, 123, 568, 275, 4, 506, 573, 1511, + /* 580 */ 562, 331, 1204, 1205, 1204, 1169, 548, 548, 1169, 261, + /* 590 */ 571, 7, 134, 134, 134, 134, 133, 133, 132, 132, + /* 600 */ 132, 131, 128, 451, 108, 533, 130, 127, 234, 1204, + /* 610 */ 448, 447, 413, 1451, 452, 983, 886, 96, 1598, 1233, + /* 620 */ 1204, 1205, 1204, 984, 1235, 1450, 565, 1204, 1205, 1204, + /* 630 */ 229, 522, 1234, 534, 1333, 1333, 137, 138, 91, 1449, + /* 640 */ 1228, 1228, 1063, 1066, 1053, 1053, 135, 135, 136, 136, + /* 650 */ 136, 136, 373, 1595, 971, 1040, 413, 1236, 418, 1236, + /* 660 */ 879, 121, 121, 948, 373, 1595, 1204, 1205, 1204, 122, + /* 670 */ 1204, 452, 577, 452, 363, 417, 1028, 882, 373, 1595, + /* 680 */ 137, 138, 91, 462, 1228, 1228, 1063, 1066, 1053, 1053, + /* 690 */ 135, 135, 136, 136, 136, 136, 134, 134, 134, 134, + /* 700 */ 133, 133, 132, 132, 132, 131, 128, 451, 1028, 1028, + /* 710 */ 1030, 1031, 35, 570, 570, 570, 197, 423, 1040, 198, + /* 720 */ 1204, 123, 568, 1204, 4, 320, 567, 1204, 1205, 1204, + /* 730 */ 40, 388, 576, 384, 882, 1029, 423, 1188, 571, 1028, + /* 740 */ 134, 134, 134, 134, 133, 133, 132, 132, 132, 131, + /* 750 */ 128, 451, 529, 1568, 1204, 19, 19, 1204, 575, 492, + /* 760 */ 413, 157, 452, 489, 1187, 1331, 1331, 5, 1204, 949, + /* 770 */ 431, 1028, 1028, 1030, 565, 22, 22, 1204, 1205, 1204, + /* 780 */ 1204, 1205, 1204, 477, 137, 138, 91, 212, 1228, 1228, + /* 790 */ 1063, 1066, 1053, 1053, 135, 135, 136, 136, 136, 136, + /* 800 */ 1188, 48, 111, 1040, 413, 1204, 213, 970, 1041, 121, + /* 810 */ 121, 1204, 1205, 1204, 1204, 1205, 1204, 122, 221, 452, + /* 820 */ 577, 452, 44, 487, 1028, 1204, 1205, 1204, 137, 138, + /* 830 */ 91, 378, 1228, 1228, 1063, 1066, 1053, 1053, 135, 135, + /* 840 */ 136, 136, 136, 136, 134, 134, 134, 134, 133, 133, + /* 850 */ 132, 132, 132, 131, 128, 451, 1028, 1028, 1030, 1031, + /* 860 */ 35, 461, 1204, 1205, 1204, 1569, 1040, 377, 214, 1149, + /* 870 */ 1657, 535, 1657, 437, 902, 320, 567, 1568, 364, 320, + /* 880 */ 567, 412, 329, 1029, 519, 1188, 3, 1028, 134, 134, + /* 890 */ 134, 134, 133, 133, 132, 132, 132, 131, 128, 451, + /* 900 */ 1659, 399, 1169, 307, 893, 307, 515, 576, 413, 214, + /* 910 */ 498, 944, 1024, 540, 903, 1169, 943, 392, 1169, 1028, + /* 920 */ 1028, 1030, 406, 298, 1204, 50, 1149, 1658, 413, 1658, + /* 930 */ 145, 145, 137, 138, 91, 293, 1228, 1228, 1063, 1066, + /* 940 */ 1053, 1053, 135, 135, 136, 136, 136, 136, 1188, 1147, + /* 950 */ 514, 1568, 137, 138, 91, 1505, 1228, 1228, 1063, 1066, + /* 960 */ 1053, 1053, 135, 135, 136, 136, 136, 136, 434, 323, + /* 970 */ 435, 539, 111, 1506, 274, 291, 372, 517, 367, 516, + /* 980 */ 262, 1204, 1205, 1204, 1574, 481, 363, 576, 7, 1569, + /* 990 */ 1568, 377, 134, 134, 134, 134, 133, 133, 132, 132, + /* 1000 */ 132, 131, 128, 451, 1568, 576, 1147, 576, 232, 576, + /* 1010 */ 19, 19, 134, 134, 134, 134, 133, 133, 132, 132, + /* 1020 */ 132, 131, 128, 451, 1169, 433, 576, 1207, 19, 19, + /* 1030 */ 19, 19, 19, 19, 1627, 576, 911, 1169, 47, 120, + /* 1040 */ 1169, 117, 413, 306, 498, 438, 1125, 206, 336, 19, + /* 1050 */ 19, 1435, 49, 449, 449, 449, 1368, 315, 81, 81, + /* 1060 */ 576, 304, 413, 1570, 207, 377, 137, 138, 91, 115, + /* 1070 */ 1228, 1228, 1063, 1066, 1053, 1053, 135, 135, 136, 136, + /* 1080 */ 136, 136, 576, 82, 82, 1207, 137, 138, 91, 1340, + /* 1090 */ 1228, 1228, 1063, 1066, 1053, 1053, 135, 135, 136, 136, + /* 1100 */ 136, 136, 1569, 386, 377, 82, 82, 463, 1126, 1552, + /* 1110 */ 333, 463, 335, 131, 128, 451, 1569, 161, 377, 16, + /* 1120 */ 317, 387, 428, 1127, 448, 447, 134, 134, 134, 134, + /* 1130 */ 133, 133, 132, 132, 132, 131, 128, 451, 1128, 576, + /* 1140 */ 1105, 10, 445, 267, 576, 1554, 134, 134, 134, 134, + /* 1150 */ 133, 133, 132, 132, 132, 131, 128, 451, 532, 576, + /* 1160 */ 922, 576, 19, 19, 576, 1573, 576, 147, 147, 7, + /* 1170 */ 923, 1236, 498, 1236, 576, 487, 413, 552, 285, 1224, + /* 1180 */ 969, 215, 82, 82, 66, 66, 1435, 67, 67, 21, + /* 1190 */ 21, 1110, 1110, 495, 334, 297, 413, 53, 53, 297, + /* 1200 */ 137, 138, 91, 119, 1228, 1228, 1063, 1066, 1053, 1053, + /* 1210 */ 135, 135, 136, 136, 136, 136, 413, 1336, 1311, 446, + /* 1220 */ 137, 138, 91, 227, 1228, 1228, 1063, 1066, 1053, 1053, + /* 1230 */ 135, 135, 136, 136, 136, 136, 574, 1224, 936, 936, + /* 1240 */ 137, 126, 91, 141, 1228, 1228, 1063, 1066, 1053, 1053, + /* 1250 */ 135, 135, 136, 136, 136, 136, 533, 429, 472, 346, + /* 1260 */ 134, 134, 134, 134, 133, 133, 132, 132, 132, 131, + /* 1270 */ 128, 451, 576, 457, 233, 343, 1435, 403, 498, 1550, + /* 1280 */ 134, 134, 134, 134, 133, 133, 132, 132, 132, 131, + /* 1290 */ 128, 451, 576, 324, 576, 82, 82, 487, 576, 969, + /* 1300 */ 134, 134, 134, 134, 133, 133, 132, 132, 132, 131, + /* 1310 */ 128, 451, 288, 288, 546, 68, 68, 54, 54, 553, + /* 1320 */ 413, 69, 69, 351, 6, 573, 944, 562, 410, 409, + /* 1330 */ 1435, 943, 450, 545, 260, 259, 258, 576, 158, 576, + /* 1340 */ 413, 222, 1180, 479, 969, 138, 91, 430, 1228, 1228, + /* 1350 */ 1063, 1066, 1053, 1053, 135, 135, 136, 136, 136, 136, + /* 1360 */ 70, 70, 71, 71, 576, 1126, 91, 576, 1228, 1228, + /* 1370 */ 1063, 1066, 1053, 1053, 135, 135, 136, 136, 136, 136, + /* 1380 */ 1127, 166, 850, 851, 852, 1282, 419, 72, 72, 108, + /* 1390 */ 73, 73, 1310, 358, 1180, 1128, 576, 305, 576, 123, + /* 1400 */ 568, 494, 4, 488, 134, 134, 134, 134, 133, 133, + /* 1410 */ 132, 132, 132, 131, 128, 451, 571, 564, 534, 55, + /* 1420 */ 55, 56, 56, 576, 134, 134, 134, 134, 133, 133, + /* 1430 */ 132, 132, 132, 131, 128, 451, 576, 1104, 233, 1104, + /* 1440 */ 452, 1602, 582, 2, 1259, 576, 57, 57, 576, 321, + /* 1450 */ 576, 155, 565, 1435, 485, 353, 576, 356, 1341, 59, + /* 1460 */ 59, 576, 44, 969, 569, 419, 576, 238, 60, 60, + /* 1470 */ 261, 74, 74, 75, 75, 287, 231, 576, 1366, 76, + /* 1480 */ 76, 1040, 420, 184, 20, 20, 576, 121, 121, 77, + /* 1490 */ 77, 97, 218, 288, 288, 122, 125, 452, 577, 452, + /* 1500 */ 143, 143, 1028, 576, 520, 576, 573, 576, 562, 144, + /* 1510 */ 144, 474, 227, 1244, 478, 123, 568, 576, 4, 320, + /* 1520 */ 567, 245, 411, 576, 443, 411, 78, 78, 62, 62, + /* 1530 */ 79, 79, 571, 319, 1028, 1028, 1030, 1031, 35, 418, + /* 1540 */ 63, 63, 576, 290, 411, 9, 80, 80, 1144, 576, + /* 1550 */ 400, 576, 486, 455, 576, 1223, 452, 576, 325, 342, + /* 1560 */ 576, 111, 576, 1188, 242, 64, 64, 473, 565, 576, + /* 1570 */ 23, 576, 170, 170, 171, 171, 576, 87, 87, 328, + /* 1580 */ 65, 65, 542, 83, 83, 146, 146, 541, 123, 568, + /* 1590 */ 341, 4, 84, 84, 168, 168, 576, 1040, 576, 148, + /* 1600 */ 148, 576, 1380, 121, 121, 571, 1021, 576, 266, 576, + /* 1610 */ 424, 122, 576, 452, 577, 452, 576, 553, 1028, 142, + /* 1620 */ 142, 169, 169, 576, 162, 162, 528, 889, 371, 452, + /* 1630 */ 152, 152, 151, 151, 1379, 149, 149, 109, 370, 150, + /* 1640 */ 150, 565, 576, 480, 576, 266, 86, 86, 576, 1092, + /* 1650 */ 1028, 1028, 1030, 1031, 35, 542, 482, 576, 266, 466, + /* 1660 */ 543, 123, 568, 1616, 4, 88, 88, 85, 85, 475, + /* 1670 */ 1040, 52, 52, 222, 901, 900, 121, 121, 571, 1188, + /* 1680 */ 58, 58, 244, 1032, 122, 889, 452, 577, 452, 908, + /* 1690 */ 909, 1028, 300, 347, 504, 111, 263, 361, 165, 111, + /* 1700 */ 111, 1088, 452, 263, 974, 1153, 266, 1092, 986, 987, + /* 1710 */ 942, 939, 125, 125, 565, 1103, 872, 1103, 159, 941, + /* 1720 */ 1309, 125, 1557, 1028, 1028, 1030, 1031, 35, 542, 337, + /* 1730 */ 1530, 205, 1529, 541, 499, 1589, 490, 348, 1376, 352, + /* 1740 */ 355, 1032, 357, 1040, 359, 1324, 1308, 366, 563, 121, + /* 1750 */ 121, 376, 1188, 1389, 1434, 1362, 280, 122, 1374, 452, + /* 1760 */ 577, 452, 167, 1439, 1028, 1289, 1280, 1268, 1267, 1269, + /* 1770 */ 1609, 1359, 312, 313, 314, 397, 12, 237, 224, 1421, + /* 1780 */ 295, 1416, 1409, 1426, 339, 484, 340, 509, 1371, 1612, + /* 1790 */ 1372, 1425, 1244, 404, 301, 228, 1028, 1028, 1030, 1031, + /* 1800 */ 35, 1601, 1192, 454, 345, 1307, 292, 369, 1502, 1501, + /* 1810 */ 270, 396, 396, 395, 277, 393, 1370, 1369, 859, 1549, + /* 1820 */ 186, 123, 568, 235, 4, 1188, 391, 210, 211, 223, + /* 1830 */ 1547, 239, 1241, 327, 422, 96, 220, 195, 571, 180, + /* 1840 */ 188, 326, 468, 469, 190, 191, 502, 192, 193, 566, + /* 1850 */ 247, 109, 1430, 491, 199, 251, 102, 281, 402, 476, + /* 1860 */ 405, 1496, 452, 497, 253, 1422, 13, 1428, 14, 1427, + /* 1870 */ 203, 1507, 241, 500, 565, 354, 407, 92, 95, 1270, + /* 1880 */ 175, 254, 518, 43, 1327, 255, 1326, 1325, 436, 1518, + /* 1890 */ 350, 1318, 104, 229, 893, 1626, 440, 441, 1625, 408, + /* 1900 */ 240, 1296, 268, 1040, 310, 269, 1297, 527, 444, 121, + /* 1910 */ 121, 368, 1295, 1594, 1624, 311, 1394, 122, 1317, 452, + /* 1920 */ 577, 452, 374, 1580, 1028, 1393, 140, 553, 11, 90, + /* 1930 */ 568, 385, 4, 116, 318, 414, 1579, 110, 1483, 537, + /* 1940 */ 320, 567, 1350, 555, 42, 579, 571, 1349, 1198, 383, + /* 1950 */ 276, 390, 216, 389, 278, 279, 1028, 1028, 1030, 1031, + /* 1960 */ 35, 172, 580, 1265, 458, 1260, 415, 416, 185, 156, + /* 1970 */ 452, 1534, 1535, 173, 1533, 1532, 89, 308, 225, 226, + /* 1980 */ 846, 174, 565, 453, 217, 1188, 322, 236, 1102, 154, + /* 1990 */ 1100, 330, 187, 176, 1223, 243, 189, 925, 338, 246, + /* 2000 */ 1116, 194, 177, 425, 178, 427, 98, 196, 99, 100, + /* 2010 */ 101, 1040, 179, 1119, 1115, 248, 249, 121, 121, 163, + /* 2020 */ 24, 250, 349, 1238, 496, 122, 1108, 452, 577, 452, + /* 2030 */ 1192, 454, 1028, 266, 292, 200, 252, 201, 861, 396, + /* 2040 */ 396, 395, 277, 393, 15, 501, 859, 370, 292, 256, + /* 2050 */ 202, 554, 505, 396, 396, 395, 277, 393, 103, 239, + /* 2060 */ 859, 327, 25, 26, 1028, 1028, 1030, 1031, 35, 326, + /* 2070 */ 362, 510, 891, 239, 365, 327, 513, 904, 105, 309, + /* 2080 */ 164, 181, 27, 326, 106, 521, 107, 1185, 1069, 1155, + /* 2090 */ 17, 1154, 230, 1188, 284, 286, 265, 204, 125, 1171, + /* 2100 */ 241, 28, 978, 972, 29, 41, 1175, 1179, 175, 1173, + /* 2110 */ 30, 43, 31, 8, 241, 1178, 32, 1160, 208, 549, + /* 2120 */ 33, 111, 175, 1083, 1070, 43, 1068, 1072, 240, 113, + /* 2130 */ 114, 34, 561, 118, 1124, 271, 1073, 36, 18, 572, + /* 2140 */ 1033, 873, 240, 124, 37, 935, 272, 273, 1617, 183, + /* 2150 */ 153, 394, 1194, 1193, 1256, 1256, 1256, 1256, 1256, 1256, + /* 2160 */ 1256, 1256, 1256, 414, 1256, 1256, 1256, 1256, 320, 567, + /* 2170 */ 1256, 1256, 1256, 1256, 1256, 1256, 1256, 414, 1256, 1256, + /* 2180 */ 1256, 1256, 320, 567, 1256, 1256, 1256, 1256, 1256, 1256, + /* 2190 */ 1256, 1256, 458, 1256, 1256, 1256, 1256, 1256, 1256, 1256, + /* 2200 */ 1256, 1256, 1256, 1256, 1256, 1256, 458, }; static const YYCODETYPE yy_lookahead[] = { - /* 0 */ 193, 193, 193, 274, 275, 276, 193, 274, 275, 276, - /* 10 */ 193, 223, 219, 225, 206, 210, 211, 212, 193, 19, - /* 20 */ 219, 233, 216, 216, 217, 216, 217, 193, 295, 216, - /* 30 */ 217, 31, 193, 216, 217, 193, 228, 213, 230, 39, - /* 40 */ 206, 216, 217, 43, 44, 45, 46, 47, 48, 49, - /* 50 */ 50, 51, 52, 53, 54, 55, 56, 57, 193, 19, - /* 60 */ 185, 186, 187, 188, 189, 190, 253, 274, 275, 276, - /* 70 */ 195, 193, 197, 193, 261, 274, 275, 276, 253, 204, - /* 80 */ 238, 204, 81, 43, 44, 45, 46, 47, 48, 49, - /* 90 */ 50, 51, 52, 53, 54, 55, 56, 57, 274, 275, - /* 100 */ 276, 262, 102, 103, 104, 105, 106, 107, 108, 109, - /* 110 */ 110, 111, 112, 113, 239, 240, 239, 240, 210, 211, - /* 120 */ 212, 314, 315, 314, 59, 316, 86, 252, 88, 252, - /* 130 */ 19, 314, 315, 256, 257, 113, 25, 72, 296, 138, - /* 140 */ 139, 266, 102, 103, 104, 105, 106, 107, 108, 109, - /* 150 */ 110, 111, 112, 113, 43, 44, 45, 46, 47, 48, - /* 160 */ 49, 50, 51, 52, 53, 54, 55, 56, 57, 81, - /* 170 */ 292, 59, 292, 298, 108, 109, 110, 111, 112, 113, - /* 180 */ 69, 116, 117, 118, 72, 106, 107, 193, 111, 112, - /* 190 */ 113, 54, 55, 56, 57, 58, 102, 103, 104, 105, - /* 200 */ 106, 107, 108, 109, 110, 111, 112, 113, 120, 25, - /* 210 */ 216, 217, 145, 102, 103, 104, 105, 106, 107, 108, - /* 220 */ 109, 110, 111, 112, 113, 231, 138, 139, 116, 117, - /* 230 */ 118, 164, 153, 19, 155, 54, 55, 56, 57, 102, - /* 240 */ 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, - /* 250 */ 113, 128, 129, 46, 47, 48, 49, 43, 44, 45, - /* 260 */ 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, - /* 270 */ 56, 57, 216, 193, 25, 59, 193, 19, 165, 166, - /* 280 */ 193, 67, 24, 102, 103, 104, 105, 106, 107, 108, - /* 290 */ 109, 110, 111, 112, 113, 73, 216, 217, 59, 216, - /* 300 */ 217, 43, 44, 45, 46, 47, 48, 49, 50, 51, - /* 310 */ 52, 53, 54, 55, 56, 57, 102, 103, 104, 105, - /* 320 */ 106, 107, 108, 109, 110, 111, 112, 113, 121, 145, - /* 330 */ 59, 193, 116, 117, 118, 119, 273, 204, 122, 123, - /* 340 */ 124, 19, 20, 134, 22, 136, 137, 19, 132, 127, - /* 350 */ 128, 129, 24, 22, 23, 116, 117, 118, 36, 193, - /* 360 */ 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, - /* 370 */ 112, 113, 239, 240, 311, 312, 215, 106, 107, 241, - /* 380 */ 19, 59, 216, 217, 223, 252, 115, 116, 117, 118, - /* 390 */ 151, 120, 26, 71, 193, 308, 309, 193, 149, 128, - /* 400 */ 313, 216, 269, 81, 43, 44, 45, 46, 47, 48, - /* 410 */ 49, 50, 51, 52, 53, 54, 55, 56, 57, 253, - /* 420 */ 216, 217, 100, 95, 153, 59, 155, 261, 106, 107, - /* 430 */ 25, 193, 101, 193, 193, 231, 114, 25, 116, 117, - /* 440 */ 118, 113, 304, 121, 193, 204, 59, 119, 120, 121, - /* 450 */ 122, 123, 124, 125, 216, 217, 193, 216, 217, 131, - /* 460 */ 138, 139, 230, 102, 103, 104, 105, 106, 107, 108, - /* 470 */ 109, 110, 111, 112, 113, 153, 154, 155, 156, 157, - /* 480 */ 239, 240, 116, 117, 118, 76, 193, 23, 19, 25, - /* 490 */ 22, 253, 23, 252, 253, 108, 87, 204, 89, 261, - /* 500 */ 198, 92, 261, 116, 117, 118, 193, 306, 307, 216, - /* 510 */ 217, 150, 43, 44, 45, 46, 47, 48, 49, 50, - /* 520 */ 51, 52, 53, 54, 55, 56, 57, 59, 193, 216, - /* 530 */ 217, 19, 239, 240, 283, 23, 106, 107, 108, 109, - /* 540 */ 110, 111, 112, 113, 73, 252, 253, 142, 308, 309, - /* 550 */ 138, 139, 81, 313, 145, 43, 44, 45, 46, 47, - /* 560 */ 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, - /* 570 */ 307, 102, 103, 104, 105, 106, 107, 108, 109, 110, - /* 580 */ 111, 112, 113, 281, 116, 117, 118, 285, 23, 193, - /* 590 */ 25, 119, 59, 193, 122, 123, 124, 59, 127, 203, - /* 600 */ 59, 205, 19, 268, 132, 25, 23, 22, 193, 138, - /* 610 */ 139, 249, 204, 251, 102, 103, 104, 105, 106, 107, - /* 620 */ 108, 109, 110, 111, 112, 113, 43, 44, 45, 46, - /* 630 */ 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, - /* 640 */ 57, 19, 22, 23, 59, 23, 25, 239, 240, 116, - /* 650 */ 117, 118, 193, 11, 116, 117, 118, 116, 117, 118, - /* 660 */ 252, 269, 22, 193, 15, 43, 44, 45, 46, 47, - /* 670 */ 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, - /* 680 */ 273, 143, 193, 118, 143, 102, 103, 104, 105, 106, - /* 690 */ 107, 108, 109, 110, 111, 112, 113, 76, 118, 59, - /* 700 */ 241, 116, 117, 118, 304, 216, 217, 292, 143, 60, - /* 710 */ 89, 241, 19, 92, 193, 193, 23, 22, 311, 312, - /* 720 */ 231, 101, 22, 143, 102, 103, 104, 105, 106, 107, - /* 730 */ 108, 109, 110, 111, 112, 113, 43, 44, 45, 46, - /* 740 */ 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, - /* 750 */ 57, 19, 193, 193, 59, 23, 116, 117, 118, 59, - /* 760 */ 201, 21, 241, 304, 193, 206, 127, 128, 129, 193, - /* 770 */ 128, 129, 235, 236, 304, 43, 44, 45, 46, 47, - /* 780 */ 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, - /* 790 */ 22, 193, 216, 217, 193, 102, 103, 104, 105, 106, - /* 800 */ 107, 108, 109, 110, 111, 112, 113, 231, 193, 193, - /* 810 */ 193, 116, 117, 118, 216, 217, 116, 117, 118, 226, - /* 820 */ 80, 193, 19, 235, 236, 304, 23, 211, 212, 231, - /* 830 */ 204, 216, 217, 205, 102, 103, 104, 105, 106, 107, - /* 840 */ 108, 109, 110, 111, 112, 113, 43, 44, 45, 46, - /* 850 */ 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, - /* 860 */ 57, 19, 193, 123, 76, 239, 240, 193, 253, 239, - /* 870 */ 240, 239, 240, 244, 106, 107, 193, 89, 252, 193, - /* 880 */ 92, 59, 252, 254, 252, 43, 44, 45, 46, 47, - /* 890 */ 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, - /* 900 */ 284, 161, 216, 217, 193, 102, 103, 104, 105, 106, - /* 910 */ 107, 108, 109, 110, 111, 112, 113, 231, 193, 244, - /* 920 */ 187, 188, 189, 190, 7, 8, 9, 309, 195, 254, - /* 930 */ 197, 313, 19, 127, 128, 129, 262, 204, 22, 117, - /* 940 */ 24, 216, 217, 273, 102, 103, 104, 105, 106, 107, - /* 950 */ 108, 109, 110, 111, 112, 113, 43, 44, 45, 46, - /* 960 */ 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, - /* 970 */ 57, 193, 239, 240, 193, 59, 19, 188, 253, 190, - /* 980 */ 193, 311, 312, 16, 195, 252, 197, 193, 19, 301, - /* 990 */ 302, 135, 193, 204, 216, 217, 140, 216, 217, 266, - /* 1000 */ 204, 159, 45, 46, 47, 48, 49, 50, 51, 52, - /* 1010 */ 53, 54, 55, 56, 57, 102, 103, 104, 105, 106, - /* 1020 */ 107, 108, 109, 110, 111, 112, 113, 12, 239, 240, - /* 1030 */ 193, 298, 238, 117, 253, 239, 240, 238, 259, 260, - /* 1040 */ 193, 252, 27, 193, 77, 193, 79, 204, 252, 262, - /* 1050 */ 193, 299, 300, 193, 100, 266, 278, 42, 204, 102, - /* 1060 */ 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, - /* 1070 */ 113, 117, 159, 216, 217, 121, 216, 217, 63, 193, - /* 1080 */ 193, 193, 239, 240, 115, 116, 193, 298, 73, 240, - /* 1090 */ 238, 231, 19, 239, 240, 252, 22, 24, 211, 212, - /* 1100 */ 263, 252, 216, 217, 216, 217, 252, 153, 154, 155, - /* 1110 */ 253, 193, 19, 144, 213, 268, 43, 44, 45, 46, - /* 1120 */ 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, - /* 1130 */ 57, 193, 19, 59, 216, 217, 43, 44, 45, 46, - /* 1140 */ 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, - /* 1150 */ 57, 193, 19, 24, 216, 217, 43, 44, 45, 46, - /* 1160 */ 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, - /* 1170 */ 57, 284, 193, 208, 209, 102, 103, 104, 105, 106, - /* 1180 */ 107, 108, 109, 110, 111, 112, 113, 286, 59, 193, - /* 1190 */ 232, 117, 291, 193, 193, 102, 103, 104, 105, 106, - /* 1200 */ 107, 108, 109, 110, 111, 112, 113, 193, 204, 22, - /* 1210 */ 23, 193, 25, 66, 193, 102, 103, 104, 105, 106, - /* 1220 */ 107, 108, 109, 110, 111, 112, 113, 193, 193, 193, - /* 1230 */ 216, 217, 85, 193, 238, 19, 16, 216, 217, 238, - /* 1240 */ 193, 94, 193, 239, 240, 231, 117, 268, 35, 116, - /* 1250 */ 216, 217, 216, 217, 22, 23, 252, 25, 208, 209, - /* 1260 */ 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, - /* 1270 */ 54, 55, 56, 57, 193, 193, 19, 5, 59, 66, - /* 1280 */ 193, 263, 10, 11, 12, 13, 14, 74, 101, 17, - /* 1290 */ 193, 46, 193, 146, 193, 76, 213, 77, 263, 79, - /* 1300 */ 12, 260, 30, 46, 32, 264, 87, 193, 89, 29, - /* 1310 */ 263, 92, 40, 33, 232, 27, 193, 108, 102, 103, - /* 1320 */ 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, - /* 1330 */ 42, 138, 139, 101, 193, 116, 117, 118, 19, 20, - /* 1340 */ 255, 22, 70, 130, 135, 65, 256, 257, 193, 140, - /* 1350 */ 78, 63, 193, 81, 193, 36, 193, 216, 217, 193, - /* 1360 */ 115, 193, 263, 193, 145, 268, 59, 48, 193, 193, - /* 1370 */ 98, 193, 115, 193, 291, 216, 217, 193, 59, 216, - /* 1380 */ 217, 161, 216, 217, 216, 217, 216, 217, 131, 193, - /* 1390 */ 71, 193, 216, 217, 216, 217, 216, 217, 193, 260, - /* 1400 */ 216, 217, 19, 264, 85, 133, 244, 100, 193, 90, - /* 1410 */ 138, 139, 216, 217, 216, 217, 254, 244, 193, 100, - /* 1420 */ 193, 216, 217, 116, 117, 106, 107, 254, 121, 193, - /* 1430 */ 115, 216, 217, 114, 162, 116, 117, 118, 115, 244, - /* 1440 */ 121, 216, 217, 216, 217, 193, 309, 193, 31, 254, - /* 1450 */ 313, 309, 216, 217, 309, 313, 39, 193, 313, 309, - /* 1460 */ 153, 154, 155, 313, 193, 150, 25, 144, 216, 217, - /* 1470 */ 216, 217, 153, 154, 155, 156, 157, 0, 1, 2, - /* 1480 */ 216, 217, 5, 149, 150, 22, 193, 10, 11, 12, - /* 1490 */ 13, 14, 193, 158, 17, 160, 193, 19, 20, 116, - /* 1500 */ 22, 25, 193, 24, 22, 193, 24, 30, 226, 32, - /* 1510 */ 19, 20, 226, 22, 36, 193, 53, 40, 193, 216, - /* 1520 */ 217, 193, 23, 193, 25, 216, 217, 36, 216, 217, - /* 1530 */ 193, 99, 193, 193, 22, 193, 193, 59, 216, 217, - /* 1540 */ 193, 216, 217, 193, 216, 217, 193, 70, 129, 71, - /* 1550 */ 59, 129, 193, 216, 217, 78, 216, 217, 81, 216, - /* 1560 */ 217, 193, 71, 85, 193, 133, 193, 126, 90, 216, - /* 1570 */ 217, 152, 258, 61, 152, 98, 85, 193, 100, 193, - /* 1580 */ 23, 90, 25, 121, 106, 107, 23, 216, 217, 216, - /* 1590 */ 217, 100, 114, 131, 116, 117, 118, 106, 107, 121, - /* 1600 */ 216, 217, 216, 217, 193, 114, 193, 116, 117, 118, - /* 1610 */ 133, 22, 121, 193, 59, 138, 139, 193, 142, 193, - /* 1620 */ 141, 23, 23, 25, 25, 120, 121, 216, 217, 216, - /* 1630 */ 217, 153, 154, 155, 156, 157, 216, 217, 19, 162, - /* 1640 */ 216, 217, 216, 217, 153, 154, 155, 156, 157, 1, - /* 1650 */ 2, 193, 59, 5, 19, 20, 318, 22, 10, 11, - /* 1660 */ 12, 13, 14, 193, 59, 17, 193, 23, 23, 25, - /* 1670 */ 25, 36, 117, 193, 216, 217, 193, 23, 30, 25, - /* 1680 */ 32, 19, 20, 23, 22, 25, 216, 217, 40, 216, - /* 1690 */ 217, 7, 8, 23, 59, 25, 83, 84, 36, 23, - /* 1700 */ 193, 25, 23, 23, 25, 25, 71, 153, 145, 155, - /* 1710 */ 117, 153, 23, 155, 25, 23, 97, 25, 70, 193, - /* 1720 */ 193, 59, 117, 236, 193, 193, 78, 193, 193, 81, - /* 1730 */ 141, 193, 193, 71, 193, 100, 288, 287, 242, 255, - /* 1740 */ 255, 106, 107, 108, 255, 255, 98, 243, 297, 114, - /* 1750 */ 214, 116, 117, 118, 245, 191, 121, 271, 293, 267, - /* 1760 */ 267, 246, 100, 246, 245, 271, 271, 293, 106, 107, - /* 1770 */ 220, 271, 229, 225, 249, 219, 114, 259, 116, 117, - /* 1780 */ 118, 133, 259, 121, 219, 219, 138, 139, 153, 154, - /* 1790 */ 155, 156, 157, 280, 249, 243, 19, 20, 245, 22, - /* 1800 */ 196, 259, 140, 259, 60, 297, 141, 297, 200, 200, - /* 1810 */ 162, 38, 200, 36, 294, 153, 154, 155, 156, 157, - /* 1820 */ 151, 150, 294, 283, 22, 43, 234, 18, 237, 200, - /* 1830 */ 270, 272, 237, 237, 237, 18, 59, 199, 270, 149, - /* 1840 */ 246, 272, 272, 200, 234, 234, 246, 246, 71, 246, - /* 1850 */ 199, 158, 290, 62, 22, 200, 19, 20, 199, 22, - /* 1860 */ 289, 221, 221, 200, 200, 199, 199, 115, 218, 64, - /* 1870 */ 218, 218, 22, 36, 227, 126, 227, 100, 165, 221, - /* 1880 */ 224, 224, 24, 106, 107, 312, 218, 305, 113, 282, - /* 1890 */ 91, 114, 220, 116, 117, 118, 59, 282, 121, 218, - /* 1900 */ 218, 218, 200, 317, 317, 82, 221, 265, 71, 148, - /* 1910 */ 145, 265, 22, 277, 200, 158, 279, 140, 147, 25, - /* 1920 */ 146, 202, 248, 250, 249, 247, 13, 250, 194, 194, - /* 1930 */ 153, 154, 155, 156, 157, 6, 303, 100, 192, 192, - /* 1940 */ 246, 213, 192, 106, 107, 207, 213, 207, 222, 213, - /* 1950 */ 213, 114, 222, 116, 117, 118, 214, 214, 121, 4, - /* 1960 */ 207, 213, 3, 22, 303, 15, 163, 16, 23, 23, - /* 1970 */ 139, 151, 130, 25, 20, 142, 24, 16, 144, 1, - /* 1980 */ 142, 130, 130, 61, 37, 53, 300, 151, 53, 53, - /* 1990 */ 153, 154, 155, 156, 157, 53, 130, 116, 34, 1, - /* 2000 */ 141, 5, 22, 115, 161, 68, 25, 68, 75, 41, - /* 2010 */ 141, 115, 24, 20, 19, 131, 125, 23, 28, 22, - /* 2020 */ 67, 22, 22, 22, 67, 59, 24, 96, 22, 67, - /* 2030 */ 23, 149, 22, 25, 23, 23, 23, 22, 34, 141, - /* 2040 */ 37, 97, 23, 23, 116, 22, 143, 25, 34, 75, - /* 2050 */ 34, 34, 34, 88, 75, 34, 86, 23, 22, 34, - /* 2060 */ 93, 24, 34, 25, 25, 142, 142, 23, 44, 23, - /* 2070 */ 23, 23, 23, 11, 23, 25, 22, 22, 22, 141, - /* 2080 */ 23, 23, 22, 22, 25, 15, 1, 23, 25, 1, - /* 2090 */ 141, 135, 319, 319, 319, 319, 319, 319, 319, 141, - /* 2100 */ 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, - /* 2110 */ 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, - /* 2120 */ 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, - /* 2130 */ 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, - /* 2140 */ 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, - /* 2150 */ 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, - /* 2160 */ 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, - /* 2170 */ 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, - /* 2180 */ 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, - /* 2190 */ 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, - /* 2200 */ 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, - /* 2210 */ 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, - /* 2220 */ 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, - /* 2230 */ 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, - /* 2240 */ 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, - /* 2250 */ 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, - /* 2260 */ 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, - /* 2270 */ 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, - /* 2280 */ 319, 319, 319, 319, 319, + /* 0 */ 277, 278, 279, 241, 242, 225, 195, 227, 195, 241, + /* 10 */ 242, 195, 217, 221, 195, 235, 254, 195, 256, 19, + /* 20 */ 225, 298, 254, 195, 256, 206, 213, 214, 206, 218, + /* 30 */ 219, 31, 206, 195, 218, 219, 195, 218, 219, 39, + /* 40 */ 218, 219, 313, 43, 44, 45, 317, 47, 48, 49, + /* 50 */ 50, 51, 52, 53, 54, 55, 56, 57, 58, 19, + /* 60 */ 241, 242, 195, 241, 242, 195, 255, 241, 242, 277, + /* 70 */ 278, 279, 234, 254, 255, 256, 254, 255, 256, 218, + /* 80 */ 254, 240, 256, 43, 44, 45, 264, 47, 48, 49, + /* 90 */ 50, 51, 52, 53, 54, 55, 56, 57, 58, 271, + /* 100 */ 287, 22, 23, 103, 104, 105, 106, 107, 108, 109, + /* 110 */ 110, 111, 112, 113, 114, 114, 47, 48, 49, 50, + /* 120 */ 187, 188, 189, 190, 191, 192, 190, 87, 192, 89, + /* 130 */ 197, 19, 199, 197, 318, 199, 320, 25, 195, 206, + /* 140 */ 299, 271, 206, 103, 104, 105, 106, 107, 108, 109, + /* 150 */ 110, 111, 112, 113, 114, 43, 44, 45, 195, 47, + /* 160 */ 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, + /* 170 */ 58, 60, 21, 195, 241, 242, 215, 241, 242, 312, + /* 180 */ 313, 102, 70, 205, 317, 207, 242, 254, 77, 256, + /* 190 */ 254, 122, 256, 55, 56, 57, 58, 59, 254, 88, + /* 200 */ 256, 90, 269, 240, 93, 269, 107, 108, 109, 110, + /* 210 */ 111, 112, 113, 114, 271, 103, 104, 105, 106, 107, + /* 220 */ 108, 109, 110, 111, 112, 113, 114, 313, 117, 118, + /* 230 */ 119, 317, 81, 195, 301, 19, 195, 301, 277, 278, + /* 240 */ 279, 103, 104, 105, 106, 107, 108, 109, 110, 111, + /* 250 */ 112, 113, 114, 55, 56, 57, 58, 146, 195, 43, + /* 260 */ 44, 45, 74, 47, 48, 49, 50, 51, 52, 53, + /* 270 */ 54, 55, 56, 57, 58, 124, 195, 60, 109, 110, + /* 280 */ 111, 112, 113, 114, 68, 195, 103, 104, 105, 106, + /* 290 */ 107, 108, 109, 110, 111, 112, 113, 114, 208, 218, + /* 300 */ 219, 103, 104, 105, 106, 107, 108, 109, 110, 111, + /* 310 */ 112, 113, 114, 162, 233, 24, 128, 129, 130, 103, + /* 320 */ 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, + /* 330 */ 114, 195, 195, 215, 117, 118, 119, 120, 195, 19, + /* 340 */ 123, 124, 125, 207, 24, 74, 246, 60, 310, 311, + /* 350 */ 133, 60, 311, 82, 22, 218, 219, 257, 195, 19, + /* 360 */ 73, 218, 219, 43, 44, 45, 206, 47, 48, 49, + /* 370 */ 50, 51, 52, 53, 54, 55, 56, 57, 58, 22, + /* 380 */ 23, 218, 219, 43, 44, 45, 54, 47, 48, 49, + /* 390 */ 50, 51, 52, 53, 54, 55, 56, 57, 58, 128, + /* 400 */ 82, 241, 242, 195, 117, 118, 119, 289, 60, 118, + /* 410 */ 139, 140, 294, 195, 254, 195, 256, 195, 255, 259, + /* 420 */ 260, 73, 22, 103, 104, 105, 106, 107, 108, 109, + /* 430 */ 110, 111, 112, 113, 114, 206, 218, 219, 218, 219, + /* 440 */ 218, 219, 234, 103, 104, 105, 106, 107, 108, 109, + /* 450 */ 110, 111, 112, 113, 114, 318, 319, 139, 140, 102, + /* 460 */ 60, 318, 319, 221, 19, 117, 118, 119, 23, 195, + /* 470 */ 241, 242, 313, 255, 206, 255, 317, 255, 206, 129, + /* 480 */ 130, 206, 264, 254, 264, 256, 264, 195, 43, 44, + /* 490 */ 45, 151, 47, 48, 49, 50, 51, 52, 53, 54, + /* 500 */ 55, 56, 57, 58, 246, 213, 214, 19, 19, 241, + /* 510 */ 242, 195, 23, 241, 242, 257, 241, 242, 118, 277, + /* 520 */ 278, 279, 254, 29, 256, 60, 254, 33, 256, 254, + /* 530 */ 206, 256, 43, 44, 45, 218, 47, 48, 49, 50, + /* 540 */ 51, 52, 53, 54, 55, 56, 57, 58, 103, 104, + /* 550 */ 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, + /* 560 */ 66, 19, 218, 60, 120, 241, 242, 123, 124, 125, + /* 570 */ 60, 232, 77, 19, 20, 26, 22, 133, 254, 287, + /* 580 */ 256, 265, 117, 118, 119, 90, 312, 313, 93, 47, + /* 590 */ 36, 317, 103, 104, 105, 106, 107, 108, 109, 110, + /* 600 */ 111, 112, 113, 114, 116, 117, 277, 278, 279, 60, + /* 610 */ 107, 108, 19, 276, 60, 31, 23, 152, 195, 116, + /* 620 */ 117, 118, 119, 39, 121, 276, 72, 117, 118, 119, + /* 630 */ 166, 167, 129, 145, 237, 238, 43, 44, 45, 276, + /* 640 */ 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, + /* 650 */ 57, 58, 315, 316, 144, 101, 19, 154, 116, 156, + /* 660 */ 23, 107, 108, 109, 315, 316, 117, 118, 119, 115, + /* 670 */ 60, 117, 118, 119, 132, 200, 122, 60, 315, 316, + /* 680 */ 43, 44, 45, 272, 47, 48, 49, 50, 51, 52, + /* 690 */ 53, 54, 55, 56, 57, 58, 103, 104, 105, 106, + /* 700 */ 107, 108, 109, 110, 111, 112, 113, 114, 154, 155, + /* 710 */ 156, 157, 158, 212, 213, 214, 22, 195, 101, 22, + /* 720 */ 60, 19, 20, 60, 22, 139, 140, 117, 118, 119, + /* 730 */ 22, 251, 195, 253, 117, 118, 195, 183, 36, 122, + /* 740 */ 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, + /* 750 */ 113, 114, 195, 195, 60, 218, 219, 60, 195, 284, + /* 760 */ 19, 25, 60, 288, 23, 237, 238, 22, 60, 109, + /* 770 */ 233, 154, 155, 156, 72, 218, 219, 117, 118, 119, + /* 780 */ 117, 118, 119, 116, 43, 44, 45, 265, 47, 48, + /* 790 */ 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, + /* 800 */ 183, 243, 25, 101, 19, 60, 265, 144, 23, 107, + /* 810 */ 108, 117, 118, 119, 117, 118, 119, 115, 151, 117, + /* 820 */ 118, 119, 82, 195, 122, 117, 118, 119, 43, 44, + /* 830 */ 45, 195, 47, 48, 49, 50, 51, 52, 53, 54, + /* 840 */ 55, 56, 57, 58, 103, 104, 105, 106, 107, 108, + /* 850 */ 109, 110, 111, 112, 113, 114, 154, 155, 156, 157, + /* 860 */ 158, 121, 117, 118, 119, 307, 101, 309, 195, 22, + /* 870 */ 23, 195, 25, 19, 35, 139, 140, 195, 24, 139, + /* 880 */ 140, 208, 195, 118, 109, 183, 22, 122, 103, 104, + /* 890 */ 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, + /* 900 */ 304, 305, 77, 230, 127, 232, 67, 195, 19, 195, + /* 910 */ 195, 136, 23, 88, 75, 90, 141, 203, 93, 154, + /* 920 */ 155, 156, 208, 295, 60, 243, 22, 23, 19, 25, + /* 930 */ 218, 219, 43, 44, 45, 100, 47, 48, 49, 50, + /* 940 */ 51, 52, 53, 54, 55, 56, 57, 58, 183, 102, + /* 950 */ 96, 195, 43, 44, 45, 240, 47, 48, 49, 50, + /* 960 */ 51, 52, 53, 54, 55, 56, 57, 58, 114, 134, + /* 970 */ 131, 146, 25, 286, 120, 121, 122, 123, 124, 125, + /* 980 */ 126, 117, 118, 119, 313, 195, 132, 195, 317, 307, + /* 990 */ 195, 309, 103, 104, 105, 106, 107, 108, 109, 110, + /* 1000 */ 111, 112, 113, 114, 195, 195, 102, 195, 195, 195, + /* 1010 */ 218, 219, 103, 104, 105, 106, 107, 108, 109, 110, + /* 1020 */ 111, 112, 113, 114, 77, 233, 195, 60, 218, 219, + /* 1030 */ 218, 219, 218, 219, 23, 195, 25, 90, 243, 159, + /* 1040 */ 93, 161, 19, 233, 195, 233, 23, 233, 16, 218, + /* 1050 */ 219, 195, 243, 212, 213, 214, 262, 263, 218, 219, + /* 1060 */ 195, 271, 19, 307, 233, 309, 43, 44, 45, 160, + /* 1070 */ 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, + /* 1080 */ 57, 58, 195, 218, 219, 118, 43, 44, 45, 240, + /* 1090 */ 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, + /* 1100 */ 57, 58, 307, 195, 309, 218, 219, 263, 12, 195, + /* 1110 */ 78, 267, 80, 112, 113, 114, 307, 22, 309, 24, + /* 1120 */ 255, 281, 266, 27, 107, 108, 103, 104, 105, 106, + /* 1130 */ 107, 108, 109, 110, 111, 112, 113, 114, 42, 195, + /* 1140 */ 11, 22, 255, 24, 195, 195, 103, 104, 105, 106, + /* 1150 */ 107, 108, 109, 110, 111, 112, 113, 114, 19, 195, + /* 1160 */ 64, 195, 218, 219, 195, 313, 195, 218, 219, 317, + /* 1170 */ 74, 154, 195, 156, 195, 195, 19, 233, 23, 60, + /* 1180 */ 25, 24, 218, 219, 218, 219, 195, 218, 219, 218, + /* 1190 */ 219, 128, 129, 130, 162, 263, 19, 218, 219, 267, + /* 1200 */ 43, 44, 45, 160, 47, 48, 49, 50, 51, 52, + /* 1210 */ 53, 54, 55, 56, 57, 58, 19, 240, 228, 255, + /* 1220 */ 43, 44, 45, 25, 47, 48, 49, 50, 51, 52, + /* 1230 */ 53, 54, 55, 56, 57, 58, 135, 118, 137, 138, + /* 1240 */ 43, 44, 45, 22, 47, 48, 49, 50, 51, 52, + /* 1250 */ 53, 54, 55, 56, 57, 58, 117, 266, 129, 130, + /* 1260 */ 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, + /* 1270 */ 113, 114, 195, 195, 119, 295, 195, 206, 195, 195, + /* 1280 */ 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, + /* 1290 */ 113, 114, 195, 195, 195, 218, 219, 195, 195, 144, + /* 1300 */ 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, + /* 1310 */ 113, 114, 241, 242, 67, 218, 219, 218, 219, 146, + /* 1320 */ 19, 218, 219, 240, 215, 254, 136, 256, 107, 108, + /* 1330 */ 195, 141, 255, 86, 128, 129, 130, 195, 165, 195, + /* 1340 */ 19, 143, 95, 272, 25, 44, 45, 266, 47, 48, + /* 1350 */ 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, + /* 1360 */ 218, 219, 218, 219, 195, 12, 45, 195, 47, 48, + /* 1370 */ 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, + /* 1380 */ 27, 23, 7, 8, 9, 210, 211, 218, 219, 116, + /* 1390 */ 218, 219, 228, 16, 147, 42, 195, 295, 195, 19, + /* 1400 */ 20, 266, 22, 294, 103, 104, 105, 106, 107, 108, + /* 1410 */ 109, 110, 111, 112, 113, 114, 36, 64, 145, 218, + /* 1420 */ 219, 218, 219, 195, 103, 104, 105, 106, 107, 108, + /* 1430 */ 109, 110, 111, 112, 113, 114, 195, 154, 119, 156, + /* 1440 */ 60, 189, 190, 191, 192, 195, 218, 219, 195, 197, + /* 1450 */ 195, 199, 72, 195, 19, 78, 195, 80, 206, 218, + /* 1460 */ 219, 195, 82, 144, 210, 211, 195, 15, 218, 219, + /* 1470 */ 47, 218, 219, 218, 219, 259, 260, 195, 261, 218, + /* 1480 */ 219, 101, 302, 303, 218, 219, 195, 107, 108, 218, + /* 1490 */ 219, 150, 151, 241, 242, 115, 25, 117, 118, 119, + /* 1500 */ 218, 219, 122, 195, 146, 195, 254, 195, 256, 218, + /* 1510 */ 219, 246, 25, 61, 246, 19, 20, 195, 22, 139, + /* 1520 */ 140, 269, 257, 195, 266, 257, 218, 219, 218, 219, + /* 1530 */ 218, 219, 36, 246, 154, 155, 156, 157, 158, 116, + /* 1540 */ 218, 219, 195, 22, 257, 49, 218, 219, 23, 195, + /* 1550 */ 25, 195, 117, 301, 195, 25, 60, 195, 195, 23, + /* 1560 */ 195, 25, 195, 183, 24, 218, 219, 130, 72, 195, + /* 1570 */ 22, 195, 218, 219, 218, 219, 195, 218, 219, 195, + /* 1580 */ 218, 219, 86, 218, 219, 218, 219, 91, 19, 20, + /* 1590 */ 153, 22, 218, 219, 218, 219, 195, 101, 195, 218, + /* 1600 */ 219, 195, 195, 107, 108, 36, 23, 195, 25, 195, + /* 1610 */ 62, 115, 195, 117, 118, 119, 195, 146, 122, 218, + /* 1620 */ 219, 218, 219, 195, 218, 219, 19, 60, 122, 60, + /* 1630 */ 218, 219, 218, 219, 195, 218, 219, 150, 132, 218, + /* 1640 */ 219, 72, 195, 23, 195, 25, 218, 219, 195, 60, + /* 1650 */ 154, 155, 156, 157, 158, 86, 23, 195, 25, 195, + /* 1660 */ 91, 19, 20, 142, 22, 218, 219, 218, 219, 130, + /* 1670 */ 101, 218, 219, 143, 121, 122, 107, 108, 36, 183, + /* 1680 */ 218, 219, 142, 60, 115, 118, 117, 118, 119, 7, + /* 1690 */ 8, 122, 153, 23, 23, 25, 25, 23, 23, 25, + /* 1700 */ 25, 23, 60, 25, 23, 98, 25, 118, 84, 85, + /* 1710 */ 23, 23, 25, 25, 72, 154, 23, 156, 25, 23, + /* 1720 */ 228, 25, 195, 154, 155, 156, 157, 158, 86, 195, + /* 1730 */ 195, 258, 195, 91, 291, 322, 195, 195, 195, 195, + /* 1740 */ 195, 118, 195, 101, 195, 195, 195, 195, 238, 107, + /* 1750 */ 108, 195, 183, 195, 195, 195, 290, 115, 195, 117, + /* 1760 */ 118, 119, 244, 195, 122, 195, 195, 195, 195, 195, + /* 1770 */ 195, 258, 258, 258, 258, 193, 245, 300, 216, 274, + /* 1780 */ 247, 270, 270, 274, 296, 296, 248, 222, 262, 198, + /* 1790 */ 262, 274, 61, 274, 248, 231, 154, 155, 156, 157, + /* 1800 */ 158, 0, 1, 2, 247, 227, 5, 221, 221, 221, + /* 1810 */ 142, 10, 11, 12, 13, 14, 262, 262, 17, 202, + /* 1820 */ 300, 19, 20, 300, 22, 183, 247, 251, 251, 245, + /* 1830 */ 202, 30, 38, 32, 202, 152, 151, 22, 36, 43, + /* 1840 */ 236, 40, 18, 202, 239, 239, 18, 239, 239, 283, + /* 1850 */ 201, 150, 236, 202, 236, 201, 159, 202, 248, 248, + /* 1860 */ 248, 248, 60, 63, 201, 275, 273, 275, 273, 275, + /* 1870 */ 22, 286, 71, 223, 72, 202, 223, 297, 297, 202, + /* 1880 */ 79, 201, 116, 82, 220, 201, 220, 220, 65, 293, + /* 1890 */ 292, 229, 22, 166, 127, 226, 24, 114, 226, 223, + /* 1900 */ 99, 222, 202, 101, 285, 92, 220, 308, 83, 107, + /* 1910 */ 108, 220, 220, 316, 220, 285, 268, 115, 229, 117, + /* 1920 */ 118, 119, 223, 321, 122, 268, 149, 146, 22, 19, + /* 1930 */ 20, 202, 22, 159, 282, 134, 321, 148, 280, 147, + /* 1940 */ 139, 140, 252, 141, 25, 204, 36, 252, 13, 251, + /* 1950 */ 196, 248, 250, 249, 196, 6, 154, 155, 156, 157, + /* 1960 */ 158, 209, 194, 194, 163, 194, 306, 306, 303, 224, + /* 1970 */ 60, 215, 215, 209, 215, 215, 215, 224, 216, 216, + /* 1980 */ 4, 209, 72, 3, 22, 183, 164, 15, 23, 16, + /* 1990 */ 23, 140, 152, 131, 25, 24, 143, 20, 16, 145, + /* 2000 */ 1, 143, 131, 62, 131, 37, 54, 152, 54, 54, + /* 2010 */ 54, 101, 131, 117, 1, 34, 142, 107, 108, 5, + /* 2020 */ 22, 116, 162, 76, 41, 115, 69, 117, 118, 119, + /* 2030 */ 1, 2, 122, 25, 5, 69, 142, 116, 20, 10, + /* 2040 */ 11, 12, 13, 14, 24, 19, 17, 132, 5, 126, + /* 2050 */ 22, 141, 68, 10, 11, 12, 13, 14, 22, 30, + /* 2060 */ 17, 32, 22, 22, 154, 155, 156, 157, 158, 40, + /* 2070 */ 23, 68, 60, 30, 24, 32, 97, 28, 22, 68, + /* 2080 */ 23, 37, 34, 40, 150, 22, 25, 23, 23, 23, + /* 2090 */ 22, 98, 142, 183, 23, 23, 34, 22, 25, 89, + /* 2100 */ 71, 34, 117, 144, 34, 22, 76, 76, 79, 87, + /* 2110 */ 34, 82, 34, 44, 71, 94, 34, 23, 25, 24, + /* 2120 */ 34, 25, 79, 23, 23, 82, 23, 23, 99, 143, + /* 2130 */ 143, 22, 25, 25, 23, 22, 11, 22, 22, 25, + /* 2140 */ 23, 23, 99, 22, 22, 136, 142, 142, 142, 25, + /* 2150 */ 23, 15, 1, 1, 323, 323, 323, 323, 323, 323, + /* 2160 */ 323, 323, 323, 134, 323, 323, 323, 323, 139, 140, + /* 2170 */ 323, 323, 323, 323, 323, 323, 323, 134, 323, 323, + /* 2180 */ 323, 323, 139, 140, 323, 323, 323, 323, 323, 323, + /* 2190 */ 323, 323, 163, 323, 323, 323, 323, 323, 323, 323, + /* 2200 */ 323, 323, 323, 323, 323, 323, 163, 323, 323, 323, + /* 2210 */ 323, 323, 323, 323, 323, 323, 323, 323, 323, 323, + /* 2220 */ 323, 323, 323, 323, 323, 323, 323, 323, 323, 323, + /* 2230 */ 323, 323, 323, 323, 323, 323, 323, 323, 323, 323, + /* 2240 */ 323, 323, 323, 323, 323, 323, 323, 323, 323, 323, + /* 2250 */ 323, 323, 323, 323, 323, 323, 323, 323, 323, 323, + /* 2260 */ 323, 323, 323, 323, 323, 323, 323, 323, 323, 323, + /* 2270 */ 323, 323, 323, 323, 323, 323, 323, 323, 323, 323, + /* 2280 */ 323, 323, 323, 323, 323, 323, 323, 323, 323, 323, + /* 2290 */ 323, 323, 323, 323, 323, 323, 323, 323, 323, 323, + /* 2300 */ 323, 323, 323, 323, 323, 323, 323, 323, 323, 323, + /* 2310 */ 323, 323, 323, 323, 323, 323, 323, 323, 323, 323, + /* 2320 */ 323, 323, 323, 323, 323, 323, 323, 323, 323, 323, + /* 2330 */ 323, 323, 323, 323, 323, 323, 323, 323, 323, 323, + /* 2340 */ 323, 187, 187, 187, 187, 187, 187, 187, 187, 187, + /* 2350 */ 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, + /* 2360 */ 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, + /* 2370 */ 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, + /* 2380 */ 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, + /* 2390 */ 187, 187, 187, 187, }; -#define YY_SHIFT_COUNT (578) +#define YY_SHIFT_COUNT (582) #define YY_SHIFT_MIN (0) -#define YY_SHIFT_MAX (2088) +#define YY_SHIFT_MAX (2152) static const unsigned short int yy_shift_ofst[] = { - /* 0 */ 1648, 1477, 1272, 322, 322, 1, 1319, 1478, 1491, 1837, - /* 10 */ 1837, 1837, 471, 0, 0, 214, 1093, 1837, 1837, 1837, - /* 20 */ 1837, 1837, 1837, 1837, 1837, 1837, 1837, 1837, 1837, 1837, - /* 30 */ 1837, 271, 271, 1219, 1219, 216, 88, 1, 1, 1, - /* 40 */ 1, 1, 40, 111, 258, 361, 469, 512, 583, 622, - /* 50 */ 693, 732, 803, 842, 913, 1073, 1093, 1093, 1093, 1093, - /* 60 */ 1093, 1093, 1093, 1093, 1093, 1093, 1093, 1093, 1093, 1093, - /* 70 */ 1093, 1093, 1093, 1093, 1113, 1093, 1216, 957, 957, 1635, - /* 80 */ 1662, 1777, 1837, 1837, 1837, 1837, 1837, 1837, 1837, 1837, - /* 90 */ 1837, 1837, 1837, 1837, 1837, 1837, 1837, 1837, 1837, 1837, - /* 100 */ 1837, 1837, 1837, 1837, 1837, 1837, 1837, 1837, 1837, 1837, - /* 110 */ 1837, 1837, 1837, 1837, 1837, 1837, 1837, 1837, 1837, 1837, - /* 120 */ 1837, 1837, 1837, 1837, 1837, 1837, 1837, 1837, 1837, 1837, - /* 130 */ 1837, 137, 181, 181, 181, 181, 181, 181, 181, 94, - /* 140 */ 430, 66, 65, 112, 366, 533, 533, 740, 1257, 533, - /* 150 */ 533, 79, 79, 533, 412, 412, 412, 77, 412, 123, - /* 160 */ 113, 113, 113, 22, 22, 2100, 2100, 328, 328, 328, - /* 170 */ 239, 468, 468, 468, 468, 1015, 1015, 409, 366, 1187, - /* 180 */ 1232, 533, 533, 533, 533, 533, 533, 533, 533, 533, - /* 190 */ 533, 533, 533, 533, 533, 533, 533, 533, 533, 533, - /* 200 */ 533, 969, 621, 621, 533, 642, 788, 788, 1133, 1133, - /* 210 */ 822, 822, 67, 1193, 2100, 2100, 2100, 2100, 2100, 2100, - /* 220 */ 2100, 1307, 954, 954, 585, 472, 640, 387, 695, 538, - /* 230 */ 541, 700, 533, 533, 533, 533, 533, 533, 533, 533, - /* 240 */ 533, 533, 222, 533, 533, 533, 533, 533, 533, 533, - /* 250 */ 533, 533, 533, 533, 533, 1213, 1213, 1213, 533, 533, - /* 260 */ 533, 565, 533, 533, 533, 916, 1147, 533, 533, 1288, - /* 270 */ 533, 533, 533, 533, 533, 533, 533, 533, 639, 1280, - /* 280 */ 209, 1129, 1129, 1129, 1129, 580, 209, 209, 1209, 768, - /* 290 */ 917, 649, 1315, 1334, 405, 1334, 1383, 249, 1315, 1315, - /* 300 */ 249, 1315, 405, 1383, 1441, 464, 1245, 1417, 1417, 1417, - /* 310 */ 1323, 1323, 1323, 1323, 184, 184, 1335, 1476, 856, 1482, - /* 320 */ 1744, 1744, 1665, 1665, 1773, 1773, 1665, 1669, 1671, 1802, - /* 330 */ 1782, 1809, 1809, 1809, 1809, 1665, 1817, 1690, 1671, 1671, - /* 340 */ 1690, 1802, 1782, 1690, 1782, 1690, 1665, 1817, 1693, 1791, - /* 350 */ 1665, 1817, 1832, 1665, 1817, 1665, 1817, 1832, 1752, 1752, - /* 360 */ 1752, 1805, 1850, 1850, 1832, 1752, 1749, 1752, 1805, 1752, - /* 370 */ 1752, 1713, 1858, 1775, 1775, 1832, 1665, 1799, 1799, 1823, - /* 380 */ 1823, 1761, 1765, 1890, 1665, 1757, 1761, 1771, 1774, 1690, - /* 390 */ 1894, 1913, 1913, 1929, 1929, 1929, 2100, 2100, 2100, 2100, - /* 400 */ 2100, 2100, 2100, 2100, 2100, 2100, 2100, 2100, 2100, 2100, - /* 410 */ 2100, 207, 1220, 331, 620, 967, 806, 1074, 1499, 1432, - /* 420 */ 1463, 1479, 1419, 1422, 1557, 1512, 1598, 1599, 1644, 1645, - /* 430 */ 1654, 1660, 1555, 1505, 1684, 1462, 1670, 1563, 1619, 1593, - /* 440 */ 1676, 1679, 1613, 1680, 1554, 1558, 1689, 1692, 1605, 1589, - /* 450 */ 1955, 1959, 1941, 1803, 1950, 1951, 1945, 1946, 1831, 1820, - /* 460 */ 1842, 1948, 1948, 1952, 1833, 1954, 1834, 1961, 1978, 1838, - /* 470 */ 1851, 1948, 1852, 1922, 1947, 1948, 1836, 1932, 1935, 1936, - /* 480 */ 1942, 1866, 1881, 1964, 1859, 1998, 1996, 1980, 1888, 1843, - /* 490 */ 1937, 1981, 1939, 1933, 1968, 1869, 1896, 1988, 1993, 1995, - /* 500 */ 1884, 1891, 1997, 1953, 1999, 2000, 1994, 2001, 1957, 1966, - /* 510 */ 2002, 1931, 1990, 2006, 1962, 2003, 2007, 2004, 1882, 2010, - /* 520 */ 2011, 2012, 2008, 2013, 2015, 1944, 1898, 2019, 2020, 1928, - /* 530 */ 2014, 2023, 1903, 2022, 2016, 2017, 2018, 2021, 1965, 1974, - /* 540 */ 1970, 2024, 1979, 1967, 2025, 2034, 2036, 2037, 2038, 2039, - /* 550 */ 2028, 1923, 1924, 2044, 2022, 2046, 2047, 2048, 2049, 2050, - /* 560 */ 2051, 2054, 2062, 2055, 2056, 2057, 2058, 2060, 2061, 2059, - /* 570 */ 1956, 1938, 1949, 1958, 2063, 2064, 2070, 2085, 2088, + /* 0 */ 2029, 1801, 2043, 1380, 1380, 318, 271, 1496, 1569, 1642, + /* 10 */ 702, 702, 702, 740, 318, 318, 318, 318, 318, 0, + /* 20 */ 0, 216, 1177, 702, 702, 702, 702, 702, 702, 702, + /* 30 */ 702, 702, 702, 702, 702, 702, 702, 702, 503, 503, + /* 40 */ 111, 111, 217, 287, 348, 610, 610, 736, 736, 736, + /* 50 */ 736, 40, 112, 320, 340, 445, 489, 593, 637, 741, + /* 60 */ 785, 889, 909, 1023, 1043, 1157, 1177, 1177, 1177, 1177, + /* 70 */ 1177, 1177, 1177, 1177, 1177, 1177, 1177, 1177, 1177, 1177, + /* 80 */ 1177, 1177, 1177, 1177, 1197, 1177, 1301, 1321, 1321, 554, + /* 90 */ 1802, 1910, 702, 702, 702, 702, 702, 702, 702, 702, + /* 100 */ 702, 702, 702, 702, 702, 702, 702, 702, 702, 702, + /* 110 */ 702, 702, 702, 702, 702, 702, 702, 702, 702, 702, + /* 120 */ 702, 702, 702, 702, 702, 702, 702, 702, 702, 702, + /* 130 */ 702, 702, 702, 702, 702, 702, 702, 702, 702, 702, + /* 140 */ 702, 702, 138, 198, 198, 198, 198, 198, 198, 198, + /* 150 */ 183, 99, 169, 549, 610, 151, 542, 610, 610, 1017, + /* 160 */ 1017, 610, 1001, 350, 464, 464, 464, 586, 1, 1, + /* 170 */ 2207, 2207, 854, 854, 854, 465, 694, 694, 694, 694, + /* 180 */ 1096, 1096, 825, 549, 847, 904, 610, 610, 610, 610, + /* 190 */ 610, 610, 610, 610, 610, 610, 610, 610, 610, 610, + /* 200 */ 610, 610, 610, 610, 610, 488, 947, 947, 610, 1129, + /* 210 */ 495, 495, 1139, 1139, 967, 967, 1173, 2207, 2207, 2207, + /* 220 */ 2207, 2207, 2207, 2207, 617, 765, 765, 697, 444, 708, + /* 230 */ 660, 745, 510, 663, 864, 610, 610, 610, 610, 610, + /* 240 */ 610, 610, 610, 610, 610, 188, 610, 610, 610, 610, + /* 250 */ 610, 610, 610, 610, 610, 610, 610, 610, 839, 839, + /* 260 */ 839, 610, 610, 610, 1155, 610, 610, 610, 1119, 1247, + /* 270 */ 610, 1353, 610, 610, 610, 610, 610, 610, 610, 610, + /* 280 */ 1063, 494, 1101, 291, 291, 291, 291, 1319, 1101, 1101, + /* 290 */ 775, 1221, 1375, 1452, 667, 1341, 1198, 1341, 1435, 1487, + /* 300 */ 667, 667, 1487, 667, 1198, 1435, 777, 1011, 1423, 584, + /* 310 */ 584, 584, 1273, 1273, 1273, 1273, 1471, 1471, 880, 1530, + /* 320 */ 1190, 1095, 1731, 1731, 1668, 1668, 1794, 1794, 1668, 1683, + /* 330 */ 1685, 1815, 1796, 1824, 1824, 1824, 1824, 1668, 1828, 1701, + /* 340 */ 1685, 1685, 1701, 1815, 1796, 1701, 1796, 1701, 1668, 1828, + /* 350 */ 1697, 1800, 1668, 1828, 1848, 1668, 1828, 1668, 1828, 1848, + /* 360 */ 1766, 1766, 1766, 1823, 1870, 1870, 1848, 1766, 1767, 1766, + /* 370 */ 1823, 1766, 1766, 1727, 1872, 1783, 1783, 1848, 1668, 1813, + /* 380 */ 1813, 1825, 1825, 1777, 1781, 1906, 1668, 1774, 1777, 1789, + /* 390 */ 1792, 1701, 1919, 1935, 1935, 1949, 1949, 1949, 2207, 2207, + /* 400 */ 2207, 2207, 2207, 2207, 2207, 2207, 2207, 2207, 2207, 2207, + /* 410 */ 2207, 2207, 2207, 69, 1032, 79, 357, 1377, 1206, 400, + /* 420 */ 1525, 835, 332, 1540, 1437, 1539, 1536, 1548, 1583, 1620, + /* 430 */ 1633, 1670, 1671, 1674, 1567, 1553, 1682, 1506, 1675, 1358, + /* 440 */ 1607, 1589, 1678, 1681, 1624, 1687, 1688, 1283, 1561, 1693, + /* 450 */ 1696, 1623, 1521, 1976, 1980, 1962, 1822, 1972, 1973, 1965, + /* 460 */ 1967, 1851, 1840, 1862, 1969, 1969, 1971, 1853, 1977, 1854, + /* 470 */ 1982, 1999, 1858, 1871, 1969, 1873, 1941, 1968, 1969, 1855, + /* 480 */ 1952, 1954, 1955, 1956, 1881, 1896, 1981, 1874, 2013, 2014, + /* 490 */ 1998, 1905, 1860, 1957, 2008, 1966, 1947, 1983, 1894, 1921, + /* 500 */ 2020, 2018, 2026, 1915, 1923, 2028, 1984, 2036, 2040, 2047, + /* 510 */ 2041, 2003, 2012, 2050, 1979, 2049, 2056, 2011, 2044, 2057, + /* 520 */ 2048, 1934, 2063, 2064, 2065, 2061, 2066, 2068, 1993, 1950, + /* 530 */ 2071, 2072, 1985, 2062, 2075, 1959, 2073, 2067, 2070, 2076, + /* 540 */ 2078, 2010, 2030, 2022, 2069, 2031, 2021, 2082, 2094, 2083, + /* 550 */ 2095, 2093, 2096, 2086, 1986, 1987, 2100, 2073, 2101, 2103, + /* 560 */ 2104, 2109, 2107, 2108, 2111, 2113, 2125, 2115, 2116, 2117, + /* 570 */ 2118, 2121, 2122, 2114, 2009, 2004, 2005, 2006, 2124, 2127, + /* 580 */ 2136, 2151, 2152, }; -#define YY_REDUCE_COUNT (410) -#define YY_REDUCE_MIN (-271) -#define YY_REDUCE_MAX (1753) +#define YY_REDUCE_COUNT (412) +#define YY_REDUCE_MIN (-277) +#define YY_REDUCE_MAX (1772) static const short yy_reduce_ofst[] = { - /* 0 */ -125, 733, 789, 241, 293, -123, -193, -191, -183, -187, - /* 10 */ 166, 238, 133, -207, -199, -267, -176, -6, 204, 489, - /* 20 */ 576, 598, -175, 686, 860, 615, 725, 1014, 778, 781, - /* 30 */ 857, 616, 887, 87, 240, -192, 408, 626, 796, 843, - /* 40 */ 854, 1004, -271, -271, -271, -271, -271, -271, -271, -271, - /* 50 */ -271, -271, -271, -271, -271, -271, -271, -271, -271, -271, - /* 60 */ -271, -271, -271, -271, -271, -271, -271, -271, -271, -271, - /* 70 */ -271, -271, -271, -271, -271, -271, -271, -271, -271, 80, - /* 80 */ 83, 313, 886, 888, 918, 938, 1021, 1034, 1036, 1141, - /* 90 */ 1159, 1163, 1166, 1168, 1170, 1176, 1178, 1180, 1184, 1196, - /* 100 */ 1198, 1205, 1215, 1225, 1227, 1236, 1252, 1254, 1264, 1303, - /* 110 */ 1309, 1312, 1322, 1325, 1328, 1337, 1340, 1343, 1353, 1371, - /* 120 */ 1373, 1384, 1386, 1411, 1413, 1420, 1424, 1426, 1458, 1470, - /* 130 */ 1473, -271, -271, -271, -271, -271, -271, -271, -271, -271, - /* 140 */ -271, -271, 138, 459, 396, -158, 470, 302, -212, 521, - /* 150 */ 201, -195, -92, 559, 630, 632, 630, -271, 632, 901, - /* 160 */ 63, 407, 670, -271, -271, -271, -271, 161, 161, 161, - /* 170 */ 251, 335, 847, 979, 1097, 537, 588, 618, 628, 688, - /* 180 */ 688, -166, -161, 674, 787, 794, 799, 852, 996, -122, - /* 190 */ 837, -120, 1018, 1035, 415, 1047, 1001, 958, 1082, 400, - /* 200 */ 1099, 779, 1137, 1142, 263, 1083, 1145, 1150, 1041, 1139, - /* 210 */ 965, 1050, 362, 849, 752, 629, 675, 1162, 1173, 1090, - /* 220 */ 1195, -194, 56, 185, -135, 232, 522, 560, 571, 601, - /* 230 */ 617, 669, 683, 711, 850, 893, 1000, 1040, 1049, 1081, - /* 240 */ 1087, 1101, 392, 1114, 1123, 1155, 1161, 1175, 1271, 1293, - /* 250 */ 1299, 1330, 1339, 1342, 1347, 593, 1282, 1286, 1350, 1359, - /* 260 */ 1368, 1314, 1480, 1483, 1507, 1085, 1338, 1526, 1527, 1487, - /* 270 */ 1531, 560, 1532, 1534, 1535, 1538, 1539, 1541, 1448, 1450, - /* 280 */ 1496, 1484, 1485, 1489, 1490, 1314, 1496, 1496, 1504, 1536, - /* 290 */ 1564, 1451, 1486, 1492, 1509, 1493, 1465, 1515, 1494, 1495, - /* 300 */ 1517, 1500, 1519, 1474, 1550, 1543, 1548, 1556, 1565, 1566, - /* 310 */ 1518, 1523, 1542, 1544, 1525, 1545, 1513, 1553, 1552, 1604, - /* 320 */ 1508, 1510, 1608, 1609, 1520, 1528, 1612, 1540, 1559, 1560, - /* 330 */ 1592, 1591, 1595, 1596, 1597, 1629, 1638, 1594, 1569, 1570, - /* 340 */ 1600, 1568, 1610, 1601, 1611, 1603, 1643, 1651, 1562, 1571, - /* 350 */ 1655, 1659, 1640, 1663, 1666, 1664, 1667, 1641, 1650, 1652, - /* 360 */ 1653, 1647, 1656, 1657, 1658, 1668, 1672, 1681, 1649, 1682, - /* 370 */ 1683, 1573, 1582, 1607, 1615, 1685, 1702, 1586, 1587, 1642, - /* 380 */ 1646, 1673, 1675, 1636, 1714, 1637, 1677, 1674, 1678, 1694, - /* 390 */ 1719, 1734, 1735, 1746, 1747, 1750, 1633, 1661, 1686, 1738, - /* 400 */ 1728, 1733, 1736, 1737, 1740, 1726, 1730, 1742, 1743, 1748, - /* 410 */ 1753, + /* 0 */ -67, 1252, -64, -178, -181, 160, 1071, 143, -184, 137, + /* 10 */ 218, 220, 222, -174, 229, 268, 272, 275, 324, -208, + /* 20 */ 242, -277, -39, 81, 537, 792, 810, 812, -189, 814, + /* 30 */ 831, 163, 865, 944, 887, 840, 964, 1077, -187, 292, + /* 40 */ -133, 274, 673, 558, 682, 795, 809, -238, -232, -238, + /* 50 */ -232, 329, 329, 329, 329, 329, 329, 329, 329, 329, + /* 60 */ 329, 329, 329, 329, 329, 329, 329, 329, 329, 329, + /* 70 */ 329, 329, 329, 329, 329, 329, 329, 329, 329, 329, + /* 80 */ 329, 329, 329, 329, 329, 329, 329, 329, 329, 557, + /* 90 */ 712, 949, 966, 969, 971, 979, 1097, 1099, 1103, 1142, + /* 100 */ 1144, 1169, 1172, 1201, 1203, 1228, 1241, 1250, 1253, 1255, + /* 110 */ 1261, 1266, 1271, 1282, 1291, 1308, 1310, 1312, 1322, 1328, + /* 120 */ 1347, 1354, 1356, 1359, 1362, 1365, 1367, 1374, 1376, 1381, + /* 130 */ 1401, 1403, 1406, 1412, 1414, 1417, 1421, 1428, 1447, 1449, + /* 140 */ 1453, 1462, 329, 329, 329, 329, 329, 329, 329, 329, + /* 150 */ 329, 329, 329, -22, -159, 475, -220, 756, 38, 501, + /* 160 */ 841, 714, 329, 118, 337, 349, 363, -56, 329, 329, + /* 170 */ 329, 329, -205, -205, -205, 687, -172, -130, -57, 790, + /* 180 */ 397, 528, -271, 136, 596, 596, 90, 316, 522, 541, + /* 190 */ -37, 715, 849, 977, 628, 856, 980, 991, 1081, 1102, + /* 200 */ 1135, 1083, -162, 208, 1258, 794, -86, 159, 41, 1109, + /* 210 */ 671, 852, 844, 932, 1175, 1254, 480, 1180, 100, 258, + /* 220 */ 1265, 1268, 1216, 1287, -139, 317, 344, 63, 339, 423, + /* 230 */ 563, 636, 676, 813, 908, 914, 950, 1078, 1084, 1098, + /* 240 */ 1363, 1384, 1407, 1439, 1464, 411, 1527, 1534, 1535, 1537, + /* 250 */ 1541, 1542, 1543, 1544, 1545, 1547, 1549, 1550, 990, 1164, + /* 260 */ 1492, 1551, 1552, 1556, 1217, 1558, 1559, 1560, 1473, 1413, + /* 270 */ 1563, 1510, 1568, 563, 1570, 1571, 1572, 1573, 1574, 1575, + /* 280 */ 1443, 1466, 1518, 1513, 1514, 1515, 1516, 1217, 1518, 1518, + /* 290 */ 1531, 1562, 1582, 1477, 1505, 1511, 1533, 1512, 1488, 1538, + /* 300 */ 1509, 1517, 1546, 1519, 1557, 1489, 1565, 1564, 1578, 1586, + /* 310 */ 1587, 1588, 1526, 1528, 1554, 1555, 1576, 1577, 1566, 1579, + /* 320 */ 1584, 1591, 1520, 1523, 1617, 1628, 1580, 1581, 1632, 1585, + /* 330 */ 1590, 1593, 1604, 1605, 1606, 1608, 1609, 1641, 1649, 1610, + /* 340 */ 1592, 1594, 1611, 1595, 1616, 1612, 1618, 1613, 1651, 1654, + /* 350 */ 1596, 1598, 1655, 1663, 1650, 1673, 1680, 1677, 1684, 1653, + /* 360 */ 1664, 1666, 1667, 1662, 1669, 1672, 1676, 1686, 1679, 1691, + /* 370 */ 1689, 1692, 1694, 1597, 1599, 1619, 1630, 1699, 1700, 1602, + /* 380 */ 1615, 1648, 1657, 1690, 1698, 1658, 1729, 1652, 1695, 1702, + /* 390 */ 1704, 1703, 1741, 1754, 1758, 1768, 1769, 1771, 1660, 1661, + /* 400 */ 1665, 1752, 1756, 1757, 1759, 1760, 1764, 1745, 1753, 1762, + /* 410 */ 1763, 1761, 1772, }; static const YYACTIONTYPE yy_default[] = { - /* 0 */ 1648, 1648, 1648, 1478, 1243, 1354, 1243, 1243, 1243, 1478, - /* 10 */ 1478, 1478, 1243, 1384, 1384, 1531, 1276, 1243, 1243, 1243, - /* 20 */ 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1477, 1243, - /* 30 */ 1243, 1243, 1243, 1564, 1564, 1243, 1243, 1243, 1243, 1243, - /* 40 */ 1243, 1243, 1243, 1393, 1243, 1400, 1243, 1243, 1243, 1243, - /* 50 */ 1243, 1479, 1480, 1243, 1243, 1243, 1530, 1532, 1495, 1407, - /* 60 */ 1406, 1405, 1404, 1513, 1372, 1398, 1391, 1395, 1474, 1475, - /* 70 */ 1473, 1626, 1480, 1479, 1243, 1394, 1442, 1458, 1441, 1243, - /* 80 */ 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1243, - /* 90 */ 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1243, - /* 100 */ 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1243, - /* 110 */ 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1243, - /* 120 */ 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1243, - /* 130 */ 1243, 1450, 1457, 1456, 1455, 1464, 1454, 1451, 1444, 1443, - /* 140 */ 1445, 1446, 1243, 1243, 1267, 1243, 1243, 1264, 1318, 1243, - /* 150 */ 1243, 1243, 1243, 1243, 1550, 1549, 1243, 1447, 1243, 1276, - /* 160 */ 1435, 1434, 1433, 1461, 1448, 1460, 1459, 1538, 1600, 1599, - /* 170 */ 1496, 1243, 1243, 1243, 1243, 1243, 1243, 1564, 1243, 1243, - /* 180 */ 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1243, - /* 190 */ 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1243, - /* 200 */ 1243, 1374, 1564, 1564, 1243, 1276, 1564, 1564, 1375, 1375, - /* 210 */ 1272, 1272, 1378, 1243, 1545, 1345, 1345, 1345, 1345, 1354, - /* 220 */ 1345, 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1243, - /* 230 */ 1243, 1243, 1243, 1243, 1243, 1243, 1535, 1533, 1243, 1243, - /* 240 */ 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1243, - /* 250 */ 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1243, - /* 260 */ 1243, 1243, 1243, 1243, 1243, 1350, 1243, 1243, 1243, 1243, - /* 270 */ 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1593, 1243, 1508, - /* 280 */ 1332, 1350, 1350, 1350, 1350, 1352, 1333, 1331, 1344, 1277, - /* 290 */ 1250, 1640, 1410, 1399, 1351, 1399, 1637, 1397, 1410, 1410, - /* 300 */ 1397, 1410, 1351, 1637, 1293, 1615, 1288, 1384, 1384, 1384, - /* 310 */ 1374, 1374, 1374, 1374, 1378, 1378, 1476, 1351, 1344, 1243, - /* 320 */ 1640, 1640, 1360, 1360, 1639, 1639, 1360, 1496, 1623, 1419, - /* 330 */ 1321, 1327, 1327, 1327, 1327, 1360, 1261, 1397, 1623, 1623, - /* 340 */ 1397, 1419, 1321, 1397, 1321, 1397, 1360, 1261, 1512, 1634, - /* 350 */ 1360, 1261, 1486, 1360, 1261, 1360, 1261, 1486, 1319, 1319, - /* 360 */ 1319, 1308, 1243, 1243, 1486, 1319, 1293, 1319, 1308, 1319, - /* 370 */ 1319, 1582, 1243, 1490, 1490, 1486, 1360, 1574, 1574, 1387, - /* 380 */ 1387, 1392, 1378, 1481, 1360, 1243, 1392, 1390, 1388, 1397, - /* 390 */ 1311, 1596, 1596, 1592, 1592, 1592, 1645, 1645, 1545, 1608, - /* 400 */ 1276, 1276, 1276, 1276, 1608, 1295, 1295, 1277, 1277, 1276, - /* 410 */ 1608, 1243, 1243, 1243, 1243, 1243, 1243, 1603, 1243, 1540, - /* 420 */ 1497, 1364, 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1243, - /* 430 */ 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1551, 1243, - /* 440 */ 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1424, - /* 450 */ 1243, 1246, 1542, 1243, 1243, 1243, 1243, 1243, 1243, 1243, - /* 460 */ 1243, 1401, 1402, 1365, 1243, 1243, 1243, 1243, 1243, 1243, - /* 470 */ 1243, 1416, 1243, 1243, 1243, 1411, 1243, 1243, 1243, 1243, - /* 480 */ 1243, 1243, 1243, 1243, 1636, 1243, 1243, 1243, 1243, 1243, - /* 490 */ 1243, 1511, 1510, 1243, 1243, 1362, 1243, 1243, 1243, 1243, - /* 500 */ 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1291, - /* 510 */ 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1243, - /* 520 */ 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1243, - /* 530 */ 1243, 1243, 1243, 1389, 1243, 1243, 1243, 1243, 1243, 1243, - /* 540 */ 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1579, 1379, - /* 550 */ 1243, 1243, 1243, 1243, 1627, 1243, 1243, 1243, 1243, 1243, - /* 560 */ 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1243, 1619, - /* 570 */ 1335, 1425, 1243, 1428, 1265, 1243, 1255, 1243, 1243, + /* 0 */ 1663, 1663, 1663, 1491, 1254, 1367, 1254, 1254, 1254, 1254, + /* 10 */ 1491, 1491, 1491, 1254, 1254, 1254, 1254, 1254, 1254, 1397, + /* 20 */ 1397, 1544, 1287, 1254, 1254, 1254, 1254, 1254, 1254, 1254, + /* 30 */ 1254, 1254, 1254, 1254, 1254, 1490, 1254, 1254, 1254, 1254, + /* 40 */ 1578, 1578, 1254, 1254, 1254, 1254, 1254, 1563, 1562, 1254, + /* 50 */ 1254, 1254, 1406, 1254, 1413, 1254, 1254, 1254, 1254, 1254, + /* 60 */ 1492, 1493, 1254, 1254, 1254, 1254, 1543, 1545, 1508, 1420, + /* 70 */ 1419, 1418, 1417, 1526, 1385, 1411, 1404, 1408, 1487, 1488, + /* 80 */ 1486, 1641, 1493, 1492, 1254, 1407, 1455, 1471, 1454, 1254, + /* 90 */ 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, + /* 100 */ 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, + /* 110 */ 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, + /* 120 */ 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, + /* 130 */ 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, + /* 140 */ 1254, 1254, 1463, 1470, 1469, 1468, 1477, 1467, 1464, 1457, + /* 150 */ 1456, 1458, 1459, 1278, 1254, 1275, 1329, 1254, 1254, 1254, + /* 160 */ 1254, 1254, 1460, 1287, 1448, 1447, 1446, 1254, 1474, 1461, + /* 170 */ 1473, 1472, 1551, 1615, 1614, 1509, 1254, 1254, 1254, 1254, + /* 180 */ 1254, 1254, 1578, 1254, 1254, 1254, 1254, 1254, 1254, 1254, + /* 190 */ 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, + /* 200 */ 1254, 1254, 1254, 1254, 1254, 1387, 1578, 1578, 1254, 1287, + /* 210 */ 1578, 1578, 1388, 1388, 1283, 1283, 1391, 1558, 1358, 1358, + /* 220 */ 1358, 1358, 1367, 1358, 1254, 1254, 1254, 1254, 1254, 1254, + /* 230 */ 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1548, + /* 240 */ 1546, 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, + /* 250 */ 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, + /* 260 */ 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1363, 1254, + /* 270 */ 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1608, + /* 280 */ 1254, 1521, 1343, 1363, 1363, 1363, 1363, 1365, 1344, 1342, + /* 290 */ 1357, 1288, 1261, 1655, 1423, 1412, 1364, 1412, 1652, 1410, + /* 300 */ 1423, 1423, 1410, 1423, 1364, 1652, 1304, 1630, 1299, 1397, + /* 310 */ 1397, 1397, 1387, 1387, 1387, 1387, 1391, 1391, 1489, 1364, + /* 320 */ 1357, 1254, 1655, 1655, 1373, 1373, 1654, 1654, 1373, 1509, + /* 330 */ 1638, 1432, 1332, 1338, 1338, 1338, 1338, 1373, 1272, 1410, + /* 340 */ 1638, 1638, 1410, 1432, 1332, 1410, 1332, 1410, 1373, 1272, + /* 350 */ 1525, 1649, 1373, 1272, 1499, 1373, 1272, 1373, 1272, 1499, + /* 360 */ 1330, 1330, 1330, 1319, 1254, 1254, 1499, 1330, 1304, 1330, + /* 370 */ 1319, 1330, 1330, 1596, 1254, 1503, 1503, 1499, 1373, 1588, + /* 380 */ 1588, 1400, 1400, 1405, 1391, 1494, 1373, 1254, 1405, 1403, + /* 390 */ 1401, 1410, 1322, 1611, 1611, 1607, 1607, 1607, 1660, 1660, + /* 400 */ 1558, 1623, 1287, 1287, 1287, 1287, 1623, 1306, 1306, 1288, + /* 410 */ 1288, 1287, 1623, 1254, 1254, 1254, 1254, 1254, 1254, 1618, + /* 420 */ 1254, 1553, 1510, 1377, 1254, 1254, 1254, 1254, 1254, 1254, + /* 430 */ 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, + /* 440 */ 1564, 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, + /* 450 */ 1254, 1254, 1437, 1254, 1257, 1555, 1254, 1254, 1254, 1254, + /* 460 */ 1254, 1254, 1254, 1254, 1414, 1415, 1378, 1254, 1254, 1254, + /* 470 */ 1254, 1254, 1254, 1254, 1429, 1254, 1254, 1254, 1424, 1254, + /* 480 */ 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1651, 1254, 1254, + /* 490 */ 1254, 1254, 1254, 1254, 1524, 1523, 1254, 1254, 1375, 1254, + /* 500 */ 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, + /* 510 */ 1254, 1254, 1302, 1254, 1254, 1254, 1254, 1254, 1254, 1254, + /* 520 */ 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, + /* 530 */ 1254, 1254, 1254, 1254, 1254, 1254, 1402, 1254, 1254, 1254, + /* 540 */ 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, + /* 550 */ 1254, 1593, 1392, 1254, 1254, 1254, 1254, 1642, 1254, 1254, + /* 560 */ 1254, 1254, 1352, 1254, 1254, 1254, 1254, 1254, 1254, 1254, + /* 570 */ 1254, 1254, 1254, 1634, 1346, 1438, 1254, 1441, 1276, 1254, + /* 580 */ 1266, 1254, 1254, }; /********** End of lemon-generated parsing tables *****************************/ @@ -172241,52 +178775,53 @@ static const YYACTIONTYPE yy_default[] = { static const YYCODETYPE yyFallback[] = { 0, /* $ => nothing */ 0, /* SEMI => nothing */ - 59, /* EXPLAIN => ID */ - 59, /* QUERY => ID */ - 59, /* PLAN => ID */ - 59, /* BEGIN => ID */ + 60, /* EXPLAIN => ID */ + 60, /* QUERY => ID */ + 60, /* PLAN => ID */ + 60, /* BEGIN => ID */ 0, /* TRANSACTION => nothing */ - 59, /* DEFERRED => ID */ - 59, /* IMMEDIATE => ID */ - 59, /* EXCLUSIVE => ID */ + 60, /* DEFERRED => ID */ + 60, /* IMMEDIATE => ID */ + 60, /* EXCLUSIVE => ID */ 0, /* COMMIT => nothing */ - 59, /* END => ID */ - 59, /* ROLLBACK => ID */ - 59, /* SAVEPOINT => ID */ - 59, /* RELEASE => ID */ + 60, /* END => ID */ + 60, /* ROLLBACK => ID */ + 60, /* SAVEPOINT => ID */ + 60, /* RELEASE => ID */ 0, /* TO => nothing */ 0, /* TABLE => nothing */ 0, /* CREATE => nothing */ - 59, /* IF => ID */ + 60, /* IF => ID */ 0, /* NOT => nothing */ 0, /* EXISTS => nothing */ - 59, /* TEMP => ID */ + 60, /* TEMP => ID */ 0, /* LP => nothing */ 0, /* RP => nothing */ 0, /* AS => nothing */ 0, /* COMMA => nothing */ - 59, /* WITHOUT => ID */ - 59, /* ABORT => ID */ - 59, /* ACTION => ID */ - 59, /* AFTER => ID */ - 59, /* ANALYZE => ID */ - 59, /* ASC => ID */ - 59, /* ATTACH => ID */ - 59, /* BEFORE => ID */ - 59, /* BY => ID */ - 59, /* CASCADE => ID */ - 59, /* CAST => ID */ - 59, /* CONFLICT => ID */ - 59, /* DATABASE => ID */ - 59, /* DESC => ID */ - 59, /* DETACH => ID */ - 59, /* EACH => ID */ - 59, /* FAIL => ID */ + 60, /* WITHOUT => ID */ + 60, /* ABORT => ID */ + 60, /* ACTION => ID */ + 60, /* AFTER => ID */ + 60, /* ANALYZE => ID */ + 60, /* ASC => ID */ + 60, /* ATTACH => ID */ + 60, /* BEFORE => ID */ + 60, /* BY => ID */ + 60, /* CASCADE => ID */ + 60, /* CAST => ID */ + 60, /* CONFLICT => ID */ + 60, /* DATABASE => ID */ + 60, /* DESC => ID */ + 60, /* DETACH => ID */ + 60, /* EACH => ID */ + 60, /* FAIL => ID */ 0, /* OR => nothing */ 0, /* AND => nothing */ 0, /* IS => nothing */ - 59, /* MATCH => ID */ - 59, /* LIKE_KW => ID */ + 0, /* ISNOT => nothing */ + 60, /* MATCH => ID */ + 60, /* LIKE_KW => ID */ 0, /* BETWEEN => nothing */ 0, /* IN => nothing */ 0, /* ISNULL => nothing */ @@ -172299,47 +178834,47 @@ static const YYCODETYPE yyFallback[] = { 0, /* GE => nothing */ 0, /* ESCAPE => nothing */ 0, /* ID => nothing */ - 59, /* COLUMNKW => ID */ - 59, /* DO => ID */ - 59, /* FOR => ID */ - 59, /* IGNORE => ID */ - 59, /* INITIALLY => ID */ - 59, /* INSTEAD => ID */ - 59, /* NO => ID */ - 59, /* KEY => ID */ - 59, /* OF => ID */ - 59, /* OFFSET => ID */ - 59, /* PRAGMA => ID */ - 59, /* RAISE => ID */ - 59, /* RECURSIVE => ID */ - 59, /* REPLACE => ID */ - 59, /* RESTRICT => ID */ - 59, /* ROW => ID */ - 59, /* ROWS => ID */ - 59, /* TRIGGER => ID */ - 59, /* VACUUM => ID */ - 59, /* VIEW => ID */ - 59, /* VIRTUAL => ID */ - 59, /* WITH => ID */ - 59, /* NULLS => ID */ - 59, /* FIRST => ID */ - 59, /* LAST => ID */ - 59, /* CURRENT => ID */ - 59, /* FOLLOWING => ID */ - 59, /* PARTITION => ID */ - 59, /* PRECEDING => ID */ - 59, /* RANGE => ID */ - 59, /* UNBOUNDED => ID */ - 59, /* EXCLUDE => ID */ - 59, /* GROUPS => ID */ - 59, /* OTHERS => ID */ - 59, /* TIES => ID */ - 59, /* GENERATED => ID */ - 59, /* ALWAYS => ID */ - 59, /* MATERIALIZED => ID */ - 59, /* REINDEX => ID */ - 59, /* RENAME => ID */ - 59, /* CTIME_KW => ID */ + 60, /* COLUMNKW => ID */ + 60, /* DO => ID */ + 60, /* FOR => ID */ + 60, /* IGNORE => ID */ + 60, /* INITIALLY => ID */ + 60, /* INSTEAD => ID */ + 60, /* NO => ID */ + 60, /* KEY => ID */ + 60, /* OF => ID */ + 60, /* OFFSET => ID */ + 60, /* PRAGMA => ID */ + 60, /* RAISE => ID */ + 60, /* RECURSIVE => ID */ + 60, /* REPLACE => ID */ + 60, /* RESTRICT => ID */ + 60, /* ROW => ID */ + 60, /* ROWS => ID */ + 60, /* TRIGGER => ID */ + 60, /* VACUUM => ID */ + 60, /* VIEW => ID */ + 60, /* VIRTUAL => ID */ + 60, /* WITH => ID */ + 60, /* NULLS => ID */ + 60, /* FIRST => ID */ + 60, /* LAST => ID */ + 60, /* CURRENT => ID */ + 60, /* FOLLOWING => ID */ + 60, /* PARTITION => ID */ + 60, /* PRECEDING => ID */ + 60, /* RANGE => ID */ + 60, /* UNBOUNDED => ID */ + 60, /* EXCLUDE => ID */ + 60, /* GROUPS => ID */ + 60, /* OTHERS => ID */ + 60, /* TIES => ID */ + 60, /* GENERATED => ID */ + 60, /* ALWAYS => ID */ + 60, /* MATERIALIZED => ID */ + 60, /* REINDEX => ID */ + 60, /* RENAME => ID */ + 60, /* CTIME_KW => ID */ 0, /* ANY => nothing */ 0, /* BITAND => nothing */ 0, /* BITOR => nothing */ @@ -172410,10 +178945,9 @@ static const YYCODETYPE yyFallback[] = { 0, /* AGG_FUNCTION => nothing */ 0, /* AGG_COLUMN => nothing */ 0, /* TRUEFALSE => nothing */ - 0, /* ISNOT => nothing */ 0, /* FUNCTION => nothing */ - 0, /* UMINUS => nothing */ 0, /* UPLUS => nothing */ + 0, /* UMINUS => nothing */ 0, /* TRUTH => nothing */ 0, /* REGISTER => nothing */ 0, /* VECTOR => nothing */ @@ -172422,7 +178956,9 @@ static const YYCODETYPE yyFallback[] = { 0, /* ASTERISK => nothing */ 0, /* SPAN => nothing */ 0, /* ERROR => nothing */ + 0, /* QNUMBER => nothing */ 0, /* SPACE => nothing */ + 0, /* COMMENT => nothing */ 0, /* ILLEGAL => nothing */ }; #endif /* YYFALLBACK */ @@ -172464,14 +179000,9 @@ struct yyParser { #endif sqlite3ParserARG_SDECL /* A place to hold %extra_argument */ sqlite3ParserCTX_SDECL /* A place to hold %extra_context */ -#if YYSTACKDEPTH<=0 - int yystksz; /* Current side of the stack */ - yyStackEntry *yystack; /* The parser's stack */ - yyStackEntry yystk0; /* First stack entry */ -#else - yyStackEntry yystack[YYSTACKDEPTH]; /* The parser's stack */ - yyStackEntry *yystackEnd; /* Last entry in the stack */ -#endif + yyStackEntry *yystackEnd; /* Last entry in the stack */ + yyStackEntry *yystack; /* The parser stack */ + yyStackEntry yystk0[YYSTACKDEPTH]; /* Initial stack space */ }; typedef struct yyParser yyParser; @@ -172558,135 +179089,135 @@ static const char *const yyTokenName[] = { /* 43 */ "OR", /* 44 */ "AND", /* 45 */ "IS", - /* 46 */ "MATCH", - /* 47 */ "LIKE_KW", - /* 48 */ "BETWEEN", - /* 49 */ "IN", - /* 50 */ "ISNULL", - /* 51 */ "NOTNULL", - /* 52 */ "NE", - /* 53 */ "EQ", - /* 54 */ "GT", - /* 55 */ "LE", - /* 56 */ "LT", - /* 57 */ "GE", - /* 58 */ "ESCAPE", - /* 59 */ "ID", - /* 60 */ "COLUMNKW", - /* 61 */ "DO", - /* 62 */ "FOR", - /* 63 */ "IGNORE", - /* 64 */ "INITIALLY", - /* 65 */ "INSTEAD", - /* 66 */ "NO", - /* 67 */ "KEY", - /* 68 */ "OF", - /* 69 */ "OFFSET", - /* 70 */ "PRAGMA", - /* 71 */ "RAISE", - /* 72 */ "RECURSIVE", - /* 73 */ "REPLACE", - /* 74 */ "RESTRICT", - /* 75 */ "ROW", - /* 76 */ "ROWS", - /* 77 */ "TRIGGER", - /* 78 */ "VACUUM", - /* 79 */ "VIEW", - /* 80 */ "VIRTUAL", - /* 81 */ "WITH", - /* 82 */ "NULLS", - /* 83 */ "FIRST", - /* 84 */ "LAST", - /* 85 */ "CURRENT", - /* 86 */ "FOLLOWING", - /* 87 */ "PARTITION", - /* 88 */ "PRECEDING", - /* 89 */ "RANGE", - /* 90 */ "UNBOUNDED", - /* 91 */ "EXCLUDE", - /* 92 */ "GROUPS", - /* 93 */ "OTHERS", - /* 94 */ "TIES", - /* 95 */ "GENERATED", - /* 96 */ "ALWAYS", - /* 97 */ "MATERIALIZED", - /* 98 */ "REINDEX", - /* 99 */ "RENAME", - /* 100 */ "CTIME_KW", - /* 101 */ "ANY", - /* 102 */ "BITAND", - /* 103 */ "BITOR", - /* 104 */ "LSHIFT", - /* 105 */ "RSHIFT", - /* 106 */ "PLUS", - /* 107 */ "MINUS", - /* 108 */ "STAR", - /* 109 */ "SLASH", - /* 110 */ "REM", - /* 111 */ "CONCAT", - /* 112 */ "PTR", - /* 113 */ "COLLATE", - /* 114 */ "BITNOT", - /* 115 */ "ON", - /* 116 */ "INDEXED", - /* 117 */ "STRING", - /* 118 */ "JOIN_KW", - /* 119 */ "CONSTRAINT", - /* 120 */ "DEFAULT", - /* 121 */ "NULL", - /* 122 */ "PRIMARY", - /* 123 */ "UNIQUE", - /* 124 */ "CHECK", - /* 125 */ "REFERENCES", - /* 126 */ "AUTOINCR", - /* 127 */ "INSERT", - /* 128 */ "DELETE", - /* 129 */ "UPDATE", - /* 130 */ "SET", - /* 131 */ "DEFERRABLE", - /* 132 */ "FOREIGN", - /* 133 */ "DROP", - /* 134 */ "UNION", - /* 135 */ "ALL", - /* 136 */ "EXCEPT", - /* 137 */ "INTERSECT", - /* 138 */ "SELECT", - /* 139 */ "VALUES", - /* 140 */ "DISTINCT", - /* 141 */ "DOT", - /* 142 */ "FROM", - /* 143 */ "JOIN", - /* 144 */ "USING", - /* 145 */ "ORDER", - /* 146 */ "GROUP", - /* 147 */ "HAVING", - /* 148 */ "LIMIT", - /* 149 */ "WHERE", - /* 150 */ "RETURNING", - /* 151 */ "INTO", - /* 152 */ "NOTHING", - /* 153 */ "FLOAT", - /* 154 */ "BLOB", - /* 155 */ "INTEGER", - /* 156 */ "VARIABLE", - /* 157 */ "CASE", - /* 158 */ "WHEN", - /* 159 */ "THEN", - /* 160 */ "ELSE", - /* 161 */ "INDEX", - /* 162 */ "ALTER", - /* 163 */ "ADD", - /* 164 */ "WINDOW", - /* 165 */ "OVER", - /* 166 */ "FILTER", - /* 167 */ "COLUMN", - /* 168 */ "AGG_FUNCTION", - /* 169 */ "AGG_COLUMN", - /* 170 */ "TRUEFALSE", - /* 171 */ "ISNOT", + /* 46 */ "ISNOT", + /* 47 */ "MATCH", + /* 48 */ "LIKE_KW", + /* 49 */ "BETWEEN", + /* 50 */ "IN", + /* 51 */ "ISNULL", + /* 52 */ "NOTNULL", + /* 53 */ "NE", + /* 54 */ "EQ", + /* 55 */ "GT", + /* 56 */ "LE", + /* 57 */ "LT", + /* 58 */ "GE", + /* 59 */ "ESCAPE", + /* 60 */ "ID", + /* 61 */ "COLUMNKW", + /* 62 */ "DO", + /* 63 */ "FOR", + /* 64 */ "IGNORE", + /* 65 */ "INITIALLY", + /* 66 */ "INSTEAD", + /* 67 */ "NO", + /* 68 */ "KEY", + /* 69 */ "OF", + /* 70 */ "OFFSET", + /* 71 */ "PRAGMA", + /* 72 */ "RAISE", + /* 73 */ "RECURSIVE", + /* 74 */ "REPLACE", + /* 75 */ "RESTRICT", + /* 76 */ "ROW", + /* 77 */ "ROWS", + /* 78 */ "TRIGGER", + /* 79 */ "VACUUM", + /* 80 */ "VIEW", + /* 81 */ "VIRTUAL", + /* 82 */ "WITH", + /* 83 */ "NULLS", + /* 84 */ "FIRST", + /* 85 */ "LAST", + /* 86 */ "CURRENT", + /* 87 */ "FOLLOWING", + /* 88 */ "PARTITION", + /* 89 */ "PRECEDING", + /* 90 */ "RANGE", + /* 91 */ "UNBOUNDED", + /* 92 */ "EXCLUDE", + /* 93 */ "GROUPS", + /* 94 */ "OTHERS", + /* 95 */ "TIES", + /* 96 */ "GENERATED", + /* 97 */ "ALWAYS", + /* 98 */ "MATERIALIZED", + /* 99 */ "REINDEX", + /* 100 */ "RENAME", + /* 101 */ "CTIME_KW", + /* 102 */ "ANY", + /* 103 */ "BITAND", + /* 104 */ "BITOR", + /* 105 */ "LSHIFT", + /* 106 */ "RSHIFT", + /* 107 */ "PLUS", + /* 108 */ "MINUS", + /* 109 */ "STAR", + /* 110 */ "SLASH", + /* 111 */ "REM", + /* 112 */ "CONCAT", + /* 113 */ "PTR", + /* 114 */ "COLLATE", + /* 115 */ "BITNOT", + /* 116 */ "ON", + /* 117 */ "INDEXED", + /* 118 */ "STRING", + /* 119 */ "JOIN_KW", + /* 120 */ "CONSTRAINT", + /* 121 */ "DEFAULT", + /* 122 */ "NULL", + /* 123 */ "PRIMARY", + /* 124 */ "UNIQUE", + /* 125 */ "CHECK", + /* 126 */ "REFERENCES", + /* 127 */ "AUTOINCR", + /* 128 */ "INSERT", + /* 129 */ "DELETE", + /* 130 */ "UPDATE", + /* 131 */ "SET", + /* 132 */ "DEFERRABLE", + /* 133 */ "FOREIGN", + /* 134 */ "DROP", + /* 135 */ "UNION", + /* 136 */ "ALL", + /* 137 */ "EXCEPT", + /* 138 */ "INTERSECT", + /* 139 */ "SELECT", + /* 140 */ "VALUES", + /* 141 */ "DISTINCT", + /* 142 */ "DOT", + /* 143 */ "FROM", + /* 144 */ "JOIN", + /* 145 */ "USING", + /* 146 */ "ORDER", + /* 147 */ "GROUP", + /* 148 */ "HAVING", + /* 149 */ "LIMIT", + /* 150 */ "WHERE", + /* 151 */ "RETURNING", + /* 152 */ "INTO", + /* 153 */ "NOTHING", + /* 154 */ "FLOAT", + /* 155 */ "BLOB", + /* 156 */ "INTEGER", + /* 157 */ "VARIABLE", + /* 158 */ "CASE", + /* 159 */ "WHEN", + /* 160 */ "THEN", + /* 161 */ "ELSE", + /* 162 */ "INDEX", + /* 163 */ "ALTER", + /* 164 */ "ADD", + /* 165 */ "WINDOW", + /* 166 */ "OVER", + /* 167 */ "FILTER", + /* 168 */ "COLUMN", + /* 169 */ "AGG_FUNCTION", + /* 170 */ "AGG_COLUMN", + /* 171 */ "TRUEFALSE", /* 172 */ "FUNCTION", - /* 173 */ "UMINUS", - /* 174 */ "UPLUS", + /* 173 */ "UPLUS", + /* 174 */ "UMINUS", /* 175 */ "TRUTH", /* 176 */ "REGISTER", /* 177 */ "VECTOR", @@ -172695,142 +179226,146 @@ static const char *const yyTokenName[] = { /* 180 */ "ASTERISK", /* 181 */ "SPAN", /* 182 */ "ERROR", - /* 183 */ "SPACE", - /* 184 */ "ILLEGAL", - /* 185 */ "input", - /* 186 */ "cmdlist", - /* 187 */ "ecmd", - /* 188 */ "cmdx", - /* 189 */ "explain", - /* 190 */ "cmd", - /* 191 */ "transtype", - /* 192 */ "trans_opt", - /* 193 */ "nm", - /* 194 */ "savepoint_opt", - /* 195 */ "create_table", - /* 196 */ "create_table_args", - /* 197 */ "createkw", - /* 198 */ "temp", - /* 199 */ "ifnotexists", - /* 200 */ "dbnm", - /* 201 */ "columnlist", - /* 202 */ "conslist_opt", - /* 203 */ "table_option_set", - /* 204 */ "select", - /* 205 */ "table_option", - /* 206 */ "columnname", - /* 207 */ "carglist", - /* 208 */ "typetoken", - /* 209 */ "typename", - /* 210 */ "signed", - /* 211 */ "plus_num", - /* 212 */ "minus_num", - /* 213 */ "scanpt", - /* 214 */ "scantok", - /* 215 */ "ccons", - /* 216 */ "term", - /* 217 */ "expr", - /* 218 */ "onconf", - /* 219 */ "sortorder", - /* 220 */ "autoinc", - /* 221 */ "eidlist_opt", - /* 222 */ "refargs", - /* 223 */ "defer_subclause", - /* 224 */ "generated", - /* 225 */ "refarg", - /* 226 */ "refact", - /* 227 */ "init_deferred_pred_opt", - /* 228 */ "conslist", - /* 229 */ "tconscomma", - /* 230 */ "tcons", - /* 231 */ "sortlist", - /* 232 */ "eidlist", - /* 233 */ "defer_subclause_opt", - /* 234 */ "orconf", - /* 235 */ "resolvetype", - /* 236 */ "raisetype", - /* 237 */ "ifexists", - /* 238 */ "fullname", - /* 239 */ "selectnowith", - /* 240 */ "oneselect", - /* 241 */ "wqlist", - /* 242 */ "multiselect_op", - /* 243 */ "distinct", - /* 244 */ "selcollist", - /* 245 */ "from", - /* 246 */ "where_opt", - /* 247 */ "groupby_opt", - /* 248 */ "having_opt", - /* 249 */ "orderby_opt", - /* 250 */ "limit_opt", - /* 251 */ "window_clause", - /* 252 */ "values", - /* 253 */ "nexprlist", - /* 254 */ "sclp", - /* 255 */ "as", - /* 256 */ "seltablist", - /* 257 */ "stl_prefix", - /* 258 */ "joinop", - /* 259 */ "on_using", - /* 260 */ "indexed_by", - /* 261 */ "exprlist", - /* 262 */ "xfullname", - /* 263 */ "idlist", - /* 264 */ "indexed_opt", - /* 265 */ "nulls", - /* 266 */ "with", - /* 267 */ "where_opt_ret", - /* 268 */ "setlist", - /* 269 */ "insert_cmd", - /* 270 */ "idlist_opt", - /* 271 */ "upsert", - /* 272 */ "returning", - /* 273 */ "filter_over", - /* 274 */ "likeop", - /* 275 */ "between_op", - /* 276 */ "in_op", - /* 277 */ "paren_exprlist", - /* 278 */ "case_operand", - /* 279 */ "case_exprlist", - /* 280 */ "case_else", - /* 281 */ "uniqueflag", - /* 282 */ "collate", - /* 283 */ "vinto", - /* 284 */ "nmnum", - /* 285 */ "trigger_decl", - /* 286 */ "trigger_cmd_list", - /* 287 */ "trigger_time", - /* 288 */ "trigger_event", - /* 289 */ "foreach_clause", - /* 290 */ "when_clause", - /* 291 */ "trigger_cmd", - /* 292 */ "trnm", - /* 293 */ "tridxby", - /* 294 */ "database_kw_opt", - /* 295 */ "key_opt", - /* 296 */ "add_column_fullname", - /* 297 */ "kwcolumn_opt", - /* 298 */ "create_vtab", - /* 299 */ "vtabarglist", - /* 300 */ "vtabarg", - /* 301 */ "vtabargtoken", - /* 302 */ "lp", - /* 303 */ "anylist", - /* 304 */ "wqitem", - /* 305 */ "wqas", - /* 306 */ "windowdefn_list", - /* 307 */ "windowdefn", - /* 308 */ "window", - /* 309 */ "frame_opt", - /* 310 */ "part_opt", - /* 311 */ "filter_clause", - /* 312 */ "over_clause", - /* 313 */ "range_or_rows", - /* 314 */ "frame_bound", - /* 315 */ "frame_bound_s", - /* 316 */ "frame_bound_e", - /* 317 */ "frame_exclude_opt", - /* 318 */ "frame_exclude", + /* 183 */ "QNUMBER", + /* 184 */ "SPACE", + /* 185 */ "COMMENT", + /* 186 */ "ILLEGAL", + /* 187 */ "input", + /* 188 */ "cmdlist", + /* 189 */ "ecmd", + /* 190 */ "cmdx", + /* 191 */ "explain", + /* 192 */ "cmd", + /* 193 */ "transtype", + /* 194 */ "trans_opt", + /* 195 */ "nm", + /* 196 */ "savepoint_opt", + /* 197 */ "create_table", + /* 198 */ "create_table_args", + /* 199 */ "createkw", + /* 200 */ "temp", + /* 201 */ "ifnotexists", + /* 202 */ "dbnm", + /* 203 */ "columnlist", + /* 204 */ "conslist_opt", + /* 205 */ "table_option_set", + /* 206 */ "select", + /* 207 */ "table_option", + /* 208 */ "columnname", + /* 209 */ "carglist", + /* 210 */ "typetoken", + /* 211 */ "typename", + /* 212 */ "signed", + /* 213 */ "plus_num", + /* 214 */ "minus_num", + /* 215 */ "scanpt", + /* 216 */ "scantok", + /* 217 */ "ccons", + /* 218 */ "term", + /* 219 */ "expr", + /* 220 */ "onconf", + /* 221 */ "sortorder", + /* 222 */ "autoinc", + /* 223 */ "eidlist_opt", + /* 224 */ "refargs", + /* 225 */ "defer_subclause", + /* 226 */ "generated", + /* 227 */ "refarg", + /* 228 */ "refact", + /* 229 */ "init_deferred_pred_opt", + /* 230 */ "conslist", + /* 231 */ "tconscomma", + /* 232 */ "tcons", + /* 233 */ "sortlist", + /* 234 */ "eidlist", + /* 235 */ "defer_subclause_opt", + /* 236 */ "orconf", + /* 237 */ "resolvetype", + /* 238 */ "raisetype", + /* 239 */ "ifexists", + /* 240 */ "fullname", + /* 241 */ "selectnowith", + /* 242 */ "oneselect", + /* 243 */ "wqlist", + /* 244 */ "multiselect_op", + /* 245 */ "distinct", + /* 246 */ "selcollist", + /* 247 */ "from", + /* 248 */ "where_opt", + /* 249 */ "groupby_opt", + /* 250 */ "having_opt", + /* 251 */ "orderby_opt", + /* 252 */ "limit_opt", + /* 253 */ "window_clause", + /* 254 */ "values", + /* 255 */ "nexprlist", + /* 256 */ "mvalues", + /* 257 */ "sclp", + /* 258 */ "as", + /* 259 */ "seltablist", + /* 260 */ "stl_prefix", + /* 261 */ "joinop", + /* 262 */ "on_using", + /* 263 */ "indexed_by", + /* 264 */ "exprlist", + /* 265 */ "xfullname", + /* 266 */ "idlist", + /* 267 */ "indexed_opt", + /* 268 */ "nulls", + /* 269 */ "with", + /* 270 */ "where_opt_ret", + /* 271 */ "setlist", + /* 272 */ "insert_cmd", + /* 273 */ "idlist_opt", + /* 274 */ "upsert", + /* 275 */ "returning", + /* 276 */ "filter_over", + /* 277 */ "likeop", + /* 278 */ "between_op", + /* 279 */ "in_op", + /* 280 */ "paren_exprlist", + /* 281 */ "case_operand", + /* 282 */ "case_exprlist", + /* 283 */ "case_else", + /* 284 */ "uniqueflag", + /* 285 */ "collate", + /* 286 */ "vinto", + /* 287 */ "nmnum", + /* 288 */ "trigger_decl", + /* 289 */ "trigger_cmd_list", + /* 290 */ "trigger_time", + /* 291 */ "trigger_event", + /* 292 */ "foreach_clause", + /* 293 */ "when_clause", + /* 294 */ "trigger_cmd", + /* 295 */ "trnm", + /* 296 */ "tridxby", + /* 297 */ "database_kw_opt", + /* 298 */ "key_opt", + /* 299 */ "add_column_fullname", + /* 300 */ "kwcolumn_opt", + /* 301 */ "create_vtab", + /* 302 */ "vtabarglist", + /* 303 */ "vtabarg", + /* 304 */ "vtabargtoken", + /* 305 */ "lp", + /* 306 */ "anylist", + /* 307 */ "wqitem", + /* 308 */ "wqas", + /* 309 */ "withnm", + /* 310 */ "windowdefn_list", + /* 311 */ "windowdefn", + /* 312 */ "window", + /* 313 */ "frame_opt", + /* 314 */ "part_opt", + /* 315 */ "filter_clause", + /* 316 */ "over_clause", + /* 317 */ "range_or_rows", + /* 318 */ "frame_bound", + /* 319 */ "frame_bound_s", + /* 320 */ "frame_bound_e", + /* 321 */ "frame_exclude_opt", + /* 322 */ "frame_exclude", }; #endif /* defined(YYCOVERAGE) || !defined(NDEBUG) */ @@ -172933,351 +179468,363 @@ static const char *const yyRuleName[] = { /* 92 */ "oneselect ::= SELECT distinct selcollist from where_opt groupby_opt having_opt orderby_opt limit_opt", /* 93 */ "oneselect ::= SELECT distinct selcollist from where_opt groupby_opt having_opt window_clause orderby_opt limit_opt", /* 94 */ "values ::= VALUES LP nexprlist RP", - /* 95 */ "values ::= values COMMA LP nexprlist RP", - /* 96 */ "distinct ::= DISTINCT", - /* 97 */ "distinct ::= ALL", - /* 98 */ "distinct ::=", - /* 99 */ "sclp ::=", - /* 100 */ "selcollist ::= sclp scanpt expr scanpt as", - /* 101 */ "selcollist ::= sclp scanpt STAR", - /* 102 */ "selcollist ::= sclp scanpt nm DOT STAR", - /* 103 */ "as ::= AS nm", - /* 104 */ "as ::=", - /* 105 */ "from ::=", - /* 106 */ "from ::= FROM seltablist", - /* 107 */ "stl_prefix ::= seltablist joinop", - /* 108 */ "stl_prefix ::=", - /* 109 */ "seltablist ::= stl_prefix nm dbnm as on_using", - /* 110 */ "seltablist ::= stl_prefix nm dbnm as indexed_by on_using", - /* 111 */ "seltablist ::= stl_prefix nm dbnm LP exprlist RP as on_using", - /* 112 */ "seltablist ::= stl_prefix LP select RP as on_using", - /* 113 */ "seltablist ::= stl_prefix LP seltablist RP as on_using", - /* 114 */ "dbnm ::=", - /* 115 */ "dbnm ::= DOT nm", - /* 116 */ "fullname ::= nm", - /* 117 */ "fullname ::= nm DOT nm", - /* 118 */ "xfullname ::= nm", - /* 119 */ "xfullname ::= nm DOT nm", - /* 120 */ "xfullname ::= nm DOT nm AS nm", - /* 121 */ "xfullname ::= nm AS nm", - /* 122 */ "joinop ::= COMMA|JOIN", - /* 123 */ "joinop ::= JOIN_KW JOIN", - /* 124 */ "joinop ::= JOIN_KW nm JOIN", - /* 125 */ "joinop ::= JOIN_KW nm nm JOIN", - /* 126 */ "on_using ::= ON expr", - /* 127 */ "on_using ::= USING LP idlist RP", - /* 128 */ "on_using ::=", - /* 129 */ "indexed_opt ::=", - /* 130 */ "indexed_by ::= INDEXED BY nm", - /* 131 */ "indexed_by ::= NOT INDEXED", - /* 132 */ "orderby_opt ::=", - /* 133 */ "orderby_opt ::= ORDER BY sortlist", - /* 134 */ "sortlist ::= sortlist COMMA expr sortorder nulls", - /* 135 */ "sortlist ::= expr sortorder nulls", - /* 136 */ "sortorder ::= ASC", - /* 137 */ "sortorder ::= DESC", - /* 138 */ "sortorder ::=", - /* 139 */ "nulls ::= NULLS FIRST", - /* 140 */ "nulls ::= NULLS LAST", - /* 141 */ "nulls ::=", - /* 142 */ "groupby_opt ::=", - /* 143 */ "groupby_opt ::= GROUP BY nexprlist", - /* 144 */ "having_opt ::=", - /* 145 */ "having_opt ::= HAVING expr", - /* 146 */ "limit_opt ::=", - /* 147 */ "limit_opt ::= LIMIT expr", - /* 148 */ "limit_opt ::= LIMIT expr OFFSET expr", - /* 149 */ "limit_opt ::= LIMIT expr COMMA expr", - /* 150 */ "cmd ::= with DELETE FROM xfullname indexed_opt where_opt_ret", - /* 151 */ "where_opt ::=", - /* 152 */ "where_opt ::= WHERE expr", - /* 153 */ "where_opt_ret ::=", - /* 154 */ "where_opt_ret ::= WHERE expr", - /* 155 */ "where_opt_ret ::= RETURNING selcollist", - /* 156 */ "where_opt_ret ::= WHERE expr RETURNING selcollist", - /* 157 */ "cmd ::= with UPDATE orconf xfullname indexed_opt SET setlist from where_opt_ret", - /* 158 */ "setlist ::= setlist COMMA nm EQ expr", - /* 159 */ "setlist ::= setlist COMMA LP idlist RP EQ expr", - /* 160 */ "setlist ::= nm EQ expr", - /* 161 */ "setlist ::= LP idlist RP EQ expr", - /* 162 */ "cmd ::= with insert_cmd INTO xfullname idlist_opt select upsert", - /* 163 */ "cmd ::= with insert_cmd INTO xfullname idlist_opt DEFAULT VALUES returning", - /* 164 */ "upsert ::=", - /* 165 */ "upsert ::= RETURNING selcollist", - /* 166 */ "upsert ::= ON CONFLICT LP sortlist RP where_opt DO UPDATE SET setlist where_opt upsert", - /* 167 */ "upsert ::= ON CONFLICT LP sortlist RP where_opt DO NOTHING upsert", - /* 168 */ "upsert ::= ON CONFLICT DO NOTHING returning", - /* 169 */ "upsert ::= ON CONFLICT DO UPDATE SET setlist where_opt returning", - /* 170 */ "returning ::= RETURNING selcollist", - /* 171 */ "insert_cmd ::= INSERT orconf", - /* 172 */ "insert_cmd ::= REPLACE", - /* 173 */ "idlist_opt ::=", - /* 174 */ "idlist_opt ::= LP idlist RP", - /* 175 */ "idlist ::= idlist COMMA nm", - /* 176 */ "idlist ::= nm", - /* 177 */ "expr ::= LP expr RP", - /* 178 */ "expr ::= ID|INDEXED|JOIN_KW", - /* 179 */ "expr ::= nm DOT nm", - /* 180 */ "expr ::= nm DOT nm DOT nm", - /* 181 */ "term ::= NULL|FLOAT|BLOB", - /* 182 */ "term ::= STRING", - /* 183 */ "term ::= INTEGER", - /* 184 */ "expr ::= VARIABLE", - /* 185 */ "expr ::= expr COLLATE ID|STRING", - /* 186 */ "expr ::= CAST LP expr AS typetoken RP", - /* 187 */ "expr ::= ID|INDEXED|JOIN_KW LP distinct exprlist RP", - /* 188 */ "expr ::= ID|INDEXED|JOIN_KW LP distinct exprlist ORDER BY sortlist RP", - /* 189 */ "expr ::= ID|INDEXED|JOIN_KW LP STAR RP", - /* 190 */ "expr ::= ID|INDEXED|JOIN_KW LP distinct exprlist RP filter_over", - /* 191 */ "expr ::= ID|INDEXED|JOIN_KW LP distinct exprlist ORDER BY sortlist RP filter_over", - /* 192 */ "expr ::= ID|INDEXED|JOIN_KW LP STAR RP filter_over", - /* 193 */ "term ::= CTIME_KW", - /* 194 */ "expr ::= LP nexprlist COMMA expr RP", - /* 195 */ "expr ::= expr AND expr", - /* 196 */ "expr ::= expr OR expr", - /* 197 */ "expr ::= expr LT|GT|GE|LE expr", - /* 198 */ "expr ::= expr EQ|NE expr", - /* 199 */ "expr ::= expr BITAND|BITOR|LSHIFT|RSHIFT expr", - /* 200 */ "expr ::= expr PLUS|MINUS expr", - /* 201 */ "expr ::= expr STAR|SLASH|REM expr", - /* 202 */ "expr ::= expr CONCAT expr", - /* 203 */ "likeop ::= NOT LIKE_KW|MATCH", - /* 204 */ "expr ::= expr likeop expr", - /* 205 */ "expr ::= expr likeop expr ESCAPE expr", - /* 206 */ "expr ::= expr ISNULL|NOTNULL", - /* 207 */ "expr ::= expr NOT NULL", - /* 208 */ "expr ::= expr IS expr", - /* 209 */ "expr ::= expr IS NOT expr", - /* 210 */ "expr ::= expr IS NOT DISTINCT FROM expr", - /* 211 */ "expr ::= expr IS DISTINCT FROM expr", - /* 212 */ "expr ::= NOT expr", - /* 213 */ "expr ::= BITNOT expr", - /* 214 */ "expr ::= PLUS|MINUS expr", - /* 215 */ "expr ::= expr PTR expr", - /* 216 */ "between_op ::= BETWEEN", - /* 217 */ "between_op ::= NOT BETWEEN", - /* 218 */ "expr ::= expr between_op expr AND expr", - /* 219 */ "in_op ::= IN", - /* 220 */ "in_op ::= NOT IN", - /* 221 */ "expr ::= expr in_op LP exprlist RP", - /* 222 */ "expr ::= LP select RP", - /* 223 */ "expr ::= expr in_op LP select RP", - /* 224 */ "expr ::= expr in_op nm dbnm paren_exprlist", - /* 225 */ "expr ::= EXISTS LP select RP", - /* 226 */ "expr ::= CASE case_operand case_exprlist case_else END", - /* 227 */ "case_exprlist ::= case_exprlist WHEN expr THEN expr", - /* 228 */ "case_exprlist ::= WHEN expr THEN expr", - /* 229 */ "case_else ::= ELSE expr", - /* 230 */ "case_else ::=", - /* 231 */ "case_operand ::=", - /* 232 */ "exprlist ::=", - /* 233 */ "nexprlist ::= nexprlist COMMA expr", - /* 234 */ "nexprlist ::= expr", - /* 235 */ "paren_exprlist ::=", - /* 236 */ "paren_exprlist ::= LP exprlist RP", - /* 237 */ "cmd ::= createkw uniqueflag INDEX ifnotexists nm dbnm ON nm LP sortlist RP where_opt", - /* 238 */ "uniqueflag ::= UNIQUE", - /* 239 */ "uniqueflag ::=", - /* 240 */ "eidlist_opt ::=", - /* 241 */ "eidlist_opt ::= LP eidlist RP", - /* 242 */ "eidlist ::= eidlist COMMA nm collate sortorder", - /* 243 */ "eidlist ::= nm collate sortorder", - /* 244 */ "collate ::=", - /* 245 */ "collate ::= COLLATE ID|STRING", - /* 246 */ "cmd ::= DROP INDEX ifexists fullname", - /* 247 */ "cmd ::= VACUUM vinto", - /* 248 */ "cmd ::= VACUUM nm vinto", - /* 249 */ "vinto ::= INTO expr", - /* 250 */ "vinto ::=", - /* 251 */ "cmd ::= PRAGMA nm dbnm", - /* 252 */ "cmd ::= PRAGMA nm dbnm EQ nmnum", - /* 253 */ "cmd ::= PRAGMA nm dbnm LP nmnum RP", - /* 254 */ "cmd ::= PRAGMA nm dbnm EQ minus_num", - /* 255 */ "cmd ::= PRAGMA nm dbnm LP minus_num RP", - /* 256 */ "plus_num ::= PLUS INTEGER|FLOAT", - /* 257 */ "minus_num ::= MINUS INTEGER|FLOAT", - /* 258 */ "cmd ::= createkw trigger_decl BEGIN trigger_cmd_list END", - /* 259 */ "trigger_decl ::= temp TRIGGER ifnotexists nm dbnm trigger_time trigger_event ON fullname foreach_clause when_clause", - /* 260 */ "trigger_time ::= BEFORE|AFTER", - /* 261 */ "trigger_time ::= INSTEAD OF", - /* 262 */ "trigger_time ::=", - /* 263 */ "trigger_event ::= DELETE|INSERT", - /* 264 */ "trigger_event ::= UPDATE", - /* 265 */ "trigger_event ::= UPDATE OF idlist", - /* 266 */ "when_clause ::=", - /* 267 */ "when_clause ::= WHEN expr", - /* 268 */ "trigger_cmd_list ::= trigger_cmd_list trigger_cmd SEMI", - /* 269 */ "trigger_cmd_list ::= trigger_cmd SEMI", - /* 270 */ "trnm ::= nm DOT nm", - /* 271 */ "tridxby ::= INDEXED BY nm", - /* 272 */ "tridxby ::= NOT INDEXED", - /* 273 */ "trigger_cmd ::= UPDATE orconf trnm tridxby SET setlist from where_opt scanpt", - /* 274 */ "trigger_cmd ::= scanpt insert_cmd INTO trnm idlist_opt select upsert scanpt", - /* 275 */ "trigger_cmd ::= DELETE FROM trnm tridxby where_opt scanpt", - /* 276 */ "trigger_cmd ::= scanpt select scanpt", - /* 277 */ "expr ::= RAISE LP IGNORE RP", - /* 278 */ "expr ::= RAISE LP raisetype COMMA nm RP", - /* 279 */ "raisetype ::= ROLLBACK", - /* 280 */ "raisetype ::= ABORT", - /* 281 */ "raisetype ::= FAIL", - /* 282 */ "cmd ::= DROP TRIGGER ifexists fullname", - /* 283 */ "cmd ::= ATTACH database_kw_opt expr AS expr key_opt", - /* 284 */ "cmd ::= DETACH database_kw_opt expr", - /* 285 */ "key_opt ::=", - /* 286 */ "key_opt ::= KEY expr", - /* 287 */ "cmd ::= REINDEX", - /* 288 */ "cmd ::= REINDEX nm dbnm", - /* 289 */ "cmd ::= ANALYZE", - /* 290 */ "cmd ::= ANALYZE nm dbnm", - /* 291 */ "cmd ::= ALTER TABLE fullname RENAME TO nm", - /* 292 */ "cmd ::= ALTER TABLE add_column_fullname ADD kwcolumn_opt columnname carglist", - /* 293 */ "cmd ::= ALTER TABLE fullname DROP kwcolumn_opt nm", - /* 294 */ "add_column_fullname ::= fullname", - /* 295 */ "cmd ::= ALTER TABLE fullname RENAME kwcolumn_opt nm TO nm", - /* 296 */ "cmd ::= create_vtab", - /* 297 */ "cmd ::= create_vtab LP vtabarglist RP", - /* 298 */ "create_vtab ::= createkw VIRTUAL TABLE ifnotexists nm dbnm USING nm", - /* 299 */ "vtabarg ::=", - /* 300 */ "vtabargtoken ::= ANY", - /* 301 */ "vtabargtoken ::= lp anylist RP", - /* 302 */ "lp ::= LP", - /* 303 */ "with ::= WITH wqlist", - /* 304 */ "with ::= WITH RECURSIVE wqlist", - /* 305 */ "wqas ::= AS", - /* 306 */ "wqas ::= AS MATERIALIZED", - /* 307 */ "wqas ::= AS NOT MATERIALIZED", - /* 308 */ "wqitem ::= nm eidlist_opt wqas LP select RP", - /* 309 */ "wqlist ::= wqitem", - /* 310 */ "wqlist ::= wqlist COMMA wqitem", - /* 311 */ "windowdefn_list ::= windowdefn_list COMMA windowdefn", - /* 312 */ "windowdefn ::= nm AS LP window RP", - /* 313 */ "window ::= PARTITION BY nexprlist orderby_opt frame_opt", - /* 314 */ "window ::= nm PARTITION BY nexprlist orderby_opt frame_opt", - /* 315 */ "window ::= ORDER BY sortlist frame_opt", - /* 316 */ "window ::= nm ORDER BY sortlist frame_opt", - /* 317 */ "window ::= nm frame_opt", - /* 318 */ "frame_opt ::=", - /* 319 */ "frame_opt ::= range_or_rows frame_bound_s frame_exclude_opt", - /* 320 */ "frame_opt ::= range_or_rows BETWEEN frame_bound_s AND frame_bound_e frame_exclude_opt", - /* 321 */ "range_or_rows ::= RANGE|ROWS|GROUPS", - /* 322 */ "frame_bound_s ::= frame_bound", - /* 323 */ "frame_bound_s ::= UNBOUNDED PRECEDING", - /* 324 */ "frame_bound_e ::= frame_bound", - /* 325 */ "frame_bound_e ::= UNBOUNDED FOLLOWING", - /* 326 */ "frame_bound ::= expr PRECEDING|FOLLOWING", - /* 327 */ "frame_bound ::= CURRENT ROW", - /* 328 */ "frame_exclude_opt ::=", - /* 329 */ "frame_exclude_opt ::= EXCLUDE frame_exclude", - /* 330 */ "frame_exclude ::= NO OTHERS", - /* 331 */ "frame_exclude ::= CURRENT ROW", - /* 332 */ "frame_exclude ::= GROUP|TIES", - /* 333 */ "window_clause ::= WINDOW windowdefn_list", - /* 334 */ "filter_over ::= filter_clause over_clause", - /* 335 */ "filter_over ::= over_clause", - /* 336 */ "filter_over ::= filter_clause", - /* 337 */ "over_clause ::= OVER LP window RP", - /* 338 */ "over_clause ::= OVER nm", - /* 339 */ "filter_clause ::= FILTER LP WHERE expr RP", - /* 340 */ "input ::= cmdlist", - /* 341 */ "cmdlist ::= cmdlist ecmd", - /* 342 */ "cmdlist ::= ecmd", - /* 343 */ "ecmd ::= SEMI", - /* 344 */ "ecmd ::= cmdx SEMI", - /* 345 */ "ecmd ::= explain cmdx SEMI", - /* 346 */ "trans_opt ::=", - /* 347 */ "trans_opt ::= TRANSACTION", - /* 348 */ "trans_opt ::= TRANSACTION nm", - /* 349 */ "savepoint_opt ::= SAVEPOINT", - /* 350 */ "savepoint_opt ::=", - /* 351 */ "cmd ::= create_table create_table_args", - /* 352 */ "table_option_set ::= table_option", - /* 353 */ "columnlist ::= columnlist COMMA columnname carglist", - /* 354 */ "columnlist ::= columnname carglist", - /* 355 */ "nm ::= ID|INDEXED|JOIN_KW", - /* 356 */ "nm ::= STRING", - /* 357 */ "typetoken ::= typename", - /* 358 */ "typename ::= ID|STRING", - /* 359 */ "signed ::= plus_num", - /* 360 */ "signed ::= minus_num", - /* 361 */ "carglist ::= carglist ccons", - /* 362 */ "carglist ::=", - /* 363 */ "ccons ::= NULL onconf", - /* 364 */ "ccons ::= GENERATED ALWAYS AS generated", - /* 365 */ "ccons ::= AS generated", - /* 366 */ "conslist_opt ::= COMMA conslist", - /* 367 */ "conslist ::= conslist tconscomma tcons", - /* 368 */ "conslist ::= tcons", - /* 369 */ "tconscomma ::=", - /* 370 */ "defer_subclause_opt ::= defer_subclause", - /* 371 */ "resolvetype ::= raisetype", - /* 372 */ "selectnowith ::= oneselect", - /* 373 */ "oneselect ::= values", - /* 374 */ "sclp ::= selcollist COMMA", - /* 375 */ "as ::= ID|STRING", - /* 376 */ "indexed_opt ::= indexed_by", - /* 377 */ "returning ::=", - /* 378 */ "expr ::= term", - /* 379 */ "likeop ::= LIKE_KW|MATCH", - /* 380 */ "case_operand ::= expr", - /* 381 */ "exprlist ::= nexprlist", - /* 382 */ "nmnum ::= plus_num", - /* 383 */ "nmnum ::= nm", - /* 384 */ "nmnum ::= ON", - /* 385 */ "nmnum ::= DELETE", - /* 386 */ "nmnum ::= DEFAULT", - /* 387 */ "plus_num ::= INTEGER|FLOAT", - /* 388 */ "foreach_clause ::=", - /* 389 */ "foreach_clause ::= FOR EACH ROW", - /* 390 */ "trnm ::= nm", - /* 391 */ "tridxby ::=", - /* 392 */ "database_kw_opt ::= DATABASE", - /* 393 */ "database_kw_opt ::=", - /* 394 */ "kwcolumn_opt ::=", - /* 395 */ "kwcolumn_opt ::= COLUMNKW", - /* 396 */ "vtabarglist ::= vtabarg", - /* 397 */ "vtabarglist ::= vtabarglist COMMA vtabarg", - /* 398 */ "vtabarg ::= vtabarg vtabargtoken", - /* 399 */ "anylist ::=", - /* 400 */ "anylist ::= anylist LP anylist RP", - /* 401 */ "anylist ::= anylist ANY", - /* 402 */ "with ::=", - /* 403 */ "windowdefn_list ::= windowdefn", - /* 404 */ "window ::= frame_opt", + /* 95 */ "oneselect ::= mvalues", + /* 96 */ "mvalues ::= values COMMA LP nexprlist RP", + /* 97 */ "mvalues ::= mvalues COMMA LP nexprlist RP", + /* 98 */ "distinct ::= DISTINCT", + /* 99 */ "distinct ::= ALL", + /* 100 */ "distinct ::=", + /* 101 */ "sclp ::=", + /* 102 */ "selcollist ::= sclp scanpt expr scanpt as", + /* 103 */ "selcollist ::= sclp scanpt STAR", + /* 104 */ "selcollist ::= sclp scanpt nm DOT STAR", + /* 105 */ "as ::= AS nm", + /* 106 */ "as ::=", + /* 107 */ "from ::=", + /* 108 */ "from ::= FROM seltablist", + /* 109 */ "stl_prefix ::= seltablist joinop", + /* 110 */ "stl_prefix ::=", + /* 111 */ "seltablist ::= stl_prefix nm dbnm as on_using", + /* 112 */ "seltablist ::= stl_prefix nm dbnm as indexed_by on_using", + /* 113 */ "seltablist ::= stl_prefix nm dbnm LP exprlist RP as on_using", + /* 114 */ "seltablist ::= stl_prefix LP select RP as on_using", + /* 115 */ "seltablist ::= stl_prefix LP seltablist RP as on_using", + /* 116 */ "dbnm ::=", + /* 117 */ "dbnm ::= DOT nm", + /* 118 */ "fullname ::= nm", + /* 119 */ "fullname ::= nm DOT nm", + /* 120 */ "xfullname ::= nm", + /* 121 */ "xfullname ::= nm DOT nm", + /* 122 */ "xfullname ::= nm DOT nm AS nm", + /* 123 */ "xfullname ::= nm AS nm", + /* 124 */ "joinop ::= COMMA|JOIN", + /* 125 */ "joinop ::= JOIN_KW JOIN", + /* 126 */ "joinop ::= JOIN_KW nm JOIN", + /* 127 */ "joinop ::= JOIN_KW nm nm JOIN", + /* 128 */ "on_using ::= ON expr", + /* 129 */ "on_using ::= USING LP idlist RP", + /* 130 */ "on_using ::=", + /* 131 */ "indexed_opt ::=", + /* 132 */ "indexed_by ::= INDEXED BY nm", + /* 133 */ "indexed_by ::= NOT INDEXED", + /* 134 */ "orderby_opt ::=", + /* 135 */ "orderby_opt ::= ORDER BY sortlist", + /* 136 */ "sortlist ::= sortlist COMMA expr sortorder nulls", + /* 137 */ "sortlist ::= expr sortorder nulls", + /* 138 */ "sortorder ::= ASC", + /* 139 */ "sortorder ::= DESC", + /* 140 */ "sortorder ::=", + /* 141 */ "nulls ::= NULLS FIRST", + /* 142 */ "nulls ::= NULLS LAST", + /* 143 */ "nulls ::=", + /* 144 */ "groupby_opt ::=", + /* 145 */ "groupby_opt ::= GROUP BY nexprlist", + /* 146 */ "having_opt ::=", + /* 147 */ "having_opt ::= HAVING expr", + /* 148 */ "limit_opt ::=", + /* 149 */ "limit_opt ::= LIMIT expr", + /* 150 */ "limit_opt ::= LIMIT expr OFFSET expr", + /* 151 */ "limit_opt ::= LIMIT expr COMMA expr", + /* 152 */ "cmd ::= with DELETE FROM xfullname indexed_opt where_opt_ret", + /* 153 */ "where_opt ::=", + /* 154 */ "where_opt ::= WHERE expr", + /* 155 */ "where_opt_ret ::=", + /* 156 */ "where_opt_ret ::= WHERE expr", + /* 157 */ "where_opt_ret ::= RETURNING selcollist", + /* 158 */ "where_opt_ret ::= WHERE expr RETURNING selcollist", + /* 159 */ "cmd ::= with UPDATE orconf xfullname indexed_opt SET setlist from where_opt_ret", + /* 160 */ "setlist ::= setlist COMMA nm EQ expr", + /* 161 */ "setlist ::= setlist COMMA LP idlist RP EQ expr", + /* 162 */ "setlist ::= nm EQ expr", + /* 163 */ "setlist ::= LP idlist RP EQ expr", + /* 164 */ "cmd ::= with insert_cmd INTO xfullname idlist_opt select upsert", + /* 165 */ "cmd ::= with insert_cmd INTO xfullname idlist_opt DEFAULT VALUES returning", + /* 166 */ "upsert ::=", + /* 167 */ "upsert ::= RETURNING selcollist", + /* 168 */ "upsert ::= ON CONFLICT LP sortlist RP where_opt DO UPDATE SET setlist where_opt upsert", + /* 169 */ "upsert ::= ON CONFLICT LP sortlist RP where_opt DO NOTHING upsert", + /* 170 */ "upsert ::= ON CONFLICT DO NOTHING returning", + /* 171 */ "upsert ::= ON CONFLICT DO UPDATE SET setlist where_opt returning", + /* 172 */ "returning ::= RETURNING selcollist", + /* 173 */ "insert_cmd ::= INSERT orconf", + /* 174 */ "insert_cmd ::= REPLACE", + /* 175 */ "idlist_opt ::=", + /* 176 */ "idlist_opt ::= LP idlist RP", + /* 177 */ "idlist ::= idlist COMMA nm", + /* 178 */ "idlist ::= nm", + /* 179 */ "expr ::= LP expr RP", + /* 180 */ "expr ::= ID|INDEXED|JOIN_KW", + /* 181 */ "expr ::= nm DOT nm", + /* 182 */ "expr ::= nm DOT nm DOT nm", + /* 183 */ "term ::= NULL|FLOAT|BLOB", + /* 184 */ "term ::= STRING", + /* 185 */ "term ::= INTEGER", + /* 186 */ "expr ::= VARIABLE", + /* 187 */ "expr ::= expr COLLATE ID|STRING", + /* 188 */ "expr ::= CAST LP expr AS typetoken RP", + /* 189 */ "expr ::= ID|INDEXED|JOIN_KW LP distinct exprlist RP", + /* 190 */ "expr ::= ID|INDEXED|JOIN_KW LP distinct exprlist ORDER BY sortlist RP", + /* 191 */ "expr ::= ID|INDEXED|JOIN_KW LP STAR RP", + /* 192 */ "expr ::= ID|INDEXED|JOIN_KW LP distinct exprlist RP filter_over", + /* 193 */ "expr ::= ID|INDEXED|JOIN_KW LP distinct exprlist ORDER BY sortlist RP filter_over", + /* 194 */ "expr ::= ID|INDEXED|JOIN_KW LP STAR RP filter_over", + /* 195 */ "term ::= CTIME_KW", + /* 196 */ "expr ::= LP nexprlist COMMA expr RP", + /* 197 */ "expr ::= expr AND expr", + /* 198 */ "expr ::= expr OR expr", + /* 199 */ "expr ::= expr LT|GT|GE|LE expr", + /* 200 */ "expr ::= expr EQ|NE expr", + /* 201 */ "expr ::= expr BITAND|BITOR|LSHIFT|RSHIFT expr", + /* 202 */ "expr ::= expr PLUS|MINUS expr", + /* 203 */ "expr ::= expr STAR|SLASH|REM expr", + /* 204 */ "expr ::= expr CONCAT expr", + /* 205 */ "likeop ::= NOT LIKE_KW|MATCH", + /* 206 */ "expr ::= expr likeop expr", + /* 207 */ "expr ::= expr likeop expr ESCAPE expr", + /* 208 */ "expr ::= expr ISNULL|NOTNULL", + /* 209 */ "expr ::= expr NOT NULL", + /* 210 */ "expr ::= expr IS expr", + /* 211 */ "expr ::= expr IS NOT expr", + /* 212 */ "expr ::= expr IS NOT DISTINCT FROM expr", + /* 213 */ "expr ::= expr IS DISTINCT FROM expr", + /* 214 */ "expr ::= NOT expr", + /* 215 */ "expr ::= BITNOT expr", + /* 216 */ "expr ::= PLUS|MINUS expr", + /* 217 */ "expr ::= expr PTR expr", + /* 218 */ "between_op ::= BETWEEN", + /* 219 */ "between_op ::= NOT BETWEEN", + /* 220 */ "expr ::= expr between_op expr AND expr", + /* 221 */ "in_op ::= IN", + /* 222 */ "in_op ::= NOT IN", + /* 223 */ "expr ::= expr in_op LP exprlist RP", + /* 224 */ "expr ::= LP select RP", + /* 225 */ "expr ::= expr in_op LP select RP", + /* 226 */ "expr ::= expr in_op nm dbnm paren_exprlist", + /* 227 */ "expr ::= EXISTS LP select RP", + /* 228 */ "expr ::= CASE case_operand case_exprlist case_else END", + /* 229 */ "case_exprlist ::= case_exprlist WHEN expr THEN expr", + /* 230 */ "case_exprlist ::= WHEN expr THEN expr", + /* 231 */ "case_else ::= ELSE expr", + /* 232 */ "case_else ::=", + /* 233 */ "case_operand ::=", + /* 234 */ "exprlist ::=", + /* 235 */ "nexprlist ::= nexprlist COMMA expr", + /* 236 */ "nexprlist ::= expr", + /* 237 */ "paren_exprlist ::=", + /* 238 */ "paren_exprlist ::= LP exprlist RP", + /* 239 */ "cmd ::= createkw uniqueflag INDEX ifnotexists nm dbnm ON nm LP sortlist RP where_opt", + /* 240 */ "uniqueflag ::= UNIQUE", + /* 241 */ "uniqueflag ::=", + /* 242 */ "eidlist_opt ::=", + /* 243 */ "eidlist_opt ::= LP eidlist RP", + /* 244 */ "eidlist ::= eidlist COMMA nm collate sortorder", + /* 245 */ "eidlist ::= nm collate sortorder", + /* 246 */ "collate ::=", + /* 247 */ "collate ::= COLLATE ID|STRING", + /* 248 */ "cmd ::= DROP INDEX ifexists fullname", + /* 249 */ "cmd ::= VACUUM vinto", + /* 250 */ "cmd ::= VACUUM nm vinto", + /* 251 */ "vinto ::= INTO expr", + /* 252 */ "vinto ::=", + /* 253 */ "cmd ::= PRAGMA nm dbnm", + /* 254 */ "cmd ::= PRAGMA nm dbnm EQ nmnum", + /* 255 */ "cmd ::= PRAGMA nm dbnm LP nmnum RP", + /* 256 */ "cmd ::= PRAGMA nm dbnm EQ minus_num", + /* 257 */ "cmd ::= PRAGMA nm dbnm LP minus_num RP", + /* 258 */ "plus_num ::= PLUS INTEGER|FLOAT", + /* 259 */ "minus_num ::= MINUS INTEGER|FLOAT", + /* 260 */ "cmd ::= createkw trigger_decl BEGIN trigger_cmd_list END", + /* 261 */ "trigger_decl ::= temp TRIGGER ifnotexists nm dbnm trigger_time trigger_event ON fullname foreach_clause when_clause", + /* 262 */ "trigger_time ::= BEFORE|AFTER", + /* 263 */ "trigger_time ::= INSTEAD OF", + /* 264 */ "trigger_time ::=", + /* 265 */ "trigger_event ::= DELETE|INSERT", + /* 266 */ "trigger_event ::= UPDATE", + /* 267 */ "trigger_event ::= UPDATE OF idlist", + /* 268 */ "when_clause ::=", + /* 269 */ "when_clause ::= WHEN expr", + /* 270 */ "trigger_cmd_list ::= trigger_cmd_list trigger_cmd SEMI", + /* 271 */ "trigger_cmd_list ::= trigger_cmd SEMI", + /* 272 */ "trnm ::= nm DOT nm", + /* 273 */ "tridxby ::= INDEXED BY nm", + /* 274 */ "tridxby ::= NOT INDEXED", + /* 275 */ "trigger_cmd ::= UPDATE orconf trnm tridxby SET setlist from where_opt scanpt", + /* 276 */ "trigger_cmd ::= scanpt insert_cmd INTO trnm idlist_opt select upsert scanpt", + /* 277 */ "trigger_cmd ::= DELETE FROM trnm tridxby where_opt scanpt", + /* 278 */ "trigger_cmd ::= scanpt select scanpt", + /* 279 */ "expr ::= RAISE LP IGNORE RP", + /* 280 */ "expr ::= RAISE LP raisetype COMMA expr RP", + /* 281 */ "raisetype ::= ROLLBACK", + /* 282 */ "raisetype ::= ABORT", + /* 283 */ "raisetype ::= FAIL", + /* 284 */ "cmd ::= DROP TRIGGER ifexists fullname", + /* 285 */ "cmd ::= ATTACH database_kw_opt expr AS expr key_opt", + /* 286 */ "cmd ::= DETACH database_kw_opt expr", + /* 287 */ "key_opt ::=", + /* 288 */ "key_opt ::= KEY expr", + /* 289 */ "cmd ::= REINDEX", + /* 290 */ "cmd ::= REINDEX nm dbnm", + /* 291 */ "cmd ::= ANALYZE", + /* 292 */ "cmd ::= ANALYZE nm dbnm", + /* 293 */ "cmd ::= ALTER TABLE fullname RENAME TO nm", + /* 294 */ "cmd ::= ALTER TABLE add_column_fullname ADD kwcolumn_opt columnname carglist", + /* 295 */ "cmd ::= ALTER TABLE fullname DROP kwcolumn_opt nm", + /* 296 */ "add_column_fullname ::= fullname", + /* 297 */ "cmd ::= ALTER TABLE fullname RENAME kwcolumn_opt nm TO nm", + /* 298 */ "cmd ::= create_vtab", + /* 299 */ "cmd ::= create_vtab LP vtabarglist RP", + /* 300 */ "create_vtab ::= createkw VIRTUAL TABLE ifnotexists nm dbnm USING nm", + /* 301 */ "vtabarg ::=", + /* 302 */ "vtabargtoken ::= ANY", + /* 303 */ "vtabargtoken ::= lp anylist RP", + /* 304 */ "lp ::= LP", + /* 305 */ "with ::= WITH wqlist", + /* 306 */ "with ::= WITH RECURSIVE wqlist", + /* 307 */ "wqas ::= AS", + /* 308 */ "wqas ::= AS MATERIALIZED", + /* 309 */ "wqas ::= AS NOT MATERIALIZED", + /* 310 */ "wqitem ::= withnm eidlist_opt wqas LP select RP", + /* 311 */ "withnm ::= nm", + /* 312 */ "wqlist ::= wqitem", + /* 313 */ "wqlist ::= wqlist COMMA wqitem", + /* 314 */ "windowdefn_list ::= windowdefn_list COMMA windowdefn", + /* 315 */ "windowdefn ::= nm AS LP window RP", + /* 316 */ "window ::= PARTITION BY nexprlist orderby_opt frame_opt", + /* 317 */ "window ::= nm PARTITION BY nexprlist orderby_opt frame_opt", + /* 318 */ "window ::= ORDER BY sortlist frame_opt", + /* 319 */ "window ::= nm ORDER BY sortlist frame_opt", + /* 320 */ "window ::= nm frame_opt", + /* 321 */ "frame_opt ::=", + /* 322 */ "frame_opt ::= range_or_rows frame_bound_s frame_exclude_opt", + /* 323 */ "frame_opt ::= range_or_rows BETWEEN frame_bound_s AND frame_bound_e frame_exclude_opt", + /* 324 */ "range_or_rows ::= RANGE|ROWS|GROUPS", + /* 325 */ "frame_bound_s ::= frame_bound", + /* 326 */ "frame_bound_s ::= UNBOUNDED PRECEDING", + /* 327 */ "frame_bound_e ::= frame_bound", + /* 328 */ "frame_bound_e ::= UNBOUNDED FOLLOWING", + /* 329 */ "frame_bound ::= expr PRECEDING|FOLLOWING", + /* 330 */ "frame_bound ::= CURRENT ROW", + /* 331 */ "frame_exclude_opt ::=", + /* 332 */ "frame_exclude_opt ::= EXCLUDE frame_exclude", + /* 333 */ "frame_exclude ::= NO OTHERS", + /* 334 */ "frame_exclude ::= CURRENT ROW", + /* 335 */ "frame_exclude ::= GROUP|TIES", + /* 336 */ "window_clause ::= WINDOW windowdefn_list", + /* 337 */ "filter_over ::= filter_clause over_clause", + /* 338 */ "filter_over ::= over_clause", + /* 339 */ "filter_over ::= filter_clause", + /* 340 */ "over_clause ::= OVER LP window RP", + /* 341 */ "over_clause ::= OVER nm", + /* 342 */ "filter_clause ::= FILTER LP WHERE expr RP", + /* 343 */ "term ::= QNUMBER", + /* 344 */ "input ::= cmdlist", + /* 345 */ "cmdlist ::= cmdlist ecmd", + /* 346 */ "cmdlist ::= ecmd", + /* 347 */ "ecmd ::= SEMI", + /* 348 */ "ecmd ::= cmdx SEMI", + /* 349 */ "ecmd ::= explain cmdx SEMI", + /* 350 */ "trans_opt ::=", + /* 351 */ "trans_opt ::= TRANSACTION", + /* 352 */ "trans_opt ::= TRANSACTION nm", + /* 353 */ "savepoint_opt ::= SAVEPOINT", + /* 354 */ "savepoint_opt ::=", + /* 355 */ "cmd ::= create_table create_table_args", + /* 356 */ "table_option_set ::= table_option", + /* 357 */ "columnlist ::= columnlist COMMA columnname carglist", + /* 358 */ "columnlist ::= columnname carglist", + /* 359 */ "nm ::= ID|INDEXED|JOIN_KW", + /* 360 */ "nm ::= STRING", + /* 361 */ "typetoken ::= typename", + /* 362 */ "typename ::= ID|STRING", + /* 363 */ "signed ::= plus_num", + /* 364 */ "signed ::= minus_num", + /* 365 */ "carglist ::= carglist ccons", + /* 366 */ "carglist ::=", + /* 367 */ "ccons ::= NULL onconf", + /* 368 */ "ccons ::= GENERATED ALWAYS AS generated", + /* 369 */ "ccons ::= AS generated", + /* 370 */ "conslist_opt ::= COMMA conslist", + /* 371 */ "conslist ::= conslist tconscomma tcons", + /* 372 */ "conslist ::= tcons", + /* 373 */ "tconscomma ::=", + /* 374 */ "defer_subclause_opt ::= defer_subclause", + /* 375 */ "resolvetype ::= raisetype", + /* 376 */ "selectnowith ::= oneselect", + /* 377 */ "oneselect ::= values", + /* 378 */ "sclp ::= selcollist COMMA", + /* 379 */ "as ::= ID|STRING", + /* 380 */ "indexed_opt ::= indexed_by", + /* 381 */ "returning ::=", + /* 382 */ "expr ::= term", + /* 383 */ "likeop ::= LIKE_KW|MATCH", + /* 384 */ "case_operand ::= expr", + /* 385 */ "exprlist ::= nexprlist", + /* 386 */ "nmnum ::= plus_num", + /* 387 */ "nmnum ::= nm", + /* 388 */ "nmnum ::= ON", + /* 389 */ "nmnum ::= DELETE", + /* 390 */ "nmnum ::= DEFAULT", + /* 391 */ "plus_num ::= INTEGER|FLOAT", + /* 392 */ "foreach_clause ::=", + /* 393 */ "foreach_clause ::= FOR EACH ROW", + /* 394 */ "trnm ::= nm", + /* 395 */ "tridxby ::=", + /* 396 */ "database_kw_opt ::= DATABASE", + /* 397 */ "database_kw_opt ::=", + /* 398 */ "kwcolumn_opt ::=", + /* 399 */ "kwcolumn_opt ::= COLUMNKW", + /* 400 */ "vtabarglist ::= vtabarg", + /* 401 */ "vtabarglist ::= vtabarglist COMMA vtabarg", + /* 402 */ "vtabarg ::= vtabarg vtabargtoken", + /* 403 */ "anylist ::=", + /* 404 */ "anylist ::= anylist LP anylist RP", + /* 405 */ "anylist ::= anylist ANY", + /* 406 */ "with ::=", + /* 407 */ "windowdefn_list ::= windowdefn", + /* 408 */ "window ::= frame_opt", }; #endif /* NDEBUG */ -#if YYSTACKDEPTH<=0 +#if YYGROWABLESTACK /* ** Try to increase the size of the parser stack. Return the number ** of errors. Return 0 on success. */ static int yyGrowStack(yyParser *p){ + int oldSize = 1 + (int)(p->yystackEnd - p->yystack); int newSize; int idx; yyStackEntry *pNew; - newSize = p->yystksz*2 + 100; - idx = p->yytos ? (int)(p->yytos - p->yystack) : 0; - if( p->yystack==&p->yystk0 ){ - pNew = malloc(newSize*sizeof(pNew[0])); - if( pNew ) pNew[0] = p->yystk0; + newSize = oldSize*2 + 100; + idx = (int)(p->yytos - p->yystack); + if( p->yystack==p->yystk0 ){ + pNew = YYREALLOC(0, newSize*sizeof(pNew[0])); + if( pNew==0 ) return 1; + memcpy(pNew, p->yystack, oldSize*sizeof(pNew[0])); }else{ - pNew = realloc(p->yystack, newSize*sizeof(pNew[0])); + pNew = YYREALLOC(p->yystack, newSize*sizeof(pNew[0])); + if( pNew==0 ) return 1; } - if( pNew ){ - p->yystack = pNew; - p->yytos = &p->yystack[idx]; + p->yystack = pNew; + p->yytos = &p->yystack[idx]; #ifndef NDEBUG - if( yyTraceFILE ){ - fprintf(yyTraceFILE,"%sStack grows from %d to %d entries.\n", - yyTracePrompt, p->yystksz, newSize); - } -#endif - p->yystksz = newSize; + if( yyTraceFILE ){ + fprintf(yyTraceFILE,"%sStack grows from %d to %d entries.\n", + yyTracePrompt, oldSize, newSize); } - return pNew==0; +#endif + p->yystackEnd = &p->yystack[newSize-1]; + return 0; } +#endif /* YYGROWABLESTACK */ + +#if !YYGROWABLESTACK +/* For builds that do no have a growable stack, yyGrowStack always +** returns an error. +*/ +# define yyGrowStack(X) 1 #endif /* Datatype of the argument to the memory allocated passed as the @@ -173297,24 +179844,14 @@ SQLITE_PRIVATE void sqlite3ParserInit(void *yypRawParser sqlite3ParserCTX_PDECL) #ifdef YYTRACKMAXSTACKDEPTH yypParser->yyhwm = 0; #endif -#if YYSTACKDEPTH<=0 - yypParser->yytos = NULL; - yypParser->yystack = NULL; - yypParser->yystksz = 0; - if( yyGrowStack(yypParser) ){ - yypParser->yystack = &yypParser->yystk0; - yypParser->yystksz = 1; - } -#endif + yypParser->yystack = yypParser->yystk0; + yypParser->yystackEnd = &yypParser->yystack[YYSTACKDEPTH-1]; #ifndef YYNOERRORRECOVERY yypParser->yyerrcnt = -1; #endif yypParser->yytos = yypParser->yystack; yypParser->yystack[0].stateno = 0; yypParser->yystack[0].major = 0; -#if YYSTACKDEPTH>0 - yypParser->yystackEnd = &yypParser->yystack[YYSTACKDEPTH-1]; -#endif } #ifndef sqlite3Parser_ENGINEALWAYSONSTACK @@ -173368,97 +179905,98 @@ static void yy_destructor( ** inside the C code. */ /********* Begin destructor definitions ***************************************/ - case 204: /* select */ - case 239: /* selectnowith */ - case 240: /* oneselect */ - case 252: /* values */ + case 206: /* select */ + case 241: /* selectnowith */ + case 242: /* oneselect */ + case 254: /* values */ + case 256: /* mvalues */ { -sqlite3SelectDelete(pParse->db, (yypminor->yy47)); -} - break; - case 216: /* term */ - case 217: /* expr */ - case 246: /* where_opt */ - case 248: /* having_opt */ - case 267: /* where_opt_ret */ - case 278: /* case_operand */ - case 280: /* case_else */ - case 283: /* vinto */ - case 290: /* when_clause */ - case 295: /* key_opt */ - case 311: /* filter_clause */ +sqlite3SelectDelete(pParse->db, (yypminor->yy637)); +} + break; + case 218: /* term */ + case 219: /* expr */ + case 248: /* where_opt */ + case 250: /* having_opt */ + case 270: /* where_opt_ret */ + case 281: /* case_operand */ + case 283: /* case_else */ + case 286: /* vinto */ + case 293: /* when_clause */ + case 298: /* key_opt */ + case 315: /* filter_clause */ { -sqlite3ExprDelete(pParse->db, (yypminor->yy528)); -} - break; - case 221: /* eidlist_opt */ - case 231: /* sortlist */ - case 232: /* eidlist */ - case 244: /* selcollist */ - case 247: /* groupby_opt */ - case 249: /* orderby_opt */ - case 253: /* nexprlist */ - case 254: /* sclp */ - case 261: /* exprlist */ - case 268: /* setlist */ - case 277: /* paren_exprlist */ - case 279: /* case_exprlist */ - case 310: /* part_opt */ +sqlite3ExprDelete(pParse->db, (yypminor->yy590)); +} + break; + case 223: /* eidlist_opt */ + case 233: /* sortlist */ + case 234: /* eidlist */ + case 246: /* selcollist */ + case 249: /* groupby_opt */ + case 251: /* orderby_opt */ + case 255: /* nexprlist */ + case 257: /* sclp */ + case 264: /* exprlist */ + case 271: /* setlist */ + case 280: /* paren_exprlist */ + case 282: /* case_exprlist */ + case 314: /* part_opt */ { -sqlite3ExprListDelete(pParse->db, (yypminor->yy322)); +sqlite3ExprListDelete(pParse->db, (yypminor->yy402)); } break; - case 238: /* fullname */ - case 245: /* from */ - case 256: /* seltablist */ - case 257: /* stl_prefix */ - case 262: /* xfullname */ + case 240: /* fullname */ + case 247: /* from */ + case 259: /* seltablist */ + case 260: /* stl_prefix */ + case 265: /* xfullname */ { -sqlite3SrcListDelete(pParse->db, (yypminor->yy131)); +sqlite3SrcListDelete(pParse->db, (yypminor->yy563)); } break; - case 241: /* wqlist */ + case 243: /* wqlist */ { -sqlite3WithDelete(pParse->db, (yypminor->yy521)); +sqlite3WithDelete(pParse->db, (yypminor->yy125)); } break; - case 251: /* window_clause */ - case 306: /* windowdefn_list */ + case 253: /* window_clause */ + case 310: /* windowdefn_list */ { -sqlite3WindowListDelete(pParse->db, (yypminor->yy41)); +sqlite3WindowListDelete(pParse->db, (yypminor->yy483)); } break; - case 263: /* idlist */ - case 270: /* idlist_opt */ + case 266: /* idlist */ + case 273: /* idlist_opt */ { -sqlite3IdListDelete(pParse->db, (yypminor->yy254)); +sqlite3IdListDelete(pParse->db, (yypminor->yy204)); } break; - case 273: /* filter_over */ - case 307: /* windowdefn */ - case 308: /* window */ - case 309: /* frame_opt */ - case 312: /* over_clause */ + case 276: /* filter_over */ + case 311: /* windowdefn */ + case 312: /* window */ + case 313: /* frame_opt */ + case 316: /* over_clause */ { -sqlite3WindowDelete(pParse->db, (yypminor->yy41)); +sqlite3WindowDelete(pParse->db, (yypminor->yy483)); } break; - case 286: /* trigger_cmd_list */ - case 291: /* trigger_cmd */ + case 289: /* trigger_cmd_list */ + case 294: /* trigger_cmd */ { -sqlite3DeleteTriggerStep(pParse->db, (yypminor->yy33)); +sqlite3DeleteTriggerStep(pParse->db, (yypminor->yy319)); } break; - case 288: /* trigger_event */ + case 291: /* trigger_event */ { -sqlite3IdListDelete(pParse->db, (yypminor->yy180).b); +sqlite3IdListDelete(pParse->db, (yypminor->yy28).b); } break; - case 314: /* frame_bound */ - case 315: /* frame_bound_s */ - case 316: /* frame_bound_e */ + case 318: /* frame_bound */ + case 319: /* frame_bound_s */ + case 320: /* frame_bound_e */ { -sqlite3ExprDelete(pParse->db, (yypminor->yy595).pExpr); +sqlite3ExprDelete(pParse->db, (yypminor->yy205).pExpr); } break; /********* End destructor definitions *****************************************/ @@ -173492,9 +180030,26 @@ static void yy_pop_parser_stack(yyParser *pParser){ */ SQLITE_PRIVATE void sqlite3ParserFinalize(void *p){ yyParser *pParser = (yyParser*)p; - while( pParser->yytos>pParser->yystack ) yy_pop_parser_stack(pParser); -#if YYSTACKDEPTH<=0 - if( pParser->yystack!=&pParser->yystk0 ) free(pParser->yystack); + + /* In-lined version of calling yy_pop_parser_stack() for each + ** element left in the stack */ + yyStackEntry *yytos = pParser->yytos; + while( yytos>pParser->yystack ){ +#ifndef NDEBUG + if( yyTraceFILE ){ + fprintf(yyTraceFILE,"%sPopping %s\n", + yyTracePrompt, + yyTokenName[yytos->major]); + } +#endif + if( yytos->major>=YY_MIN_DSTRCTR ){ + yy_destructor(pParser, yytos->major, &yytos->minor); + } + yytos--; + } + +#if YYGROWABLESTACK + if( pParser->yystack!=pParser->yystk0 ) YYFREE(pParser->yystack); #endif } @@ -173677,7 +180232,7 @@ static void yyStackOverflow(yyParser *yypParser){ ** stack every overflows */ /******** Begin %stack_overflow code ******************************************/ - sqlite3ErrorMsg(pParse, "parser stack overflow"); + sqlite3OomFault(pParse->db); /******** End %stack_overflow code ********************************************/ sqlite3ParserARG_STORE /* Suppress warning about unused %extra_argument var */ sqlite3ParserCTX_STORE @@ -173721,25 +180276,19 @@ static void yy_shift( assert( yypParser->yyhwm == (int)(yypParser->yytos - yypParser->yystack) ); } #endif -#if YYSTACKDEPTH>0 - if( yypParser->yytos>yypParser->yystackEnd ){ - yypParser->yytos--; - yyStackOverflow(yypParser); - return; - } -#else - if( yypParser->yytos>=&yypParser->yystack[yypParser->yystksz] ){ + yytos = yypParser->yytos; + if( yytos>yypParser->yystackEnd ){ if( yyGrowStack(yypParser) ){ yypParser->yytos--; yyStackOverflow(yypParser); return; } + yytos = yypParser->yytos; + assert( yytos <= yypParser->yystackEnd ); } -#endif if( yyNewState > YY_MAX_SHIFT ){ yyNewState += YY_MIN_REDUCE - YY_MIN_SHIFTREDUCE; } - yytos = yypParser->yytos; yytos->stateno = yyNewState; yytos->major = yyMajor; yytos->minor.yy0 = yyMinor; @@ -173749,411 +180298,415 @@ static void yy_shift( /* For rule J, yyRuleInfoLhs[J] contains the symbol on the left-hand side ** of that rule */ static const YYCODETYPE yyRuleInfoLhs[] = { - 189, /* (0) explain ::= EXPLAIN */ - 189, /* (1) explain ::= EXPLAIN QUERY PLAN */ - 188, /* (2) cmdx ::= cmd */ - 190, /* (3) cmd ::= BEGIN transtype trans_opt */ - 191, /* (4) transtype ::= */ - 191, /* (5) transtype ::= DEFERRED */ - 191, /* (6) transtype ::= IMMEDIATE */ - 191, /* (7) transtype ::= EXCLUSIVE */ - 190, /* (8) cmd ::= COMMIT|END trans_opt */ - 190, /* (9) cmd ::= ROLLBACK trans_opt */ - 190, /* (10) cmd ::= SAVEPOINT nm */ - 190, /* (11) cmd ::= RELEASE savepoint_opt nm */ - 190, /* (12) cmd ::= ROLLBACK trans_opt TO savepoint_opt nm */ - 195, /* (13) create_table ::= createkw temp TABLE ifnotexists nm dbnm */ - 197, /* (14) createkw ::= CREATE */ - 199, /* (15) ifnotexists ::= */ - 199, /* (16) ifnotexists ::= IF NOT EXISTS */ - 198, /* (17) temp ::= TEMP */ - 198, /* (18) temp ::= */ - 196, /* (19) create_table_args ::= LP columnlist conslist_opt RP table_option_set */ - 196, /* (20) create_table_args ::= AS select */ - 203, /* (21) table_option_set ::= */ - 203, /* (22) table_option_set ::= table_option_set COMMA table_option */ - 205, /* (23) table_option ::= WITHOUT nm */ - 205, /* (24) table_option ::= nm */ - 206, /* (25) columnname ::= nm typetoken */ - 208, /* (26) typetoken ::= */ - 208, /* (27) typetoken ::= typename LP signed RP */ - 208, /* (28) typetoken ::= typename LP signed COMMA signed RP */ - 209, /* (29) typename ::= typename ID|STRING */ - 213, /* (30) scanpt ::= */ - 214, /* (31) scantok ::= */ - 215, /* (32) ccons ::= CONSTRAINT nm */ - 215, /* (33) ccons ::= DEFAULT scantok term */ - 215, /* (34) ccons ::= DEFAULT LP expr RP */ - 215, /* (35) ccons ::= DEFAULT PLUS scantok term */ - 215, /* (36) ccons ::= DEFAULT MINUS scantok term */ - 215, /* (37) ccons ::= DEFAULT scantok ID|INDEXED */ - 215, /* (38) ccons ::= NOT NULL onconf */ - 215, /* (39) ccons ::= PRIMARY KEY sortorder onconf autoinc */ - 215, /* (40) ccons ::= UNIQUE onconf */ - 215, /* (41) ccons ::= CHECK LP expr RP */ - 215, /* (42) ccons ::= REFERENCES nm eidlist_opt refargs */ - 215, /* (43) ccons ::= defer_subclause */ - 215, /* (44) ccons ::= COLLATE ID|STRING */ - 224, /* (45) generated ::= LP expr RP */ - 224, /* (46) generated ::= LP expr RP ID */ - 220, /* (47) autoinc ::= */ - 220, /* (48) autoinc ::= AUTOINCR */ - 222, /* (49) refargs ::= */ - 222, /* (50) refargs ::= refargs refarg */ - 225, /* (51) refarg ::= MATCH nm */ - 225, /* (52) refarg ::= ON INSERT refact */ - 225, /* (53) refarg ::= ON DELETE refact */ - 225, /* (54) refarg ::= ON UPDATE refact */ - 226, /* (55) refact ::= SET NULL */ - 226, /* (56) refact ::= SET DEFAULT */ - 226, /* (57) refact ::= CASCADE */ - 226, /* (58) refact ::= RESTRICT */ - 226, /* (59) refact ::= NO ACTION */ - 223, /* (60) defer_subclause ::= NOT DEFERRABLE init_deferred_pred_opt */ - 223, /* (61) defer_subclause ::= DEFERRABLE init_deferred_pred_opt */ - 227, /* (62) init_deferred_pred_opt ::= */ - 227, /* (63) init_deferred_pred_opt ::= INITIALLY DEFERRED */ - 227, /* (64) init_deferred_pred_opt ::= INITIALLY IMMEDIATE */ - 202, /* (65) conslist_opt ::= */ - 229, /* (66) tconscomma ::= COMMA */ - 230, /* (67) tcons ::= CONSTRAINT nm */ - 230, /* (68) tcons ::= PRIMARY KEY LP sortlist autoinc RP onconf */ - 230, /* (69) tcons ::= UNIQUE LP sortlist RP onconf */ - 230, /* (70) tcons ::= CHECK LP expr RP onconf */ - 230, /* (71) tcons ::= FOREIGN KEY LP eidlist RP REFERENCES nm eidlist_opt refargs defer_subclause_opt */ - 233, /* (72) defer_subclause_opt ::= */ - 218, /* (73) onconf ::= */ - 218, /* (74) onconf ::= ON CONFLICT resolvetype */ - 234, /* (75) orconf ::= */ - 234, /* (76) orconf ::= OR resolvetype */ - 235, /* (77) resolvetype ::= IGNORE */ - 235, /* (78) resolvetype ::= REPLACE */ - 190, /* (79) cmd ::= DROP TABLE ifexists fullname */ - 237, /* (80) ifexists ::= IF EXISTS */ - 237, /* (81) ifexists ::= */ - 190, /* (82) cmd ::= createkw temp VIEW ifnotexists nm dbnm eidlist_opt AS select */ - 190, /* (83) cmd ::= DROP VIEW ifexists fullname */ - 190, /* (84) cmd ::= select */ - 204, /* (85) select ::= WITH wqlist selectnowith */ - 204, /* (86) select ::= WITH RECURSIVE wqlist selectnowith */ - 204, /* (87) select ::= selectnowith */ - 239, /* (88) selectnowith ::= selectnowith multiselect_op oneselect */ - 242, /* (89) multiselect_op ::= UNION */ - 242, /* (90) multiselect_op ::= UNION ALL */ - 242, /* (91) multiselect_op ::= EXCEPT|INTERSECT */ - 240, /* (92) oneselect ::= SELECT distinct selcollist from where_opt groupby_opt having_opt orderby_opt limit_opt */ - 240, /* (93) oneselect ::= SELECT distinct selcollist from where_opt groupby_opt having_opt window_clause orderby_opt limit_opt */ - 252, /* (94) values ::= VALUES LP nexprlist RP */ - 252, /* (95) values ::= values COMMA LP nexprlist RP */ - 243, /* (96) distinct ::= DISTINCT */ - 243, /* (97) distinct ::= ALL */ - 243, /* (98) distinct ::= */ - 254, /* (99) sclp ::= */ - 244, /* (100) selcollist ::= sclp scanpt expr scanpt as */ - 244, /* (101) selcollist ::= sclp scanpt STAR */ - 244, /* (102) selcollist ::= sclp scanpt nm DOT STAR */ - 255, /* (103) as ::= AS nm */ - 255, /* (104) as ::= */ - 245, /* (105) from ::= */ - 245, /* (106) from ::= FROM seltablist */ - 257, /* (107) stl_prefix ::= seltablist joinop */ - 257, /* (108) stl_prefix ::= */ - 256, /* (109) seltablist ::= stl_prefix nm dbnm as on_using */ - 256, /* (110) seltablist ::= stl_prefix nm dbnm as indexed_by on_using */ - 256, /* (111) seltablist ::= stl_prefix nm dbnm LP exprlist RP as on_using */ - 256, /* (112) seltablist ::= stl_prefix LP select RP as on_using */ - 256, /* (113) seltablist ::= stl_prefix LP seltablist RP as on_using */ - 200, /* (114) dbnm ::= */ - 200, /* (115) dbnm ::= DOT nm */ - 238, /* (116) fullname ::= nm */ - 238, /* (117) fullname ::= nm DOT nm */ - 262, /* (118) xfullname ::= nm */ - 262, /* (119) xfullname ::= nm DOT nm */ - 262, /* (120) xfullname ::= nm DOT nm AS nm */ - 262, /* (121) xfullname ::= nm AS nm */ - 258, /* (122) joinop ::= COMMA|JOIN */ - 258, /* (123) joinop ::= JOIN_KW JOIN */ - 258, /* (124) joinop ::= JOIN_KW nm JOIN */ - 258, /* (125) joinop ::= JOIN_KW nm nm JOIN */ - 259, /* (126) on_using ::= ON expr */ - 259, /* (127) on_using ::= USING LP idlist RP */ - 259, /* (128) on_using ::= */ - 264, /* (129) indexed_opt ::= */ - 260, /* (130) indexed_by ::= INDEXED BY nm */ - 260, /* (131) indexed_by ::= NOT INDEXED */ - 249, /* (132) orderby_opt ::= */ - 249, /* (133) orderby_opt ::= ORDER BY sortlist */ - 231, /* (134) sortlist ::= sortlist COMMA expr sortorder nulls */ - 231, /* (135) sortlist ::= expr sortorder nulls */ - 219, /* (136) sortorder ::= ASC */ - 219, /* (137) sortorder ::= DESC */ - 219, /* (138) sortorder ::= */ - 265, /* (139) nulls ::= NULLS FIRST */ - 265, /* (140) nulls ::= NULLS LAST */ - 265, /* (141) nulls ::= */ - 247, /* (142) groupby_opt ::= */ - 247, /* (143) groupby_opt ::= GROUP BY nexprlist */ - 248, /* (144) having_opt ::= */ - 248, /* (145) having_opt ::= HAVING expr */ - 250, /* (146) limit_opt ::= */ - 250, /* (147) limit_opt ::= LIMIT expr */ - 250, /* (148) limit_opt ::= LIMIT expr OFFSET expr */ - 250, /* (149) limit_opt ::= LIMIT expr COMMA expr */ - 190, /* (150) cmd ::= with DELETE FROM xfullname indexed_opt where_opt_ret */ - 246, /* (151) where_opt ::= */ - 246, /* (152) where_opt ::= WHERE expr */ - 267, /* (153) where_opt_ret ::= */ - 267, /* (154) where_opt_ret ::= WHERE expr */ - 267, /* (155) where_opt_ret ::= RETURNING selcollist */ - 267, /* (156) where_opt_ret ::= WHERE expr RETURNING selcollist */ - 190, /* (157) cmd ::= with UPDATE orconf xfullname indexed_opt SET setlist from where_opt_ret */ - 268, /* (158) setlist ::= setlist COMMA nm EQ expr */ - 268, /* (159) setlist ::= setlist COMMA LP idlist RP EQ expr */ - 268, /* (160) setlist ::= nm EQ expr */ - 268, /* (161) setlist ::= LP idlist RP EQ expr */ - 190, /* (162) cmd ::= with insert_cmd INTO xfullname idlist_opt select upsert */ - 190, /* (163) cmd ::= with insert_cmd INTO xfullname idlist_opt DEFAULT VALUES returning */ - 271, /* (164) upsert ::= */ - 271, /* (165) upsert ::= RETURNING selcollist */ - 271, /* (166) upsert ::= ON CONFLICT LP sortlist RP where_opt DO UPDATE SET setlist where_opt upsert */ - 271, /* (167) upsert ::= ON CONFLICT LP sortlist RP where_opt DO NOTHING upsert */ - 271, /* (168) upsert ::= ON CONFLICT DO NOTHING returning */ - 271, /* (169) upsert ::= ON CONFLICT DO UPDATE SET setlist where_opt returning */ - 272, /* (170) returning ::= RETURNING selcollist */ - 269, /* (171) insert_cmd ::= INSERT orconf */ - 269, /* (172) insert_cmd ::= REPLACE */ - 270, /* (173) idlist_opt ::= */ - 270, /* (174) idlist_opt ::= LP idlist RP */ - 263, /* (175) idlist ::= idlist COMMA nm */ - 263, /* (176) idlist ::= nm */ - 217, /* (177) expr ::= LP expr RP */ - 217, /* (178) expr ::= ID|INDEXED|JOIN_KW */ - 217, /* (179) expr ::= nm DOT nm */ - 217, /* (180) expr ::= nm DOT nm DOT nm */ - 216, /* (181) term ::= NULL|FLOAT|BLOB */ - 216, /* (182) term ::= STRING */ - 216, /* (183) term ::= INTEGER */ - 217, /* (184) expr ::= VARIABLE */ - 217, /* (185) expr ::= expr COLLATE ID|STRING */ - 217, /* (186) expr ::= CAST LP expr AS typetoken RP */ - 217, /* (187) expr ::= ID|INDEXED|JOIN_KW LP distinct exprlist RP */ - 217, /* (188) expr ::= ID|INDEXED|JOIN_KW LP distinct exprlist ORDER BY sortlist RP */ - 217, /* (189) expr ::= ID|INDEXED|JOIN_KW LP STAR RP */ - 217, /* (190) expr ::= ID|INDEXED|JOIN_KW LP distinct exprlist RP filter_over */ - 217, /* (191) expr ::= ID|INDEXED|JOIN_KW LP distinct exprlist ORDER BY sortlist RP filter_over */ - 217, /* (192) expr ::= ID|INDEXED|JOIN_KW LP STAR RP filter_over */ - 216, /* (193) term ::= CTIME_KW */ - 217, /* (194) expr ::= LP nexprlist COMMA expr RP */ - 217, /* (195) expr ::= expr AND expr */ - 217, /* (196) expr ::= expr OR expr */ - 217, /* (197) expr ::= expr LT|GT|GE|LE expr */ - 217, /* (198) expr ::= expr EQ|NE expr */ - 217, /* (199) expr ::= expr BITAND|BITOR|LSHIFT|RSHIFT expr */ - 217, /* (200) expr ::= expr PLUS|MINUS expr */ - 217, /* (201) expr ::= expr STAR|SLASH|REM expr */ - 217, /* (202) expr ::= expr CONCAT expr */ - 274, /* (203) likeop ::= NOT LIKE_KW|MATCH */ - 217, /* (204) expr ::= expr likeop expr */ - 217, /* (205) expr ::= expr likeop expr ESCAPE expr */ - 217, /* (206) expr ::= expr ISNULL|NOTNULL */ - 217, /* (207) expr ::= expr NOT NULL */ - 217, /* (208) expr ::= expr IS expr */ - 217, /* (209) expr ::= expr IS NOT expr */ - 217, /* (210) expr ::= expr IS NOT DISTINCT FROM expr */ - 217, /* (211) expr ::= expr IS DISTINCT FROM expr */ - 217, /* (212) expr ::= NOT expr */ - 217, /* (213) expr ::= BITNOT expr */ - 217, /* (214) expr ::= PLUS|MINUS expr */ - 217, /* (215) expr ::= expr PTR expr */ - 275, /* (216) between_op ::= BETWEEN */ - 275, /* (217) between_op ::= NOT BETWEEN */ - 217, /* (218) expr ::= expr between_op expr AND expr */ - 276, /* (219) in_op ::= IN */ - 276, /* (220) in_op ::= NOT IN */ - 217, /* (221) expr ::= expr in_op LP exprlist RP */ - 217, /* (222) expr ::= LP select RP */ - 217, /* (223) expr ::= expr in_op LP select RP */ - 217, /* (224) expr ::= expr in_op nm dbnm paren_exprlist */ - 217, /* (225) expr ::= EXISTS LP select RP */ - 217, /* (226) expr ::= CASE case_operand case_exprlist case_else END */ - 279, /* (227) case_exprlist ::= case_exprlist WHEN expr THEN expr */ - 279, /* (228) case_exprlist ::= WHEN expr THEN expr */ - 280, /* (229) case_else ::= ELSE expr */ - 280, /* (230) case_else ::= */ - 278, /* (231) case_operand ::= */ - 261, /* (232) exprlist ::= */ - 253, /* (233) nexprlist ::= nexprlist COMMA expr */ - 253, /* (234) nexprlist ::= expr */ - 277, /* (235) paren_exprlist ::= */ - 277, /* (236) paren_exprlist ::= LP exprlist RP */ - 190, /* (237) cmd ::= createkw uniqueflag INDEX ifnotexists nm dbnm ON nm LP sortlist RP where_opt */ - 281, /* (238) uniqueflag ::= UNIQUE */ - 281, /* (239) uniqueflag ::= */ - 221, /* (240) eidlist_opt ::= */ - 221, /* (241) eidlist_opt ::= LP eidlist RP */ - 232, /* (242) eidlist ::= eidlist COMMA nm collate sortorder */ - 232, /* (243) eidlist ::= nm collate sortorder */ - 282, /* (244) collate ::= */ - 282, /* (245) collate ::= COLLATE ID|STRING */ - 190, /* (246) cmd ::= DROP INDEX ifexists fullname */ - 190, /* (247) cmd ::= VACUUM vinto */ - 190, /* (248) cmd ::= VACUUM nm vinto */ - 283, /* (249) vinto ::= INTO expr */ - 283, /* (250) vinto ::= */ - 190, /* (251) cmd ::= PRAGMA nm dbnm */ - 190, /* (252) cmd ::= PRAGMA nm dbnm EQ nmnum */ - 190, /* (253) cmd ::= PRAGMA nm dbnm LP nmnum RP */ - 190, /* (254) cmd ::= PRAGMA nm dbnm EQ minus_num */ - 190, /* (255) cmd ::= PRAGMA nm dbnm LP minus_num RP */ - 211, /* (256) plus_num ::= PLUS INTEGER|FLOAT */ - 212, /* (257) minus_num ::= MINUS INTEGER|FLOAT */ - 190, /* (258) cmd ::= createkw trigger_decl BEGIN trigger_cmd_list END */ - 285, /* (259) trigger_decl ::= temp TRIGGER ifnotexists nm dbnm trigger_time trigger_event ON fullname foreach_clause when_clause */ - 287, /* (260) trigger_time ::= BEFORE|AFTER */ - 287, /* (261) trigger_time ::= INSTEAD OF */ - 287, /* (262) trigger_time ::= */ - 288, /* (263) trigger_event ::= DELETE|INSERT */ - 288, /* (264) trigger_event ::= UPDATE */ - 288, /* (265) trigger_event ::= UPDATE OF idlist */ - 290, /* (266) when_clause ::= */ - 290, /* (267) when_clause ::= WHEN expr */ - 286, /* (268) trigger_cmd_list ::= trigger_cmd_list trigger_cmd SEMI */ - 286, /* (269) trigger_cmd_list ::= trigger_cmd SEMI */ - 292, /* (270) trnm ::= nm DOT nm */ - 293, /* (271) tridxby ::= INDEXED BY nm */ - 293, /* (272) tridxby ::= NOT INDEXED */ - 291, /* (273) trigger_cmd ::= UPDATE orconf trnm tridxby SET setlist from where_opt scanpt */ - 291, /* (274) trigger_cmd ::= scanpt insert_cmd INTO trnm idlist_opt select upsert scanpt */ - 291, /* (275) trigger_cmd ::= DELETE FROM trnm tridxby where_opt scanpt */ - 291, /* (276) trigger_cmd ::= scanpt select scanpt */ - 217, /* (277) expr ::= RAISE LP IGNORE RP */ - 217, /* (278) expr ::= RAISE LP raisetype COMMA nm RP */ - 236, /* (279) raisetype ::= ROLLBACK */ - 236, /* (280) raisetype ::= ABORT */ - 236, /* (281) raisetype ::= FAIL */ - 190, /* (282) cmd ::= DROP TRIGGER ifexists fullname */ - 190, /* (283) cmd ::= ATTACH database_kw_opt expr AS expr key_opt */ - 190, /* (284) cmd ::= DETACH database_kw_opt expr */ - 295, /* (285) key_opt ::= */ - 295, /* (286) key_opt ::= KEY expr */ - 190, /* (287) cmd ::= REINDEX */ - 190, /* (288) cmd ::= REINDEX nm dbnm */ - 190, /* (289) cmd ::= ANALYZE */ - 190, /* (290) cmd ::= ANALYZE nm dbnm */ - 190, /* (291) cmd ::= ALTER TABLE fullname RENAME TO nm */ - 190, /* (292) cmd ::= ALTER TABLE add_column_fullname ADD kwcolumn_opt columnname carglist */ - 190, /* (293) cmd ::= ALTER TABLE fullname DROP kwcolumn_opt nm */ - 296, /* (294) add_column_fullname ::= fullname */ - 190, /* (295) cmd ::= ALTER TABLE fullname RENAME kwcolumn_opt nm TO nm */ - 190, /* (296) cmd ::= create_vtab */ - 190, /* (297) cmd ::= create_vtab LP vtabarglist RP */ - 298, /* (298) create_vtab ::= createkw VIRTUAL TABLE ifnotexists nm dbnm USING nm */ - 300, /* (299) vtabarg ::= */ - 301, /* (300) vtabargtoken ::= ANY */ - 301, /* (301) vtabargtoken ::= lp anylist RP */ - 302, /* (302) lp ::= LP */ - 266, /* (303) with ::= WITH wqlist */ - 266, /* (304) with ::= WITH RECURSIVE wqlist */ - 305, /* (305) wqas ::= AS */ - 305, /* (306) wqas ::= AS MATERIALIZED */ - 305, /* (307) wqas ::= AS NOT MATERIALIZED */ - 304, /* (308) wqitem ::= nm eidlist_opt wqas LP select RP */ - 241, /* (309) wqlist ::= wqitem */ - 241, /* (310) wqlist ::= wqlist COMMA wqitem */ - 306, /* (311) windowdefn_list ::= windowdefn_list COMMA windowdefn */ - 307, /* (312) windowdefn ::= nm AS LP window RP */ - 308, /* (313) window ::= PARTITION BY nexprlist orderby_opt frame_opt */ - 308, /* (314) window ::= nm PARTITION BY nexprlist orderby_opt frame_opt */ - 308, /* (315) window ::= ORDER BY sortlist frame_opt */ - 308, /* (316) window ::= nm ORDER BY sortlist frame_opt */ - 308, /* (317) window ::= nm frame_opt */ - 309, /* (318) frame_opt ::= */ - 309, /* (319) frame_opt ::= range_or_rows frame_bound_s frame_exclude_opt */ - 309, /* (320) frame_opt ::= range_or_rows BETWEEN frame_bound_s AND frame_bound_e frame_exclude_opt */ - 313, /* (321) range_or_rows ::= RANGE|ROWS|GROUPS */ - 315, /* (322) frame_bound_s ::= frame_bound */ - 315, /* (323) frame_bound_s ::= UNBOUNDED PRECEDING */ - 316, /* (324) frame_bound_e ::= frame_bound */ - 316, /* (325) frame_bound_e ::= UNBOUNDED FOLLOWING */ - 314, /* (326) frame_bound ::= expr PRECEDING|FOLLOWING */ - 314, /* (327) frame_bound ::= CURRENT ROW */ - 317, /* (328) frame_exclude_opt ::= */ - 317, /* (329) frame_exclude_opt ::= EXCLUDE frame_exclude */ - 318, /* (330) frame_exclude ::= NO OTHERS */ - 318, /* (331) frame_exclude ::= CURRENT ROW */ - 318, /* (332) frame_exclude ::= GROUP|TIES */ - 251, /* (333) window_clause ::= WINDOW windowdefn_list */ - 273, /* (334) filter_over ::= filter_clause over_clause */ - 273, /* (335) filter_over ::= over_clause */ - 273, /* (336) filter_over ::= filter_clause */ - 312, /* (337) over_clause ::= OVER LP window RP */ - 312, /* (338) over_clause ::= OVER nm */ - 311, /* (339) filter_clause ::= FILTER LP WHERE expr RP */ - 185, /* (340) input ::= cmdlist */ - 186, /* (341) cmdlist ::= cmdlist ecmd */ - 186, /* (342) cmdlist ::= ecmd */ - 187, /* (343) ecmd ::= SEMI */ - 187, /* (344) ecmd ::= cmdx SEMI */ - 187, /* (345) ecmd ::= explain cmdx SEMI */ - 192, /* (346) trans_opt ::= */ - 192, /* (347) trans_opt ::= TRANSACTION */ - 192, /* (348) trans_opt ::= TRANSACTION nm */ - 194, /* (349) savepoint_opt ::= SAVEPOINT */ - 194, /* (350) savepoint_opt ::= */ - 190, /* (351) cmd ::= create_table create_table_args */ - 203, /* (352) table_option_set ::= table_option */ - 201, /* (353) columnlist ::= columnlist COMMA columnname carglist */ - 201, /* (354) columnlist ::= columnname carglist */ - 193, /* (355) nm ::= ID|INDEXED|JOIN_KW */ - 193, /* (356) nm ::= STRING */ - 208, /* (357) typetoken ::= typename */ - 209, /* (358) typename ::= ID|STRING */ - 210, /* (359) signed ::= plus_num */ - 210, /* (360) signed ::= minus_num */ - 207, /* (361) carglist ::= carglist ccons */ - 207, /* (362) carglist ::= */ - 215, /* (363) ccons ::= NULL onconf */ - 215, /* (364) ccons ::= GENERATED ALWAYS AS generated */ - 215, /* (365) ccons ::= AS generated */ - 202, /* (366) conslist_opt ::= COMMA conslist */ - 228, /* (367) conslist ::= conslist tconscomma tcons */ - 228, /* (368) conslist ::= tcons */ - 229, /* (369) tconscomma ::= */ - 233, /* (370) defer_subclause_opt ::= defer_subclause */ - 235, /* (371) resolvetype ::= raisetype */ - 239, /* (372) selectnowith ::= oneselect */ - 240, /* (373) oneselect ::= values */ - 254, /* (374) sclp ::= selcollist COMMA */ - 255, /* (375) as ::= ID|STRING */ - 264, /* (376) indexed_opt ::= indexed_by */ - 272, /* (377) returning ::= */ - 217, /* (378) expr ::= term */ - 274, /* (379) likeop ::= LIKE_KW|MATCH */ - 278, /* (380) case_operand ::= expr */ - 261, /* (381) exprlist ::= nexprlist */ - 284, /* (382) nmnum ::= plus_num */ - 284, /* (383) nmnum ::= nm */ - 284, /* (384) nmnum ::= ON */ - 284, /* (385) nmnum ::= DELETE */ - 284, /* (386) nmnum ::= DEFAULT */ - 211, /* (387) plus_num ::= INTEGER|FLOAT */ - 289, /* (388) foreach_clause ::= */ - 289, /* (389) foreach_clause ::= FOR EACH ROW */ - 292, /* (390) trnm ::= nm */ - 293, /* (391) tridxby ::= */ - 294, /* (392) database_kw_opt ::= DATABASE */ - 294, /* (393) database_kw_opt ::= */ - 297, /* (394) kwcolumn_opt ::= */ - 297, /* (395) kwcolumn_opt ::= COLUMNKW */ - 299, /* (396) vtabarglist ::= vtabarg */ - 299, /* (397) vtabarglist ::= vtabarglist COMMA vtabarg */ - 300, /* (398) vtabarg ::= vtabarg vtabargtoken */ - 303, /* (399) anylist ::= */ - 303, /* (400) anylist ::= anylist LP anylist RP */ - 303, /* (401) anylist ::= anylist ANY */ - 266, /* (402) with ::= */ - 306, /* (403) windowdefn_list ::= windowdefn */ - 308, /* (404) window ::= frame_opt */ + 191, /* (0) explain ::= EXPLAIN */ + 191, /* (1) explain ::= EXPLAIN QUERY PLAN */ + 190, /* (2) cmdx ::= cmd */ + 192, /* (3) cmd ::= BEGIN transtype trans_opt */ + 193, /* (4) transtype ::= */ + 193, /* (5) transtype ::= DEFERRED */ + 193, /* (6) transtype ::= IMMEDIATE */ + 193, /* (7) transtype ::= EXCLUSIVE */ + 192, /* (8) cmd ::= COMMIT|END trans_opt */ + 192, /* (9) cmd ::= ROLLBACK trans_opt */ + 192, /* (10) cmd ::= SAVEPOINT nm */ + 192, /* (11) cmd ::= RELEASE savepoint_opt nm */ + 192, /* (12) cmd ::= ROLLBACK trans_opt TO savepoint_opt nm */ + 197, /* (13) create_table ::= createkw temp TABLE ifnotexists nm dbnm */ + 199, /* (14) createkw ::= CREATE */ + 201, /* (15) ifnotexists ::= */ + 201, /* (16) ifnotexists ::= IF NOT EXISTS */ + 200, /* (17) temp ::= TEMP */ + 200, /* (18) temp ::= */ + 198, /* (19) create_table_args ::= LP columnlist conslist_opt RP table_option_set */ + 198, /* (20) create_table_args ::= AS select */ + 205, /* (21) table_option_set ::= */ + 205, /* (22) table_option_set ::= table_option_set COMMA table_option */ + 207, /* (23) table_option ::= WITHOUT nm */ + 207, /* (24) table_option ::= nm */ + 208, /* (25) columnname ::= nm typetoken */ + 210, /* (26) typetoken ::= */ + 210, /* (27) typetoken ::= typename LP signed RP */ + 210, /* (28) typetoken ::= typename LP signed COMMA signed RP */ + 211, /* (29) typename ::= typename ID|STRING */ + 215, /* (30) scanpt ::= */ + 216, /* (31) scantok ::= */ + 217, /* (32) ccons ::= CONSTRAINT nm */ + 217, /* (33) ccons ::= DEFAULT scantok term */ + 217, /* (34) ccons ::= DEFAULT LP expr RP */ + 217, /* (35) ccons ::= DEFAULT PLUS scantok term */ + 217, /* (36) ccons ::= DEFAULT MINUS scantok term */ + 217, /* (37) ccons ::= DEFAULT scantok ID|INDEXED */ + 217, /* (38) ccons ::= NOT NULL onconf */ + 217, /* (39) ccons ::= PRIMARY KEY sortorder onconf autoinc */ + 217, /* (40) ccons ::= UNIQUE onconf */ + 217, /* (41) ccons ::= CHECK LP expr RP */ + 217, /* (42) ccons ::= REFERENCES nm eidlist_opt refargs */ + 217, /* (43) ccons ::= defer_subclause */ + 217, /* (44) ccons ::= COLLATE ID|STRING */ + 226, /* (45) generated ::= LP expr RP */ + 226, /* (46) generated ::= LP expr RP ID */ + 222, /* (47) autoinc ::= */ + 222, /* (48) autoinc ::= AUTOINCR */ + 224, /* (49) refargs ::= */ + 224, /* (50) refargs ::= refargs refarg */ + 227, /* (51) refarg ::= MATCH nm */ + 227, /* (52) refarg ::= ON INSERT refact */ + 227, /* (53) refarg ::= ON DELETE refact */ + 227, /* (54) refarg ::= ON UPDATE refact */ + 228, /* (55) refact ::= SET NULL */ + 228, /* (56) refact ::= SET DEFAULT */ + 228, /* (57) refact ::= CASCADE */ + 228, /* (58) refact ::= RESTRICT */ + 228, /* (59) refact ::= NO ACTION */ + 225, /* (60) defer_subclause ::= NOT DEFERRABLE init_deferred_pred_opt */ + 225, /* (61) defer_subclause ::= DEFERRABLE init_deferred_pred_opt */ + 229, /* (62) init_deferred_pred_opt ::= */ + 229, /* (63) init_deferred_pred_opt ::= INITIALLY DEFERRED */ + 229, /* (64) init_deferred_pred_opt ::= INITIALLY IMMEDIATE */ + 204, /* (65) conslist_opt ::= */ + 231, /* (66) tconscomma ::= COMMA */ + 232, /* (67) tcons ::= CONSTRAINT nm */ + 232, /* (68) tcons ::= PRIMARY KEY LP sortlist autoinc RP onconf */ + 232, /* (69) tcons ::= UNIQUE LP sortlist RP onconf */ + 232, /* (70) tcons ::= CHECK LP expr RP onconf */ + 232, /* (71) tcons ::= FOREIGN KEY LP eidlist RP REFERENCES nm eidlist_opt refargs defer_subclause_opt */ + 235, /* (72) defer_subclause_opt ::= */ + 220, /* (73) onconf ::= */ + 220, /* (74) onconf ::= ON CONFLICT resolvetype */ + 236, /* (75) orconf ::= */ + 236, /* (76) orconf ::= OR resolvetype */ + 237, /* (77) resolvetype ::= IGNORE */ + 237, /* (78) resolvetype ::= REPLACE */ + 192, /* (79) cmd ::= DROP TABLE ifexists fullname */ + 239, /* (80) ifexists ::= IF EXISTS */ + 239, /* (81) ifexists ::= */ + 192, /* (82) cmd ::= createkw temp VIEW ifnotexists nm dbnm eidlist_opt AS select */ + 192, /* (83) cmd ::= DROP VIEW ifexists fullname */ + 192, /* (84) cmd ::= select */ + 206, /* (85) select ::= WITH wqlist selectnowith */ + 206, /* (86) select ::= WITH RECURSIVE wqlist selectnowith */ + 206, /* (87) select ::= selectnowith */ + 241, /* (88) selectnowith ::= selectnowith multiselect_op oneselect */ + 244, /* (89) multiselect_op ::= UNION */ + 244, /* (90) multiselect_op ::= UNION ALL */ + 244, /* (91) multiselect_op ::= EXCEPT|INTERSECT */ + 242, /* (92) oneselect ::= SELECT distinct selcollist from where_opt groupby_opt having_opt orderby_opt limit_opt */ + 242, /* (93) oneselect ::= SELECT distinct selcollist from where_opt groupby_opt having_opt window_clause orderby_opt limit_opt */ + 254, /* (94) values ::= VALUES LP nexprlist RP */ + 242, /* (95) oneselect ::= mvalues */ + 256, /* (96) mvalues ::= values COMMA LP nexprlist RP */ + 256, /* (97) mvalues ::= mvalues COMMA LP nexprlist RP */ + 245, /* (98) distinct ::= DISTINCT */ + 245, /* (99) distinct ::= ALL */ + 245, /* (100) distinct ::= */ + 257, /* (101) sclp ::= */ + 246, /* (102) selcollist ::= sclp scanpt expr scanpt as */ + 246, /* (103) selcollist ::= sclp scanpt STAR */ + 246, /* (104) selcollist ::= sclp scanpt nm DOT STAR */ + 258, /* (105) as ::= AS nm */ + 258, /* (106) as ::= */ + 247, /* (107) from ::= */ + 247, /* (108) from ::= FROM seltablist */ + 260, /* (109) stl_prefix ::= seltablist joinop */ + 260, /* (110) stl_prefix ::= */ + 259, /* (111) seltablist ::= stl_prefix nm dbnm as on_using */ + 259, /* (112) seltablist ::= stl_prefix nm dbnm as indexed_by on_using */ + 259, /* (113) seltablist ::= stl_prefix nm dbnm LP exprlist RP as on_using */ + 259, /* (114) seltablist ::= stl_prefix LP select RP as on_using */ + 259, /* (115) seltablist ::= stl_prefix LP seltablist RP as on_using */ + 202, /* (116) dbnm ::= */ + 202, /* (117) dbnm ::= DOT nm */ + 240, /* (118) fullname ::= nm */ + 240, /* (119) fullname ::= nm DOT nm */ + 265, /* (120) xfullname ::= nm */ + 265, /* (121) xfullname ::= nm DOT nm */ + 265, /* (122) xfullname ::= nm DOT nm AS nm */ + 265, /* (123) xfullname ::= nm AS nm */ + 261, /* (124) joinop ::= COMMA|JOIN */ + 261, /* (125) joinop ::= JOIN_KW JOIN */ + 261, /* (126) joinop ::= JOIN_KW nm JOIN */ + 261, /* (127) joinop ::= JOIN_KW nm nm JOIN */ + 262, /* (128) on_using ::= ON expr */ + 262, /* (129) on_using ::= USING LP idlist RP */ + 262, /* (130) on_using ::= */ + 267, /* (131) indexed_opt ::= */ + 263, /* (132) indexed_by ::= INDEXED BY nm */ + 263, /* (133) indexed_by ::= NOT INDEXED */ + 251, /* (134) orderby_opt ::= */ + 251, /* (135) orderby_opt ::= ORDER BY sortlist */ + 233, /* (136) sortlist ::= sortlist COMMA expr sortorder nulls */ + 233, /* (137) sortlist ::= expr sortorder nulls */ + 221, /* (138) sortorder ::= ASC */ + 221, /* (139) sortorder ::= DESC */ + 221, /* (140) sortorder ::= */ + 268, /* (141) nulls ::= NULLS FIRST */ + 268, /* (142) nulls ::= NULLS LAST */ + 268, /* (143) nulls ::= */ + 249, /* (144) groupby_opt ::= */ + 249, /* (145) groupby_opt ::= GROUP BY nexprlist */ + 250, /* (146) having_opt ::= */ + 250, /* (147) having_opt ::= HAVING expr */ + 252, /* (148) limit_opt ::= */ + 252, /* (149) limit_opt ::= LIMIT expr */ + 252, /* (150) limit_opt ::= LIMIT expr OFFSET expr */ + 252, /* (151) limit_opt ::= LIMIT expr COMMA expr */ + 192, /* (152) cmd ::= with DELETE FROM xfullname indexed_opt where_opt_ret */ + 248, /* (153) where_opt ::= */ + 248, /* (154) where_opt ::= WHERE expr */ + 270, /* (155) where_opt_ret ::= */ + 270, /* (156) where_opt_ret ::= WHERE expr */ + 270, /* (157) where_opt_ret ::= RETURNING selcollist */ + 270, /* (158) where_opt_ret ::= WHERE expr RETURNING selcollist */ + 192, /* (159) cmd ::= with UPDATE orconf xfullname indexed_opt SET setlist from where_opt_ret */ + 271, /* (160) setlist ::= setlist COMMA nm EQ expr */ + 271, /* (161) setlist ::= setlist COMMA LP idlist RP EQ expr */ + 271, /* (162) setlist ::= nm EQ expr */ + 271, /* (163) setlist ::= LP idlist RP EQ expr */ + 192, /* (164) cmd ::= with insert_cmd INTO xfullname idlist_opt select upsert */ + 192, /* (165) cmd ::= with insert_cmd INTO xfullname idlist_opt DEFAULT VALUES returning */ + 274, /* (166) upsert ::= */ + 274, /* (167) upsert ::= RETURNING selcollist */ + 274, /* (168) upsert ::= ON CONFLICT LP sortlist RP where_opt DO UPDATE SET setlist where_opt upsert */ + 274, /* (169) upsert ::= ON CONFLICT LP sortlist RP where_opt DO NOTHING upsert */ + 274, /* (170) upsert ::= ON CONFLICT DO NOTHING returning */ + 274, /* (171) upsert ::= ON CONFLICT DO UPDATE SET setlist where_opt returning */ + 275, /* (172) returning ::= RETURNING selcollist */ + 272, /* (173) insert_cmd ::= INSERT orconf */ + 272, /* (174) insert_cmd ::= REPLACE */ + 273, /* (175) idlist_opt ::= */ + 273, /* (176) idlist_opt ::= LP idlist RP */ + 266, /* (177) idlist ::= idlist COMMA nm */ + 266, /* (178) idlist ::= nm */ + 219, /* (179) expr ::= LP expr RP */ + 219, /* (180) expr ::= ID|INDEXED|JOIN_KW */ + 219, /* (181) expr ::= nm DOT nm */ + 219, /* (182) expr ::= nm DOT nm DOT nm */ + 218, /* (183) term ::= NULL|FLOAT|BLOB */ + 218, /* (184) term ::= STRING */ + 218, /* (185) term ::= INTEGER */ + 219, /* (186) expr ::= VARIABLE */ + 219, /* (187) expr ::= expr COLLATE ID|STRING */ + 219, /* (188) expr ::= CAST LP expr AS typetoken RP */ + 219, /* (189) expr ::= ID|INDEXED|JOIN_KW LP distinct exprlist RP */ + 219, /* (190) expr ::= ID|INDEXED|JOIN_KW LP distinct exprlist ORDER BY sortlist RP */ + 219, /* (191) expr ::= ID|INDEXED|JOIN_KW LP STAR RP */ + 219, /* (192) expr ::= ID|INDEXED|JOIN_KW LP distinct exprlist RP filter_over */ + 219, /* (193) expr ::= ID|INDEXED|JOIN_KW LP distinct exprlist ORDER BY sortlist RP filter_over */ + 219, /* (194) expr ::= ID|INDEXED|JOIN_KW LP STAR RP filter_over */ + 218, /* (195) term ::= CTIME_KW */ + 219, /* (196) expr ::= LP nexprlist COMMA expr RP */ + 219, /* (197) expr ::= expr AND expr */ + 219, /* (198) expr ::= expr OR expr */ + 219, /* (199) expr ::= expr LT|GT|GE|LE expr */ + 219, /* (200) expr ::= expr EQ|NE expr */ + 219, /* (201) expr ::= expr BITAND|BITOR|LSHIFT|RSHIFT expr */ + 219, /* (202) expr ::= expr PLUS|MINUS expr */ + 219, /* (203) expr ::= expr STAR|SLASH|REM expr */ + 219, /* (204) expr ::= expr CONCAT expr */ + 277, /* (205) likeop ::= NOT LIKE_KW|MATCH */ + 219, /* (206) expr ::= expr likeop expr */ + 219, /* (207) expr ::= expr likeop expr ESCAPE expr */ + 219, /* (208) expr ::= expr ISNULL|NOTNULL */ + 219, /* (209) expr ::= expr NOT NULL */ + 219, /* (210) expr ::= expr IS expr */ + 219, /* (211) expr ::= expr IS NOT expr */ + 219, /* (212) expr ::= expr IS NOT DISTINCT FROM expr */ + 219, /* (213) expr ::= expr IS DISTINCT FROM expr */ + 219, /* (214) expr ::= NOT expr */ + 219, /* (215) expr ::= BITNOT expr */ + 219, /* (216) expr ::= PLUS|MINUS expr */ + 219, /* (217) expr ::= expr PTR expr */ + 278, /* (218) between_op ::= BETWEEN */ + 278, /* (219) between_op ::= NOT BETWEEN */ + 219, /* (220) expr ::= expr between_op expr AND expr */ + 279, /* (221) in_op ::= IN */ + 279, /* (222) in_op ::= NOT IN */ + 219, /* (223) expr ::= expr in_op LP exprlist RP */ + 219, /* (224) expr ::= LP select RP */ + 219, /* (225) expr ::= expr in_op LP select RP */ + 219, /* (226) expr ::= expr in_op nm dbnm paren_exprlist */ + 219, /* (227) expr ::= EXISTS LP select RP */ + 219, /* (228) expr ::= CASE case_operand case_exprlist case_else END */ + 282, /* (229) case_exprlist ::= case_exprlist WHEN expr THEN expr */ + 282, /* (230) case_exprlist ::= WHEN expr THEN expr */ + 283, /* (231) case_else ::= ELSE expr */ + 283, /* (232) case_else ::= */ + 281, /* (233) case_operand ::= */ + 264, /* (234) exprlist ::= */ + 255, /* (235) nexprlist ::= nexprlist COMMA expr */ + 255, /* (236) nexprlist ::= expr */ + 280, /* (237) paren_exprlist ::= */ + 280, /* (238) paren_exprlist ::= LP exprlist RP */ + 192, /* (239) cmd ::= createkw uniqueflag INDEX ifnotexists nm dbnm ON nm LP sortlist RP where_opt */ + 284, /* (240) uniqueflag ::= UNIQUE */ + 284, /* (241) uniqueflag ::= */ + 223, /* (242) eidlist_opt ::= */ + 223, /* (243) eidlist_opt ::= LP eidlist RP */ + 234, /* (244) eidlist ::= eidlist COMMA nm collate sortorder */ + 234, /* (245) eidlist ::= nm collate sortorder */ + 285, /* (246) collate ::= */ + 285, /* (247) collate ::= COLLATE ID|STRING */ + 192, /* (248) cmd ::= DROP INDEX ifexists fullname */ + 192, /* (249) cmd ::= VACUUM vinto */ + 192, /* (250) cmd ::= VACUUM nm vinto */ + 286, /* (251) vinto ::= INTO expr */ + 286, /* (252) vinto ::= */ + 192, /* (253) cmd ::= PRAGMA nm dbnm */ + 192, /* (254) cmd ::= PRAGMA nm dbnm EQ nmnum */ + 192, /* (255) cmd ::= PRAGMA nm dbnm LP nmnum RP */ + 192, /* (256) cmd ::= PRAGMA nm dbnm EQ minus_num */ + 192, /* (257) cmd ::= PRAGMA nm dbnm LP minus_num RP */ + 213, /* (258) plus_num ::= PLUS INTEGER|FLOAT */ + 214, /* (259) minus_num ::= MINUS INTEGER|FLOAT */ + 192, /* (260) cmd ::= createkw trigger_decl BEGIN trigger_cmd_list END */ + 288, /* (261) trigger_decl ::= temp TRIGGER ifnotexists nm dbnm trigger_time trigger_event ON fullname foreach_clause when_clause */ + 290, /* (262) trigger_time ::= BEFORE|AFTER */ + 290, /* (263) trigger_time ::= INSTEAD OF */ + 290, /* (264) trigger_time ::= */ + 291, /* (265) trigger_event ::= DELETE|INSERT */ + 291, /* (266) trigger_event ::= UPDATE */ + 291, /* (267) trigger_event ::= UPDATE OF idlist */ + 293, /* (268) when_clause ::= */ + 293, /* (269) when_clause ::= WHEN expr */ + 289, /* (270) trigger_cmd_list ::= trigger_cmd_list trigger_cmd SEMI */ + 289, /* (271) trigger_cmd_list ::= trigger_cmd SEMI */ + 295, /* (272) trnm ::= nm DOT nm */ + 296, /* (273) tridxby ::= INDEXED BY nm */ + 296, /* (274) tridxby ::= NOT INDEXED */ + 294, /* (275) trigger_cmd ::= UPDATE orconf trnm tridxby SET setlist from where_opt scanpt */ + 294, /* (276) trigger_cmd ::= scanpt insert_cmd INTO trnm idlist_opt select upsert scanpt */ + 294, /* (277) trigger_cmd ::= DELETE FROM trnm tridxby where_opt scanpt */ + 294, /* (278) trigger_cmd ::= scanpt select scanpt */ + 219, /* (279) expr ::= RAISE LP IGNORE RP */ + 219, /* (280) expr ::= RAISE LP raisetype COMMA expr RP */ + 238, /* (281) raisetype ::= ROLLBACK */ + 238, /* (282) raisetype ::= ABORT */ + 238, /* (283) raisetype ::= FAIL */ + 192, /* (284) cmd ::= DROP TRIGGER ifexists fullname */ + 192, /* (285) cmd ::= ATTACH database_kw_opt expr AS expr key_opt */ + 192, /* (286) cmd ::= DETACH database_kw_opt expr */ + 298, /* (287) key_opt ::= */ + 298, /* (288) key_opt ::= KEY expr */ + 192, /* (289) cmd ::= REINDEX */ + 192, /* (290) cmd ::= REINDEX nm dbnm */ + 192, /* (291) cmd ::= ANALYZE */ + 192, /* (292) cmd ::= ANALYZE nm dbnm */ + 192, /* (293) cmd ::= ALTER TABLE fullname RENAME TO nm */ + 192, /* (294) cmd ::= ALTER TABLE add_column_fullname ADD kwcolumn_opt columnname carglist */ + 192, /* (295) cmd ::= ALTER TABLE fullname DROP kwcolumn_opt nm */ + 299, /* (296) add_column_fullname ::= fullname */ + 192, /* (297) cmd ::= ALTER TABLE fullname RENAME kwcolumn_opt nm TO nm */ + 192, /* (298) cmd ::= create_vtab */ + 192, /* (299) cmd ::= create_vtab LP vtabarglist RP */ + 301, /* (300) create_vtab ::= createkw VIRTUAL TABLE ifnotexists nm dbnm USING nm */ + 303, /* (301) vtabarg ::= */ + 304, /* (302) vtabargtoken ::= ANY */ + 304, /* (303) vtabargtoken ::= lp anylist RP */ + 305, /* (304) lp ::= LP */ + 269, /* (305) with ::= WITH wqlist */ + 269, /* (306) with ::= WITH RECURSIVE wqlist */ + 308, /* (307) wqas ::= AS */ + 308, /* (308) wqas ::= AS MATERIALIZED */ + 308, /* (309) wqas ::= AS NOT MATERIALIZED */ + 307, /* (310) wqitem ::= withnm eidlist_opt wqas LP select RP */ + 309, /* (311) withnm ::= nm */ + 243, /* (312) wqlist ::= wqitem */ + 243, /* (313) wqlist ::= wqlist COMMA wqitem */ + 310, /* (314) windowdefn_list ::= windowdefn_list COMMA windowdefn */ + 311, /* (315) windowdefn ::= nm AS LP window RP */ + 312, /* (316) window ::= PARTITION BY nexprlist orderby_opt frame_opt */ + 312, /* (317) window ::= nm PARTITION BY nexprlist orderby_opt frame_opt */ + 312, /* (318) window ::= ORDER BY sortlist frame_opt */ + 312, /* (319) window ::= nm ORDER BY sortlist frame_opt */ + 312, /* (320) window ::= nm frame_opt */ + 313, /* (321) frame_opt ::= */ + 313, /* (322) frame_opt ::= range_or_rows frame_bound_s frame_exclude_opt */ + 313, /* (323) frame_opt ::= range_or_rows BETWEEN frame_bound_s AND frame_bound_e frame_exclude_opt */ + 317, /* (324) range_or_rows ::= RANGE|ROWS|GROUPS */ + 319, /* (325) frame_bound_s ::= frame_bound */ + 319, /* (326) frame_bound_s ::= UNBOUNDED PRECEDING */ + 320, /* (327) frame_bound_e ::= frame_bound */ + 320, /* (328) frame_bound_e ::= UNBOUNDED FOLLOWING */ + 318, /* (329) frame_bound ::= expr PRECEDING|FOLLOWING */ + 318, /* (330) frame_bound ::= CURRENT ROW */ + 321, /* (331) frame_exclude_opt ::= */ + 321, /* (332) frame_exclude_opt ::= EXCLUDE frame_exclude */ + 322, /* (333) frame_exclude ::= NO OTHERS */ + 322, /* (334) frame_exclude ::= CURRENT ROW */ + 322, /* (335) frame_exclude ::= GROUP|TIES */ + 253, /* (336) window_clause ::= WINDOW windowdefn_list */ + 276, /* (337) filter_over ::= filter_clause over_clause */ + 276, /* (338) filter_over ::= over_clause */ + 276, /* (339) filter_over ::= filter_clause */ + 316, /* (340) over_clause ::= OVER LP window RP */ + 316, /* (341) over_clause ::= OVER nm */ + 315, /* (342) filter_clause ::= FILTER LP WHERE expr RP */ + 218, /* (343) term ::= QNUMBER */ + 187, /* (344) input ::= cmdlist */ + 188, /* (345) cmdlist ::= cmdlist ecmd */ + 188, /* (346) cmdlist ::= ecmd */ + 189, /* (347) ecmd ::= SEMI */ + 189, /* (348) ecmd ::= cmdx SEMI */ + 189, /* (349) ecmd ::= explain cmdx SEMI */ + 194, /* (350) trans_opt ::= */ + 194, /* (351) trans_opt ::= TRANSACTION */ + 194, /* (352) trans_opt ::= TRANSACTION nm */ + 196, /* (353) savepoint_opt ::= SAVEPOINT */ + 196, /* (354) savepoint_opt ::= */ + 192, /* (355) cmd ::= create_table create_table_args */ + 205, /* (356) table_option_set ::= table_option */ + 203, /* (357) columnlist ::= columnlist COMMA columnname carglist */ + 203, /* (358) columnlist ::= columnname carglist */ + 195, /* (359) nm ::= ID|INDEXED|JOIN_KW */ + 195, /* (360) nm ::= STRING */ + 210, /* (361) typetoken ::= typename */ + 211, /* (362) typename ::= ID|STRING */ + 212, /* (363) signed ::= plus_num */ + 212, /* (364) signed ::= minus_num */ + 209, /* (365) carglist ::= carglist ccons */ + 209, /* (366) carglist ::= */ + 217, /* (367) ccons ::= NULL onconf */ + 217, /* (368) ccons ::= GENERATED ALWAYS AS generated */ + 217, /* (369) ccons ::= AS generated */ + 204, /* (370) conslist_opt ::= COMMA conslist */ + 230, /* (371) conslist ::= conslist tconscomma tcons */ + 230, /* (372) conslist ::= tcons */ + 231, /* (373) tconscomma ::= */ + 235, /* (374) defer_subclause_opt ::= defer_subclause */ + 237, /* (375) resolvetype ::= raisetype */ + 241, /* (376) selectnowith ::= oneselect */ + 242, /* (377) oneselect ::= values */ + 257, /* (378) sclp ::= selcollist COMMA */ + 258, /* (379) as ::= ID|STRING */ + 267, /* (380) indexed_opt ::= indexed_by */ + 275, /* (381) returning ::= */ + 219, /* (382) expr ::= term */ + 277, /* (383) likeop ::= LIKE_KW|MATCH */ + 281, /* (384) case_operand ::= expr */ + 264, /* (385) exprlist ::= nexprlist */ + 287, /* (386) nmnum ::= plus_num */ + 287, /* (387) nmnum ::= nm */ + 287, /* (388) nmnum ::= ON */ + 287, /* (389) nmnum ::= DELETE */ + 287, /* (390) nmnum ::= DEFAULT */ + 213, /* (391) plus_num ::= INTEGER|FLOAT */ + 292, /* (392) foreach_clause ::= */ + 292, /* (393) foreach_clause ::= FOR EACH ROW */ + 295, /* (394) trnm ::= nm */ + 296, /* (395) tridxby ::= */ + 297, /* (396) database_kw_opt ::= DATABASE */ + 297, /* (397) database_kw_opt ::= */ + 300, /* (398) kwcolumn_opt ::= */ + 300, /* (399) kwcolumn_opt ::= COLUMNKW */ + 302, /* (400) vtabarglist ::= vtabarg */ + 302, /* (401) vtabarglist ::= vtabarglist COMMA vtabarg */ + 303, /* (402) vtabarg ::= vtabarg vtabargtoken */ + 306, /* (403) anylist ::= */ + 306, /* (404) anylist ::= anylist LP anylist RP */ + 306, /* (405) anylist ::= anylist ANY */ + 269, /* (406) with ::= */ + 310, /* (407) windowdefn_list ::= windowdefn */ + 312, /* (408) window ::= frame_opt */ }; /* For rule J, yyRuleInfoNRhs[J] contains the negative of the number @@ -174254,316 +180807,320 @@ static const signed char yyRuleInfoNRhs[] = { -9, /* (92) oneselect ::= SELECT distinct selcollist from where_opt groupby_opt having_opt orderby_opt limit_opt */ -10, /* (93) oneselect ::= SELECT distinct selcollist from where_opt groupby_opt having_opt window_clause orderby_opt limit_opt */ -4, /* (94) values ::= VALUES LP nexprlist RP */ - -5, /* (95) values ::= values COMMA LP nexprlist RP */ - -1, /* (96) distinct ::= DISTINCT */ - -1, /* (97) distinct ::= ALL */ - 0, /* (98) distinct ::= */ - 0, /* (99) sclp ::= */ - -5, /* (100) selcollist ::= sclp scanpt expr scanpt as */ - -3, /* (101) selcollist ::= sclp scanpt STAR */ - -5, /* (102) selcollist ::= sclp scanpt nm DOT STAR */ - -2, /* (103) as ::= AS nm */ - 0, /* (104) as ::= */ - 0, /* (105) from ::= */ - -2, /* (106) from ::= FROM seltablist */ - -2, /* (107) stl_prefix ::= seltablist joinop */ - 0, /* (108) stl_prefix ::= */ - -5, /* (109) seltablist ::= stl_prefix nm dbnm as on_using */ - -6, /* (110) seltablist ::= stl_prefix nm dbnm as indexed_by on_using */ - -8, /* (111) seltablist ::= stl_prefix nm dbnm LP exprlist RP as on_using */ - -6, /* (112) seltablist ::= stl_prefix LP select RP as on_using */ - -6, /* (113) seltablist ::= stl_prefix LP seltablist RP as on_using */ - 0, /* (114) dbnm ::= */ - -2, /* (115) dbnm ::= DOT nm */ - -1, /* (116) fullname ::= nm */ - -3, /* (117) fullname ::= nm DOT nm */ - -1, /* (118) xfullname ::= nm */ - -3, /* (119) xfullname ::= nm DOT nm */ - -5, /* (120) xfullname ::= nm DOT nm AS nm */ - -3, /* (121) xfullname ::= nm AS nm */ - -1, /* (122) joinop ::= COMMA|JOIN */ - -2, /* (123) joinop ::= JOIN_KW JOIN */ - -3, /* (124) joinop ::= JOIN_KW nm JOIN */ - -4, /* (125) joinop ::= JOIN_KW nm nm JOIN */ - -2, /* (126) on_using ::= ON expr */ - -4, /* (127) on_using ::= USING LP idlist RP */ - 0, /* (128) on_using ::= */ - 0, /* (129) indexed_opt ::= */ - -3, /* (130) indexed_by ::= INDEXED BY nm */ - -2, /* (131) indexed_by ::= NOT INDEXED */ - 0, /* (132) orderby_opt ::= */ - -3, /* (133) orderby_opt ::= ORDER BY sortlist */ - -5, /* (134) sortlist ::= sortlist COMMA expr sortorder nulls */ - -3, /* (135) sortlist ::= expr sortorder nulls */ - -1, /* (136) sortorder ::= ASC */ - -1, /* (137) sortorder ::= DESC */ - 0, /* (138) sortorder ::= */ - -2, /* (139) nulls ::= NULLS FIRST */ - -2, /* (140) nulls ::= NULLS LAST */ - 0, /* (141) nulls ::= */ - 0, /* (142) groupby_opt ::= */ - -3, /* (143) groupby_opt ::= GROUP BY nexprlist */ - 0, /* (144) having_opt ::= */ - -2, /* (145) having_opt ::= HAVING expr */ - 0, /* (146) limit_opt ::= */ - -2, /* (147) limit_opt ::= LIMIT expr */ - -4, /* (148) limit_opt ::= LIMIT expr OFFSET expr */ - -4, /* (149) limit_opt ::= LIMIT expr COMMA expr */ - -6, /* (150) cmd ::= with DELETE FROM xfullname indexed_opt where_opt_ret */ - 0, /* (151) where_opt ::= */ - -2, /* (152) where_opt ::= WHERE expr */ - 0, /* (153) where_opt_ret ::= */ - -2, /* (154) where_opt_ret ::= WHERE expr */ - -2, /* (155) where_opt_ret ::= RETURNING selcollist */ - -4, /* (156) where_opt_ret ::= WHERE expr RETURNING selcollist */ - -9, /* (157) cmd ::= with UPDATE orconf xfullname indexed_opt SET setlist from where_opt_ret */ - -5, /* (158) setlist ::= setlist COMMA nm EQ expr */ - -7, /* (159) setlist ::= setlist COMMA LP idlist RP EQ expr */ - -3, /* (160) setlist ::= nm EQ expr */ - -5, /* (161) setlist ::= LP idlist RP EQ expr */ - -7, /* (162) cmd ::= with insert_cmd INTO xfullname idlist_opt select upsert */ - -8, /* (163) cmd ::= with insert_cmd INTO xfullname idlist_opt DEFAULT VALUES returning */ - 0, /* (164) upsert ::= */ - -2, /* (165) upsert ::= RETURNING selcollist */ - -12, /* (166) upsert ::= ON CONFLICT LP sortlist RP where_opt DO UPDATE SET setlist where_opt upsert */ - -9, /* (167) upsert ::= ON CONFLICT LP sortlist RP where_opt DO NOTHING upsert */ - -5, /* (168) upsert ::= ON CONFLICT DO NOTHING returning */ - -8, /* (169) upsert ::= ON CONFLICT DO UPDATE SET setlist where_opt returning */ - -2, /* (170) returning ::= RETURNING selcollist */ - -2, /* (171) insert_cmd ::= INSERT orconf */ - -1, /* (172) insert_cmd ::= REPLACE */ - 0, /* (173) idlist_opt ::= */ - -3, /* (174) idlist_opt ::= LP idlist RP */ - -3, /* (175) idlist ::= idlist COMMA nm */ - -1, /* (176) idlist ::= nm */ - -3, /* (177) expr ::= LP expr RP */ - -1, /* (178) expr ::= ID|INDEXED|JOIN_KW */ - -3, /* (179) expr ::= nm DOT nm */ - -5, /* (180) expr ::= nm DOT nm DOT nm */ - -1, /* (181) term ::= NULL|FLOAT|BLOB */ - -1, /* (182) term ::= STRING */ - -1, /* (183) term ::= INTEGER */ - -1, /* (184) expr ::= VARIABLE */ - -3, /* (185) expr ::= expr COLLATE ID|STRING */ - -6, /* (186) expr ::= CAST LP expr AS typetoken RP */ - -5, /* (187) expr ::= ID|INDEXED|JOIN_KW LP distinct exprlist RP */ - -8, /* (188) expr ::= ID|INDEXED|JOIN_KW LP distinct exprlist ORDER BY sortlist RP */ - -4, /* (189) expr ::= ID|INDEXED|JOIN_KW LP STAR RP */ - -6, /* (190) expr ::= ID|INDEXED|JOIN_KW LP distinct exprlist RP filter_over */ - -9, /* (191) expr ::= ID|INDEXED|JOIN_KW LP distinct exprlist ORDER BY sortlist RP filter_over */ - -5, /* (192) expr ::= ID|INDEXED|JOIN_KW LP STAR RP filter_over */ - -1, /* (193) term ::= CTIME_KW */ - -5, /* (194) expr ::= LP nexprlist COMMA expr RP */ - -3, /* (195) expr ::= expr AND expr */ - -3, /* (196) expr ::= expr OR expr */ - -3, /* (197) expr ::= expr LT|GT|GE|LE expr */ - -3, /* (198) expr ::= expr EQ|NE expr */ - -3, /* (199) expr ::= expr BITAND|BITOR|LSHIFT|RSHIFT expr */ - -3, /* (200) expr ::= expr PLUS|MINUS expr */ - -3, /* (201) expr ::= expr STAR|SLASH|REM expr */ - -3, /* (202) expr ::= expr CONCAT expr */ - -2, /* (203) likeop ::= NOT LIKE_KW|MATCH */ - -3, /* (204) expr ::= expr likeop expr */ - -5, /* (205) expr ::= expr likeop expr ESCAPE expr */ - -2, /* (206) expr ::= expr ISNULL|NOTNULL */ - -3, /* (207) expr ::= expr NOT NULL */ - -3, /* (208) expr ::= expr IS expr */ - -4, /* (209) expr ::= expr IS NOT expr */ - -6, /* (210) expr ::= expr IS NOT DISTINCT FROM expr */ - -5, /* (211) expr ::= expr IS DISTINCT FROM expr */ - -2, /* (212) expr ::= NOT expr */ - -2, /* (213) expr ::= BITNOT expr */ - -2, /* (214) expr ::= PLUS|MINUS expr */ - -3, /* (215) expr ::= expr PTR expr */ - -1, /* (216) between_op ::= BETWEEN */ - -2, /* (217) between_op ::= NOT BETWEEN */ - -5, /* (218) expr ::= expr between_op expr AND expr */ - -1, /* (219) in_op ::= IN */ - -2, /* (220) in_op ::= NOT IN */ - -5, /* (221) expr ::= expr in_op LP exprlist RP */ - -3, /* (222) expr ::= LP select RP */ - -5, /* (223) expr ::= expr in_op LP select RP */ - -5, /* (224) expr ::= expr in_op nm dbnm paren_exprlist */ - -4, /* (225) expr ::= EXISTS LP select RP */ - -5, /* (226) expr ::= CASE case_operand case_exprlist case_else END */ - -5, /* (227) case_exprlist ::= case_exprlist WHEN expr THEN expr */ - -4, /* (228) case_exprlist ::= WHEN expr THEN expr */ - -2, /* (229) case_else ::= ELSE expr */ - 0, /* (230) case_else ::= */ - 0, /* (231) case_operand ::= */ - 0, /* (232) exprlist ::= */ - -3, /* (233) nexprlist ::= nexprlist COMMA expr */ - -1, /* (234) nexprlist ::= expr */ - 0, /* (235) paren_exprlist ::= */ - -3, /* (236) paren_exprlist ::= LP exprlist RP */ - -12, /* (237) cmd ::= createkw uniqueflag INDEX ifnotexists nm dbnm ON nm LP sortlist RP where_opt */ - -1, /* (238) uniqueflag ::= UNIQUE */ - 0, /* (239) uniqueflag ::= */ - 0, /* (240) eidlist_opt ::= */ - -3, /* (241) eidlist_opt ::= LP eidlist RP */ - -5, /* (242) eidlist ::= eidlist COMMA nm collate sortorder */ - -3, /* (243) eidlist ::= nm collate sortorder */ - 0, /* (244) collate ::= */ - -2, /* (245) collate ::= COLLATE ID|STRING */ - -4, /* (246) cmd ::= DROP INDEX ifexists fullname */ - -2, /* (247) cmd ::= VACUUM vinto */ - -3, /* (248) cmd ::= VACUUM nm vinto */ - -2, /* (249) vinto ::= INTO expr */ - 0, /* (250) vinto ::= */ - -3, /* (251) cmd ::= PRAGMA nm dbnm */ - -5, /* (252) cmd ::= PRAGMA nm dbnm EQ nmnum */ - -6, /* (253) cmd ::= PRAGMA nm dbnm LP nmnum RP */ - -5, /* (254) cmd ::= PRAGMA nm dbnm EQ minus_num */ - -6, /* (255) cmd ::= PRAGMA nm dbnm LP minus_num RP */ - -2, /* (256) plus_num ::= PLUS INTEGER|FLOAT */ - -2, /* (257) minus_num ::= MINUS INTEGER|FLOAT */ - -5, /* (258) cmd ::= createkw trigger_decl BEGIN trigger_cmd_list END */ - -11, /* (259) trigger_decl ::= temp TRIGGER ifnotexists nm dbnm trigger_time trigger_event ON fullname foreach_clause when_clause */ - -1, /* (260) trigger_time ::= BEFORE|AFTER */ - -2, /* (261) trigger_time ::= INSTEAD OF */ - 0, /* (262) trigger_time ::= */ - -1, /* (263) trigger_event ::= DELETE|INSERT */ - -1, /* (264) trigger_event ::= UPDATE */ - -3, /* (265) trigger_event ::= UPDATE OF idlist */ - 0, /* (266) when_clause ::= */ - -2, /* (267) when_clause ::= WHEN expr */ - -3, /* (268) trigger_cmd_list ::= trigger_cmd_list trigger_cmd SEMI */ - -2, /* (269) trigger_cmd_list ::= trigger_cmd SEMI */ - -3, /* (270) trnm ::= nm DOT nm */ - -3, /* (271) tridxby ::= INDEXED BY nm */ - -2, /* (272) tridxby ::= NOT INDEXED */ - -9, /* (273) trigger_cmd ::= UPDATE orconf trnm tridxby SET setlist from where_opt scanpt */ - -8, /* (274) trigger_cmd ::= scanpt insert_cmd INTO trnm idlist_opt select upsert scanpt */ - -6, /* (275) trigger_cmd ::= DELETE FROM trnm tridxby where_opt scanpt */ - -3, /* (276) trigger_cmd ::= scanpt select scanpt */ - -4, /* (277) expr ::= RAISE LP IGNORE RP */ - -6, /* (278) expr ::= RAISE LP raisetype COMMA nm RP */ - -1, /* (279) raisetype ::= ROLLBACK */ - -1, /* (280) raisetype ::= ABORT */ - -1, /* (281) raisetype ::= FAIL */ - -4, /* (282) cmd ::= DROP TRIGGER ifexists fullname */ - -6, /* (283) cmd ::= ATTACH database_kw_opt expr AS expr key_opt */ - -3, /* (284) cmd ::= DETACH database_kw_opt expr */ - 0, /* (285) key_opt ::= */ - -2, /* (286) key_opt ::= KEY expr */ - -1, /* (287) cmd ::= REINDEX */ - -3, /* (288) cmd ::= REINDEX nm dbnm */ - -1, /* (289) cmd ::= ANALYZE */ - -3, /* (290) cmd ::= ANALYZE nm dbnm */ - -6, /* (291) cmd ::= ALTER TABLE fullname RENAME TO nm */ - -7, /* (292) cmd ::= ALTER TABLE add_column_fullname ADD kwcolumn_opt columnname carglist */ - -6, /* (293) cmd ::= ALTER TABLE fullname DROP kwcolumn_opt nm */ - -1, /* (294) add_column_fullname ::= fullname */ - -8, /* (295) cmd ::= ALTER TABLE fullname RENAME kwcolumn_opt nm TO nm */ - -1, /* (296) cmd ::= create_vtab */ - -4, /* (297) cmd ::= create_vtab LP vtabarglist RP */ - -8, /* (298) create_vtab ::= createkw VIRTUAL TABLE ifnotexists nm dbnm USING nm */ - 0, /* (299) vtabarg ::= */ - -1, /* (300) vtabargtoken ::= ANY */ - -3, /* (301) vtabargtoken ::= lp anylist RP */ - -1, /* (302) lp ::= LP */ - -2, /* (303) with ::= WITH wqlist */ - -3, /* (304) with ::= WITH RECURSIVE wqlist */ - -1, /* (305) wqas ::= AS */ - -2, /* (306) wqas ::= AS MATERIALIZED */ - -3, /* (307) wqas ::= AS NOT MATERIALIZED */ - -6, /* (308) wqitem ::= nm eidlist_opt wqas LP select RP */ - -1, /* (309) wqlist ::= wqitem */ - -3, /* (310) wqlist ::= wqlist COMMA wqitem */ - -3, /* (311) windowdefn_list ::= windowdefn_list COMMA windowdefn */ - -5, /* (312) windowdefn ::= nm AS LP window RP */ - -5, /* (313) window ::= PARTITION BY nexprlist orderby_opt frame_opt */ - -6, /* (314) window ::= nm PARTITION BY nexprlist orderby_opt frame_opt */ - -4, /* (315) window ::= ORDER BY sortlist frame_opt */ - -5, /* (316) window ::= nm ORDER BY sortlist frame_opt */ - -2, /* (317) window ::= nm frame_opt */ - 0, /* (318) frame_opt ::= */ - -3, /* (319) frame_opt ::= range_or_rows frame_bound_s frame_exclude_opt */ - -6, /* (320) frame_opt ::= range_or_rows BETWEEN frame_bound_s AND frame_bound_e frame_exclude_opt */ - -1, /* (321) range_or_rows ::= RANGE|ROWS|GROUPS */ - -1, /* (322) frame_bound_s ::= frame_bound */ - -2, /* (323) frame_bound_s ::= UNBOUNDED PRECEDING */ - -1, /* (324) frame_bound_e ::= frame_bound */ - -2, /* (325) frame_bound_e ::= UNBOUNDED FOLLOWING */ - -2, /* (326) frame_bound ::= expr PRECEDING|FOLLOWING */ - -2, /* (327) frame_bound ::= CURRENT ROW */ - 0, /* (328) frame_exclude_opt ::= */ - -2, /* (329) frame_exclude_opt ::= EXCLUDE frame_exclude */ - -2, /* (330) frame_exclude ::= NO OTHERS */ - -2, /* (331) frame_exclude ::= CURRENT ROW */ - -1, /* (332) frame_exclude ::= GROUP|TIES */ - -2, /* (333) window_clause ::= WINDOW windowdefn_list */ - -2, /* (334) filter_over ::= filter_clause over_clause */ - -1, /* (335) filter_over ::= over_clause */ - -1, /* (336) filter_over ::= filter_clause */ - -4, /* (337) over_clause ::= OVER LP window RP */ - -2, /* (338) over_clause ::= OVER nm */ - -5, /* (339) filter_clause ::= FILTER LP WHERE expr RP */ - -1, /* (340) input ::= cmdlist */ - -2, /* (341) cmdlist ::= cmdlist ecmd */ - -1, /* (342) cmdlist ::= ecmd */ - -1, /* (343) ecmd ::= SEMI */ - -2, /* (344) ecmd ::= cmdx SEMI */ - -3, /* (345) ecmd ::= explain cmdx SEMI */ - 0, /* (346) trans_opt ::= */ - -1, /* (347) trans_opt ::= TRANSACTION */ - -2, /* (348) trans_opt ::= TRANSACTION nm */ - -1, /* (349) savepoint_opt ::= SAVEPOINT */ - 0, /* (350) savepoint_opt ::= */ - -2, /* (351) cmd ::= create_table create_table_args */ - -1, /* (352) table_option_set ::= table_option */ - -4, /* (353) columnlist ::= columnlist COMMA columnname carglist */ - -2, /* (354) columnlist ::= columnname carglist */ - -1, /* (355) nm ::= ID|INDEXED|JOIN_KW */ - -1, /* (356) nm ::= STRING */ - -1, /* (357) typetoken ::= typename */ - -1, /* (358) typename ::= ID|STRING */ - -1, /* (359) signed ::= plus_num */ - -1, /* (360) signed ::= minus_num */ - -2, /* (361) carglist ::= carglist ccons */ - 0, /* (362) carglist ::= */ - -2, /* (363) ccons ::= NULL onconf */ - -4, /* (364) ccons ::= GENERATED ALWAYS AS generated */ - -2, /* (365) ccons ::= AS generated */ - -2, /* (366) conslist_opt ::= COMMA conslist */ - -3, /* (367) conslist ::= conslist tconscomma tcons */ - -1, /* (368) conslist ::= tcons */ - 0, /* (369) tconscomma ::= */ - -1, /* (370) defer_subclause_opt ::= defer_subclause */ - -1, /* (371) resolvetype ::= raisetype */ - -1, /* (372) selectnowith ::= oneselect */ - -1, /* (373) oneselect ::= values */ - -2, /* (374) sclp ::= selcollist COMMA */ - -1, /* (375) as ::= ID|STRING */ - -1, /* (376) indexed_opt ::= indexed_by */ - 0, /* (377) returning ::= */ - -1, /* (378) expr ::= term */ - -1, /* (379) likeop ::= LIKE_KW|MATCH */ - -1, /* (380) case_operand ::= expr */ - -1, /* (381) exprlist ::= nexprlist */ - -1, /* (382) nmnum ::= plus_num */ - -1, /* (383) nmnum ::= nm */ - -1, /* (384) nmnum ::= ON */ - -1, /* (385) nmnum ::= DELETE */ - -1, /* (386) nmnum ::= DEFAULT */ - -1, /* (387) plus_num ::= INTEGER|FLOAT */ - 0, /* (388) foreach_clause ::= */ - -3, /* (389) foreach_clause ::= FOR EACH ROW */ - -1, /* (390) trnm ::= nm */ - 0, /* (391) tridxby ::= */ - -1, /* (392) database_kw_opt ::= DATABASE */ - 0, /* (393) database_kw_opt ::= */ - 0, /* (394) kwcolumn_opt ::= */ - -1, /* (395) kwcolumn_opt ::= COLUMNKW */ - -1, /* (396) vtabarglist ::= vtabarg */ - -3, /* (397) vtabarglist ::= vtabarglist COMMA vtabarg */ - -2, /* (398) vtabarg ::= vtabarg vtabargtoken */ - 0, /* (399) anylist ::= */ - -4, /* (400) anylist ::= anylist LP anylist RP */ - -2, /* (401) anylist ::= anylist ANY */ - 0, /* (402) with ::= */ - -1, /* (403) windowdefn_list ::= windowdefn */ - -1, /* (404) window ::= frame_opt */ + -1, /* (95) oneselect ::= mvalues */ + -5, /* (96) mvalues ::= values COMMA LP nexprlist RP */ + -5, /* (97) mvalues ::= mvalues COMMA LP nexprlist RP */ + -1, /* (98) distinct ::= DISTINCT */ + -1, /* (99) distinct ::= ALL */ + 0, /* (100) distinct ::= */ + 0, /* (101) sclp ::= */ + -5, /* (102) selcollist ::= sclp scanpt expr scanpt as */ + -3, /* (103) selcollist ::= sclp scanpt STAR */ + -5, /* (104) selcollist ::= sclp scanpt nm DOT STAR */ + -2, /* (105) as ::= AS nm */ + 0, /* (106) as ::= */ + 0, /* (107) from ::= */ + -2, /* (108) from ::= FROM seltablist */ + -2, /* (109) stl_prefix ::= seltablist joinop */ + 0, /* (110) stl_prefix ::= */ + -5, /* (111) seltablist ::= stl_prefix nm dbnm as on_using */ + -6, /* (112) seltablist ::= stl_prefix nm dbnm as indexed_by on_using */ + -8, /* (113) seltablist ::= stl_prefix nm dbnm LP exprlist RP as on_using */ + -6, /* (114) seltablist ::= stl_prefix LP select RP as on_using */ + -6, /* (115) seltablist ::= stl_prefix LP seltablist RP as on_using */ + 0, /* (116) dbnm ::= */ + -2, /* (117) dbnm ::= DOT nm */ + -1, /* (118) fullname ::= nm */ + -3, /* (119) fullname ::= nm DOT nm */ + -1, /* (120) xfullname ::= nm */ + -3, /* (121) xfullname ::= nm DOT nm */ + -5, /* (122) xfullname ::= nm DOT nm AS nm */ + -3, /* (123) xfullname ::= nm AS nm */ + -1, /* (124) joinop ::= COMMA|JOIN */ + -2, /* (125) joinop ::= JOIN_KW JOIN */ + -3, /* (126) joinop ::= JOIN_KW nm JOIN */ + -4, /* (127) joinop ::= JOIN_KW nm nm JOIN */ + -2, /* (128) on_using ::= ON expr */ + -4, /* (129) on_using ::= USING LP idlist RP */ + 0, /* (130) on_using ::= */ + 0, /* (131) indexed_opt ::= */ + -3, /* (132) indexed_by ::= INDEXED BY nm */ + -2, /* (133) indexed_by ::= NOT INDEXED */ + 0, /* (134) orderby_opt ::= */ + -3, /* (135) orderby_opt ::= ORDER BY sortlist */ + -5, /* (136) sortlist ::= sortlist COMMA expr sortorder nulls */ + -3, /* (137) sortlist ::= expr sortorder nulls */ + -1, /* (138) sortorder ::= ASC */ + -1, /* (139) sortorder ::= DESC */ + 0, /* (140) sortorder ::= */ + -2, /* (141) nulls ::= NULLS FIRST */ + -2, /* (142) nulls ::= NULLS LAST */ + 0, /* (143) nulls ::= */ + 0, /* (144) groupby_opt ::= */ + -3, /* (145) groupby_opt ::= GROUP BY nexprlist */ + 0, /* (146) having_opt ::= */ + -2, /* (147) having_opt ::= HAVING expr */ + 0, /* (148) limit_opt ::= */ + -2, /* (149) limit_opt ::= LIMIT expr */ + -4, /* (150) limit_opt ::= LIMIT expr OFFSET expr */ + -4, /* (151) limit_opt ::= LIMIT expr COMMA expr */ + -6, /* (152) cmd ::= with DELETE FROM xfullname indexed_opt where_opt_ret */ + 0, /* (153) where_opt ::= */ + -2, /* (154) where_opt ::= WHERE expr */ + 0, /* (155) where_opt_ret ::= */ + -2, /* (156) where_opt_ret ::= WHERE expr */ + -2, /* (157) where_opt_ret ::= RETURNING selcollist */ + -4, /* (158) where_opt_ret ::= WHERE expr RETURNING selcollist */ + -9, /* (159) cmd ::= with UPDATE orconf xfullname indexed_opt SET setlist from where_opt_ret */ + -5, /* (160) setlist ::= setlist COMMA nm EQ expr */ + -7, /* (161) setlist ::= setlist COMMA LP idlist RP EQ expr */ + -3, /* (162) setlist ::= nm EQ expr */ + -5, /* (163) setlist ::= LP idlist RP EQ expr */ + -7, /* (164) cmd ::= with insert_cmd INTO xfullname idlist_opt select upsert */ + -8, /* (165) cmd ::= with insert_cmd INTO xfullname idlist_opt DEFAULT VALUES returning */ + 0, /* (166) upsert ::= */ + -2, /* (167) upsert ::= RETURNING selcollist */ + -12, /* (168) upsert ::= ON CONFLICT LP sortlist RP where_opt DO UPDATE SET setlist where_opt upsert */ + -9, /* (169) upsert ::= ON CONFLICT LP sortlist RP where_opt DO NOTHING upsert */ + -5, /* (170) upsert ::= ON CONFLICT DO NOTHING returning */ + -8, /* (171) upsert ::= ON CONFLICT DO UPDATE SET setlist where_opt returning */ + -2, /* (172) returning ::= RETURNING selcollist */ + -2, /* (173) insert_cmd ::= INSERT orconf */ + -1, /* (174) insert_cmd ::= REPLACE */ + 0, /* (175) idlist_opt ::= */ + -3, /* (176) idlist_opt ::= LP idlist RP */ + -3, /* (177) idlist ::= idlist COMMA nm */ + -1, /* (178) idlist ::= nm */ + -3, /* (179) expr ::= LP expr RP */ + -1, /* (180) expr ::= ID|INDEXED|JOIN_KW */ + -3, /* (181) expr ::= nm DOT nm */ + -5, /* (182) expr ::= nm DOT nm DOT nm */ + -1, /* (183) term ::= NULL|FLOAT|BLOB */ + -1, /* (184) term ::= STRING */ + -1, /* (185) term ::= INTEGER */ + -1, /* (186) expr ::= VARIABLE */ + -3, /* (187) expr ::= expr COLLATE ID|STRING */ + -6, /* (188) expr ::= CAST LP expr AS typetoken RP */ + -5, /* (189) expr ::= ID|INDEXED|JOIN_KW LP distinct exprlist RP */ + -8, /* (190) expr ::= ID|INDEXED|JOIN_KW LP distinct exprlist ORDER BY sortlist RP */ + -4, /* (191) expr ::= ID|INDEXED|JOIN_KW LP STAR RP */ + -6, /* (192) expr ::= ID|INDEXED|JOIN_KW LP distinct exprlist RP filter_over */ + -9, /* (193) expr ::= ID|INDEXED|JOIN_KW LP distinct exprlist ORDER BY sortlist RP filter_over */ + -5, /* (194) expr ::= ID|INDEXED|JOIN_KW LP STAR RP filter_over */ + -1, /* (195) term ::= CTIME_KW */ + -5, /* (196) expr ::= LP nexprlist COMMA expr RP */ + -3, /* (197) expr ::= expr AND expr */ + -3, /* (198) expr ::= expr OR expr */ + -3, /* (199) expr ::= expr LT|GT|GE|LE expr */ + -3, /* (200) expr ::= expr EQ|NE expr */ + -3, /* (201) expr ::= expr BITAND|BITOR|LSHIFT|RSHIFT expr */ + -3, /* (202) expr ::= expr PLUS|MINUS expr */ + -3, /* (203) expr ::= expr STAR|SLASH|REM expr */ + -3, /* (204) expr ::= expr CONCAT expr */ + -2, /* (205) likeop ::= NOT LIKE_KW|MATCH */ + -3, /* (206) expr ::= expr likeop expr */ + -5, /* (207) expr ::= expr likeop expr ESCAPE expr */ + -2, /* (208) expr ::= expr ISNULL|NOTNULL */ + -3, /* (209) expr ::= expr NOT NULL */ + -3, /* (210) expr ::= expr IS expr */ + -4, /* (211) expr ::= expr IS NOT expr */ + -6, /* (212) expr ::= expr IS NOT DISTINCT FROM expr */ + -5, /* (213) expr ::= expr IS DISTINCT FROM expr */ + -2, /* (214) expr ::= NOT expr */ + -2, /* (215) expr ::= BITNOT expr */ + -2, /* (216) expr ::= PLUS|MINUS expr */ + -3, /* (217) expr ::= expr PTR expr */ + -1, /* (218) between_op ::= BETWEEN */ + -2, /* (219) between_op ::= NOT BETWEEN */ + -5, /* (220) expr ::= expr between_op expr AND expr */ + -1, /* (221) in_op ::= IN */ + -2, /* (222) in_op ::= NOT IN */ + -5, /* (223) expr ::= expr in_op LP exprlist RP */ + -3, /* (224) expr ::= LP select RP */ + -5, /* (225) expr ::= expr in_op LP select RP */ + -5, /* (226) expr ::= expr in_op nm dbnm paren_exprlist */ + -4, /* (227) expr ::= EXISTS LP select RP */ + -5, /* (228) expr ::= CASE case_operand case_exprlist case_else END */ + -5, /* (229) case_exprlist ::= case_exprlist WHEN expr THEN expr */ + -4, /* (230) case_exprlist ::= WHEN expr THEN expr */ + -2, /* (231) case_else ::= ELSE expr */ + 0, /* (232) case_else ::= */ + 0, /* (233) case_operand ::= */ + 0, /* (234) exprlist ::= */ + -3, /* (235) nexprlist ::= nexprlist COMMA expr */ + -1, /* (236) nexprlist ::= expr */ + 0, /* (237) paren_exprlist ::= */ + -3, /* (238) paren_exprlist ::= LP exprlist RP */ + -12, /* (239) cmd ::= createkw uniqueflag INDEX ifnotexists nm dbnm ON nm LP sortlist RP where_opt */ + -1, /* (240) uniqueflag ::= UNIQUE */ + 0, /* (241) uniqueflag ::= */ + 0, /* (242) eidlist_opt ::= */ + -3, /* (243) eidlist_opt ::= LP eidlist RP */ + -5, /* (244) eidlist ::= eidlist COMMA nm collate sortorder */ + -3, /* (245) eidlist ::= nm collate sortorder */ + 0, /* (246) collate ::= */ + -2, /* (247) collate ::= COLLATE ID|STRING */ + -4, /* (248) cmd ::= DROP INDEX ifexists fullname */ + -2, /* (249) cmd ::= VACUUM vinto */ + -3, /* (250) cmd ::= VACUUM nm vinto */ + -2, /* (251) vinto ::= INTO expr */ + 0, /* (252) vinto ::= */ + -3, /* (253) cmd ::= PRAGMA nm dbnm */ + -5, /* (254) cmd ::= PRAGMA nm dbnm EQ nmnum */ + -6, /* (255) cmd ::= PRAGMA nm dbnm LP nmnum RP */ + -5, /* (256) cmd ::= PRAGMA nm dbnm EQ minus_num */ + -6, /* (257) cmd ::= PRAGMA nm dbnm LP minus_num RP */ + -2, /* (258) plus_num ::= PLUS INTEGER|FLOAT */ + -2, /* (259) minus_num ::= MINUS INTEGER|FLOAT */ + -5, /* (260) cmd ::= createkw trigger_decl BEGIN trigger_cmd_list END */ + -11, /* (261) trigger_decl ::= temp TRIGGER ifnotexists nm dbnm trigger_time trigger_event ON fullname foreach_clause when_clause */ + -1, /* (262) trigger_time ::= BEFORE|AFTER */ + -2, /* (263) trigger_time ::= INSTEAD OF */ + 0, /* (264) trigger_time ::= */ + -1, /* (265) trigger_event ::= DELETE|INSERT */ + -1, /* (266) trigger_event ::= UPDATE */ + -3, /* (267) trigger_event ::= UPDATE OF idlist */ + 0, /* (268) when_clause ::= */ + -2, /* (269) when_clause ::= WHEN expr */ + -3, /* (270) trigger_cmd_list ::= trigger_cmd_list trigger_cmd SEMI */ + -2, /* (271) trigger_cmd_list ::= trigger_cmd SEMI */ + -3, /* (272) trnm ::= nm DOT nm */ + -3, /* (273) tridxby ::= INDEXED BY nm */ + -2, /* (274) tridxby ::= NOT INDEXED */ + -9, /* (275) trigger_cmd ::= UPDATE orconf trnm tridxby SET setlist from where_opt scanpt */ + -8, /* (276) trigger_cmd ::= scanpt insert_cmd INTO trnm idlist_opt select upsert scanpt */ + -6, /* (277) trigger_cmd ::= DELETE FROM trnm tridxby where_opt scanpt */ + -3, /* (278) trigger_cmd ::= scanpt select scanpt */ + -4, /* (279) expr ::= RAISE LP IGNORE RP */ + -6, /* (280) expr ::= RAISE LP raisetype COMMA expr RP */ + -1, /* (281) raisetype ::= ROLLBACK */ + -1, /* (282) raisetype ::= ABORT */ + -1, /* (283) raisetype ::= FAIL */ + -4, /* (284) cmd ::= DROP TRIGGER ifexists fullname */ + -6, /* (285) cmd ::= ATTACH database_kw_opt expr AS expr key_opt */ + -3, /* (286) cmd ::= DETACH database_kw_opt expr */ + 0, /* (287) key_opt ::= */ + -2, /* (288) key_opt ::= KEY expr */ + -1, /* (289) cmd ::= REINDEX */ + -3, /* (290) cmd ::= REINDEX nm dbnm */ + -1, /* (291) cmd ::= ANALYZE */ + -3, /* (292) cmd ::= ANALYZE nm dbnm */ + -6, /* (293) cmd ::= ALTER TABLE fullname RENAME TO nm */ + -7, /* (294) cmd ::= ALTER TABLE add_column_fullname ADD kwcolumn_opt columnname carglist */ + -6, /* (295) cmd ::= ALTER TABLE fullname DROP kwcolumn_opt nm */ + -1, /* (296) add_column_fullname ::= fullname */ + -8, /* (297) cmd ::= ALTER TABLE fullname RENAME kwcolumn_opt nm TO nm */ + -1, /* (298) cmd ::= create_vtab */ + -4, /* (299) cmd ::= create_vtab LP vtabarglist RP */ + -8, /* (300) create_vtab ::= createkw VIRTUAL TABLE ifnotexists nm dbnm USING nm */ + 0, /* (301) vtabarg ::= */ + -1, /* (302) vtabargtoken ::= ANY */ + -3, /* (303) vtabargtoken ::= lp anylist RP */ + -1, /* (304) lp ::= LP */ + -2, /* (305) with ::= WITH wqlist */ + -3, /* (306) with ::= WITH RECURSIVE wqlist */ + -1, /* (307) wqas ::= AS */ + -2, /* (308) wqas ::= AS MATERIALIZED */ + -3, /* (309) wqas ::= AS NOT MATERIALIZED */ + -6, /* (310) wqitem ::= withnm eidlist_opt wqas LP select RP */ + -1, /* (311) withnm ::= nm */ + -1, /* (312) wqlist ::= wqitem */ + -3, /* (313) wqlist ::= wqlist COMMA wqitem */ + -3, /* (314) windowdefn_list ::= windowdefn_list COMMA windowdefn */ + -5, /* (315) windowdefn ::= nm AS LP window RP */ + -5, /* (316) window ::= PARTITION BY nexprlist orderby_opt frame_opt */ + -6, /* (317) window ::= nm PARTITION BY nexprlist orderby_opt frame_opt */ + -4, /* (318) window ::= ORDER BY sortlist frame_opt */ + -5, /* (319) window ::= nm ORDER BY sortlist frame_opt */ + -2, /* (320) window ::= nm frame_opt */ + 0, /* (321) frame_opt ::= */ + -3, /* (322) frame_opt ::= range_or_rows frame_bound_s frame_exclude_opt */ + -6, /* (323) frame_opt ::= range_or_rows BETWEEN frame_bound_s AND frame_bound_e frame_exclude_opt */ + -1, /* (324) range_or_rows ::= RANGE|ROWS|GROUPS */ + -1, /* (325) frame_bound_s ::= frame_bound */ + -2, /* (326) frame_bound_s ::= UNBOUNDED PRECEDING */ + -1, /* (327) frame_bound_e ::= frame_bound */ + -2, /* (328) frame_bound_e ::= UNBOUNDED FOLLOWING */ + -2, /* (329) frame_bound ::= expr PRECEDING|FOLLOWING */ + -2, /* (330) frame_bound ::= CURRENT ROW */ + 0, /* (331) frame_exclude_opt ::= */ + -2, /* (332) frame_exclude_opt ::= EXCLUDE frame_exclude */ + -2, /* (333) frame_exclude ::= NO OTHERS */ + -2, /* (334) frame_exclude ::= CURRENT ROW */ + -1, /* (335) frame_exclude ::= GROUP|TIES */ + -2, /* (336) window_clause ::= WINDOW windowdefn_list */ + -2, /* (337) filter_over ::= filter_clause over_clause */ + -1, /* (338) filter_over ::= over_clause */ + -1, /* (339) filter_over ::= filter_clause */ + -4, /* (340) over_clause ::= OVER LP window RP */ + -2, /* (341) over_clause ::= OVER nm */ + -5, /* (342) filter_clause ::= FILTER LP WHERE expr RP */ + -1, /* (343) term ::= QNUMBER */ + -1, /* (344) input ::= cmdlist */ + -2, /* (345) cmdlist ::= cmdlist ecmd */ + -1, /* (346) cmdlist ::= ecmd */ + -1, /* (347) ecmd ::= SEMI */ + -2, /* (348) ecmd ::= cmdx SEMI */ + -3, /* (349) ecmd ::= explain cmdx SEMI */ + 0, /* (350) trans_opt ::= */ + -1, /* (351) trans_opt ::= TRANSACTION */ + -2, /* (352) trans_opt ::= TRANSACTION nm */ + -1, /* (353) savepoint_opt ::= SAVEPOINT */ + 0, /* (354) savepoint_opt ::= */ + -2, /* (355) cmd ::= create_table create_table_args */ + -1, /* (356) table_option_set ::= table_option */ + -4, /* (357) columnlist ::= columnlist COMMA columnname carglist */ + -2, /* (358) columnlist ::= columnname carglist */ + -1, /* (359) nm ::= ID|INDEXED|JOIN_KW */ + -1, /* (360) nm ::= STRING */ + -1, /* (361) typetoken ::= typename */ + -1, /* (362) typename ::= ID|STRING */ + -1, /* (363) signed ::= plus_num */ + -1, /* (364) signed ::= minus_num */ + -2, /* (365) carglist ::= carglist ccons */ + 0, /* (366) carglist ::= */ + -2, /* (367) ccons ::= NULL onconf */ + -4, /* (368) ccons ::= GENERATED ALWAYS AS generated */ + -2, /* (369) ccons ::= AS generated */ + -2, /* (370) conslist_opt ::= COMMA conslist */ + -3, /* (371) conslist ::= conslist tconscomma tcons */ + -1, /* (372) conslist ::= tcons */ + 0, /* (373) tconscomma ::= */ + -1, /* (374) defer_subclause_opt ::= defer_subclause */ + -1, /* (375) resolvetype ::= raisetype */ + -1, /* (376) selectnowith ::= oneselect */ + -1, /* (377) oneselect ::= values */ + -2, /* (378) sclp ::= selcollist COMMA */ + -1, /* (379) as ::= ID|STRING */ + -1, /* (380) indexed_opt ::= indexed_by */ + 0, /* (381) returning ::= */ + -1, /* (382) expr ::= term */ + -1, /* (383) likeop ::= LIKE_KW|MATCH */ + -1, /* (384) case_operand ::= expr */ + -1, /* (385) exprlist ::= nexprlist */ + -1, /* (386) nmnum ::= plus_num */ + -1, /* (387) nmnum ::= nm */ + -1, /* (388) nmnum ::= ON */ + -1, /* (389) nmnum ::= DELETE */ + -1, /* (390) nmnum ::= DEFAULT */ + -1, /* (391) plus_num ::= INTEGER|FLOAT */ + 0, /* (392) foreach_clause ::= */ + -3, /* (393) foreach_clause ::= FOR EACH ROW */ + -1, /* (394) trnm ::= nm */ + 0, /* (395) tridxby ::= */ + -1, /* (396) database_kw_opt ::= DATABASE */ + 0, /* (397) database_kw_opt ::= */ + 0, /* (398) kwcolumn_opt ::= */ + -1, /* (399) kwcolumn_opt ::= COLUMNKW */ + -1, /* (400) vtabarglist ::= vtabarg */ + -3, /* (401) vtabarglist ::= vtabarglist COMMA vtabarg */ + -2, /* (402) vtabarg ::= vtabarg vtabargtoken */ + 0, /* (403) anylist ::= */ + -4, /* (404) anylist ::= anylist LP anylist RP */ + -2, /* (405) anylist ::= anylist ANY */ + 0, /* (406) with ::= */ + -1, /* (407) windowdefn_list ::= windowdefn */ + -1, /* (408) window ::= frame_opt */ }; static void yy_accept(yyParser*); /* Forward Declaration */ @@ -174615,16 +181172,16 @@ static YYACTIONTYPE yy_reduce( { sqlite3FinishCoding(pParse); } break; case 3: /* cmd ::= BEGIN transtype trans_opt */ -{sqlite3BeginTransaction(pParse, yymsp[-1].minor.yy394);} +{sqlite3BeginTransaction(pParse, yymsp[-1].minor.yy502);} break; case 4: /* transtype ::= */ -{yymsp[1].minor.yy394 = TK_DEFERRED;} +{yymsp[1].minor.yy502 = TK_DEFERRED;} break; case 5: /* transtype ::= DEFERRED */ case 6: /* transtype ::= IMMEDIATE */ yytestcase(yyruleno==6); case 7: /* transtype ::= EXCLUSIVE */ yytestcase(yyruleno==7); - case 321: /* range_or_rows ::= RANGE|ROWS|GROUPS */ yytestcase(yyruleno==321); -{yymsp[0].minor.yy394 = yymsp[0].major; /*A-overwrites-X*/} + case 324: /* range_or_rows ::= RANGE|ROWS|GROUPS */ yytestcase(yyruleno==324); +{yymsp[0].minor.yy502 = yymsp[0].major; /*A-overwrites-X*/} break; case 8: /* cmd ::= COMMIT|END trans_opt */ case 9: /* cmd ::= ROLLBACK trans_opt */ yytestcase(yyruleno==9); @@ -174647,11 +181204,13 @@ static YYACTIONTYPE yy_reduce( break; case 13: /* create_table ::= createkw temp TABLE ifnotexists nm dbnm */ { - sqlite3StartTable(pParse,&yymsp[-1].minor.yy0,&yymsp[0].minor.yy0,yymsp[-4].minor.yy394,0,0,yymsp[-2].minor.yy394); + sqlite3StartTable(pParse,&yymsp[-1].minor.yy0,&yymsp[0].minor.yy0,yymsp[-4].minor.yy502,0,0,yymsp[-2].minor.yy502); } break; case 14: /* createkw ::= CREATE */ -{disableLookaside(pParse);} +{ + disableLookaside(pParse); +} break; case 15: /* ifnotexists ::= */ case 18: /* temp ::= */ yytestcase(yyruleno==18); @@ -174659,40 +181218,40 @@ static YYACTIONTYPE yy_reduce( case 62: /* init_deferred_pred_opt ::= */ yytestcase(yyruleno==62); case 72: /* defer_subclause_opt ::= */ yytestcase(yyruleno==72); case 81: /* ifexists ::= */ yytestcase(yyruleno==81); - case 98: /* distinct ::= */ yytestcase(yyruleno==98); - case 244: /* collate ::= */ yytestcase(yyruleno==244); -{yymsp[1].minor.yy394 = 0;} + case 100: /* distinct ::= */ yytestcase(yyruleno==100); + case 246: /* collate ::= */ yytestcase(yyruleno==246); +{yymsp[1].minor.yy502 = 0;} break; case 16: /* ifnotexists ::= IF NOT EXISTS */ -{yymsp[-2].minor.yy394 = 1;} +{yymsp[-2].minor.yy502 = 1;} break; case 17: /* temp ::= TEMP */ -{yymsp[0].minor.yy394 = pParse->db->init.busy==0;} +{yymsp[0].minor.yy502 = pParse->db->init.busy==0;} break; case 19: /* create_table_args ::= LP columnlist conslist_opt RP table_option_set */ { - sqlite3EndTable(pParse,&yymsp[-2].minor.yy0,&yymsp[-1].minor.yy0,yymsp[0].minor.yy285,0); + sqlite3EndTable(pParse,&yymsp[-2].minor.yy0,&yymsp[-1].minor.yy0,yymsp[0].minor.yy9,0); } break; case 20: /* create_table_args ::= AS select */ { - sqlite3EndTable(pParse,0,0,0,yymsp[0].minor.yy47); - sqlite3SelectDelete(pParse->db, yymsp[0].minor.yy47); + sqlite3EndTable(pParse,0,0,0,yymsp[0].minor.yy637); + sqlite3SelectDelete(pParse->db, yymsp[0].minor.yy637); } break; case 21: /* table_option_set ::= */ -{yymsp[1].minor.yy285 = 0;} +{yymsp[1].minor.yy9 = 0;} break; case 22: /* table_option_set ::= table_option_set COMMA table_option */ -{yylhsminor.yy285 = yymsp[-2].minor.yy285|yymsp[0].minor.yy285;} - yymsp[-2].minor.yy285 = yylhsminor.yy285; +{yylhsminor.yy9 = yymsp[-2].minor.yy9|yymsp[0].minor.yy9;} + yymsp[-2].minor.yy9 = yylhsminor.yy9; break; case 23: /* table_option ::= WITHOUT nm */ { if( yymsp[0].minor.yy0.n==5 && sqlite3_strnicmp(yymsp[0].minor.yy0.z,"rowid",5)==0 ){ - yymsp[-1].minor.yy285 = TF_WithoutRowid | TF_NoVisibleRowid; + yymsp[-1].minor.yy9 = TF_WithoutRowid | TF_NoVisibleRowid; }else{ - yymsp[-1].minor.yy285 = 0; + yymsp[-1].minor.yy9 = 0; sqlite3ErrorMsg(pParse, "unknown table option: %.*s", yymsp[0].minor.yy0.n, yymsp[0].minor.yy0.z); } } @@ -174700,20 +181259,20 @@ static YYACTIONTYPE yy_reduce( case 24: /* table_option ::= nm */ { if( yymsp[0].minor.yy0.n==6 && sqlite3_strnicmp(yymsp[0].minor.yy0.z,"strict",6)==0 ){ - yylhsminor.yy285 = TF_Strict; + yylhsminor.yy9 = TF_Strict; }else{ - yylhsminor.yy285 = 0; + yylhsminor.yy9 = 0; sqlite3ErrorMsg(pParse, "unknown table option: %.*s", yymsp[0].minor.yy0.n, yymsp[0].minor.yy0.z); } } - yymsp[0].minor.yy285 = yylhsminor.yy285; + yymsp[0].minor.yy9 = yylhsminor.yy9; break; case 25: /* columnname ::= nm typetoken */ {sqlite3AddColumn(pParse,yymsp[-1].minor.yy0,yymsp[0].minor.yy0);} break; case 26: /* typetoken ::= */ case 65: /* conslist_opt ::= */ yytestcase(yyruleno==65); - case 104: /* as ::= */ yytestcase(yyruleno==104); + case 106: /* as ::= */ yytestcase(yyruleno==106); {yymsp[1].minor.yy0.n = 0; yymsp[1].minor.yy0.z = 0;} break; case 27: /* typetoken ::= typename LP signed RP */ @@ -174732,7 +181291,7 @@ static YYACTIONTYPE yy_reduce( case 30: /* scanpt ::= */ { assert( yyLookahead!=YYNOCODE ); - yymsp[1].minor.yy522 = yyLookaheadToken.z; + yymsp[1].minor.yy342 = yyLookaheadToken.z; } break; case 31: /* scantok ::= */ @@ -174743,20 +181302,20 @@ static YYACTIONTYPE yy_reduce( break; case 32: /* ccons ::= CONSTRAINT nm */ case 67: /* tcons ::= CONSTRAINT nm */ yytestcase(yyruleno==67); -{pParse->constraintName = yymsp[0].minor.yy0;} +{ASSERT_IS_CREATE; pParse->u1.cr.constraintName = yymsp[0].minor.yy0;} break; case 33: /* ccons ::= DEFAULT scantok term */ -{sqlite3AddDefaultValue(pParse,yymsp[0].minor.yy528,yymsp[-1].minor.yy0.z,&yymsp[-1].minor.yy0.z[yymsp[-1].minor.yy0.n]);} +{sqlite3AddDefaultValue(pParse,yymsp[0].minor.yy590,yymsp[-1].minor.yy0.z,&yymsp[-1].minor.yy0.z[yymsp[-1].minor.yy0.n]);} break; case 34: /* ccons ::= DEFAULT LP expr RP */ -{sqlite3AddDefaultValue(pParse,yymsp[-1].minor.yy528,yymsp[-2].minor.yy0.z+1,yymsp[0].minor.yy0.z);} +{sqlite3AddDefaultValue(pParse,yymsp[-1].minor.yy590,yymsp[-2].minor.yy0.z+1,yymsp[0].minor.yy0.z);} break; case 35: /* ccons ::= DEFAULT PLUS scantok term */ -{sqlite3AddDefaultValue(pParse,yymsp[0].minor.yy528,yymsp[-2].minor.yy0.z,&yymsp[-1].minor.yy0.z[yymsp[-1].minor.yy0.n]);} +{sqlite3AddDefaultValue(pParse,yymsp[0].minor.yy590,yymsp[-2].minor.yy0.z,&yymsp[-1].minor.yy0.z[yymsp[-1].minor.yy0.n]);} break; case 36: /* ccons ::= DEFAULT MINUS scantok term */ { - Expr *p = sqlite3PExpr(pParse, TK_UMINUS, yymsp[0].minor.yy528, 0); + Expr *p = sqlite3PExpr(pParse, TK_UMINUS, yymsp[0].minor.yy590, 0); sqlite3AddDefaultValue(pParse,p,yymsp[-2].minor.yy0.z,&yymsp[-1].minor.yy0.z[yymsp[-1].minor.yy0.n]); } break; @@ -174771,151 +181330,155 @@ static YYACTIONTYPE yy_reduce( } break; case 38: /* ccons ::= NOT NULL onconf */ -{sqlite3AddNotNull(pParse, yymsp[0].minor.yy394);} +{sqlite3AddNotNull(pParse, yymsp[0].minor.yy502);} break; case 39: /* ccons ::= PRIMARY KEY sortorder onconf autoinc */ -{sqlite3AddPrimaryKey(pParse,0,yymsp[-1].minor.yy394,yymsp[0].minor.yy394,yymsp[-2].minor.yy394);} +{sqlite3AddPrimaryKey(pParse,0,yymsp[-1].minor.yy502,yymsp[0].minor.yy502,yymsp[-2].minor.yy502);} break; case 40: /* ccons ::= UNIQUE onconf */ -{sqlite3CreateIndex(pParse,0,0,0,0,yymsp[0].minor.yy394,0,0,0,0, +{sqlite3CreateIndex(pParse,0,0,0,0,yymsp[0].minor.yy502,0,0,0,0, SQLITE_IDXTYPE_UNIQUE);} break; case 41: /* ccons ::= CHECK LP expr RP */ -{sqlite3AddCheckConstraint(pParse,yymsp[-1].minor.yy528,yymsp[-2].minor.yy0.z,yymsp[0].minor.yy0.z);} +{sqlite3AddCheckConstraint(pParse,yymsp[-1].minor.yy590,yymsp[-2].minor.yy0.z,yymsp[0].minor.yy0.z);} break; case 42: /* ccons ::= REFERENCES nm eidlist_opt refargs */ -{sqlite3CreateForeignKey(pParse,0,&yymsp[-2].minor.yy0,yymsp[-1].minor.yy322,yymsp[0].minor.yy394);} +{sqlite3CreateForeignKey(pParse,0,&yymsp[-2].minor.yy0,yymsp[-1].minor.yy402,yymsp[0].minor.yy502);} break; case 43: /* ccons ::= defer_subclause */ -{sqlite3DeferForeignKey(pParse,yymsp[0].minor.yy394);} +{sqlite3DeferForeignKey(pParse,yymsp[0].minor.yy502);} break; case 44: /* ccons ::= COLLATE ID|STRING */ {sqlite3AddCollateType(pParse, &yymsp[0].minor.yy0);} break; case 45: /* generated ::= LP expr RP */ -{sqlite3AddGenerated(pParse,yymsp[-1].minor.yy528,0);} +{sqlite3AddGenerated(pParse,yymsp[-1].minor.yy590,0);} break; case 46: /* generated ::= LP expr RP ID */ -{sqlite3AddGenerated(pParse,yymsp[-2].minor.yy528,&yymsp[0].minor.yy0);} +{sqlite3AddGenerated(pParse,yymsp[-2].minor.yy590,&yymsp[0].minor.yy0);} break; case 48: /* autoinc ::= AUTOINCR */ -{yymsp[0].minor.yy394 = 1;} +{yymsp[0].minor.yy502 = 1;} break; case 49: /* refargs ::= */ -{ yymsp[1].minor.yy394 = OE_None*0x0101; /* EV: R-19803-45884 */} +{ yymsp[1].minor.yy502 = OE_None*0x0101; /* EV: R-19803-45884 */} break; case 50: /* refargs ::= refargs refarg */ -{ yymsp[-1].minor.yy394 = (yymsp[-1].minor.yy394 & ~yymsp[0].minor.yy231.mask) | yymsp[0].minor.yy231.value; } +{ yymsp[-1].minor.yy502 = (yymsp[-1].minor.yy502 & ~yymsp[0].minor.yy481.mask) | yymsp[0].minor.yy481.value; } break; case 51: /* refarg ::= MATCH nm */ -{ yymsp[-1].minor.yy231.value = 0; yymsp[-1].minor.yy231.mask = 0x000000; } +{ yymsp[-1].minor.yy481.value = 0; yymsp[-1].minor.yy481.mask = 0x000000; } break; case 52: /* refarg ::= ON INSERT refact */ -{ yymsp[-2].minor.yy231.value = 0; yymsp[-2].minor.yy231.mask = 0x000000; } +{ yymsp[-2].minor.yy481.value = 0; yymsp[-2].minor.yy481.mask = 0x000000; } break; case 53: /* refarg ::= ON DELETE refact */ -{ yymsp[-2].minor.yy231.value = yymsp[0].minor.yy394; yymsp[-2].minor.yy231.mask = 0x0000ff; } +{ yymsp[-2].minor.yy481.value = yymsp[0].minor.yy502; yymsp[-2].minor.yy481.mask = 0x0000ff; } break; case 54: /* refarg ::= ON UPDATE refact */ -{ yymsp[-2].minor.yy231.value = yymsp[0].minor.yy394<<8; yymsp[-2].minor.yy231.mask = 0x00ff00; } +{ yymsp[-2].minor.yy481.value = yymsp[0].minor.yy502<<8; yymsp[-2].minor.yy481.mask = 0x00ff00; } break; case 55: /* refact ::= SET NULL */ -{ yymsp[-1].minor.yy394 = OE_SetNull; /* EV: R-33326-45252 */} +{ yymsp[-1].minor.yy502 = OE_SetNull; /* EV: R-33326-45252 */} break; case 56: /* refact ::= SET DEFAULT */ -{ yymsp[-1].minor.yy394 = OE_SetDflt; /* EV: R-33326-45252 */} +{ yymsp[-1].minor.yy502 = OE_SetDflt; /* EV: R-33326-45252 */} break; case 57: /* refact ::= CASCADE */ -{ yymsp[0].minor.yy394 = OE_Cascade; /* EV: R-33326-45252 */} +{ yymsp[0].minor.yy502 = OE_Cascade; /* EV: R-33326-45252 */} break; case 58: /* refact ::= RESTRICT */ -{ yymsp[0].minor.yy394 = OE_Restrict; /* EV: R-33326-45252 */} +{ yymsp[0].minor.yy502 = OE_Restrict; /* EV: R-33326-45252 */} break; case 59: /* refact ::= NO ACTION */ -{ yymsp[-1].minor.yy394 = OE_None; /* EV: R-33326-45252 */} +{ yymsp[-1].minor.yy502 = OE_None; /* EV: R-33326-45252 */} break; case 60: /* defer_subclause ::= NOT DEFERRABLE init_deferred_pred_opt */ -{yymsp[-2].minor.yy394 = 0;} +{yymsp[-2].minor.yy502 = 0;} break; case 61: /* defer_subclause ::= DEFERRABLE init_deferred_pred_opt */ case 76: /* orconf ::= OR resolvetype */ yytestcase(yyruleno==76); - case 171: /* insert_cmd ::= INSERT orconf */ yytestcase(yyruleno==171); -{yymsp[-1].minor.yy394 = yymsp[0].minor.yy394;} + case 173: /* insert_cmd ::= INSERT orconf */ yytestcase(yyruleno==173); +{yymsp[-1].minor.yy502 = yymsp[0].minor.yy502;} break; case 63: /* init_deferred_pred_opt ::= INITIALLY DEFERRED */ case 80: /* ifexists ::= IF EXISTS */ yytestcase(yyruleno==80); - case 217: /* between_op ::= NOT BETWEEN */ yytestcase(yyruleno==217); - case 220: /* in_op ::= NOT IN */ yytestcase(yyruleno==220); - case 245: /* collate ::= COLLATE ID|STRING */ yytestcase(yyruleno==245); -{yymsp[-1].minor.yy394 = 1;} + case 219: /* between_op ::= NOT BETWEEN */ yytestcase(yyruleno==219); + case 222: /* in_op ::= NOT IN */ yytestcase(yyruleno==222); + case 247: /* collate ::= COLLATE ID|STRING */ yytestcase(yyruleno==247); +{yymsp[-1].minor.yy502 = 1;} break; case 64: /* init_deferred_pred_opt ::= INITIALLY IMMEDIATE */ -{yymsp[-1].minor.yy394 = 0;} +{yymsp[-1].minor.yy502 = 0;} break; case 66: /* tconscomma ::= COMMA */ -{pParse->constraintName.n = 0;} +{ASSERT_IS_CREATE; pParse->u1.cr.constraintName.n = 0;} break; case 68: /* tcons ::= PRIMARY KEY LP sortlist autoinc RP onconf */ -{sqlite3AddPrimaryKey(pParse,yymsp[-3].minor.yy322,yymsp[0].minor.yy394,yymsp[-2].minor.yy394,0);} +{sqlite3AddPrimaryKey(pParse,yymsp[-3].minor.yy402,yymsp[0].minor.yy502,yymsp[-2].minor.yy502,0);} break; case 69: /* tcons ::= UNIQUE LP sortlist RP onconf */ -{sqlite3CreateIndex(pParse,0,0,0,yymsp[-2].minor.yy322,yymsp[0].minor.yy394,0,0,0,0, +{sqlite3CreateIndex(pParse,0,0,0,yymsp[-2].minor.yy402,yymsp[0].minor.yy502,0,0,0,0, SQLITE_IDXTYPE_UNIQUE);} break; case 70: /* tcons ::= CHECK LP expr RP onconf */ -{sqlite3AddCheckConstraint(pParse,yymsp[-2].minor.yy528,yymsp[-3].minor.yy0.z,yymsp[-1].minor.yy0.z);} +{sqlite3AddCheckConstraint(pParse,yymsp[-2].minor.yy590,yymsp[-3].minor.yy0.z,yymsp[-1].minor.yy0.z);} break; case 71: /* tcons ::= FOREIGN KEY LP eidlist RP REFERENCES nm eidlist_opt refargs defer_subclause_opt */ { - sqlite3CreateForeignKey(pParse, yymsp[-6].minor.yy322, &yymsp[-3].minor.yy0, yymsp[-2].minor.yy322, yymsp[-1].minor.yy394); - sqlite3DeferForeignKey(pParse, yymsp[0].minor.yy394); + sqlite3CreateForeignKey(pParse, yymsp[-6].minor.yy402, &yymsp[-3].minor.yy0, yymsp[-2].minor.yy402, yymsp[-1].minor.yy502); + sqlite3DeferForeignKey(pParse, yymsp[0].minor.yy502); } break; case 73: /* onconf ::= */ case 75: /* orconf ::= */ yytestcase(yyruleno==75); -{yymsp[1].minor.yy394 = OE_Default;} +{yymsp[1].minor.yy502 = OE_Default;} break; case 74: /* onconf ::= ON CONFLICT resolvetype */ -{yymsp[-2].minor.yy394 = yymsp[0].minor.yy394;} +{yymsp[-2].minor.yy502 = yymsp[0].minor.yy502;} break; case 77: /* resolvetype ::= IGNORE */ -{yymsp[0].minor.yy394 = OE_Ignore;} +{yymsp[0].minor.yy502 = OE_Ignore;} break; case 78: /* resolvetype ::= REPLACE */ - case 172: /* insert_cmd ::= REPLACE */ yytestcase(yyruleno==172); -{yymsp[0].minor.yy394 = OE_Replace;} + case 174: /* insert_cmd ::= REPLACE */ yytestcase(yyruleno==174); +{yymsp[0].minor.yy502 = OE_Replace;} break; case 79: /* cmd ::= DROP TABLE ifexists fullname */ { - sqlite3DropTable(pParse, yymsp[0].minor.yy131, 0, yymsp[-1].minor.yy394); + sqlite3DropTable(pParse, yymsp[0].minor.yy563, 0, yymsp[-1].minor.yy502); } break; case 82: /* cmd ::= createkw temp VIEW ifnotexists nm dbnm eidlist_opt AS select */ { - sqlite3CreateView(pParse, &yymsp[-8].minor.yy0, &yymsp[-4].minor.yy0, &yymsp[-3].minor.yy0, yymsp[-2].minor.yy322, yymsp[0].minor.yy47, yymsp[-7].minor.yy394, yymsp[-5].minor.yy394); + sqlite3CreateView(pParse, &yymsp[-8].minor.yy0, &yymsp[-4].minor.yy0, &yymsp[-3].minor.yy0, yymsp[-2].minor.yy402, yymsp[0].minor.yy637, yymsp[-7].minor.yy502, yymsp[-5].minor.yy502); } break; case 83: /* cmd ::= DROP VIEW ifexists fullname */ { - sqlite3DropTable(pParse, yymsp[0].minor.yy131, 1, yymsp[-1].minor.yy394); + sqlite3DropTable(pParse, yymsp[0].minor.yy563, 1, yymsp[-1].minor.yy502); } break; case 84: /* cmd ::= select */ { SelectDest dest = {SRT_Output, 0, 0, 0, 0, 0, 0}; - sqlite3Select(pParse, yymsp[0].minor.yy47, &dest); - sqlite3SelectDelete(pParse->db, yymsp[0].minor.yy47); + if( (pParse->db->mDbFlags & DBFLAG_EncodingFixed)!=0 + || sqlite3ReadSchema(pParse)==SQLITE_OK + ){ + sqlite3Select(pParse, yymsp[0].minor.yy637, &dest); + } + sqlite3SelectDelete(pParse->db, yymsp[0].minor.yy637); } break; case 85: /* select ::= WITH wqlist selectnowith */ -{yymsp[-2].minor.yy47 = attachWithToSelect(pParse,yymsp[0].minor.yy47,yymsp[-1].minor.yy521);} +{yymsp[-2].minor.yy637 = attachWithToSelect(pParse,yymsp[0].minor.yy637,yymsp[-1].minor.yy125);} break; case 86: /* select ::= WITH RECURSIVE wqlist selectnowith */ -{yymsp[-3].minor.yy47 = attachWithToSelect(pParse,yymsp[0].minor.yy47,yymsp[-1].minor.yy521);} +{yymsp[-3].minor.yy637 = attachWithToSelect(pParse,yymsp[0].minor.yy637,yymsp[-1].minor.yy125);} break; case 87: /* select ::= selectnowith */ { - Select *p = yymsp[0].minor.yy47; + Select *p = yymsp[0].minor.yy637; if( p ){ parserDoubleLinkSelect(pParse, p); } @@ -174923,8 +181486,8 @@ static YYACTIONTYPE yy_reduce( break; case 88: /* selectnowith ::= selectnowith multiselect_op oneselect */ { - Select *pRhs = yymsp[0].minor.yy47; - Select *pLhs = yymsp[-2].minor.yy47; + Select *pRhs = yymsp[0].minor.yy637; + Select *pLhs = yymsp[-2].minor.yy637; if( pRhs && pRhs->pPrior ){ SrcList *pFrom; Token x; @@ -174934,153 +181497,160 @@ static YYACTIONTYPE yy_reduce( pRhs = sqlite3SelectNew(pParse,0,pFrom,0,0,0,0,0,0); } if( pRhs ){ - pRhs->op = (u8)yymsp[-1].minor.yy394; + pRhs->op = (u8)yymsp[-1].minor.yy502; pRhs->pPrior = pLhs; - if( ALWAYS(pLhs) ) pLhs->selFlags &= ~SF_MultiValue; - pRhs->selFlags &= ~SF_MultiValue; - if( yymsp[-1].minor.yy394!=TK_ALL ) pParse->hasCompound = 1; + if( ALWAYS(pLhs) ) pLhs->selFlags &= ~(u32)SF_MultiValue; + pRhs->selFlags &= ~(u32)SF_MultiValue; + if( yymsp[-1].minor.yy502!=TK_ALL ) pParse->hasCompound = 1; }else{ sqlite3SelectDelete(pParse->db, pLhs); } - yymsp[-2].minor.yy47 = pRhs; + yymsp[-2].minor.yy637 = pRhs; } break; case 89: /* multiselect_op ::= UNION */ case 91: /* multiselect_op ::= EXCEPT|INTERSECT */ yytestcase(yyruleno==91); -{yymsp[0].minor.yy394 = yymsp[0].major; /*A-overwrites-OP*/} +{yymsp[0].minor.yy502 = yymsp[0].major; /*A-overwrites-OP*/} break; case 90: /* multiselect_op ::= UNION ALL */ -{yymsp[-1].minor.yy394 = TK_ALL;} +{yymsp[-1].minor.yy502 = TK_ALL;} break; case 92: /* oneselect ::= SELECT distinct selcollist from where_opt groupby_opt having_opt orderby_opt limit_opt */ { - yymsp[-8].minor.yy47 = sqlite3SelectNew(pParse,yymsp[-6].minor.yy322,yymsp[-5].minor.yy131,yymsp[-4].minor.yy528,yymsp[-3].minor.yy322,yymsp[-2].minor.yy528,yymsp[-1].minor.yy322,yymsp[-7].minor.yy394,yymsp[0].minor.yy528); + yymsp[-8].minor.yy637 = sqlite3SelectNew(pParse,yymsp[-6].minor.yy402,yymsp[-5].minor.yy563,yymsp[-4].minor.yy590,yymsp[-3].minor.yy402,yymsp[-2].minor.yy590,yymsp[-1].minor.yy402,yymsp[-7].minor.yy502,yymsp[0].minor.yy590); } break; case 93: /* oneselect ::= SELECT distinct selcollist from where_opt groupby_opt having_opt window_clause orderby_opt limit_opt */ { - yymsp[-9].minor.yy47 = sqlite3SelectNew(pParse,yymsp[-7].minor.yy322,yymsp[-6].minor.yy131,yymsp[-5].minor.yy528,yymsp[-4].minor.yy322,yymsp[-3].minor.yy528,yymsp[-1].minor.yy322,yymsp[-8].minor.yy394,yymsp[0].minor.yy528); - if( yymsp[-9].minor.yy47 ){ - yymsp[-9].minor.yy47->pWinDefn = yymsp[-2].minor.yy41; + yymsp[-9].minor.yy637 = sqlite3SelectNew(pParse,yymsp[-7].minor.yy402,yymsp[-6].minor.yy563,yymsp[-5].minor.yy590,yymsp[-4].minor.yy402,yymsp[-3].minor.yy590,yymsp[-1].minor.yy402,yymsp[-8].minor.yy502,yymsp[0].minor.yy590); + if( yymsp[-9].minor.yy637 ){ + yymsp[-9].minor.yy637->pWinDefn = yymsp[-2].minor.yy483; }else{ - sqlite3WindowListDelete(pParse->db, yymsp[-2].minor.yy41); + sqlite3WindowListDelete(pParse->db, yymsp[-2].minor.yy483); } } break; case 94: /* values ::= VALUES LP nexprlist RP */ { - yymsp[-3].minor.yy47 = sqlite3SelectNew(pParse,yymsp[-1].minor.yy322,0,0,0,0,0,SF_Values,0); + yymsp[-3].minor.yy637 = sqlite3SelectNew(pParse,yymsp[-1].minor.yy402,0,0,0,0,0,SF_Values,0); } break; - case 95: /* values ::= values COMMA LP nexprlist RP */ + case 95: /* oneselect ::= mvalues */ { - Select *pRight, *pLeft = yymsp[-4].minor.yy47; - pRight = sqlite3SelectNew(pParse,yymsp[-1].minor.yy322,0,0,0,0,0,SF_Values|SF_MultiValue,0); - if( ALWAYS(pLeft) ) pLeft->selFlags &= ~SF_MultiValue; - if( pRight ){ - pRight->op = TK_ALL; - pRight->pPrior = pLeft; - yymsp[-4].minor.yy47 = pRight; - }else{ - yymsp[-4].minor.yy47 = pLeft; - } + sqlite3MultiValuesEnd(pParse, yymsp[0].minor.yy637); } break; - case 96: /* distinct ::= DISTINCT */ -{yymsp[0].minor.yy394 = SF_Distinct;} + case 96: /* mvalues ::= values COMMA LP nexprlist RP */ + case 97: /* mvalues ::= mvalues COMMA LP nexprlist RP */ yytestcase(yyruleno==97); +{ + yymsp[-4].minor.yy637 = sqlite3MultiValues(pParse, yymsp[-4].minor.yy637, yymsp[-1].minor.yy402); +} break; - case 97: /* distinct ::= ALL */ -{yymsp[0].minor.yy394 = SF_All;} + case 98: /* distinct ::= DISTINCT */ +{yymsp[0].minor.yy502 = SF_Distinct;} break; - case 99: /* sclp ::= */ - case 132: /* orderby_opt ::= */ yytestcase(yyruleno==132); - case 142: /* groupby_opt ::= */ yytestcase(yyruleno==142); - case 232: /* exprlist ::= */ yytestcase(yyruleno==232); - case 235: /* paren_exprlist ::= */ yytestcase(yyruleno==235); - case 240: /* eidlist_opt ::= */ yytestcase(yyruleno==240); -{yymsp[1].minor.yy322 = 0;} + case 99: /* distinct ::= ALL */ +{yymsp[0].minor.yy502 = SF_All;} break; - case 100: /* selcollist ::= sclp scanpt expr scanpt as */ + case 101: /* sclp ::= */ + case 134: /* orderby_opt ::= */ yytestcase(yyruleno==134); + case 144: /* groupby_opt ::= */ yytestcase(yyruleno==144); + case 234: /* exprlist ::= */ yytestcase(yyruleno==234); + case 237: /* paren_exprlist ::= */ yytestcase(yyruleno==237); + case 242: /* eidlist_opt ::= */ yytestcase(yyruleno==242); +{yymsp[1].minor.yy402 = 0;} + break; + case 102: /* selcollist ::= sclp scanpt expr scanpt as */ { - yymsp[-4].minor.yy322 = sqlite3ExprListAppend(pParse, yymsp[-4].minor.yy322, yymsp[-2].minor.yy528); - if( yymsp[0].minor.yy0.n>0 ) sqlite3ExprListSetName(pParse, yymsp[-4].minor.yy322, &yymsp[0].minor.yy0, 1); - sqlite3ExprListSetSpan(pParse,yymsp[-4].minor.yy322,yymsp[-3].minor.yy522,yymsp[-1].minor.yy522); + yymsp[-4].minor.yy402 = sqlite3ExprListAppend(pParse, yymsp[-4].minor.yy402, yymsp[-2].minor.yy590); + if( yymsp[0].minor.yy0.n>0 ) sqlite3ExprListSetName(pParse, yymsp[-4].minor.yy402, &yymsp[0].minor.yy0, 1); + sqlite3ExprListSetSpan(pParse,yymsp[-4].minor.yy402,yymsp[-3].minor.yy342,yymsp[-1].minor.yy342); } break; - case 101: /* selcollist ::= sclp scanpt STAR */ + case 103: /* selcollist ::= sclp scanpt STAR */ { Expr *p = sqlite3Expr(pParse->db, TK_ASTERISK, 0); sqlite3ExprSetErrorOffset(p, (int)(yymsp[0].minor.yy0.z - pParse->zTail)); - yymsp[-2].minor.yy322 = sqlite3ExprListAppend(pParse, yymsp[-2].minor.yy322, p); + yymsp[-2].minor.yy402 = sqlite3ExprListAppend(pParse, yymsp[-2].minor.yy402, p); } break; - case 102: /* selcollist ::= sclp scanpt nm DOT STAR */ + case 104: /* selcollist ::= sclp scanpt nm DOT STAR */ { Expr *pRight, *pLeft, *pDot; pRight = sqlite3PExpr(pParse, TK_ASTERISK, 0, 0); sqlite3ExprSetErrorOffset(pRight, (int)(yymsp[0].minor.yy0.z - pParse->zTail)); pLeft = tokenExpr(pParse, TK_ID, yymsp[-2].minor.yy0); pDot = sqlite3PExpr(pParse, TK_DOT, pLeft, pRight); - yymsp[-4].minor.yy322 = sqlite3ExprListAppend(pParse,yymsp[-4].minor.yy322, pDot); + yymsp[-4].minor.yy402 = sqlite3ExprListAppend(pParse,yymsp[-4].minor.yy402, pDot); } break; - case 103: /* as ::= AS nm */ - case 115: /* dbnm ::= DOT nm */ yytestcase(yyruleno==115); - case 256: /* plus_num ::= PLUS INTEGER|FLOAT */ yytestcase(yyruleno==256); - case 257: /* minus_num ::= MINUS INTEGER|FLOAT */ yytestcase(yyruleno==257); + case 105: /* as ::= AS nm */ + case 117: /* dbnm ::= DOT nm */ yytestcase(yyruleno==117); + case 258: /* plus_num ::= PLUS INTEGER|FLOAT */ yytestcase(yyruleno==258); + case 259: /* minus_num ::= MINUS INTEGER|FLOAT */ yytestcase(yyruleno==259); {yymsp[-1].minor.yy0 = yymsp[0].minor.yy0;} break; - case 105: /* from ::= */ - case 108: /* stl_prefix ::= */ yytestcase(yyruleno==108); -{yymsp[1].minor.yy131 = 0;} + case 107: /* from ::= */ + case 110: /* stl_prefix ::= */ yytestcase(yyruleno==110); +{yymsp[1].minor.yy563 = 0;} break; - case 106: /* from ::= FROM seltablist */ + case 108: /* from ::= FROM seltablist */ { - yymsp[-1].minor.yy131 = yymsp[0].minor.yy131; - sqlite3SrcListShiftJoinType(pParse,yymsp[-1].minor.yy131); + yymsp[-1].minor.yy563 = yymsp[0].minor.yy563; + sqlite3SrcListShiftJoinType(pParse,yymsp[-1].minor.yy563); } break; - case 107: /* stl_prefix ::= seltablist joinop */ + case 109: /* stl_prefix ::= seltablist joinop */ { - if( ALWAYS(yymsp[-1].minor.yy131 && yymsp[-1].minor.yy131->nSrc>0) ) yymsp[-1].minor.yy131->a[yymsp[-1].minor.yy131->nSrc-1].fg.jointype = (u8)yymsp[0].minor.yy394; + if( ALWAYS(yymsp[-1].minor.yy563 && yymsp[-1].minor.yy563->nSrc>0) ) yymsp[-1].minor.yy563->a[yymsp[-1].minor.yy563->nSrc-1].fg.jointype = (u8)yymsp[0].minor.yy502; } break; - case 109: /* seltablist ::= stl_prefix nm dbnm as on_using */ + case 111: /* seltablist ::= stl_prefix nm dbnm as on_using */ { - yymsp[-4].minor.yy131 = sqlite3SrcListAppendFromTerm(pParse,yymsp[-4].minor.yy131,&yymsp[-3].minor.yy0,&yymsp[-2].minor.yy0,&yymsp[-1].minor.yy0,0,&yymsp[0].minor.yy561); + yymsp[-4].minor.yy563 = sqlite3SrcListAppendFromTerm(pParse,yymsp[-4].minor.yy563,&yymsp[-3].minor.yy0,&yymsp[-2].minor.yy0,&yymsp[-1].minor.yy0,0,&yymsp[0].minor.yy421); } break; - case 110: /* seltablist ::= stl_prefix nm dbnm as indexed_by on_using */ + case 112: /* seltablist ::= stl_prefix nm dbnm as indexed_by on_using */ { - yymsp[-5].minor.yy131 = sqlite3SrcListAppendFromTerm(pParse,yymsp[-5].minor.yy131,&yymsp[-4].minor.yy0,&yymsp[-3].minor.yy0,&yymsp[-2].minor.yy0,0,&yymsp[0].minor.yy561); - sqlite3SrcListIndexedBy(pParse, yymsp[-5].minor.yy131, &yymsp[-1].minor.yy0); + yymsp[-5].minor.yy563 = sqlite3SrcListAppendFromTerm(pParse,yymsp[-5].minor.yy563,&yymsp[-4].minor.yy0,&yymsp[-3].minor.yy0,&yymsp[-2].minor.yy0,0,&yymsp[0].minor.yy421); + sqlite3SrcListIndexedBy(pParse, yymsp[-5].minor.yy563, &yymsp[-1].minor.yy0); } break; - case 111: /* seltablist ::= stl_prefix nm dbnm LP exprlist RP as on_using */ + case 113: /* seltablist ::= stl_prefix nm dbnm LP exprlist RP as on_using */ { - yymsp[-7].minor.yy131 = sqlite3SrcListAppendFromTerm(pParse,yymsp[-7].minor.yy131,&yymsp[-6].minor.yy0,&yymsp[-5].minor.yy0,&yymsp[-1].minor.yy0,0,&yymsp[0].minor.yy561); - sqlite3SrcListFuncArgs(pParse, yymsp[-7].minor.yy131, yymsp[-3].minor.yy322); + yymsp[-7].minor.yy563 = sqlite3SrcListAppendFromTerm(pParse,yymsp[-7].minor.yy563,&yymsp[-6].minor.yy0,&yymsp[-5].minor.yy0,&yymsp[-1].minor.yy0,0,&yymsp[0].minor.yy421); + sqlite3SrcListFuncArgs(pParse, yymsp[-7].minor.yy563, yymsp[-3].minor.yy402); } break; - case 112: /* seltablist ::= stl_prefix LP select RP as on_using */ + case 114: /* seltablist ::= stl_prefix LP select RP as on_using */ { - yymsp[-5].minor.yy131 = sqlite3SrcListAppendFromTerm(pParse,yymsp[-5].minor.yy131,0,0,&yymsp[-1].minor.yy0,yymsp[-3].minor.yy47,&yymsp[0].minor.yy561); + yymsp[-5].minor.yy563 = sqlite3SrcListAppendFromTerm(pParse,yymsp[-5].minor.yy563,0,0,&yymsp[-1].minor.yy0,yymsp[-3].minor.yy637,&yymsp[0].minor.yy421); } break; - case 113: /* seltablist ::= stl_prefix LP seltablist RP as on_using */ + case 115: /* seltablist ::= stl_prefix LP seltablist RP as on_using */ { - if( yymsp[-5].minor.yy131==0 && yymsp[-1].minor.yy0.n==0 && yymsp[0].minor.yy561.pOn==0 && yymsp[0].minor.yy561.pUsing==0 ){ - yymsp[-5].minor.yy131 = yymsp[-3].minor.yy131; - }else if( ALWAYS(yymsp[-3].minor.yy131!=0) && yymsp[-3].minor.yy131->nSrc==1 ){ - yymsp[-5].minor.yy131 = sqlite3SrcListAppendFromTerm(pParse,yymsp[-5].minor.yy131,0,0,&yymsp[-1].minor.yy0,0,&yymsp[0].minor.yy561); - if( yymsp[-5].minor.yy131 ){ - SrcItem *pNew = &yymsp[-5].minor.yy131->a[yymsp[-5].minor.yy131->nSrc-1]; - SrcItem *pOld = yymsp[-3].minor.yy131->a; + if( yymsp[-5].minor.yy563==0 && yymsp[-1].minor.yy0.n==0 && yymsp[0].minor.yy421.pOn==0 && yymsp[0].minor.yy421.pUsing==0 ){ + yymsp[-5].minor.yy563 = yymsp[-3].minor.yy563; + }else if( ALWAYS(yymsp[-3].minor.yy563!=0) && yymsp[-3].minor.yy563->nSrc==1 ){ + yymsp[-5].minor.yy563 = sqlite3SrcListAppendFromTerm(pParse,yymsp[-5].minor.yy563,0,0,&yymsp[-1].minor.yy0,0,&yymsp[0].minor.yy421); + if( yymsp[-5].minor.yy563 ){ + SrcItem *pNew = &yymsp[-5].minor.yy563->a[yymsp[-5].minor.yy563->nSrc-1]; + SrcItem *pOld = yymsp[-3].minor.yy563->a; + assert( pOld->fg.fixedSchema==0 ); pNew->zName = pOld->zName; - pNew->zDatabase = pOld->zDatabase; - pNew->pSelect = pOld->pSelect; - if( pNew->pSelect && (pNew->pSelect->selFlags & SF_NestedFrom)!=0 ){ - pNew->fg.isNestedFrom = 1; + assert( pOld->fg.fixedSchema==0 ); + if( pOld->fg.isSubquery ){ + pNew->fg.isSubquery = 1; + pNew->u4.pSubq = pOld->u4.pSubq; + pOld->u4.pSubq = 0; + pOld->fg.isSubquery = 0; + assert( pNew->u4.pSubq!=0 && pNew->u4.pSubq->pSelect!=0 ); + if( (pNew->u4.pSubq->pSelect->selFlags & SF_NestedFrom)!=0 ){ + pNew->fg.isNestedFrom = 1; + } + }else{ + pNew->u4.zDatabase = pOld->u4.zDatabase; + pOld->u4.zDatabase = 0; } if( pOld->fg.isTabFunc ){ pNew->u1.pFuncArg = pOld->u1.pFuncArg; @@ -175088,156 +181658,155 @@ static YYACTIONTYPE yy_reduce( pOld->fg.isTabFunc = 0; pNew->fg.isTabFunc = 1; } - pOld->zName = pOld->zDatabase = 0; - pOld->pSelect = 0; + pOld->zName = 0; } - sqlite3SrcListDelete(pParse->db, yymsp[-3].minor.yy131); + sqlite3SrcListDelete(pParse->db, yymsp[-3].minor.yy563); }else{ Select *pSubquery; - sqlite3SrcListShiftJoinType(pParse,yymsp[-3].minor.yy131); - pSubquery = sqlite3SelectNew(pParse,0,yymsp[-3].minor.yy131,0,0,0,0,SF_NestedFrom,0); - yymsp[-5].minor.yy131 = sqlite3SrcListAppendFromTerm(pParse,yymsp[-5].minor.yy131,0,0,&yymsp[-1].minor.yy0,pSubquery,&yymsp[0].minor.yy561); + sqlite3SrcListShiftJoinType(pParse,yymsp[-3].minor.yy563); + pSubquery = sqlite3SelectNew(pParse,0,yymsp[-3].minor.yy563,0,0,0,0,SF_NestedFrom,0); + yymsp[-5].minor.yy563 = sqlite3SrcListAppendFromTerm(pParse,yymsp[-5].minor.yy563,0,0,&yymsp[-1].minor.yy0,pSubquery,&yymsp[0].minor.yy421); } } break; - case 114: /* dbnm ::= */ - case 129: /* indexed_opt ::= */ yytestcase(yyruleno==129); + case 116: /* dbnm ::= */ + case 131: /* indexed_opt ::= */ yytestcase(yyruleno==131); {yymsp[1].minor.yy0.z=0; yymsp[1].minor.yy0.n=0;} break; - case 116: /* fullname ::= nm */ + case 118: /* fullname ::= nm */ { - yylhsminor.yy131 = sqlite3SrcListAppend(pParse,0,&yymsp[0].minor.yy0,0); - if( IN_RENAME_OBJECT && yylhsminor.yy131 ) sqlite3RenameTokenMap(pParse, yylhsminor.yy131->a[0].zName, &yymsp[0].minor.yy0); + yylhsminor.yy563 = sqlite3SrcListAppend(pParse,0,&yymsp[0].minor.yy0,0); + if( IN_RENAME_OBJECT && yylhsminor.yy563 ) sqlite3RenameTokenMap(pParse, yylhsminor.yy563->a[0].zName, &yymsp[0].minor.yy0); } - yymsp[0].minor.yy131 = yylhsminor.yy131; + yymsp[0].minor.yy563 = yylhsminor.yy563; break; - case 117: /* fullname ::= nm DOT nm */ + case 119: /* fullname ::= nm DOT nm */ { - yylhsminor.yy131 = sqlite3SrcListAppend(pParse,0,&yymsp[-2].minor.yy0,&yymsp[0].minor.yy0); - if( IN_RENAME_OBJECT && yylhsminor.yy131 ) sqlite3RenameTokenMap(pParse, yylhsminor.yy131->a[0].zName, &yymsp[0].minor.yy0); + yylhsminor.yy563 = sqlite3SrcListAppend(pParse,0,&yymsp[-2].minor.yy0,&yymsp[0].minor.yy0); + if( IN_RENAME_OBJECT && yylhsminor.yy563 ) sqlite3RenameTokenMap(pParse, yylhsminor.yy563->a[0].zName, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy131 = yylhsminor.yy131; + yymsp[-2].minor.yy563 = yylhsminor.yy563; break; - case 118: /* xfullname ::= nm */ -{yymsp[0].minor.yy131 = sqlite3SrcListAppend(pParse,0,&yymsp[0].minor.yy0,0); /*A-overwrites-X*/} + case 120: /* xfullname ::= nm */ +{yymsp[0].minor.yy563 = sqlite3SrcListAppend(pParse,0,&yymsp[0].minor.yy0,0); /*A-overwrites-X*/} break; - case 119: /* xfullname ::= nm DOT nm */ -{yymsp[-2].minor.yy131 = sqlite3SrcListAppend(pParse,0,&yymsp[-2].minor.yy0,&yymsp[0].minor.yy0); /*A-overwrites-X*/} + case 121: /* xfullname ::= nm DOT nm */ +{yymsp[-2].minor.yy563 = sqlite3SrcListAppend(pParse,0,&yymsp[-2].minor.yy0,&yymsp[0].minor.yy0); /*A-overwrites-X*/} break; - case 120: /* xfullname ::= nm DOT nm AS nm */ + case 122: /* xfullname ::= nm DOT nm AS nm */ { - yymsp[-4].minor.yy131 = sqlite3SrcListAppend(pParse,0,&yymsp[-4].minor.yy0,&yymsp[-2].minor.yy0); /*A-overwrites-X*/ - if( yymsp[-4].minor.yy131 ) yymsp[-4].minor.yy131->a[0].zAlias = sqlite3NameFromToken(pParse->db, &yymsp[0].minor.yy0); + yymsp[-4].minor.yy563 = sqlite3SrcListAppend(pParse,0,&yymsp[-4].minor.yy0,&yymsp[-2].minor.yy0); /*A-overwrites-X*/ + if( yymsp[-4].minor.yy563 ) yymsp[-4].minor.yy563->a[0].zAlias = sqlite3NameFromToken(pParse->db, &yymsp[0].minor.yy0); } break; - case 121: /* xfullname ::= nm AS nm */ + case 123: /* xfullname ::= nm AS nm */ { - yymsp[-2].minor.yy131 = sqlite3SrcListAppend(pParse,0,&yymsp[-2].minor.yy0,0); /*A-overwrites-X*/ - if( yymsp[-2].minor.yy131 ) yymsp[-2].minor.yy131->a[0].zAlias = sqlite3NameFromToken(pParse->db, &yymsp[0].minor.yy0); + yymsp[-2].minor.yy563 = sqlite3SrcListAppend(pParse,0,&yymsp[-2].minor.yy0,0); /*A-overwrites-X*/ + if( yymsp[-2].minor.yy563 ) yymsp[-2].minor.yy563->a[0].zAlias = sqlite3NameFromToken(pParse->db, &yymsp[0].minor.yy0); } break; - case 122: /* joinop ::= COMMA|JOIN */ -{ yymsp[0].minor.yy394 = JT_INNER; } + case 124: /* joinop ::= COMMA|JOIN */ +{ yymsp[0].minor.yy502 = JT_INNER; } break; - case 123: /* joinop ::= JOIN_KW JOIN */ -{yymsp[-1].minor.yy394 = sqlite3JoinType(pParse,&yymsp[-1].minor.yy0,0,0); /*X-overwrites-A*/} + case 125: /* joinop ::= JOIN_KW JOIN */ +{yymsp[-1].minor.yy502 = sqlite3JoinType(pParse,&yymsp[-1].minor.yy0,0,0); /*X-overwrites-A*/} break; - case 124: /* joinop ::= JOIN_KW nm JOIN */ -{yymsp[-2].minor.yy394 = sqlite3JoinType(pParse,&yymsp[-2].minor.yy0,&yymsp[-1].minor.yy0,0); /*X-overwrites-A*/} + case 126: /* joinop ::= JOIN_KW nm JOIN */ +{yymsp[-2].minor.yy502 = sqlite3JoinType(pParse,&yymsp[-2].minor.yy0,&yymsp[-1].minor.yy0,0); /*X-overwrites-A*/} break; - case 125: /* joinop ::= JOIN_KW nm nm JOIN */ -{yymsp[-3].minor.yy394 = sqlite3JoinType(pParse,&yymsp[-3].minor.yy0,&yymsp[-2].minor.yy0,&yymsp[-1].minor.yy0);/*X-overwrites-A*/} + case 127: /* joinop ::= JOIN_KW nm nm JOIN */ +{yymsp[-3].minor.yy502 = sqlite3JoinType(pParse,&yymsp[-3].minor.yy0,&yymsp[-2].minor.yy0,&yymsp[-1].minor.yy0);/*X-overwrites-A*/} break; - case 126: /* on_using ::= ON expr */ -{yymsp[-1].minor.yy561.pOn = yymsp[0].minor.yy528; yymsp[-1].minor.yy561.pUsing = 0;} + case 128: /* on_using ::= ON expr */ +{yymsp[-1].minor.yy421.pOn = yymsp[0].minor.yy590; yymsp[-1].minor.yy421.pUsing = 0;} break; - case 127: /* on_using ::= USING LP idlist RP */ -{yymsp[-3].minor.yy561.pOn = 0; yymsp[-3].minor.yy561.pUsing = yymsp[-1].minor.yy254;} + case 129: /* on_using ::= USING LP idlist RP */ +{yymsp[-3].minor.yy421.pOn = 0; yymsp[-3].minor.yy421.pUsing = yymsp[-1].minor.yy204;} break; - case 128: /* on_using ::= */ -{yymsp[1].minor.yy561.pOn = 0; yymsp[1].minor.yy561.pUsing = 0;} + case 130: /* on_using ::= */ +{yymsp[1].minor.yy421.pOn = 0; yymsp[1].minor.yy421.pUsing = 0;} break; - case 130: /* indexed_by ::= INDEXED BY nm */ + case 132: /* indexed_by ::= INDEXED BY nm */ {yymsp[-2].minor.yy0 = yymsp[0].minor.yy0;} break; - case 131: /* indexed_by ::= NOT INDEXED */ + case 133: /* indexed_by ::= NOT INDEXED */ {yymsp[-1].minor.yy0.z=0; yymsp[-1].minor.yy0.n=1;} break; - case 133: /* orderby_opt ::= ORDER BY sortlist */ - case 143: /* groupby_opt ::= GROUP BY nexprlist */ yytestcase(yyruleno==143); -{yymsp[-2].minor.yy322 = yymsp[0].minor.yy322;} + case 135: /* orderby_opt ::= ORDER BY sortlist */ + case 145: /* groupby_opt ::= GROUP BY nexprlist */ yytestcase(yyruleno==145); +{yymsp[-2].minor.yy402 = yymsp[0].minor.yy402;} break; - case 134: /* sortlist ::= sortlist COMMA expr sortorder nulls */ + case 136: /* sortlist ::= sortlist COMMA expr sortorder nulls */ { - yymsp[-4].minor.yy322 = sqlite3ExprListAppend(pParse,yymsp[-4].minor.yy322,yymsp[-2].minor.yy528); - sqlite3ExprListSetSortOrder(yymsp[-4].minor.yy322,yymsp[-1].minor.yy394,yymsp[0].minor.yy394); + yymsp[-4].minor.yy402 = sqlite3ExprListAppend(pParse,yymsp[-4].minor.yy402,yymsp[-2].minor.yy590); + sqlite3ExprListSetSortOrder(yymsp[-4].minor.yy402,yymsp[-1].minor.yy502,yymsp[0].minor.yy502); } break; - case 135: /* sortlist ::= expr sortorder nulls */ + case 137: /* sortlist ::= expr sortorder nulls */ { - yymsp[-2].minor.yy322 = sqlite3ExprListAppend(pParse,0,yymsp[-2].minor.yy528); /*A-overwrites-Y*/ - sqlite3ExprListSetSortOrder(yymsp[-2].minor.yy322,yymsp[-1].minor.yy394,yymsp[0].minor.yy394); + yymsp[-2].minor.yy402 = sqlite3ExprListAppend(pParse,0,yymsp[-2].minor.yy590); /*A-overwrites-Y*/ + sqlite3ExprListSetSortOrder(yymsp[-2].minor.yy402,yymsp[-1].minor.yy502,yymsp[0].minor.yy502); } break; - case 136: /* sortorder ::= ASC */ -{yymsp[0].minor.yy394 = SQLITE_SO_ASC;} + case 138: /* sortorder ::= ASC */ +{yymsp[0].minor.yy502 = SQLITE_SO_ASC;} break; - case 137: /* sortorder ::= DESC */ -{yymsp[0].minor.yy394 = SQLITE_SO_DESC;} + case 139: /* sortorder ::= DESC */ +{yymsp[0].minor.yy502 = SQLITE_SO_DESC;} break; - case 138: /* sortorder ::= */ - case 141: /* nulls ::= */ yytestcase(yyruleno==141); -{yymsp[1].minor.yy394 = SQLITE_SO_UNDEFINED;} + case 140: /* sortorder ::= */ + case 143: /* nulls ::= */ yytestcase(yyruleno==143); +{yymsp[1].minor.yy502 = SQLITE_SO_UNDEFINED;} break; - case 139: /* nulls ::= NULLS FIRST */ -{yymsp[-1].minor.yy394 = SQLITE_SO_ASC;} + case 141: /* nulls ::= NULLS FIRST */ +{yymsp[-1].minor.yy502 = SQLITE_SO_ASC;} break; - case 140: /* nulls ::= NULLS LAST */ -{yymsp[-1].minor.yy394 = SQLITE_SO_DESC;} + case 142: /* nulls ::= NULLS LAST */ +{yymsp[-1].minor.yy502 = SQLITE_SO_DESC;} break; - case 144: /* having_opt ::= */ - case 146: /* limit_opt ::= */ yytestcase(yyruleno==146); - case 151: /* where_opt ::= */ yytestcase(yyruleno==151); - case 153: /* where_opt_ret ::= */ yytestcase(yyruleno==153); - case 230: /* case_else ::= */ yytestcase(yyruleno==230); - case 231: /* case_operand ::= */ yytestcase(yyruleno==231); - case 250: /* vinto ::= */ yytestcase(yyruleno==250); -{yymsp[1].minor.yy528 = 0;} + case 146: /* having_opt ::= */ + case 148: /* limit_opt ::= */ yytestcase(yyruleno==148); + case 153: /* where_opt ::= */ yytestcase(yyruleno==153); + case 155: /* where_opt_ret ::= */ yytestcase(yyruleno==155); + case 232: /* case_else ::= */ yytestcase(yyruleno==232); + case 233: /* case_operand ::= */ yytestcase(yyruleno==233); + case 252: /* vinto ::= */ yytestcase(yyruleno==252); +{yymsp[1].minor.yy590 = 0;} break; - case 145: /* having_opt ::= HAVING expr */ - case 152: /* where_opt ::= WHERE expr */ yytestcase(yyruleno==152); - case 154: /* where_opt_ret ::= WHERE expr */ yytestcase(yyruleno==154); - case 229: /* case_else ::= ELSE expr */ yytestcase(yyruleno==229); - case 249: /* vinto ::= INTO expr */ yytestcase(yyruleno==249); -{yymsp[-1].minor.yy528 = yymsp[0].minor.yy528;} + case 147: /* having_opt ::= HAVING expr */ + case 154: /* where_opt ::= WHERE expr */ yytestcase(yyruleno==154); + case 156: /* where_opt_ret ::= WHERE expr */ yytestcase(yyruleno==156); + case 231: /* case_else ::= ELSE expr */ yytestcase(yyruleno==231); + case 251: /* vinto ::= INTO expr */ yytestcase(yyruleno==251); +{yymsp[-1].minor.yy590 = yymsp[0].minor.yy590;} break; - case 147: /* limit_opt ::= LIMIT expr */ -{yymsp[-1].minor.yy528 = sqlite3PExpr(pParse,TK_LIMIT,yymsp[0].minor.yy528,0);} + case 149: /* limit_opt ::= LIMIT expr */ +{yymsp[-1].minor.yy590 = sqlite3PExpr(pParse,TK_LIMIT,yymsp[0].minor.yy590,0);} break; - case 148: /* limit_opt ::= LIMIT expr OFFSET expr */ -{yymsp[-3].minor.yy528 = sqlite3PExpr(pParse,TK_LIMIT,yymsp[-2].minor.yy528,yymsp[0].minor.yy528);} + case 150: /* limit_opt ::= LIMIT expr OFFSET expr */ +{yymsp[-3].minor.yy590 = sqlite3PExpr(pParse,TK_LIMIT,yymsp[-2].minor.yy590,yymsp[0].minor.yy590);} break; - case 149: /* limit_opt ::= LIMIT expr COMMA expr */ -{yymsp[-3].minor.yy528 = sqlite3PExpr(pParse,TK_LIMIT,yymsp[0].minor.yy528,yymsp[-2].minor.yy528);} + case 151: /* limit_opt ::= LIMIT expr COMMA expr */ +{yymsp[-3].minor.yy590 = sqlite3PExpr(pParse,TK_LIMIT,yymsp[0].minor.yy590,yymsp[-2].minor.yy590);} break; - case 150: /* cmd ::= with DELETE FROM xfullname indexed_opt where_opt_ret */ + case 152: /* cmd ::= with DELETE FROM xfullname indexed_opt where_opt_ret */ { - sqlite3SrcListIndexedBy(pParse, yymsp[-2].minor.yy131, &yymsp[-1].minor.yy0); - sqlite3DeleteFrom(pParse,yymsp[-2].minor.yy131,yymsp[0].minor.yy528,0,0); + sqlite3SrcListIndexedBy(pParse, yymsp[-2].minor.yy563, &yymsp[-1].minor.yy0); + sqlite3DeleteFrom(pParse,yymsp[-2].minor.yy563,yymsp[0].minor.yy590,0,0); } break; - case 155: /* where_opt_ret ::= RETURNING selcollist */ -{sqlite3AddReturning(pParse,yymsp[0].minor.yy322); yymsp[-1].minor.yy528 = 0;} + case 157: /* where_opt_ret ::= RETURNING selcollist */ +{sqlite3AddReturning(pParse,yymsp[0].minor.yy402); yymsp[-1].minor.yy590 = 0;} break; - case 156: /* where_opt_ret ::= WHERE expr RETURNING selcollist */ -{sqlite3AddReturning(pParse,yymsp[0].minor.yy322); yymsp[-3].minor.yy528 = yymsp[-2].minor.yy528;} + case 158: /* where_opt_ret ::= WHERE expr RETURNING selcollist */ +{sqlite3AddReturning(pParse,yymsp[0].minor.yy402); yymsp[-3].minor.yy590 = yymsp[-2].minor.yy590;} break; - case 157: /* cmd ::= with UPDATE orconf xfullname indexed_opt SET setlist from where_opt_ret */ + case 159: /* cmd ::= with UPDATE orconf xfullname indexed_opt SET setlist from where_opt_ret */ { - sqlite3SrcListIndexedBy(pParse, yymsp[-5].minor.yy131, &yymsp[-4].minor.yy0); - sqlite3ExprListCheckLength(pParse,yymsp[-2].minor.yy322,"set list"); - if( yymsp[-1].minor.yy131 ){ - SrcList *pFromClause = yymsp[-1].minor.yy131; + sqlite3SrcListIndexedBy(pParse, yymsp[-5].minor.yy563, &yymsp[-4].minor.yy0); + sqlite3ExprListCheckLength(pParse,yymsp[-2].minor.yy402,"set list"); + if( yymsp[-1].minor.yy563 ){ + SrcList *pFromClause = yymsp[-1].minor.yy563; if( pFromClause->nSrc>1 ){ Select *pSubquery; Token as; @@ -175246,92 +181815,92 @@ static YYACTIONTYPE yy_reduce( as.z = 0; pFromClause = sqlite3SrcListAppendFromTerm(pParse,0,0,0,&as,pSubquery,0); } - yymsp[-5].minor.yy131 = sqlite3SrcListAppendList(pParse, yymsp[-5].minor.yy131, pFromClause); + yymsp[-5].minor.yy563 = sqlite3SrcListAppendList(pParse, yymsp[-5].minor.yy563, pFromClause); } - sqlite3Update(pParse,yymsp[-5].minor.yy131,yymsp[-2].minor.yy322,yymsp[0].minor.yy528,yymsp[-6].minor.yy394,0,0,0); + sqlite3Update(pParse,yymsp[-5].minor.yy563,yymsp[-2].minor.yy402,yymsp[0].minor.yy590,yymsp[-6].minor.yy502,0,0,0); } break; - case 158: /* setlist ::= setlist COMMA nm EQ expr */ + case 160: /* setlist ::= setlist COMMA nm EQ expr */ { - yymsp[-4].minor.yy322 = sqlite3ExprListAppend(pParse, yymsp[-4].minor.yy322, yymsp[0].minor.yy528); - sqlite3ExprListSetName(pParse, yymsp[-4].minor.yy322, &yymsp[-2].minor.yy0, 1); + yymsp[-4].minor.yy402 = sqlite3ExprListAppend(pParse, yymsp[-4].minor.yy402, yymsp[0].minor.yy590); + sqlite3ExprListSetName(pParse, yymsp[-4].minor.yy402, &yymsp[-2].minor.yy0, 1); } break; - case 159: /* setlist ::= setlist COMMA LP idlist RP EQ expr */ + case 161: /* setlist ::= setlist COMMA LP idlist RP EQ expr */ { - yymsp[-6].minor.yy322 = sqlite3ExprListAppendVector(pParse, yymsp[-6].minor.yy322, yymsp[-3].minor.yy254, yymsp[0].minor.yy528); + yymsp[-6].minor.yy402 = sqlite3ExprListAppendVector(pParse, yymsp[-6].minor.yy402, yymsp[-3].minor.yy204, yymsp[0].minor.yy590); } break; - case 160: /* setlist ::= nm EQ expr */ + case 162: /* setlist ::= nm EQ expr */ { - yylhsminor.yy322 = sqlite3ExprListAppend(pParse, 0, yymsp[0].minor.yy528); - sqlite3ExprListSetName(pParse, yylhsminor.yy322, &yymsp[-2].minor.yy0, 1); + yylhsminor.yy402 = sqlite3ExprListAppend(pParse, 0, yymsp[0].minor.yy590); + sqlite3ExprListSetName(pParse, yylhsminor.yy402, &yymsp[-2].minor.yy0, 1); } - yymsp[-2].minor.yy322 = yylhsminor.yy322; + yymsp[-2].minor.yy402 = yylhsminor.yy402; break; - case 161: /* setlist ::= LP idlist RP EQ expr */ + case 163: /* setlist ::= LP idlist RP EQ expr */ { - yymsp[-4].minor.yy322 = sqlite3ExprListAppendVector(pParse, 0, yymsp[-3].minor.yy254, yymsp[0].minor.yy528); + yymsp[-4].minor.yy402 = sqlite3ExprListAppendVector(pParse, 0, yymsp[-3].minor.yy204, yymsp[0].minor.yy590); } break; - case 162: /* cmd ::= with insert_cmd INTO xfullname idlist_opt select upsert */ + case 164: /* cmd ::= with insert_cmd INTO xfullname idlist_opt select upsert */ { - sqlite3Insert(pParse, yymsp[-3].minor.yy131, yymsp[-1].minor.yy47, yymsp[-2].minor.yy254, yymsp[-5].minor.yy394, yymsp[0].minor.yy444); + sqlite3Insert(pParse, yymsp[-3].minor.yy563, yymsp[-1].minor.yy637, yymsp[-2].minor.yy204, yymsp[-5].minor.yy502, yymsp[0].minor.yy403); } break; - case 163: /* cmd ::= with insert_cmd INTO xfullname idlist_opt DEFAULT VALUES returning */ + case 165: /* cmd ::= with insert_cmd INTO xfullname idlist_opt DEFAULT VALUES returning */ { - sqlite3Insert(pParse, yymsp[-4].minor.yy131, 0, yymsp[-3].minor.yy254, yymsp[-6].minor.yy394, 0); + sqlite3Insert(pParse, yymsp[-4].minor.yy563, 0, yymsp[-3].minor.yy204, yymsp[-6].minor.yy502, 0); } break; - case 164: /* upsert ::= */ -{ yymsp[1].minor.yy444 = 0; } + case 166: /* upsert ::= */ +{ yymsp[1].minor.yy403 = 0; } break; - case 165: /* upsert ::= RETURNING selcollist */ -{ yymsp[-1].minor.yy444 = 0; sqlite3AddReturning(pParse,yymsp[0].minor.yy322); } + case 167: /* upsert ::= RETURNING selcollist */ +{ yymsp[-1].minor.yy403 = 0; sqlite3AddReturning(pParse,yymsp[0].minor.yy402); } break; - case 166: /* upsert ::= ON CONFLICT LP sortlist RP where_opt DO UPDATE SET setlist where_opt upsert */ -{ yymsp[-11].minor.yy444 = sqlite3UpsertNew(pParse->db,yymsp[-8].minor.yy322,yymsp[-6].minor.yy528,yymsp[-2].minor.yy322,yymsp[-1].minor.yy528,yymsp[0].minor.yy444);} + case 168: /* upsert ::= ON CONFLICT LP sortlist RP where_opt DO UPDATE SET setlist where_opt upsert */ +{ yymsp[-11].minor.yy403 = sqlite3UpsertNew(pParse->db,yymsp[-8].minor.yy402,yymsp[-6].minor.yy590,yymsp[-2].minor.yy402,yymsp[-1].minor.yy590,yymsp[0].minor.yy403);} break; - case 167: /* upsert ::= ON CONFLICT LP sortlist RP where_opt DO NOTHING upsert */ -{ yymsp[-8].minor.yy444 = sqlite3UpsertNew(pParse->db,yymsp[-5].minor.yy322,yymsp[-3].minor.yy528,0,0,yymsp[0].minor.yy444); } + case 169: /* upsert ::= ON CONFLICT LP sortlist RP where_opt DO NOTHING upsert */ +{ yymsp[-8].minor.yy403 = sqlite3UpsertNew(pParse->db,yymsp[-5].minor.yy402,yymsp[-3].minor.yy590,0,0,yymsp[0].minor.yy403); } break; - case 168: /* upsert ::= ON CONFLICT DO NOTHING returning */ -{ yymsp[-4].minor.yy444 = sqlite3UpsertNew(pParse->db,0,0,0,0,0); } + case 170: /* upsert ::= ON CONFLICT DO NOTHING returning */ +{ yymsp[-4].minor.yy403 = sqlite3UpsertNew(pParse->db,0,0,0,0,0); } break; - case 169: /* upsert ::= ON CONFLICT DO UPDATE SET setlist where_opt returning */ -{ yymsp[-7].minor.yy444 = sqlite3UpsertNew(pParse->db,0,0,yymsp[-2].minor.yy322,yymsp[-1].minor.yy528,0);} + case 171: /* upsert ::= ON CONFLICT DO UPDATE SET setlist where_opt returning */ +{ yymsp[-7].minor.yy403 = sqlite3UpsertNew(pParse->db,0,0,yymsp[-2].minor.yy402,yymsp[-1].minor.yy590,0);} break; - case 170: /* returning ::= RETURNING selcollist */ -{sqlite3AddReturning(pParse,yymsp[0].minor.yy322);} + case 172: /* returning ::= RETURNING selcollist */ +{sqlite3AddReturning(pParse,yymsp[0].minor.yy402);} break; - case 173: /* idlist_opt ::= */ -{yymsp[1].minor.yy254 = 0;} + case 175: /* idlist_opt ::= */ +{yymsp[1].minor.yy204 = 0;} break; - case 174: /* idlist_opt ::= LP idlist RP */ -{yymsp[-2].minor.yy254 = yymsp[-1].minor.yy254;} + case 176: /* idlist_opt ::= LP idlist RP */ +{yymsp[-2].minor.yy204 = yymsp[-1].minor.yy204;} break; - case 175: /* idlist ::= idlist COMMA nm */ -{yymsp[-2].minor.yy254 = sqlite3IdListAppend(pParse,yymsp[-2].minor.yy254,&yymsp[0].minor.yy0);} + case 177: /* idlist ::= idlist COMMA nm */ +{yymsp[-2].minor.yy204 = sqlite3IdListAppend(pParse,yymsp[-2].minor.yy204,&yymsp[0].minor.yy0);} break; - case 176: /* idlist ::= nm */ -{yymsp[0].minor.yy254 = sqlite3IdListAppend(pParse,0,&yymsp[0].minor.yy0); /*A-overwrites-Y*/} + case 178: /* idlist ::= nm */ +{yymsp[0].minor.yy204 = sqlite3IdListAppend(pParse,0,&yymsp[0].minor.yy0); /*A-overwrites-Y*/} break; - case 177: /* expr ::= LP expr RP */ -{yymsp[-2].minor.yy528 = yymsp[-1].minor.yy528;} + case 179: /* expr ::= LP expr RP */ +{yymsp[-2].minor.yy590 = yymsp[-1].minor.yy590;} break; - case 178: /* expr ::= ID|INDEXED|JOIN_KW */ -{yymsp[0].minor.yy528=tokenExpr(pParse,TK_ID,yymsp[0].minor.yy0); /*A-overwrites-X*/} + case 180: /* expr ::= ID|INDEXED|JOIN_KW */ +{yymsp[0].minor.yy590=tokenExpr(pParse,TK_ID,yymsp[0].minor.yy0); /*A-overwrites-X*/} break; - case 179: /* expr ::= nm DOT nm */ + case 181: /* expr ::= nm DOT nm */ { Expr *temp1 = tokenExpr(pParse,TK_ID,yymsp[-2].minor.yy0); Expr *temp2 = tokenExpr(pParse,TK_ID,yymsp[0].minor.yy0); - yylhsminor.yy528 = sqlite3PExpr(pParse, TK_DOT, temp1, temp2); + yylhsminor.yy590 = sqlite3PExpr(pParse, TK_DOT, temp1, temp2); } - yymsp[-2].minor.yy528 = yylhsminor.yy528; + yymsp[-2].minor.yy590 = yylhsminor.yy590; break; - case 180: /* expr ::= nm DOT nm DOT nm */ + case 182: /* expr ::= nm DOT nm DOT nm */ { Expr *temp1 = tokenExpr(pParse,TK_ID,yymsp[-4].minor.yy0); Expr *temp2 = tokenExpr(pParse,TK_ID,yymsp[-2].minor.yy0); @@ -175340,27 +181909,27 @@ static YYACTIONTYPE yy_reduce( if( IN_RENAME_OBJECT ){ sqlite3RenameTokenRemap(pParse, 0, temp1); } - yylhsminor.yy528 = sqlite3PExpr(pParse, TK_DOT, temp1, temp4); + yylhsminor.yy590 = sqlite3PExpr(pParse, TK_DOT, temp1, temp4); } - yymsp[-4].minor.yy528 = yylhsminor.yy528; + yymsp[-4].minor.yy590 = yylhsminor.yy590; break; - case 181: /* term ::= NULL|FLOAT|BLOB */ - case 182: /* term ::= STRING */ yytestcase(yyruleno==182); -{yymsp[0].minor.yy528=tokenExpr(pParse,yymsp[0].major,yymsp[0].minor.yy0); /*A-overwrites-X*/} + case 183: /* term ::= NULL|FLOAT|BLOB */ + case 184: /* term ::= STRING */ yytestcase(yyruleno==184); +{yymsp[0].minor.yy590=tokenExpr(pParse,yymsp[0].major,yymsp[0].minor.yy0); /*A-overwrites-X*/} break; - case 183: /* term ::= INTEGER */ + case 185: /* term ::= INTEGER */ { - yylhsminor.yy528 = sqlite3ExprAlloc(pParse->db, TK_INTEGER, &yymsp[0].minor.yy0, 1); - if( yylhsminor.yy528 ) yylhsminor.yy528->w.iOfst = (int)(yymsp[0].minor.yy0.z - pParse->zTail); + yylhsminor.yy590 = sqlite3ExprAlloc(pParse->db, TK_INTEGER, &yymsp[0].minor.yy0, 1); + if( yylhsminor.yy590 ) yylhsminor.yy590->w.iOfst = (int)(yymsp[0].minor.yy0.z - pParse->zTail); } - yymsp[0].minor.yy528 = yylhsminor.yy528; + yymsp[0].minor.yy590 = yylhsminor.yy590; break; - case 184: /* expr ::= VARIABLE */ + case 186: /* expr ::= VARIABLE */ { if( !(yymsp[0].minor.yy0.z[0]=='#' && sqlite3Isdigit(yymsp[0].minor.yy0.z[1])) ){ u32 n = yymsp[0].minor.yy0.n; - yymsp[0].minor.yy528 = tokenExpr(pParse, TK_VARIABLE, yymsp[0].minor.yy0); - sqlite3ExprAssignVarNumber(pParse, yymsp[0].minor.yy528, n); + yymsp[0].minor.yy590 = tokenExpr(pParse, TK_VARIABLE, yymsp[0].minor.yy0); + sqlite3ExprAssignVarNumber(pParse, yymsp[0].minor.yy590, n); }else{ /* When doing a nested parse, one can include terms in an expression ** that look like this: #1 #2 ... These terms refer to registers @@ -175368,405 +181937,427 @@ static YYACTIONTYPE yy_reduce( Token t = yymsp[0].minor.yy0; /*A-overwrites-X*/ assert( t.n>=2 ); if( pParse->nested==0 ){ - sqlite3ErrorMsg(pParse, "near \"%T\": syntax error", &t); - yymsp[0].minor.yy528 = 0; + parserSyntaxError(pParse, &t); + yymsp[0].minor.yy590 = 0; }else{ - yymsp[0].minor.yy528 = sqlite3PExpr(pParse, TK_REGISTER, 0, 0); - if( yymsp[0].minor.yy528 ) sqlite3GetInt32(&t.z[1], &yymsp[0].minor.yy528->iTable); + yymsp[0].minor.yy590 = sqlite3PExpr(pParse, TK_REGISTER, 0, 0); + if( yymsp[0].minor.yy590 ) sqlite3GetInt32(&t.z[1], &yymsp[0].minor.yy590->iTable); } } } break; - case 185: /* expr ::= expr COLLATE ID|STRING */ + case 187: /* expr ::= expr COLLATE ID|STRING */ { - yymsp[-2].minor.yy528 = sqlite3ExprAddCollateToken(pParse, yymsp[-2].minor.yy528, &yymsp[0].minor.yy0, 1); + yymsp[-2].minor.yy590 = sqlite3ExprAddCollateToken(pParse, yymsp[-2].minor.yy590, &yymsp[0].minor.yy0, 1); } break; - case 186: /* expr ::= CAST LP expr AS typetoken RP */ + case 188: /* expr ::= CAST LP expr AS typetoken RP */ { - yymsp[-5].minor.yy528 = sqlite3ExprAlloc(pParse->db, TK_CAST, &yymsp[-1].minor.yy0, 1); - sqlite3ExprAttachSubtrees(pParse->db, yymsp[-5].minor.yy528, yymsp[-3].minor.yy528, 0); + yymsp[-5].minor.yy590 = sqlite3ExprAlloc(pParse->db, TK_CAST, &yymsp[-1].minor.yy0, 1); + sqlite3ExprAttachSubtrees(pParse->db, yymsp[-5].minor.yy590, yymsp[-3].minor.yy590, 0); } break; - case 187: /* expr ::= ID|INDEXED|JOIN_KW LP distinct exprlist RP */ + case 189: /* expr ::= ID|INDEXED|JOIN_KW LP distinct exprlist RP */ { - yylhsminor.yy528 = sqlite3ExprFunction(pParse, yymsp[-1].minor.yy322, &yymsp[-4].minor.yy0, yymsp[-2].minor.yy394); + yylhsminor.yy590 = sqlite3ExprFunction(pParse, yymsp[-1].minor.yy402, &yymsp[-4].minor.yy0, yymsp[-2].minor.yy502); } - yymsp[-4].minor.yy528 = yylhsminor.yy528; + yymsp[-4].minor.yy590 = yylhsminor.yy590; break; - case 188: /* expr ::= ID|INDEXED|JOIN_KW LP distinct exprlist ORDER BY sortlist RP */ + case 190: /* expr ::= ID|INDEXED|JOIN_KW LP distinct exprlist ORDER BY sortlist RP */ { - yylhsminor.yy528 = sqlite3ExprFunction(pParse, yymsp[-4].minor.yy322, &yymsp[-7].minor.yy0, yymsp[-5].minor.yy394); - sqlite3ExprAddFunctionOrderBy(pParse, yylhsminor.yy528, yymsp[-1].minor.yy322); + yylhsminor.yy590 = sqlite3ExprFunction(pParse, yymsp[-4].minor.yy402, &yymsp[-7].minor.yy0, yymsp[-5].minor.yy502); + sqlite3ExprAddFunctionOrderBy(pParse, yylhsminor.yy590, yymsp[-1].minor.yy402); } - yymsp[-7].minor.yy528 = yylhsminor.yy528; + yymsp[-7].minor.yy590 = yylhsminor.yy590; break; - case 189: /* expr ::= ID|INDEXED|JOIN_KW LP STAR RP */ + case 191: /* expr ::= ID|INDEXED|JOIN_KW LP STAR RP */ { - yylhsminor.yy528 = sqlite3ExprFunction(pParse, 0, &yymsp[-3].minor.yy0, 0); + yylhsminor.yy590 = sqlite3ExprFunction(pParse, 0, &yymsp[-3].minor.yy0, 0); } - yymsp[-3].minor.yy528 = yylhsminor.yy528; + yymsp[-3].minor.yy590 = yylhsminor.yy590; break; - case 190: /* expr ::= ID|INDEXED|JOIN_KW LP distinct exprlist RP filter_over */ + case 192: /* expr ::= ID|INDEXED|JOIN_KW LP distinct exprlist RP filter_over */ { - yylhsminor.yy528 = sqlite3ExprFunction(pParse, yymsp[-2].minor.yy322, &yymsp[-5].minor.yy0, yymsp[-3].minor.yy394); - sqlite3WindowAttach(pParse, yylhsminor.yy528, yymsp[0].minor.yy41); + yylhsminor.yy590 = sqlite3ExprFunction(pParse, yymsp[-2].minor.yy402, &yymsp[-5].minor.yy0, yymsp[-3].minor.yy502); + sqlite3WindowAttach(pParse, yylhsminor.yy590, yymsp[0].minor.yy483); } - yymsp[-5].minor.yy528 = yylhsminor.yy528; + yymsp[-5].minor.yy590 = yylhsminor.yy590; break; - case 191: /* expr ::= ID|INDEXED|JOIN_KW LP distinct exprlist ORDER BY sortlist RP filter_over */ + case 193: /* expr ::= ID|INDEXED|JOIN_KW LP distinct exprlist ORDER BY sortlist RP filter_over */ { - yylhsminor.yy528 = sqlite3ExprFunction(pParse, yymsp[-5].minor.yy322, &yymsp[-8].minor.yy0, yymsp[-6].minor.yy394); - sqlite3WindowAttach(pParse, yylhsminor.yy528, yymsp[0].minor.yy41); - sqlite3ExprAddFunctionOrderBy(pParse, yylhsminor.yy528, yymsp[-2].minor.yy322); + yylhsminor.yy590 = sqlite3ExprFunction(pParse, yymsp[-5].minor.yy402, &yymsp[-8].minor.yy0, yymsp[-6].minor.yy502); + sqlite3WindowAttach(pParse, yylhsminor.yy590, yymsp[0].minor.yy483); + sqlite3ExprAddFunctionOrderBy(pParse, yylhsminor.yy590, yymsp[-2].minor.yy402); } - yymsp[-8].minor.yy528 = yylhsminor.yy528; + yymsp[-8].minor.yy590 = yylhsminor.yy590; break; - case 192: /* expr ::= ID|INDEXED|JOIN_KW LP STAR RP filter_over */ + case 194: /* expr ::= ID|INDEXED|JOIN_KW LP STAR RP filter_over */ { - yylhsminor.yy528 = sqlite3ExprFunction(pParse, 0, &yymsp[-4].minor.yy0, 0); - sqlite3WindowAttach(pParse, yylhsminor.yy528, yymsp[0].minor.yy41); + yylhsminor.yy590 = sqlite3ExprFunction(pParse, 0, &yymsp[-4].minor.yy0, 0); + sqlite3WindowAttach(pParse, yylhsminor.yy590, yymsp[0].minor.yy483); } - yymsp[-4].minor.yy528 = yylhsminor.yy528; + yymsp[-4].minor.yy590 = yylhsminor.yy590; break; - case 193: /* term ::= CTIME_KW */ + case 195: /* term ::= CTIME_KW */ { - yylhsminor.yy528 = sqlite3ExprFunction(pParse, 0, &yymsp[0].minor.yy0, 0); + yylhsminor.yy590 = sqlite3ExprFunction(pParse, 0, &yymsp[0].minor.yy0, 0); } - yymsp[0].minor.yy528 = yylhsminor.yy528; + yymsp[0].minor.yy590 = yylhsminor.yy590; break; - case 194: /* expr ::= LP nexprlist COMMA expr RP */ + case 196: /* expr ::= LP nexprlist COMMA expr RP */ { - ExprList *pList = sqlite3ExprListAppend(pParse, yymsp[-3].minor.yy322, yymsp[-1].minor.yy528); - yymsp[-4].minor.yy528 = sqlite3PExpr(pParse, TK_VECTOR, 0, 0); - if( yymsp[-4].minor.yy528 ){ - yymsp[-4].minor.yy528->x.pList = pList; + ExprList *pList = sqlite3ExprListAppend(pParse, yymsp[-3].minor.yy402, yymsp[-1].minor.yy590); + yymsp[-4].minor.yy590 = sqlite3PExpr(pParse, TK_VECTOR, 0, 0); + if( yymsp[-4].minor.yy590 ){ + yymsp[-4].minor.yy590->x.pList = pList; if( ALWAYS(pList->nExpr) ){ - yymsp[-4].minor.yy528->flags |= pList->a[0].pExpr->flags & EP_Propagate; + yymsp[-4].minor.yy590->flags |= pList->a[0].pExpr->flags & EP_Propagate; } }else{ sqlite3ExprListDelete(pParse->db, pList); } } break; - case 195: /* expr ::= expr AND expr */ -{yymsp[-2].minor.yy528=sqlite3ExprAnd(pParse,yymsp[-2].minor.yy528,yymsp[0].minor.yy528);} + case 197: /* expr ::= expr AND expr */ +{yymsp[-2].minor.yy590=sqlite3ExprAnd(pParse,yymsp[-2].minor.yy590,yymsp[0].minor.yy590);} break; - case 196: /* expr ::= expr OR expr */ - case 197: /* expr ::= expr LT|GT|GE|LE expr */ yytestcase(yyruleno==197); - case 198: /* expr ::= expr EQ|NE expr */ yytestcase(yyruleno==198); - case 199: /* expr ::= expr BITAND|BITOR|LSHIFT|RSHIFT expr */ yytestcase(yyruleno==199); - case 200: /* expr ::= expr PLUS|MINUS expr */ yytestcase(yyruleno==200); - case 201: /* expr ::= expr STAR|SLASH|REM expr */ yytestcase(yyruleno==201); - case 202: /* expr ::= expr CONCAT expr */ yytestcase(yyruleno==202); -{yymsp[-2].minor.yy528=sqlite3PExpr(pParse,yymsp[-1].major,yymsp[-2].minor.yy528,yymsp[0].minor.yy528);} + case 198: /* expr ::= expr OR expr */ + case 199: /* expr ::= expr LT|GT|GE|LE expr */ yytestcase(yyruleno==199); + case 200: /* expr ::= expr EQ|NE expr */ yytestcase(yyruleno==200); + case 201: /* expr ::= expr BITAND|BITOR|LSHIFT|RSHIFT expr */ yytestcase(yyruleno==201); + case 202: /* expr ::= expr PLUS|MINUS expr */ yytestcase(yyruleno==202); + case 203: /* expr ::= expr STAR|SLASH|REM expr */ yytestcase(yyruleno==203); + case 204: /* expr ::= expr CONCAT expr */ yytestcase(yyruleno==204); +{yymsp[-2].minor.yy590=sqlite3PExpr(pParse,yymsp[-1].major,yymsp[-2].minor.yy590,yymsp[0].minor.yy590);} break; - case 203: /* likeop ::= NOT LIKE_KW|MATCH */ + case 205: /* likeop ::= NOT LIKE_KW|MATCH */ {yymsp[-1].minor.yy0=yymsp[0].minor.yy0; yymsp[-1].minor.yy0.n|=0x80000000; /*yymsp[-1].minor.yy0-overwrite-yymsp[0].minor.yy0*/} break; - case 204: /* expr ::= expr likeop expr */ + case 206: /* expr ::= expr likeop expr */ { ExprList *pList; int bNot = yymsp[-1].minor.yy0.n & 0x80000000; yymsp[-1].minor.yy0.n &= 0x7fffffff; - pList = sqlite3ExprListAppend(pParse,0, yymsp[0].minor.yy528); - pList = sqlite3ExprListAppend(pParse,pList, yymsp[-2].minor.yy528); - yymsp[-2].minor.yy528 = sqlite3ExprFunction(pParse, pList, &yymsp[-1].minor.yy0, 0); - if( bNot ) yymsp[-2].minor.yy528 = sqlite3PExpr(pParse, TK_NOT, yymsp[-2].minor.yy528, 0); - if( yymsp[-2].minor.yy528 ) yymsp[-2].minor.yy528->flags |= EP_InfixFunc; + pList = sqlite3ExprListAppend(pParse,0, yymsp[0].minor.yy590); + pList = sqlite3ExprListAppend(pParse,pList, yymsp[-2].minor.yy590); + yymsp[-2].minor.yy590 = sqlite3ExprFunction(pParse, pList, &yymsp[-1].minor.yy0, 0); + if( bNot ) yymsp[-2].minor.yy590 = sqlite3PExpr(pParse, TK_NOT, yymsp[-2].minor.yy590, 0); + if( yymsp[-2].minor.yy590 ) yymsp[-2].minor.yy590->flags |= EP_InfixFunc; } break; - case 205: /* expr ::= expr likeop expr ESCAPE expr */ + case 207: /* expr ::= expr likeop expr ESCAPE expr */ { ExprList *pList; int bNot = yymsp[-3].minor.yy0.n & 0x80000000; yymsp[-3].minor.yy0.n &= 0x7fffffff; - pList = sqlite3ExprListAppend(pParse,0, yymsp[-2].minor.yy528); - pList = sqlite3ExprListAppend(pParse,pList, yymsp[-4].minor.yy528); - pList = sqlite3ExprListAppend(pParse,pList, yymsp[0].minor.yy528); - yymsp[-4].minor.yy528 = sqlite3ExprFunction(pParse, pList, &yymsp[-3].minor.yy0, 0); - if( bNot ) yymsp[-4].minor.yy528 = sqlite3PExpr(pParse, TK_NOT, yymsp[-4].minor.yy528, 0); - if( yymsp[-4].minor.yy528 ) yymsp[-4].minor.yy528->flags |= EP_InfixFunc; + pList = sqlite3ExprListAppend(pParse,0, yymsp[-2].minor.yy590); + pList = sqlite3ExprListAppend(pParse,pList, yymsp[-4].minor.yy590); + pList = sqlite3ExprListAppend(pParse,pList, yymsp[0].minor.yy590); + yymsp[-4].minor.yy590 = sqlite3ExprFunction(pParse, pList, &yymsp[-3].minor.yy0, 0); + if( bNot ) yymsp[-4].minor.yy590 = sqlite3PExpr(pParse, TK_NOT, yymsp[-4].minor.yy590, 0); + if( yymsp[-4].minor.yy590 ) yymsp[-4].minor.yy590->flags |= EP_InfixFunc; } break; - case 206: /* expr ::= expr ISNULL|NOTNULL */ -{yymsp[-1].minor.yy528 = sqlite3PExpr(pParse,yymsp[0].major,yymsp[-1].minor.yy528,0);} + case 208: /* expr ::= expr ISNULL|NOTNULL */ +{yymsp[-1].minor.yy590 = sqlite3PExpr(pParse,yymsp[0].major,yymsp[-1].minor.yy590,0);} break; - case 207: /* expr ::= expr NOT NULL */ -{yymsp[-2].minor.yy528 = sqlite3PExpr(pParse,TK_NOTNULL,yymsp[-2].minor.yy528,0);} + case 209: /* expr ::= expr NOT NULL */ +{yymsp[-2].minor.yy590 = sqlite3PExpr(pParse,TK_NOTNULL,yymsp[-2].minor.yy590,0);} break; - case 208: /* expr ::= expr IS expr */ + case 210: /* expr ::= expr IS expr */ { - yymsp[-2].minor.yy528 = sqlite3PExpr(pParse,TK_IS,yymsp[-2].minor.yy528,yymsp[0].minor.yy528); - binaryToUnaryIfNull(pParse, yymsp[0].minor.yy528, yymsp[-2].minor.yy528, TK_ISNULL); + yymsp[-2].minor.yy590 = sqlite3PExpr(pParse,TK_IS,yymsp[-2].minor.yy590,yymsp[0].minor.yy590); + binaryToUnaryIfNull(pParse, yymsp[0].minor.yy590, yymsp[-2].minor.yy590, TK_ISNULL); } break; - case 209: /* expr ::= expr IS NOT expr */ + case 211: /* expr ::= expr IS NOT expr */ { - yymsp[-3].minor.yy528 = sqlite3PExpr(pParse,TK_ISNOT,yymsp[-3].minor.yy528,yymsp[0].minor.yy528); - binaryToUnaryIfNull(pParse, yymsp[0].minor.yy528, yymsp[-3].minor.yy528, TK_NOTNULL); + yymsp[-3].minor.yy590 = sqlite3PExpr(pParse,TK_ISNOT,yymsp[-3].minor.yy590,yymsp[0].minor.yy590); + binaryToUnaryIfNull(pParse, yymsp[0].minor.yy590, yymsp[-3].minor.yy590, TK_NOTNULL); } break; - case 210: /* expr ::= expr IS NOT DISTINCT FROM expr */ + case 212: /* expr ::= expr IS NOT DISTINCT FROM expr */ { - yymsp[-5].minor.yy528 = sqlite3PExpr(pParse,TK_IS,yymsp[-5].minor.yy528,yymsp[0].minor.yy528); - binaryToUnaryIfNull(pParse, yymsp[0].minor.yy528, yymsp[-5].minor.yy528, TK_ISNULL); + yymsp[-5].minor.yy590 = sqlite3PExpr(pParse,TK_IS,yymsp[-5].minor.yy590,yymsp[0].minor.yy590); + binaryToUnaryIfNull(pParse, yymsp[0].minor.yy590, yymsp[-5].minor.yy590, TK_ISNULL); } break; - case 211: /* expr ::= expr IS DISTINCT FROM expr */ + case 213: /* expr ::= expr IS DISTINCT FROM expr */ { - yymsp[-4].minor.yy528 = sqlite3PExpr(pParse,TK_ISNOT,yymsp[-4].minor.yy528,yymsp[0].minor.yy528); - binaryToUnaryIfNull(pParse, yymsp[0].minor.yy528, yymsp[-4].minor.yy528, TK_NOTNULL); + yymsp[-4].minor.yy590 = sqlite3PExpr(pParse,TK_ISNOT,yymsp[-4].minor.yy590,yymsp[0].minor.yy590); + binaryToUnaryIfNull(pParse, yymsp[0].minor.yy590, yymsp[-4].minor.yy590, TK_NOTNULL); } break; - case 212: /* expr ::= NOT expr */ - case 213: /* expr ::= BITNOT expr */ yytestcase(yyruleno==213); -{yymsp[-1].minor.yy528 = sqlite3PExpr(pParse, yymsp[-1].major, yymsp[0].minor.yy528, 0);/*A-overwrites-B*/} + case 214: /* expr ::= NOT expr */ + case 215: /* expr ::= BITNOT expr */ yytestcase(yyruleno==215); +{yymsp[-1].minor.yy590 = sqlite3PExpr(pParse, yymsp[-1].major, yymsp[0].minor.yy590, 0);/*A-overwrites-B*/} break; - case 214: /* expr ::= PLUS|MINUS expr */ + case 216: /* expr ::= PLUS|MINUS expr */ { - yymsp[-1].minor.yy528 = sqlite3PExpr(pParse, yymsp[-1].major==TK_PLUS ? TK_UPLUS : TK_UMINUS, yymsp[0].minor.yy528, 0); - /*A-overwrites-B*/ + Expr *p = yymsp[0].minor.yy590; + u8 op = yymsp[-1].major + (TK_UPLUS-TK_PLUS); + assert( TK_UPLUS>TK_PLUS ); + assert( TK_UMINUS == TK_MINUS + (TK_UPLUS - TK_PLUS) ); + if( p && p->op==TK_UPLUS ){ + p->op = op; + yymsp[-1].minor.yy590 = p; + }else{ + yymsp[-1].minor.yy590 = sqlite3PExpr(pParse, op, p, 0); + /*A-overwrites-B*/ + } } break; - case 215: /* expr ::= expr PTR expr */ + case 217: /* expr ::= expr PTR expr */ { - ExprList *pList = sqlite3ExprListAppend(pParse, 0, yymsp[-2].minor.yy528); - pList = sqlite3ExprListAppend(pParse, pList, yymsp[0].minor.yy528); - yylhsminor.yy528 = sqlite3ExprFunction(pParse, pList, &yymsp[-1].minor.yy0, 0); + ExprList *pList = sqlite3ExprListAppend(pParse, 0, yymsp[-2].minor.yy590); + pList = sqlite3ExprListAppend(pParse, pList, yymsp[0].minor.yy590); + yylhsminor.yy590 = sqlite3ExprFunction(pParse, pList, &yymsp[-1].minor.yy0, 0); } - yymsp[-2].minor.yy528 = yylhsminor.yy528; + yymsp[-2].minor.yy590 = yylhsminor.yy590; break; - case 216: /* between_op ::= BETWEEN */ - case 219: /* in_op ::= IN */ yytestcase(yyruleno==219); -{yymsp[0].minor.yy394 = 0;} + case 218: /* between_op ::= BETWEEN */ + case 221: /* in_op ::= IN */ yytestcase(yyruleno==221); +{yymsp[0].minor.yy502 = 0;} break; - case 218: /* expr ::= expr between_op expr AND expr */ + case 220: /* expr ::= expr between_op expr AND expr */ { - ExprList *pList = sqlite3ExprListAppend(pParse,0, yymsp[-2].minor.yy528); - pList = sqlite3ExprListAppend(pParse,pList, yymsp[0].minor.yy528); - yymsp[-4].minor.yy528 = sqlite3PExpr(pParse, TK_BETWEEN, yymsp[-4].minor.yy528, 0); - if( yymsp[-4].minor.yy528 ){ - yymsp[-4].minor.yy528->x.pList = pList; + ExprList *pList = sqlite3ExprListAppend(pParse,0, yymsp[-2].minor.yy590); + pList = sqlite3ExprListAppend(pParse,pList, yymsp[0].minor.yy590); + yymsp[-4].minor.yy590 = sqlite3PExpr(pParse, TK_BETWEEN, yymsp[-4].minor.yy590, 0); + if( yymsp[-4].minor.yy590 ){ + yymsp[-4].minor.yy590->x.pList = pList; }else{ sqlite3ExprListDelete(pParse->db, pList); } - if( yymsp[-3].minor.yy394 ) yymsp[-4].minor.yy528 = sqlite3PExpr(pParse, TK_NOT, yymsp[-4].minor.yy528, 0); + if( yymsp[-3].minor.yy502 ) yymsp[-4].minor.yy590 = sqlite3PExpr(pParse, TK_NOT, yymsp[-4].minor.yy590, 0); } break; - case 221: /* expr ::= expr in_op LP exprlist RP */ + case 223: /* expr ::= expr in_op LP exprlist RP */ { - if( yymsp[-1].minor.yy322==0 ){ + if( yymsp[-1].minor.yy402==0 ){ /* Expressions of the form ** ** expr1 IN () ** expr1 NOT IN () ** - ** simplify to constants 0 (false) and 1 (true), respectively, - ** regardless of the value of expr1. + ** simplify to constants 0 (false) and 1 (true), respectively. + ** + ** Except, do not apply this optimization if expr1 contains a function + ** because that function might be an aggregate (we don't know yet whether + ** it is or not) and if it is an aggregate, that could change the meaning + ** of the whole query. */ - sqlite3ExprUnmapAndDelete(pParse, yymsp[-4].minor.yy528); - yymsp[-4].minor.yy528 = sqlite3Expr(pParse->db, TK_STRING, yymsp[-3].minor.yy394 ? "true" : "false"); - if( yymsp[-4].minor.yy528 ) sqlite3ExprIdToTrueFalse(yymsp[-4].minor.yy528); - }else{ - Expr *pRHS = yymsp[-1].minor.yy322->a[0].pExpr; - if( yymsp[-1].minor.yy322->nExpr==1 && sqlite3ExprIsConstant(pRHS) && yymsp[-4].minor.yy528->op!=TK_VECTOR ){ - yymsp[-1].minor.yy322->a[0].pExpr = 0; - sqlite3ExprListDelete(pParse->db, yymsp[-1].minor.yy322); + Expr *pB = sqlite3Expr(pParse->db, TK_STRING, yymsp[-3].minor.yy502 ? "true" : "false"); + if( pB ) sqlite3ExprIdToTrueFalse(pB); + if( !ExprHasProperty(yymsp[-4].minor.yy590, EP_HasFunc) ){ + sqlite3ExprUnmapAndDelete(pParse, yymsp[-4].minor.yy590); + yymsp[-4].minor.yy590 = pB; + }else{ + yymsp[-4].minor.yy590 = sqlite3PExpr(pParse, yymsp[-3].minor.yy502 ? TK_OR : TK_AND, pB, yymsp[-4].minor.yy590); + } + }else{ + Expr *pRHS = yymsp[-1].minor.yy402->a[0].pExpr; + if( yymsp[-1].minor.yy402->nExpr==1 && sqlite3ExprIsConstant(pParse,pRHS) && yymsp[-4].minor.yy590->op!=TK_VECTOR ){ + yymsp[-1].minor.yy402->a[0].pExpr = 0; + sqlite3ExprListDelete(pParse->db, yymsp[-1].minor.yy402); pRHS = sqlite3PExpr(pParse, TK_UPLUS, pRHS, 0); - yymsp[-4].minor.yy528 = sqlite3PExpr(pParse, TK_EQ, yymsp[-4].minor.yy528, pRHS); - }else if( yymsp[-1].minor.yy322->nExpr==1 && pRHS->op==TK_SELECT ){ - yymsp[-4].minor.yy528 = sqlite3PExpr(pParse, TK_IN, yymsp[-4].minor.yy528, 0); - sqlite3PExprAddSelect(pParse, yymsp[-4].minor.yy528, pRHS->x.pSelect); + yymsp[-4].minor.yy590 = sqlite3PExpr(pParse, TK_EQ, yymsp[-4].minor.yy590, pRHS); + }else if( yymsp[-1].minor.yy402->nExpr==1 && pRHS->op==TK_SELECT ){ + yymsp[-4].minor.yy590 = sqlite3PExpr(pParse, TK_IN, yymsp[-4].minor.yy590, 0); + sqlite3PExprAddSelect(pParse, yymsp[-4].minor.yy590, pRHS->x.pSelect); pRHS->x.pSelect = 0; - sqlite3ExprListDelete(pParse->db, yymsp[-1].minor.yy322); - }else{ - yymsp[-4].minor.yy528 = sqlite3PExpr(pParse, TK_IN, yymsp[-4].minor.yy528, 0); - if( yymsp[-4].minor.yy528==0 ){ - sqlite3ExprListDelete(pParse->db, yymsp[-1].minor.yy322); - }else if( yymsp[-4].minor.yy528->pLeft->op==TK_VECTOR ){ - int nExpr = yymsp[-4].minor.yy528->pLeft->x.pList->nExpr; - Select *pSelectRHS = sqlite3ExprListToValues(pParse, nExpr, yymsp[-1].minor.yy322); + sqlite3ExprListDelete(pParse->db, yymsp[-1].minor.yy402); + }else{ + yymsp[-4].minor.yy590 = sqlite3PExpr(pParse, TK_IN, yymsp[-4].minor.yy590, 0); + if( yymsp[-4].minor.yy590==0 ){ + sqlite3ExprListDelete(pParse->db, yymsp[-1].minor.yy402); + }else if( yymsp[-4].minor.yy590->pLeft->op==TK_VECTOR ){ + int nExpr = yymsp[-4].minor.yy590->pLeft->x.pList->nExpr; + Select *pSelectRHS = sqlite3ExprListToValues(pParse, nExpr, yymsp[-1].minor.yy402); if( pSelectRHS ){ parserDoubleLinkSelect(pParse, pSelectRHS); - sqlite3PExprAddSelect(pParse, yymsp[-4].minor.yy528, pSelectRHS); + sqlite3PExprAddSelect(pParse, yymsp[-4].minor.yy590, pSelectRHS); } }else{ - yymsp[-4].minor.yy528->x.pList = yymsp[-1].minor.yy322; - sqlite3ExprSetHeightAndFlags(pParse, yymsp[-4].minor.yy528); + yymsp[-4].minor.yy590->x.pList = yymsp[-1].minor.yy402; + sqlite3ExprSetHeightAndFlags(pParse, yymsp[-4].minor.yy590); } } - if( yymsp[-3].minor.yy394 ) yymsp[-4].minor.yy528 = sqlite3PExpr(pParse, TK_NOT, yymsp[-4].minor.yy528, 0); + if( yymsp[-3].minor.yy502 ) yymsp[-4].minor.yy590 = sqlite3PExpr(pParse, TK_NOT, yymsp[-4].minor.yy590, 0); } } break; - case 222: /* expr ::= LP select RP */ + case 224: /* expr ::= LP select RP */ { - yymsp[-2].minor.yy528 = sqlite3PExpr(pParse, TK_SELECT, 0, 0); - sqlite3PExprAddSelect(pParse, yymsp[-2].minor.yy528, yymsp[-1].minor.yy47); + yymsp[-2].minor.yy590 = sqlite3PExpr(pParse, TK_SELECT, 0, 0); + sqlite3PExprAddSelect(pParse, yymsp[-2].minor.yy590, yymsp[-1].minor.yy637); } break; - case 223: /* expr ::= expr in_op LP select RP */ + case 225: /* expr ::= expr in_op LP select RP */ { - yymsp[-4].minor.yy528 = sqlite3PExpr(pParse, TK_IN, yymsp[-4].minor.yy528, 0); - sqlite3PExprAddSelect(pParse, yymsp[-4].minor.yy528, yymsp[-1].minor.yy47); - if( yymsp[-3].minor.yy394 ) yymsp[-4].minor.yy528 = sqlite3PExpr(pParse, TK_NOT, yymsp[-4].minor.yy528, 0); + yymsp[-4].minor.yy590 = sqlite3PExpr(pParse, TK_IN, yymsp[-4].minor.yy590, 0); + sqlite3PExprAddSelect(pParse, yymsp[-4].minor.yy590, yymsp[-1].minor.yy637); + if( yymsp[-3].minor.yy502 ) yymsp[-4].minor.yy590 = sqlite3PExpr(pParse, TK_NOT, yymsp[-4].minor.yy590, 0); } break; - case 224: /* expr ::= expr in_op nm dbnm paren_exprlist */ + case 226: /* expr ::= expr in_op nm dbnm paren_exprlist */ { SrcList *pSrc = sqlite3SrcListAppend(pParse, 0,&yymsp[-2].minor.yy0,&yymsp[-1].minor.yy0); Select *pSelect = sqlite3SelectNew(pParse, 0,pSrc,0,0,0,0,0,0); - if( yymsp[0].minor.yy322 ) sqlite3SrcListFuncArgs(pParse, pSelect ? pSrc : 0, yymsp[0].minor.yy322); - yymsp[-4].minor.yy528 = sqlite3PExpr(pParse, TK_IN, yymsp[-4].minor.yy528, 0); - sqlite3PExprAddSelect(pParse, yymsp[-4].minor.yy528, pSelect); - if( yymsp[-3].minor.yy394 ) yymsp[-4].minor.yy528 = sqlite3PExpr(pParse, TK_NOT, yymsp[-4].minor.yy528, 0); + if( yymsp[0].minor.yy402 ) sqlite3SrcListFuncArgs(pParse, pSelect ? pSrc : 0, yymsp[0].minor.yy402); + yymsp[-4].minor.yy590 = sqlite3PExpr(pParse, TK_IN, yymsp[-4].minor.yy590, 0); + sqlite3PExprAddSelect(pParse, yymsp[-4].minor.yy590, pSelect); + if( yymsp[-3].minor.yy502 ) yymsp[-4].minor.yy590 = sqlite3PExpr(pParse, TK_NOT, yymsp[-4].minor.yy590, 0); } break; - case 225: /* expr ::= EXISTS LP select RP */ + case 227: /* expr ::= EXISTS LP select RP */ { Expr *p; - p = yymsp[-3].minor.yy528 = sqlite3PExpr(pParse, TK_EXISTS, 0, 0); - sqlite3PExprAddSelect(pParse, p, yymsp[-1].minor.yy47); + p = yymsp[-3].minor.yy590 = sqlite3PExpr(pParse, TK_EXISTS, 0, 0); + sqlite3PExprAddSelect(pParse, p, yymsp[-1].minor.yy637); } break; - case 226: /* expr ::= CASE case_operand case_exprlist case_else END */ + case 228: /* expr ::= CASE case_operand case_exprlist case_else END */ { - yymsp[-4].minor.yy528 = sqlite3PExpr(pParse, TK_CASE, yymsp[-3].minor.yy528, 0); - if( yymsp[-4].minor.yy528 ){ - yymsp[-4].minor.yy528->x.pList = yymsp[-1].minor.yy528 ? sqlite3ExprListAppend(pParse,yymsp[-2].minor.yy322,yymsp[-1].minor.yy528) : yymsp[-2].minor.yy322; - sqlite3ExprSetHeightAndFlags(pParse, yymsp[-4].minor.yy528); + yymsp[-4].minor.yy590 = sqlite3PExpr(pParse, TK_CASE, yymsp[-3].minor.yy590, 0); + if( yymsp[-4].minor.yy590 ){ + yymsp[-4].minor.yy590->x.pList = yymsp[-1].minor.yy590 ? sqlite3ExprListAppend(pParse,yymsp[-2].minor.yy402,yymsp[-1].minor.yy590) : yymsp[-2].minor.yy402; + sqlite3ExprSetHeightAndFlags(pParse, yymsp[-4].minor.yy590); }else{ - sqlite3ExprListDelete(pParse->db, yymsp[-2].minor.yy322); - sqlite3ExprDelete(pParse->db, yymsp[-1].minor.yy528); + sqlite3ExprListDelete(pParse->db, yymsp[-2].minor.yy402); + sqlite3ExprDelete(pParse->db, yymsp[-1].minor.yy590); } } break; - case 227: /* case_exprlist ::= case_exprlist WHEN expr THEN expr */ + case 229: /* case_exprlist ::= case_exprlist WHEN expr THEN expr */ { - yymsp[-4].minor.yy322 = sqlite3ExprListAppend(pParse,yymsp[-4].minor.yy322, yymsp[-2].minor.yy528); - yymsp[-4].minor.yy322 = sqlite3ExprListAppend(pParse,yymsp[-4].minor.yy322, yymsp[0].minor.yy528); + yymsp[-4].minor.yy402 = sqlite3ExprListAppend(pParse,yymsp[-4].minor.yy402, yymsp[-2].minor.yy590); + yymsp[-4].minor.yy402 = sqlite3ExprListAppend(pParse,yymsp[-4].minor.yy402, yymsp[0].minor.yy590); } break; - case 228: /* case_exprlist ::= WHEN expr THEN expr */ + case 230: /* case_exprlist ::= WHEN expr THEN expr */ { - yymsp[-3].minor.yy322 = sqlite3ExprListAppend(pParse,0, yymsp[-2].minor.yy528); - yymsp[-3].minor.yy322 = sqlite3ExprListAppend(pParse,yymsp[-3].minor.yy322, yymsp[0].minor.yy528); + yymsp[-3].minor.yy402 = sqlite3ExprListAppend(pParse,0, yymsp[-2].minor.yy590); + yymsp[-3].minor.yy402 = sqlite3ExprListAppend(pParse,yymsp[-3].minor.yy402, yymsp[0].minor.yy590); } break; - case 233: /* nexprlist ::= nexprlist COMMA expr */ -{yymsp[-2].minor.yy322 = sqlite3ExprListAppend(pParse,yymsp[-2].minor.yy322,yymsp[0].minor.yy528);} + case 235: /* nexprlist ::= nexprlist COMMA expr */ +{yymsp[-2].minor.yy402 = sqlite3ExprListAppend(pParse,yymsp[-2].minor.yy402,yymsp[0].minor.yy590);} break; - case 234: /* nexprlist ::= expr */ -{yymsp[0].minor.yy322 = sqlite3ExprListAppend(pParse,0,yymsp[0].minor.yy528); /*A-overwrites-Y*/} + case 236: /* nexprlist ::= expr */ +{yymsp[0].minor.yy402 = sqlite3ExprListAppend(pParse,0,yymsp[0].minor.yy590); /*A-overwrites-Y*/} break; - case 236: /* paren_exprlist ::= LP exprlist RP */ - case 241: /* eidlist_opt ::= LP eidlist RP */ yytestcase(yyruleno==241); -{yymsp[-2].minor.yy322 = yymsp[-1].minor.yy322;} + case 238: /* paren_exprlist ::= LP exprlist RP */ + case 243: /* eidlist_opt ::= LP eidlist RP */ yytestcase(yyruleno==243); +{yymsp[-2].minor.yy402 = yymsp[-1].minor.yy402;} break; - case 237: /* cmd ::= createkw uniqueflag INDEX ifnotexists nm dbnm ON nm LP sortlist RP where_opt */ + case 239: /* cmd ::= createkw uniqueflag INDEX ifnotexists nm dbnm ON nm LP sortlist RP where_opt */ { sqlite3CreateIndex(pParse, &yymsp[-7].minor.yy0, &yymsp[-6].minor.yy0, - sqlite3SrcListAppend(pParse,0,&yymsp[-4].minor.yy0,0), yymsp[-2].minor.yy322, yymsp[-10].minor.yy394, - &yymsp[-11].minor.yy0, yymsp[0].minor.yy528, SQLITE_SO_ASC, yymsp[-8].minor.yy394, SQLITE_IDXTYPE_APPDEF); + sqlite3SrcListAppend(pParse,0,&yymsp[-4].minor.yy0,0), yymsp[-2].minor.yy402, yymsp[-10].minor.yy502, + &yymsp[-11].minor.yy0, yymsp[0].minor.yy590, SQLITE_SO_ASC, yymsp[-8].minor.yy502, SQLITE_IDXTYPE_APPDEF); if( IN_RENAME_OBJECT && pParse->pNewIndex ){ sqlite3RenameTokenMap(pParse, pParse->pNewIndex->zName, &yymsp[-4].minor.yy0); } } break; - case 238: /* uniqueflag ::= UNIQUE */ - case 280: /* raisetype ::= ABORT */ yytestcase(yyruleno==280); -{yymsp[0].minor.yy394 = OE_Abort;} + case 240: /* uniqueflag ::= UNIQUE */ + case 282: /* raisetype ::= ABORT */ yytestcase(yyruleno==282); +{yymsp[0].minor.yy502 = OE_Abort;} break; - case 239: /* uniqueflag ::= */ -{yymsp[1].minor.yy394 = OE_None;} + case 241: /* uniqueflag ::= */ +{yymsp[1].minor.yy502 = OE_None;} break; - case 242: /* eidlist ::= eidlist COMMA nm collate sortorder */ + case 244: /* eidlist ::= eidlist COMMA nm collate sortorder */ { - yymsp[-4].minor.yy322 = parserAddExprIdListTerm(pParse, yymsp[-4].minor.yy322, &yymsp[-2].minor.yy0, yymsp[-1].minor.yy394, yymsp[0].minor.yy394); + yymsp[-4].minor.yy402 = parserAddExprIdListTerm(pParse, yymsp[-4].minor.yy402, &yymsp[-2].minor.yy0, yymsp[-1].minor.yy502, yymsp[0].minor.yy502); } break; - case 243: /* eidlist ::= nm collate sortorder */ + case 245: /* eidlist ::= nm collate sortorder */ { - yymsp[-2].minor.yy322 = parserAddExprIdListTerm(pParse, 0, &yymsp[-2].minor.yy0, yymsp[-1].minor.yy394, yymsp[0].minor.yy394); /*A-overwrites-Y*/ + yymsp[-2].minor.yy402 = parserAddExprIdListTerm(pParse, 0, &yymsp[-2].minor.yy0, yymsp[-1].minor.yy502, yymsp[0].minor.yy502); /*A-overwrites-Y*/ } break; - case 246: /* cmd ::= DROP INDEX ifexists fullname */ -{sqlite3DropIndex(pParse, yymsp[0].minor.yy131, yymsp[-1].minor.yy394);} + case 248: /* cmd ::= DROP INDEX ifexists fullname */ +{sqlite3DropIndex(pParse, yymsp[0].minor.yy563, yymsp[-1].minor.yy502);} break; - case 247: /* cmd ::= VACUUM vinto */ -{sqlite3Vacuum(pParse,0,yymsp[0].minor.yy528);} + case 249: /* cmd ::= VACUUM vinto */ +{sqlite3Vacuum(pParse,0,yymsp[0].minor.yy590);} break; - case 248: /* cmd ::= VACUUM nm vinto */ -{sqlite3Vacuum(pParse,&yymsp[-1].minor.yy0,yymsp[0].minor.yy528);} + case 250: /* cmd ::= VACUUM nm vinto */ +{sqlite3Vacuum(pParse,&yymsp[-1].minor.yy0,yymsp[0].minor.yy590);} break; - case 251: /* cmd ::= PRAGMA nm dbnm */ + case 253: /* cmd ::= PRAGMA nm dbnm */ {sqlite3Pragma(pParse,&yymsp[-1].minor.yy0,&yymsp[0].minor.yy0,0,0);} break; - case 252: /* cmd ::= PRAGMA nm dbnm EQ nmnum */ + case 254: /* cmd ::= PRAGMA nm dbnm EQ nmnum */ {sqlite3Pragma(pParse,&yymsp[-3].minor.yy0,&yymsp[-2].minor.yy0,&yymsp[0].minor.yy0,0);} break; - case 253: /* cmd ::= PRAGMA nm dbnm LP nmnum RP */ + case 255: /* cmd ::= PRAGMA nm dbnm LP nmnum RP */ {sqlite3Pragma(pParse,&yymsp[-4].minor.yy0,&yymsp[-3].minor.yy0,&yymsp[-1].minor.yy0,0);} break; - case 254: /* cmd ::= PRAGMA nm dbnm EQ minus_num */ + case 256: /* cmd ::= PRAGMA nm dbnm EQ minus_num */ {sqlite3Pragma(pParse,&yymsp[-3].minor.yy0,&yymsp[-2].minor.yy0,&yymsp[0].minor.yy0,1);} break; - case 255: /* cmd ::= PRAGMA nm dbnm LP minus_num RP */ + case 257: /* cmd ::= PRAGMA nm dbnm LP minus_num RP */ {sqlite3Pragma(pParse,&yymsp[-4].minor.yy0,&yymsp[-3].minor.yy0,&yymsp[-1].minor.yy0,1);} break; - case 258: /* cmd ::= createkw trigger_decl BEGIN trigger_cmd_list END */ + case 260: /* cmd ::= createkw trigger_decl BEGIN trigger_cmd_list END */ { Token all; all.z = yymsp[-3].minor.yy0.z; all.n = (int)(yymsp[0].minor.yy0.z - yymsp[-3].minor.yy0.z) + yymsp[0].minor.yy0.n; - sqlite3FinishTrigger(pParse, yymsp[-1].minor.yy33, &all); + sqlite3FinishTrigger(pParse, yymsp[-1].minor.yy319, &all); } break; - case 259: /* trigger_decl ::= temp TRIGGER ifnotexists nm dbnm trigger_time trigger_event ON fullname foreach_clause when_clause */ + case 261: /* trigger_decl ::= temp TRIGGER ifnotexists nm dbnm trigger_time trigger_event ON fullname foreach_clause when_clause */ { - sqlite3BeginTrigger(pParse, &yymsp[-7].minor.yy0, &yymsp[-6].minor.yy0, yymsp[-5].minor.yy394, yymsp[-4].minor.yy180.a, yymsp[-4].minor.yy180.b, yymsp[-2].minor.yy131, yymsp[0].minor.yy528, yymsp[-10].minor.yy394, yymsp[-8].minor.yy394); + sqlite3BeginTrigger(pParse, &yymsp[-7].minor.yy0, &yymsp[-6].minor.yy0, yymsp[-5].minor.yy502, yymsp[-4].minor.yy28.a, yymsp[-4].minor.yy28.b, yymsp[-2].minor.yy563, yymsp[0].minor.yy590, yymsp[-10].minor.yy502, yymsp[-8].minor.yy502); yymsp[-10].minor.yy0 = (yymsp[-6].minor.yy0.n==0?yymsp[-7].minor.yy0:yymsp[-6].minor.yy0); /*A-overwrites-T*/ +#ifdef SQLITE_DEBUG + assert( pParse->isCreate ); /* Set by createkw reduce action */ + pParse->isCreate = 0; /* But, should not be set for CREATE TRIGGER */ +#endif } break; - case 260: /* trigger_time ::= BEFORE|AFTER */ -{ yymsp[0].minor.yy394 = yymsp[0].major; /*A-overwrites-X*/ } + case 262: /* trigger_time ::= BEFORE|AFTER */ +{ yymsp[0].minor.yy502 = yymsp[0].major; /*A-overwrites-X*/ } break; - case 261: /* trigger_time ::= INSTEAD OF */ -{ yymsp[-1].minor.yy394 = TK_INSTEAD;} + case 263: /* trigger_time ::= INSTEAD OF */ +{ yymsp[-1].minor.yy502 = TK_INSTEAD;} break; - case 262: /* trigger_time ::= */ -{ yymsp[1].minor.yy394 = TK_BEFORE; } + case 264: /* trigger_time ::= */ +{ yymsp[1].minor.yy502 = TK_BEFORE; } break; - case 263: /* trigger_event ::= DELETE|INSERT */ - case 264: /* trigger_event ::= UPDATE */ yytestcase(yyruleno==264); -{yymsp[0].minor.yy180.a = yymsp[0].major; /*A-overwrites-X*/ yymsp[0].minor.yy180.b = 0;} + case 265: /* trigger_event ::= DELETE|INSERT */ + case 266: /* trigger_event ::= UPDATE */ yytestcase(yyruleno==266); +{yymsp[0].minor.yy28.a = yymsp[0].major; /*A-overwrites-X*/ yymsp[0].minor.yy28.b = 0;} break; - case 265: /* trigger_event ::= UPDATE OF idlist */ -{yymsp[-2].minor.yy180.a = TK_UPDATE; yymsp[-2].minor.yy180.b = yymsp[0].minor.yy254;} + case 267: /* trigger_event ::= UPDATE OF idlist */ +{yymsp[-2].minor.yy28.a = TK_UPDATE; yymsp[-2].minor.yy28.b = yymsp[0].minor.yy204;} break; - case 266: /* when_clause ::= */ - case 285: /* key_opt ::= */ yytestcase(yyruleno==285); -{ yymsp[1].minor.yy528 = 0; } + case 268: /* when_clause ::= */ + case 287: /* key_opt ::= */ yytestcase(yyruleno==287); +{ yymsp[1].minor.yy590 = 0; } break; - case 267: /* when_clause ::= WHEN expr */ - case 286: /* key_opt ::= KEY expr */ yytestcase(yyruleno==286); -{ yymsp[-1].minor.yy528 = yymsp[0].minor.yy528; } + case 269: /* when_clause ::= WHEN expr */ + case 288: /* key_opt ::= KEY expr */ yytestcase(yyruleno==288); +{ yymsp[-1].minor.yy590 = yymsp[0].minor.yy590; } break; - case 268: /* trigger_cmd_list ::= trigger_cmd_list trigger_cmd SEMI */ + case 270: /* trigger_cmd_list ::= trigger_cmd_list trigger_cmd SEMI */ { - assert( yymsp[-2].minor.yy33!=0 ); - yymsp[-2].minor.yy33->pLast->pNext = yymsp[-1].minor.yy33; - yymsp[-2].minor.yy33->pLast = yymsp[-1].minor.yy33; + assert( yymsp[-2].minor.yy319!=0 ); + yymsp[-2].minor.yy319->pLast->pNext = yymsp[-1].minor.yy319; + yymsp[-2].minor.yy319->pLast = yymsp[-1].minor.yy319; } break; - case 269: /* trigger_cmd_list ::= trigger_cmd SEMI */ + case 271: /* trigger_cmd_list ::= trigger_cmd SEMI */ { - assert( yymsp[-1].minor.yy33!=0 ); - yymsp[-1].minor.yy33->pLast = yymsp[-1].minor.yy33; + assert( yymsp[-1].minor.yy319!=0 ); + yymsp[-1].minor.yy319->pLast = yymsp[-1].minor.yy319; } break; - case 270: /* trnm ::= nm DOT nm */ + case 272: /* trnm ::= nm DOT nm */ { yymsp[-2].minor.yy0 = yymsp[0].minor.yy0; sqlite3ErrorMsg(pParse, @@ -175774,367 +182365,377 @@ static YYACTIONTYPE yy_reduce( "statements within triggers"); } break; - case 271: /* tridxby ::= INDEXED BY nm */ + case 273: /* tridxby ::= INDEXED BY nm */ { sqlite3ErrorMsg(pParse, "the INDEXED BY clause is not allowed on UPDATE or DELETE statements " "within triggers"); } break; - case 272: /* tridxby ::= NOT INDEXED */ + case 274: /* tridxby ::= NOT INDEXED */ { sqlite3ErrorMsg(pParse, "the NOT INDEXED clause is not allowed on UPDATE or DELETE statements " "within triggers"); } break; - case 273: /* trigger_cmd ::= UPDATE orconf trnm tridxby SET setlist from where_opt scanpt */ -{yylhsminor.yy33 = sqlite3TriggerUpdateStep(pParse, &yymsp[-6].minor.yy0, yymsp[-2].minor.yy131, yymsp[-3].minor.yy322, yymsp[-1].minor.yy528, yymsp[-7].minor.yy394, yymsp[-8].minor.yy0.z, yymsp[0].minor.yy522);} - yymsp[-8].minor.yy33 = yylhsminor.yy33; + case 275: /* trigger_cmd ::= UPDATE orconf trnm tridxby SET setlist from where_opt scanpt */ +{yylhsminor.yy319 = sqlite3TriggerUpdateStep(pParse, &yymsp[-6].minor.yy0, yymsp[-2].minor.yy563, yymsp[-3].minor.yy402, yymsp[-1].minor.yy590, yymsp[-7].minor.yy502, yymsp[-8].minor.yy0.z, yymsp[0].minor.yy342);} + yymsp[-8].minor.yy319 = yylhsminor.yy319; break; - case 274: /* trigger_cmd ::= scanpt insert_cmd INTO trnm idlist_opt select upsert scanpt */ + case 276: /* trigger_cmd ::= scanpt insert_cmd INTO trnm idlist_opt select upsert scanpt */ { - yylhsminor.yy33 = sqlite3TriggerInsertStep(pParse,&yymsp[-4].minor.yy0,yymsp[-3].minor.yy254,yymsp[-2].minor.yy47,yymsp[-6].minor.yy394,yymsp[-1].minor.yy444,yymsp[-7].minor.yy522,yymsp[0].minor.yy522);/*yylhsminor.yy33-overwrites-yymsp[-6].minor.yy394*/ + yylhsminor.yy319 = sqlite3TriggerInsertStep(pParse,&yymsp[-4].minor.yy0,yymsp[-3].minor.yy204,yymsp[-2].minor.yy637,yymsp[-6].minor.yy502,yymsp[-1].minor.yy403,yymsp[-7].minor.yy342,yymsp[0].minor.yy342);/*yylhsminor.yy319-overwrites-yymsp[-6].minor.yy502*/ } - yymsp[-7].minor.yy33 = yylhsminor.yy33; + yymsp[-7].minor.yy319 = yylhsminor.yy319; break; - case 275: /* trigger_cmd ::= DELETE FROM trnm tridxby where_opt scanpt */ -{yylhsminor.yy33 = sqlite3TriggerDeleteStep(pParse, &yymsp[-3].minor.yy0, yymsp[-1].minor.yy528, yymsp[-5].minor.yy0.z, yymsp[0].minor.yy522);} - yymsp[-5].minor.yy33 = yylhsminor.yy33; + case 277: /* trigger_cmd ::= DELETE FROM trnm tridxby where_opt scanpt */ +{yylhsminor.yy319 = sqlite3TriggerDeleteStep(pParse, &yymsp[-3].minor.yy0, yymsp[-1].minor.yy590, yymsp[-5].minor.yy0.z, yymsp[0].minor.yy342);} + yymsp[-5].minor.yy319 = yylhsminor.yy319; break; - case 276: /* trigger_cmd ::= scanpt select scanpt */ -{yylhsminor.yy33 = sqlite3TriggerSelectStep(pParse->db, yymsp[-1].minor.yy47, yymsp[-2].minor.yy522, yymsp[0].minor.yy522); /*yylhsminor.yy33-overwrites-yymsp[-1].minor.yy47*/} - yymsp[-2].minor.yy33 = yylhsminor.yy33; + case 278: /* trigger_cmd ::= scanpt select scanpt */ +{yylhsminor.yy319 = sqlite3TriggerSelectStep(pParse->db, yymsp[-1].minor.yy637, yymsp[-2].minor.yy342, yymsp[0].minor.yy342); /*yylhsminor.yy319-overwrites-yymsp[-1].minor.yy637*/} + yymsp[-2].minor.yy319 = yylhsminor.yy319; break; - case 277: /* expr ::= RAISE LP IGNORE RP */ + case 279: /* expr ::= RAISE LP IGNORE RP */ { - yymsp[-3].minor.yy528 = sqlite3PExpr(pParse, TK_RAISE, 0, 0); - if( yymsp[-3].minor.yy528 ){ - yymsp[-3].minor.yy528->affExpr = OE_Ignore; + yymsp[-3].minor.yy590 = sqlite3PExpr(pParse, TK_RAISE, 0, 0); + if( yymsp[-3].minor.yy590 ){ + yymsp[-3].minor.yy590->affExpr = OE_Ignore; } } break; - case 278: /* expr ::= RAISE LP raisetype COMMA nm RP */ + case 280: /* expr ::= RAISE LP raisetype COMMA expr RP */ { - yymsp[-5].minor.yy528 = sqlite3ExprAlloc(pParse->db, TK_RAISE, &yymsp[-1].minor.yy0, 1); - if( yymsp[-5].minor.yy528 ) { - yymsp[-5].minor.yy528->affExpr = (char)yymsp[-3].minor.yy394; + yymsp[-5].minor.yy590 = sqlite3PExpr(pParse, TK_RAISE, yymsp[-1].minor.yy590, 0); + if( yymsp[-5].minor.yy590 ) { + yymsp[-5].minor.yy590->affExpr = (char)yymsp[-3].minor.yy502; } } break; - case 279: /* raisetype ::= ROLLBACK */ -{yymsp[0].minor.yy394 = OE_Rollback;} + case 281: /* raisetype ::= ROLLBACK */ +{yymsp[0].minor.yy502 = OE_Rollback;} break; - case 281: /* raisetype ::= FAIL */ -{yymsp[0].minor.yy394 = OE_Fail;} + case 283: /* raisetype ::= FAIL */ +{yymsp[0].minor.yy502 = OE_Fail;} break; - case 282: /* cmd ::= DROP TRIGGER ifexists fullname */ + case 284: /* cmd ::= DROP TRIGGER ifexists fullname */ { - sqlite3DropTrigger(pParse,yymsp[0].minor.yy131,yymsp[-1].minor.yy394); + sqlite3DropTrigger(pParse,yymsp[0].minor.yy563,yymsp[-1].minor.yy502); } break; - case 283: /* cmd ::= ATTACH database_kw_opt expr AS expr key_opt */ + case 285: /* cmd ::= ATTACH database_kw_opt expr AS expr key_opt */ { - sqlite3Attach(pParse, yymsp[-3].minor.yy528, yymsp[-1].minor.yy528, yymsp[0].minor.yy528); + sqlite3Attach(pParse, yymsp[-3].minor.yy590, yymsp[-1].minor.yy590, yymsp[0].minor.yy590); } break; - case 284: /* cmd ::= DETACH database_kw_opt expr */ + case 286: /* cmd ::= DETACH database_kw_opt expr */ { - sqlite3Detach(pParse, yymsp[0].minor.yy528); + sqlite3Detach(pParse, yymsp[0].minor.yy590); } break; - case 287: /* cmd ::= REINDEX */ + case 289: /* cmd ::= REINDEX */ {sqlite3Reindex(pParse, 0, 0);} break; - case 288: /* cmd ::= REINDEX nm dbnm */ + case 290: /* cmd ::= REINDEX nm dbnm */ {sqlite3Reindex(pParse, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0);} break; - case 289: /* cmd ::= ANALYZE */ + case 291: /* cmd ::= ANALYZE */ {sqlite3Analyze(pParse, 0, 0);} break; - case 290: /* cmd ::= ANALYZE nm dbnm */ + case 292: /* cmd ::= ANALYZE nm dbnm */ {sqlite3Analyze(pParse, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0);} break; - case 291: /* cmd ::= ALTER TABLE fullname RENAME TO nm */ + case 293: /* cmd ::= ALTER TABLE fullname RENAME TO nm */ { - sqlite3AlterRenameTable(pParse,yymsp[-3].minor.yy131,&yymsp[0].minor.yy0); + sqlite3AlterRenameTable(pParse,yymsp[-3].minor.yy563,&yymsp[0].minor.yy0); } break; - case 292: /* cmd ::= ALTER TABLE add_column_fullname ADD kwcolumn_opt columnname carglist */ + case 294: /* cmd ::= ALTER TABLE add_column_fullname ADD kwcolumn_opt columnname carglist */ { yymsp[-1].minor.yy0.n = (int)(pParse->sLastToken.z-yymsp[-1].minor.yy0.z) + pParse->sLastToken.n; sqlite3AlterFinishAddColumn(pParse, &yymsp[-1].minor.yy0); } break; - case 293: /* cmd ::= ALTER TABLE fullname DROP kwcolumn_opt nm */ + case 295: /* cmd ::= ALTER TABLE fullname DROP kwcolumn_opt nm */ { - sqlite3AlterDropColumn(pParse, yymsp[-3].minor.yy131, &yymsp[0].minor.yy0); + sqlite3AlterDropColumn(pParse, yymsp[-3].minor.yy563, &yymsp[0].minor.yy0); } break; - case 294: /* add_column_fullname ::= fullname */ + case 296: /* add_column_fullname ::= fullname */ { disableLookaside(pParse); - sqlite3AlterBeginAddColumn(pParse, yymsp[0].minor.yy131); + sqlite3AlterBeginAddColumn(pParse, yymsp[0].minor.yy563); } break; - case 295: /* cmd ::= ALTER TABLE fullname RENAME kwcolumn_opt nm TO nm */ + case 297: /* cmd ::= ALTER TABLE fullname RENAME kwcolumn_opt nm TO nm */ { - sqlite3AlterRenameColumn(pParse, yymsp[-5].minor.yy131, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0); + sqlite3AlterRenameColumn(pParse, yymsp[-5].minor.yy563, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0); } break; - case 296: /* cmd ::= create_vtab */ + case 298: /* cmd ::= create_vtab */ {sqlite3VtabFinishParse(pParse,0);} break; - case 297: /* cmd ::= create_vtab LP vtabarglist RP */ + case 299: /* cmd ::= create_vtab LP vtabarglist RP */ {sqlite3VtabFinishParse(pParse,&yymsp[0].minor.yy0);} break; - case 298: /* create_vtab ::= createkw VIRTUAL TABLE ifnotexists nm dbnm USING nm */ + case 300: /* create_vtab ::= createkw VIRTUAL TABLE ifnotexists nm dbnm USING nm */ { - sqlite3VtabBeginParse(pParse, &yymsp[-3].minor.yy0, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, yymsp[-4].minor.yy394); + sqlite3VtabBeginParse(pParse, &yymsp[-3].minor.yy0, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, yymsp[-4].minor.yy502); } break; - case 299: /* vtabarg ::= */ + case 301: /* vtabarg ::= */ {sqlite3VtabArgInit(pParse);} break; - case 300: /* vtabargtoken ::= ANY */ - case 301: /* vtabargtoken ::= lp anylist RP */ yytestcase(yyruleno==301); - case 302: /* lp ::= LP */ yytestcase(yyruleno==302); + case 302: /* vtabargtoken ::= ANY */ + case 303: /* vtabargtoken ::= lp anylist RP */ yytestcase(yyruleno==303); + case 304: /* lp ::= LP */ yytestcase(yyruleno==304); {sqlite3VtabArgExtend(pParse,&yymsp[0].minor.yy0);} break; - case 303: /* with ::= WITH wqlist */ - case 304: /* with ::= WITH RECURSIVE wqlist */ yytestcase(yyruleno==304); -{ sqlite3WithPush(pParse, yymsp[0].minor.yy521, 1); } + case 305: /* with ::= WITH wqlist */ + case 306: /* with ::= WITH RECURSIVE wqlist */ yytestcase(yyruleno==306); +{ sqlite3WithPush(pParse, yymsp[0].minor.yy125, 1); } break; - case 305: /* wqas ::= AS */ -{yymsp[0].minor.yy516 = M10d_Any;} + case 307: /* wqas ::= AS */ +{yymsp[0].minor.yy444 = M10d_Any;} break; - case 306: /* wqas ::= AS MATERIALIZED */ -{yymsp[-1].minor.yy516 = M10d_Yes;} + case 308: /* wqas ::= AS MATERIALIZED */ +{yymsp[-1].minor.yy444 = M10d_Yes;} break; - case 307: /* wqas ::= AS NOT MATERIALIZED */ -{yymsp[-2].minor.yy516 = M10d_No;} + case 309: /* wqas ::= AS NOT MATERIALIZED */ +{yymsp[-2].minor.yy444 = M10d_No;} break; - case 308: /* wqitem ::= nm eidlist_opt wqas LP select RP */ + case 310: /* wqitem ::= withnm eidlist_opt wqas LP select RP */ { - yymsp[-5].minor.yy385 = sqlite3CteNew(pParse, &yymsp[-5].minor.yy0, yymsp[-4].minor.yy322, yymsp[-1].minor.yy47, yymsp[-3].minor.yy516); /*A-overwrites-X*/ + yymsp[-5].minor.yy361 = sqlite3CteNew(pParse, &yymsp[-5].minor.yy0, yymsp[-4].minor.yy402, yymsp[-1].minor.yy637, yymsp[-3].minor.yy444); /*A-overwrites-X*/ } break; - case 309: /* wqlist ::= wqitem */ + case 311: /* withnm ::= nm */ +{pParse->bHasWith = 1;} + break; + case 312: /* wqlist ::= wqitem */ { - yymsp[0].minor.yy521 = sqlite3WithAdd(pParse, 0, yymsp[0].minor.yy385); /*A-overwrites-X*/ + yymsp[0].minor.yy125 = sqlite3WithAdd(pParse, 0, yymsp[0].minor.yy361); /*A-overwrites-X*/ } break; - case 310: /* wqlist ::= wqlist COMMA wqitem */ + case 313: /* wqlist ::= wqlist COMMA wqitem */ { - yymsp[-2].minor.yy521 = sqlite3WithAdd(pParse, yymsp[-2].minor.yy521, yymsp[0].minor.yy385); + yymsp[-2].minor.yy125 = sqlite3WithAdd(pParse, yymsp[-2].minor.yy125, yymsp[0].minor.yy361); } break; - case 311: /* windowdefn_list ::= windowdefn_list COMMA windowdefn */ + case 314: /* windowdefn_list ::= windowdefn_list COMMA windowdefn */ { - assert( yymsp[0].minor.yy41!=0 ); - sqlite3WindowChain(pParse, yymsp[0].minor.yy41, yymsp[-2].minor.yy41); - yymsp[0].minor.yy41->pNextWin = yymsp[-2].minor.yy41; - yylhsminor.yy41 = yymsp[0].minor.yy41; + assert( yymsp[0].minor.yy483!=0 ); + sqlite3WindowChain(pParse, yymsp[0].minor.yy483, yymsp[-2].minor.yy483); + yymsp[0].minor.yy483->pNextWin = yymsp[-2].minor.yy483; + yylhsminor.yy483 = yymsp[0].minor.yy483; } - yymsp[-2].minor.yy41 = yylhsminor.yy41; + yymsp[-2].minor.yy483 = yylhsminor.yy483; break; - case 312: /* windowdefn ::= nm AS LP window RP */ + case 315: /* windowdefn ::= nm AS LP window RP */ { - if( ALWAYS(yymsp[-1].minor.yy41) ){ - yymsp[-1].minor.yy41->zName = sqlite3DbStrNDup(pParse->db, yymsp[-4].minor.yy0.z, yymsp[-4].minor.yy0.n); + if( ALWAYS(yymsp[-1].minor.yy483) ){ + yymsp[-1].minor.yy483->zName = sqlite3DbStrNDup(pParse->db, yymsp[-4].minor.yy0.z, yymsp[-4].minor.yy0.n); } - yylhsminor.yy41 = yymsp[-1].minor.yy41; + yylhsminor.yy483 = yymsp[-1].minor.yy483; } - yymsp[-4].minor.yy41 = yylhsminor.yy41; + yymsp[-4].minor.yy483 = yylhsminor.yy483; break; - case 313: /* window ::= PARTITION BY nexprlist orderby_opt frame_opt */ + case 316: /* window ::= PARTITION BY nexprlist orderby_opt frame_opt */ { - yymsp[-4].minor.yy41 = sqlite3WindowAssemble(pParse, yymsp[0].minor.yy41, yymsp[-2].minor.yy322, yymsp[-1].minor.yy322, 0); + yymsp[-4].minor.yy483 = sqlite3WindowAssemble(pParse, yymsp[0].minor.yy483, yymsp[-2].minor.yy402, yymsp[-1].minor.yy402, 0); } break; - case 314: /* window ::= nm PARTITION BY nexprlist orderby_opt frame_opt */ + case 317: /* window ::= nm PARTITION BY nexprlist orderby_opt frame_opt */ { - yylhsminor.yy41 = sqlite3WindowAssemble(pParse, yymsp[0].minor.yy41, yymsp[-2].minor.yy322, yymsp[-1].minor.yy322, &yymsp[-5].minor.yy0); + yylhsminor.yy483 = sqlite3WindowAssemble(pParse, yymsp[0].minor.yy483, yymsp[-2].minor.yy402, yymsp[-1].minor.yy402, &yymsp[-5].minor.yy0); } - yymsp[-5].minor.yy41 = yylhsminor.yy41; + yymsp[-5].minor.yy483 = yylhsminor.yy483; break; - case 315: /* window ::= ORDER BY sortlist frame_opt */ + case 318: /* window ::= ORDER BY sortlist frame_opt */ { - yymsp[-3].minor.yy41 = sqlite3WindowAssemble(pParse, yymsp[0].minor.yy41, 0, yymsp[-1].minor.yy322, 0); + yymsp[-3].minor.yy483 = sqlite3WindowAssemble(pParse, yymsp[0].minor.yy483, 0, yymsp[-1].minor.yy402, 0); } break; - case 316: /* window ::= nm ORDER BY sortlist frame_opt */ + case 319: /* window ::= nm ORDER BY sortlist frame_opt */ { - yylhsminor.yy41 = sqlite3WindowAssemble(pParse, yymsp[0].minor.yy41, 0, yymsp[-1].minor.yy322, &yymsp[-4].minor.yy0); + yylhsminor.yy483 = sqlite3WindowAssemble(pParse, yymsp[0].minor.yy483, 0, yymsp[-1].minor.yy402, &yymsp[-4].minor.yy0); } - yymsp[-4].minor.yy41 = yylhsminor.yy41; + yymsp[-4].minor.yy483 = yylhsminor.yy483; break; - case 317: /* window ::= nm frame_opt */ + case 320: /* window ::= nm frame_opt */ { - yylhsminor.yy41 = sqlite3WindowAssemble(pParse, yymsp[0].minor.yy41, 0, 0, &yymsp[-1].minor.yy0); + yylhsminor.yy483 = sqlite3WindowAssemble(pParse, yymsp[0].minor.yy483, 0, 0, &yymsp[-1].minor.yy0); } - yymsp[-1].minor.yy41 = yylhsminor.yy41; + yymsp[-1].minor.yy483 = yylhsminor.yy483; break; - case 318: /* frame_opt ::= */ + case 321: /* frame_opt ::= */ { - yymsp[1].minor.yy41 = sqlite3WindowAlloc(pParse, 0, TK_UNBOUNDED, 0, TK_CURRENT, 0, 0); + yymsp[1].minor.yy483 = sqlite3WindowAlloc(pParse, 0, TK_UNBOUNDED, 0, TK_CURRENT, 0, 0); } break; - case 319: /* frame_opt ::= range_or_rows frame_bound_s frame_exclude_opt */ + case 322: /* frame_opt ::= range_or_rows frame_bound_s frame_exclude_opt */ { - yylhsminor.yy41 = sqlite3WindowAlloc(pParse, yymsp[-2].minor.yy394, yymsp[-1].minor.yy595.eType, yymsp[-1].minor.yy595.pExpr, TK_CURRENT, 0, yymsp[0].minor.yy516); + yylhsminor.yy483 = sqlite3WindowAlloc(pParse, yymsp[-2].minor.yy502, yymsp[-1].minor.yy205.eType, yymsp[-1].minor.yy205.pExpr, TK_CURRENT, 0, yymsp[0].minor.yy444); } - yymsp[-2].minor.yy41 = yylhsminor.yy41; + yymsp[-2].minor.yy483 = yylhsminor.yy483; break; - case 320: /* frame_opt ::= range_or_rows BETWEEN frame_bound_s AND frame_bound_e frame_exclude_opt */ + case 323: /* frame_opt ::= range_or_rows BETWEEN frame_bound_s AND frame_bound_e frame_exclude_opt */ { - yylhsminor.yy41 = sqlite3WindowAlloc(pParse, yymsp[-5].minor.yy394, yymsp[-3].minor.yy595.eType, yymsp[-3].minor.yy595.pExpr, yymsp[-1].minor.yy595.eType, yymsp[-1].minor.yy595.pExpr, yymsp[0].minor.yy516); + yylhsminor.yy483 = sqlite3WindowAlloc(pParse, yymsp[-5].minor.yy502, yymsp[-3].minor.yy205.eType, yymsp[-3].minor.yy205.pExpr, yymsp[-1].minor.yy205.eType, yymsp[-1].minor.yy205.pExpr, yymsp[0].minor.yy444); } - yymsp[-5].minor.yy41 = yylhsminor.yy41; + yymsp[-5].minor.yy483 = yylhsminor.yy483; break; - case 322: /* frame_bound_s ::= frame_bound */ - case 324: /* frame_bound_e ::= frame_bound */ yytestcase(yyruleno==324); -{yylhsminor.yy595 = yymsp[0].minor.yy595;} - yymsp[0].minor.yy595 = yylhsminor.yy595; + case 325: /* frame_bound_s ::= frame_bound */ + case 327: /* frame_bound_e ::= frame_bound */ yytestcase(yyruleno==327); +{yylhsminor.yy205 = yymsp[0].minor.yy205;} + yymsp[0].minor.yy205 = yylhsminor.yy205; break; - case 323: /* frame_bound_s ::= UNBOUNDED PRECEDING */ - case 325: /* frame_bound_e ::= UNBOUNDED FOLLOWING */ yytestcase(yyruleno==325); - case 327: /* frame_bound ::= CURRENT ROW */ yytestcase(yyruleno==327); -{yylhsminor.yy595.eType = yymsp[-1].major; yylhsminor.yy595.pExpr = 0;} - yymsp[-1].minor.yy595 = yylhsminor.yy595; + case 326: /* frame_bound_s ::= UNBOUNDED PRECEDING */ + case 328: /* frame_bound_e ::= UNBOUNDED FOLLOWING */ yytestcase(yyruleno==328); + case 330: /* frame_bound ::= CURRENT ROW */ yytestcase(yyruleno==330); +{yylhsminor.yy205.eType = yymsp[-1].major; yylhsminor.yy205.pExpr = 0;} + yymsp[-1].minor.yy205 = yylhsminor.yy205; break; - case 326: /* frame_bound ::= expr PRECEDING|FOLLOWING */ -{yylhsminor.yy595.eType = yymsp[0].major; yylhsminor.yy595.pExpr = yymsp[-1].minor.yy528;} - yymsp[-1].minor.yy595 = yylhsminor.yy595; + case 329: /* frame_bound ::= expr PRECEDING|FOLLOWING */ +{yylhsminor.yy205.eType = yymsp[0].major; yylhsminor.yy205.pExpr = yymsp[-1].minor.yy590;} + yymsp[-1].minor.yy205 = yylhsminor.yy205; break; - case 328: /* frame_exclude_opt ::= */ -{yymsp[1].minor.yy516 = 0;} + case 331: /* frame_exclude_opt ::= */ +{yymsp[1].minor.yy444 = 0;} break; - case 329: /* frame_exclude_opt ::= EXCLUDE frame_exclude */ -{yymsp[-1].minor.yy516 = yymsp[0].minor.yy516;} + case 332: /* frame_exclude_opt ::= EXCLUDE frame_exclude */ +{yymsp[-1].minor.yy444 = yymsp[0].minor.yy444;} break; - case 330: /* frame_exclude ::= NO OTHERS */ - case 331: /* frame_exclude ::= CURRENT ROW */ yytestcase(yyruleno==331); -{yymsp[-1].minor.yy516 = yymsp[-1].major; /*A-overwrites-X*/} + case 333: /* frame_exclude ::= NO OTHERS */ + case 334: /* frame_exclude ::= CURRENT ROW */ yytestcase(yyruleno==334); +{yymsp[-1].minor.yy444 = yymsp[-1].major; /*A-overwrites-X*/} break; - case 332: /* frame_exclude ::= GROUP|TIES */ -{yymsp[0].minor.yy516 = yymsp[0].major; /*A-overwrites-X*/} + case 335: /* frame_exclude ::= GROUP|TIES */ +{yymsp[0].minor.yy444 = yymsp[0].major; /*A-overwrites-X*/} break; - case 333: /* window_clause ::= WINDOW windowdefn_list */ -{ yymsp[-1].minor.yy41 = yymsp[0].minor.yy41; } + case 336: /* window_clause ::= WINDOW windowdefn_list */ +{ yymsp[-1].minor.yy483 = yymsp[0].minor.yy483; } break; - case 334: /* filter_over ::= filter_clause over_clause */ + case 337: /* filter_over ::= filter_clause over_clause */ { - if( yymsp[0].minor.yy41 ){ - yymsp[0].minor.yy41->pFilter = yymsp[-1].minor.yy528; + if( yymsp[0].minor.yy483 ){ + yymsp[0].minor.yy483->pFilter = yymsp[-1].minor.yy590; }else{ - sqlite3ExprDelete(pParse->db, yymsp[-1].minor.yy528); + sqlite3ExprDelete(pParse->db, yymsp[-1].minor.yy590); } - yylhsminor.yy41 = yymsp[0].minor.yy41; + yylhsminor.yy483 = yymsp[0].minor.yy483; } - yymsp[-1].minor.yy41 = yylhsminor.yy41; + yymsp[-1].minor.yy483 = yylhsminor.yy483; break; - case 335: /* filter_over ::= over_clause */ + case 338: /* filter_over ::= over_clause */ { - yylhsminor.yy41 = yymsp[0].minor.yy41; + yylhsminor.yy483 = yymsp[0].minor.yy483; } - yymsp[0].minor.yy41 = yylhsminor.yy41; + yymsp[0].minor.yy483 = yylhsminor.yy483; break; - case 336: /* filter_over ::= filter_clause */ + case 339: /* filter_over ::= filter_clause */ { - yylhsminor.yy41 = (Window*)sqlite3DbMallocZero(pParse->db, sizeof(Window)); - if( yylhsminor.yy41 ){ - yylhsminor.yy41->eFrmType = TK_FILTER; - yylhsminor.yy41->pFilter = yymsp[0].minor.yy528; + yylhsminor.yy483 = (Window*)sqlite3DbMallocZero(pParse->db, sizeof(Window)); + if( yylhsminor.yy483 ){ + yylhsminor.yy483->eFrmType = TK_FILTER; + yylhsminor.yy483->pFilter = yymsp[0].minor.yy590; }else{ - sqlite3ExprDelete(pParse->db, yymsp[0].minor.yy528); + sqlite3ExprDelete(pParse->db, yymsp[0].minor.yy590); } } - yymsp[0].minor.yy41 = yylhsminor.yy41; + yymsp[0].minor.yy483 = yylhsminor.yy483; break; - case 337: /* over_clause ::= OVER LP window RP */ + case 340: /* over_clause ::= OVER LP window RP */ { - yymsp[-3].minor.yy41 = yymsp[-1].minor.yy41; - assert( yymsp[-3].minor.yy41!=0 ); + yymsp[-3].minor.yy483 = yymsp[-1].minor.yy483; + assert( yymsp[-3].minor.yy483!=0 ); } break; - case 338: /* over_clause ::= OVER nm */ + case 341: /* over_clause ::= OVER nm */ { - yymsp[-1].minor.yy41 = (Window*)sqlite3DbMallocZero(pParse->db, sizeof(Window)); - if( yymsp[-1].minor.yy41 ){ - yymsp[-1].minor.yy41->zName = sqlite3DbStrNDup(pParse->db, yymsp[0].minor.yy0.z, yymsp[0].minor.yy0.n); + yymsp[-1].minor.yy483 = (Window*)sqlite3DbMallocZero(pParse->db, sizeof(Window)); + if( yymsp[-1].minor.yy483 ){ + yymsp[-1].minor.yy483->zName = sqlite3DbStrNDup(pParse->db, yymsp[0].minor.yy0.z, yymsp[0].minor.yy0.n); } } break; - case 339: /* filter_clause ::= FILTER LP WHERE expr RP */ -{ yymsp[-4].minor.yy528 = yymsp[-1].minor.yy528; } + case 342: /* filter_clause ::= FILTER LP WHERE expr RP */ +{ yymsp[-4].minor.yy590 = yymsp[-1].minor.yy590; } + break; + case 343: /* term ::= QNUMBER */ +{ + yylhsminor.yy590=tokenExpr(pParse,yymsp[0].major,yymsp[0].minor.yy0); + sqlite3DequoteNumber(pParse, yylhsminor.yy590); +} + yymsp[0].minor.yy590 = yylhsminor.yy590; break; default: - /* (340) input ::= cmdlist */ yytestcase(yyruleno==340); - /* (341) cmdlist ::= cmdlist ecmd */ yytestcase(yyruleno==341); - /* (342) cmdlist ::= ecmd (OPTIMIZED OUT) */ assert(yyruleno!=342); - /* (343) ecmd ::= SEMI */ yytestcase(yyruleno==343); - /* (344) ecmd ::= cmdx SEMI */ yytestcase(yyruleno==344); - /* (345) ecmd ::= explain cmdx SEMI (NEVER REDUCES) */ assert(yyruleno!=345); - /* (346) trans_opt ::= */ yytestcase(yyruleno==346); - /* (347) trans_opt ::= TRANSACTION */ yytestcase(yyruleno==347); - /* (348) trans_opt ::= TRANSACTION nm */ yytestcase(yyruleno==348); - /* (349) savepoint_opt ::= SAVEPOINT */ yytestcase(yyruleno==349); - /* (350) savepoint_opt ::= */ yytestcase(yyruleno==350); - /* (351) cmd ::= create_table create_table_args */ yytestcase(yyruleno==351); - /* (352) table_option_set ::= table_option (OPTIMIZED OUT) */ assert(yyruleno!=352); - /* (353) columnlist ::= columnlist COMMA columnname carglist */ yytestcase(yyruleno==353); - /* (354) columnlist ::= columnname carglist */ yytestcase(yyruleno==354); - /* (355) nm ::= ID|INDEXED|JOIN_KW */ yytestcase(yyruleno==355); - /* (356) nm ::= STRING */ yytestcase(yyruleno==356); - /* (357) typetoken ::= typename */ yytestcase(yyruleno==357); - /* (358) typename ::= ID|STRING */ yytestcase(yyruleno==358); - /* (359) signed ::= plus_num (OPTIMIZED OUT) */ assert(yyruleno!=359); - /* (360) signed ::= minus_num (OPTIMIZED OUT) */ assert(yyruleno!=360); - /* (361) carglist ::= carglist ccons */ yytestcase(yyruleno==361); - /* (362) carglist ::= */ yytestcase(yyruleno==362); - /* (363) ccons ::= NULL onconf */ yytestcase(yyruleno==363); - /* (364) ccons ::= GENERATED ALWAYS AS generated */ yytestcase(yyruleno==364); - /* (365) ccons ::= AS generated */ yytestcase(yyruleno==365); - /* (366) conslist_opt ::= COMMA conslist */ yytestcase(yyruleno==366); - /* (367) conslist ::= conslist tconscomma tcons */ yytestcase(yyruleno==367); - /* (368) conslist ::= tcons (OPTIMIZED OUT) */ assert(yyruleno!=368); - /* (369) tconscomma ::= */ yytestcase(yyruleno==369); - /* (370) defer_subclause_opt ::= defer_subclause (OPTIMIZED OUT) */ assert(yyruleno!=370); - /* (371) resolvetype ::= raisetype (OPTIMIZED OUT) */ assert(yyruleno!=371); - /* (372) selectnowith ::= oneselect (OPTIMIZED OUT) */ assert(yyruleno!=372); - /* (373) oneselect ::= values */ yytestcase(yyruleno==373); - /* (374) sclp ::= selcollist COMMA */ yytestcase(yyruleno==374); - /* (375) as ::= ID|STRING */ yytestcase(yyruleno==375); - /* (376) indexed_opt ::= indexed_by (OPTIMIZED OUT) */ assert(yyruleno!=376); - /* (377) returning ::= */ yytestcase(yyruleno==377); - /* (378) expr ::= term (OPTIMIZED OUT) */ assert(yyruleno!=378); - /* (379) likeop ::= LIKE_KW|MATCH */ yytestcase(yyruleno==379); - /* (380) case_operand ::= expr */ yytestcase(yyruleno==380); - /* (381) exprlist ::= nexprlist */ yytestcase(yyruleno==381); - /* (382) nmnum ::= plus_num (OPTIMIZED OUT) */ assert(yyruleno!=382); - /* (383) nmnum ::= nm (OPTIMIZED OUT) */ assert(yyruleno!=383); - /* (384) nmnum ::= ON */ yytestcase(yyruleno==384); - /* (385) nmnum ::= DELETE */ yytestcase(yyruleno==385); - /* (386) nmnum ::= DEFAULT */ yytestcase(yyruleno==386); - /* (387) plus_num ::= INTEGER|FLOAT */ yytestcase(yyruleno==387); - /* (388) foreach_clause ::= */ yytestcase(yyruleno==388); - /* (389) foreach_clause ::= FOR EACH ROW */ yytestcase(yyruleno==389); - /* (390) trnm ::= nm */ yytestcase(yyruleno==390); - /* (391) tridxby ::= */ yytestcase(yyruleno==391); - /* (392) database_kw_opt ::= DATABASE */ yytestcase(yyruleno==392); - /* (393) database_kw_opt ::= */ yytestcase(yyruleno==393); - /* (394) kwcolumn_opt ::= */ yytestcase(yyruleno==394); - /* (395) kwcolumn_opt ::= COLUMNKW */ yytestcase(yyruleno==395); - /* (396) vtabarglist ::= vtabarg */ yytestcase(yyruleno==396); - /* (397) vtabarglist ::= vtabarglist COMMA vtabarg */ yytestcase(yyruleno==397); - /* (398) vtabarg ::= vtabarg vtabargtoken */ yytestcase(yyruleno==398); - /* (399) anylist ::= */ yytestcase(yyruleno==399); - /* (400) anylist ::= anylist LP anylist RP */ yytestcase(yyruleno==400); - /* (401) anylist ::= anylist ANY */ yytestcase(yyruleno==401); - /* (402) with ::= */ yytestcase(yyruleno==402); - /* (403) windowdefn_list ::= windowdefn (OPTIMIZED OUT) */ assert(yyruleno!=403); - /* (404) window ::= frame_opt (OPTIMIZED OUT) */ assert(yyruleno!=404); + /* (344) input ::= cmdlist */ yytestcase(yyruleno==344); + /* (345) cmdlist ::= cmdlist ecmd */ yytestcase(yyruleno==345); + /* (346) cmdlist ::= ecmd (OPTIMIZED OUT) */ assert(yyruleno!=346); + /* (347) ecmd ::= SEMI */ yytestcase(yyruleno==347); + /* (348) ecmd ::= cmdx SEMI */ yytestcase(yyruleno==348); + /* (349) ecmd ::= explain cmdx SEMI (NEVER REDUCES) */ assert(yyruleno!=349); + /* (350) trans_opt ::= */ yytestcase(yyruleno==350); + /* (351) trans_opt ::= TRANSACTION */ yytestcase(yyruleno==351); + /* (352) trans_opt ::= TRANSACTION nm */ yytestcase(yyruleno==352); + /* (353) savepoint_opt ::= SAVEPOINT */ yytestcase(yyruleno==353); + /* (354) savepoint_opt ::= */ yytestcase(yyruleno==354); + /* (355) cmd ::= create_table create_table_args */ yytestcase(yyruleno==355); + /* (356) table_option_set ::= table_option (OPTIMIZED OUT) */ assert(yyruleno!=356); + /* (357) columnlist ::= columnlist COMMA columnname carglist */ yytestcase(yyruleno==357); + /* (358) columnlist ::= columnname carglist */ yytestcase(yyruleno==358); + /* (359) nm ::= ID|INDEXED|JOIN_KW */ yytestcase(yyruleno==359); + /* (360) nm ::= STRING */ yytestcase(yyruleno==360); + /* (361) typetoken ::= typename */ yytestcase(yyruleno==361); + /* (362) typename ::= ID|STRING */ yytestcase(yyruleno==362); + /* (363) signed ::= plus_num (OPTIMIZED OUT) */ assert(yyruleno!=363); + /* (364) signed ::= minus_num (OPTIMIZED OUT) */ assert(yyruleno!=364); + /* (365) carglist ::= carglist ccons */ yytestcase(yyruleno==365); + /* (366) carglist ::= */ yytestcase(yyruleno==366); + /* (367) ccons ::= NULL onconf */ yytestcase(yyruleno==367); + /* (368) ccons ::= GENERATED ALWAYS AS generated */ yytestcase(yyruleno==368); + /* (369) ccons ::= AS generated */ yytestcase(yyruleno==369); + /* (370) conslist_opt ::= COMMA conslist */ yytestcase(yyruleno==370); + /* (371) conslist ::= conslist tconscomma tcons */ yytestcase(yyruleno==371); + /* (372) conslist ::= tcons (OPTIMIZED OUT) */ assert(yyruleno!=372); + /* (373) tconscomma ::= */ yytestcase(yyruleno==373); + /* (374) defer_subclause_opt ::= defer_subclause (OPTIMIZED OUT) */ assert(yyruleno!=374); + /* (375) resolvetype ::= raisetype (OPTIMIZED OUT) */ assert(yyruleno!=375); + /* (376) selectnowith ::= oneselect (OPTIMIZED OUT) */ assert(yyruleno!=376); + /* (377) oneselect ::= values */ yytestcase(yyruleno==377); + /* (378) sclp ::= selcollist COMMA */ yytestcase(yyruleno==378); + /* (379) as ::= ID|STRING */ yytestcase(yyruleno==379); + /* (380) indexed_opt ::= indexed_by (OPTIMIZED OUT) */ assert(yyruleno!=380); + /* (381) returning ::= */ yytestcase(yyruleno==381); + /* (382) expr ::= term (OPTIMIZED OUT) */ assert(yyruleno!=382); + /* (383) likeop ::= LIKE_KW|MATCH */ yytestcase(yyruleno==383); + /* (384) case_operand ::= expr */ yytestcase(yyruleno==384); + /* (385) exprlist ::= nexprlist */ yytestcase(yyruleno==385); + /* (386) nmnum ::= plus_num (OPTIMIZED OUT) */ assert(yyruleno!=386); + /* (387) nmnum ::= nm (OPTIMIZED OUT) */ assert(yyruleno!=387); + /* (388) nmnum ::= ON */ yytestcase(yyruleno==388); + /* (389) nmnum ::= DELETE */ yytestcase(yyruleno==389); + /* (390) nmnum ::= DEFAULT */ yytestcase(yyruleno==390); + /* (391) plus_num ::= INTEGER|FLOAT */ yytestcase(yyruleno==391); + /* (392) foreach_clause ::= */ yytestcase(yyruleno==392); + /* (393) foreach_clause ::= FOR EACH ROW */ yytestcase(yyruleno==393); + /* (394) trnm ::= nm */ yytestcase(yyruleno==394); + /* (395) tridxby ::= */ yytestcase(yyruleno==395); + /* (396) database_kw_opt ::= DATABASE */ yytestcase(yyruleno==396); + /* (397) database_kw_opt ::= */ yytestcase(yyruleno==397); + /* (398) kwcolumn_opt ::= */ yytestcase(yyruleno==398); + /* (399) kwcolumn_opt ::= COLUMNKW */ yytestcase(yyruleno==399); + /* (400) vtabarglist ::= vtabarg */ yytestcase(yyruleno==400); + /* (401) vtabarglist ::= vtabarglist COMMA vtabarg */ yytestcase(yyruleno==401); + /* (402) vtabarg ::= vtabarg vtabargtoken */ yytestcase(yyruleno==402); + /* (403) anylist ::= */ yytestcase(yyruleno==403); + /* (404) anylist ::= anylist LP anylist RP */ yytestcase(yyruleno==404); + /* (405) anylist ::= anylist ANY */ yytestcase(yyruleno==405); + /* (406) with ::= */ yytestcase(yyruleno==406); + /* (407) windowdefn_list ::= windowdefn (OPTIMIZED OUT) */ assert(yyruleno!=407); + /* (408) window ::= frame_opt (OPTIMIZED OUT) */ assert(yyruleno!=408); break; /********** End reduce actions ************************************************/ }; @@ -176197,7 +182798,7 @@ static void yy_syntax_error( UNUSED_PARAMETER(yymajor); /* Silence some compiler warnings */ if( TOKEN.z[0] ){ - sqlite3ErrorMsg(pParse, "near \"%T\": syntax error", &TOKEN); + parserSyntaxError(pParse, &TOKEN); }else{ sqlite3ErrorMsg(pParse, "incomplete input"); } @@ -176321,19 +182922,12 @@ SQLITE_PRIVATE void sqlite3Parser( (int)(yypParser->yytos - yypParser->yystack)); } #endif -#if YYSTACKDEPTH>0 if( yypParser->yytos>=yypParser->yystackEnd ){ - yyStackOverflow(yypParser); - break; - } -#else - if( yypParser->yytos>=&yypParser->yystack[yypParser->yystksz-1] ){ if( yyGrowStack(yypParser) ){ yyStackOverflow(yypParser); break; } } -#endif } yyact = yy_reduce(yypParser,yyruleno,yymajor,yyminor sqlite3ParserCTX_PARAM); }else if( yyact <= YY_MAX_SHIFTREDUCE ){ @@ -177166,7 +183760,7 @@ static int getToken(const unsigned char **pz){ int t; /* Token type to return */ do { z += sqlite3GetToken(z, &t); - }while( t==TK_SPACE ); + }while( t==TK_SPACE || t==TK_COMMENT ); if( t==TK_ID || t==TK_STRING || t==TK_JOIN_KW @@ -177237,8 +183831,9 @@ static int analyzeFilterKeyword(const unsigned char *z, int lastToken){ ** Return the length (in bytes) of the token that begins at z[0]. ** Store the token type in *tokenType before returning. */ -SQLITE_PRIVATE int sqlite3GetToken(const unsigned char *z, int *tokenType){ - int i, c; +SQLITE_PRIVATE i64 sqlite3GetToken(const unsigned char *z, int *tokenType){ + i64 i; + int c; switch( aiClass[*z] ){ /* Switch on the character-class of the first byte ** of the token. See the comment on the CC_ defines ** above. */ @@ -177255,7 +183850,7 @@ SQLITE_PRIVATE int sqlite3GetToken(const unsigned char *z, int *tokenType){ case CC_MINUS: { if( z[1]=='-' ){ for(i=2; (c=z[i])!=0 && c!='\n'; i++){} - *tokenType = TK_SPACE; /* IMP: R-22934-25134 */ + *tokenType = TK_COMMENT; return i; }else if( z[1]=='>' ){ *tokenType = TK_PTR; @@ -177291,7 +183886,7 @@ SQLITE_PRIVATE int sqlite3GetToken(const unsigned char *z, int *tokenType){ } for(i=3, c=z[2]; (c!='*' || z[i]!='/') && (c=z[i])!=0; i++){} if( c ) i++; - *tokenType = TK_SPACE; /* IMP: R-22934-25134 */ + *tokenType = TK_COMMENT; return i; } case CC_PERCENT: { @@ -177404,27 +183999,58 @@ SQLITE_PRIVATE int sqlite3GetToken(const unsigned char *z, int *tokenType){ *tokenType = TK_INTEGER; #ifndef SQLITE_OMIT_HEX_INTEGER if( z[0]=='0' && (z[1]=='x' || z[1]=='X') && sqlite3Isxdigit(z[2]) ){ - for(i=3; sqlite3Isxdigit(z[i]); i++){} - return i; - } + for(i=3; 1; i++){ + if( sqlite3Isxdigit(z[i])==0 ){ + if( z[i]==SQLITE_DIGIT_SEPARATOR ){ + *tokenType = TK_QNUMBER; + }else{ + break; + } + } + } + }else #endif - for(i=0; sqlite3Isdigit(z[i]); i++){} + { + for(i=0; 1; i++){ + if( sqlite3Isdigit(z[i])==0 ){ + if( z[i]==SQLITE_DIGIT_SEPARATOR ){ + *tokenType = TK_QNUMBER; + }else{ + break; + } + } + } #ifndef SQLITE_OMIT_FLOATING_POINT - if( z[i]=='.' ){ - i++; - while( sqlite3Isdigit(z[i]) ){ i++; } - *tokenType = TK_FLOAT; - } - if( (z[i]=='e' || z[i]=='E') && - ( sqlite3Isdigit(z[i+1]) - || ((z[i+1]=='+' || z[i+1]=='-') && sqlite3Isdigit(z[i+2])) - ) - ){ - i += 2; - while( sqlite3Isdigit(z[i]) ){ i++; } - *tokenType = TK_FLOAT; - } + if( z[i]=='.' ){ + if( *tokenType==TK_INTEGER ) *tokenType = TK_FLOAT; + for(i++; 1; i++){ + if( sqlite3Isdigit(z[i])==0 ){ + if( z[i]==SQLITE_DIGIT_SEPARATOR ){ + *tokenType = TK_QNUMBER; + }else{ + break; + } + } + } + } + if( (z[i]=='e' || z[i]=='E') && + ( sqlite3Isdigit(z[i+1]) + || ((z[i+1]=='+' || z[i+1]=='-') && sqlite3Isdigit(z[i+2])) + ) + ){ + if( *tokenType==TK_INTEGER ) *tokenType = TK_FLOAT; + for(i+=2; 1; i++){ + if( sqlite3Isdigit(z[i])==0 ){ + if( z[i]==SQLITE_DIGIT_SEPARATOR ){ + *tokenType = TK_QNUMBER; + }else{ + break; + } + } + } + } #endif + } while( IdChar(z[i]) ){ *tokenType = TK_ILLEGAL; i++; @@ -177535,7 +184161,7 @@ SQLITE_PRIVATE int sqlite3GetToken(const unsigned char *z, int *tokenType){ SQLITE_PRIVATE int sqlite3RunParser(Parse *pParse, const char *zSql){ int nErr = 0; /* Number of errors encountered */ void *pEngine; /* The LEMON-generated LALR(1) parser */ - int n = 0; /* Length of the next token token */ + i64 n = 0; /* Length of the next token token */ int tokenType; /* type of the next token */ int lastTokenParsed = -1; /* type of the previous token */ sqlite3 *db = pParse->db; /* The database connection */ @@ -177589,10 +184215,13 @@ SQLITE_PRIVATE int sqlite3RunParser(Parse *pParse, const char *zSql){ if( tokenType>=TK_WINDOW ){ assert( tokenType==TK_SPACE || tokenType==TK_OVER || tokenType==TK_FILTER || tokenType==TK_ILLEGAL || tokenType==TK_WINDOW + || tokenType==TK_QNUMBER || tokenType==TK_COMMENT ); #else if( tokenType>=TK_SPACE ){ - assert( tokenType==TK_SPACE || tokenType==TK_ILLEGAL ); + assert( tokenType==TK_SPACE || tokenType==TK_ILLEGAL + || tokenType==TK_QNUMBER || tokenType==TK_COMMENT + ); #endif /* SQLITE_OMIT_WINDOWFUNC */ if( AtomicLoad(&db->u1.isInterrupted) ){ pParse->rc = SQLITE_INTERRUPT; @@ -177625,16 +184254,23 @@ SQLITE_PRIVATE int sqlite3RunParser(Parse *pParse, const char *zSql){ assert( n==6 ); tokenType = analyzeFilterKeyword((const u8*)&zSql[6], lastTokenParsed); #endif /* SQLITE_OMIT_WINDOWFUNC */ - }else{ + }else if( tokenType==TK_COMMENT + && (db->init.busy || (db->flags & SQLITE_Comments)!=0) + ){ + /* Ignore SQL comments if either (1) we are reparsing the schema or + ** (2) SQLITE_DBCONFIG_ENABLE_COMMENTS is turned on (the default). */ + zSql += n; + continue; + }else if( tokenType!=TK_QNUMBER ){ Token x; x.z = zSql; - x.n = n; + x.n = (u32)n; sqlite3ErrorMsg(pParse, "unrecognized token: \"%T\"", &x); break; } } pParse->sLastToken.z = zSql; - pParse->sLastToken.n = n; + pParse->sLastToken.n = (u32)n; sqlite3Parser(pEngine, tokenType, pParse->sLastToken); lastTokenParsed = tokenType; zSql += n; @@ -177661,7 +184297,9 @@ SQLITE_PRIVATE int sqlite3RunParser(Parse *pParse, const char *zSql){ if( pParse->zErrMsg==0 ){ pParse->zErrMsg = sqlite3MPrintf(db, "%s", sqlite3ErrStr(pParse->rc)); } - sqlite3_log(pParse->rc, "%s in \"%s\"", pParse->zErrMsg, pParse->zTail); + if( (pParse->prepFlags & SQLITE_PREPARE_DONT_LOG)==0 ){ + sqlite3_log(pParse->rc, "%s in \"%s\"", pParse->zErrMsg, pParse->zTail); + } nErr++; } pParse->zTail = zSql; @@ -177708,7 +184346,7 @@ SQLITE_PRIVATE char *sqlite3Normalize( ){ sqlite3 *db; /* The database connection */ int i; /* Next unread byte of zSql[] */ - int n; /* length of current token */ + i64 n; /* length of current token */ int tokenType; /* type of current token */ int prevType = 0; /* Previous non-whitespace token */ int nParen; /* Number of nested levels of parentheses */ @@ -177729,6 +184367,7 @@ SQLITE_PRIVATE char *sqlite3Normalize( n = sqlite3GetToken((unsigned char*)zSql+i, &tokenType); if( NEVER(n<=0) ) break; switch( tokenType ){ + case TK_COMMENT: case TK_SPACE: { break; } @@ -178285,9 +184924,6 @@ static int (*const sqlite3BuiltinExtensions[])(sqlite3*) = { sqlite3DbstatRegister, #endif sqlite3TestExtInit, -#if !defined(SQLITE_OMIT_VIRTUALTABLE) && !defined(SQLITE_OMIT_JSON) - sqlite3JsonTableFunctions, -#endif #ifdef SQLITE_ENABLE_STMTVTAB sqlite3StmtVtabInit, #endif @@ -178370,32 +185006,6 @@ SQLITE_API char *sqlite3_temp_directory = 0; */ SQLITE_API char *sqlite3_data_directory = 0; -/* -** Determine whether or not high-precision (long double) floating point -** math works correctly on CPU currently running. -*/ -static SQLITE_NOINLINE int hasHighPrecisionDouble(int rc){ - if( sizeof(LONGDOUBLE_TYPE)<=8 ){ - /* If the size of "long double" is not more than 8, then - ** high-precision math is not possible. */ - return 0; - }else{ - /* Just because sizeof(long double)>8 does not mean that the underlying - ** hardware actually supports high-precision floating point. For example, - ** clearing the 0x100 bit in the floating-point control word on Intel - ** processors will make long double work like double, even though long - ** double takes up more space. The only way to determine if long double - ** actually works is to run an experiment. */ - LONGDOUBLE_TYPE a, b, c; - rc++; - a = 1.0+rc*0.1; - b = 1.0e+18+rc*25.0; - c = a+b; - return b!=c; - } -} - - /* ** Initialize SQLite. ** @@ -178540,6 +185150,14 @@ SQLITE_API int sqlite3_initialize(void){ if( rc==SQLITE_OK ){ sqlite3PCacheBufferSetup( sqlite3GlobalConfig.pPage, sqlite3GlobalConfig.szPage, sqlite3GlobalConfig.nPage); +#ifdef SQLITE_EXTRA_INIT_MUTEXED + { + int SQLITE_EXTRA_INIT_MUTEXED(const char*); + rc = SQLITE_EXTRA_INIT_MUTEXED(0); + } +#endif + } + if( rc==SQLITE_OK ){ sqlite3MemoryBarrier(); sqlite3GlobalConfig.isInit = 1; #ifdef SQLITE_EXTRA_INIT @@ -178590,13 +185208,6 @@ SQLITE_API int sqlite3_initialize(void){ rc = SQLITE_EXTRA_INIT(0); } #endif - - /* Experimentally determine if high-precision floating point is - ** available. */ -#ifndef SQLITE_OMIT_WSD - sqlite3Config.bUseLongDouble = hasHighPrecisionDouble(rc); -#endif - return rc; } @@ -178976,6 +185587,18 @@ SQLITE_API int sqlite3_config(int op, ...){ } #endif /* SQLITE_OMIT_DESERIALIZE */ + case SQLITE_CONFIG_ROWID_IN_VIEW: { + int *pVal = va_arg(ap,int*); +#ifdef SQLITE_ALLOW_ROWID_IN_VIEW + if( 0==*pVal ) sqlite3GlobalConfig.mNoVisibleRowid = TF_NoVisibleRowid; + if( 1==*pVal ) sqlite3GlobalConfig.mNoVisibleRowid = 0; + *pVal = (sqlite3GlobalConfig.mNoVisibleRowid==0); +#else + *pVal = 0; +#endif + break; + } + default: { rc = SQLITE_ERROR; break; @@ -178991,17 +185614,22 @@ SQLITE_API int sqlite3_config(int op, ...){ ** If lookaside is already active, return SQLITE_BUSY. ** ** The sz parameter is the number of bytes in each lookaside slot. -** The cnt parameter is the number of slots. If pStart is NULL the -** space for the lookaside memory is obtained from sqlite3_malloc(). -** If pStart is not NULL then it is sz*cnt bytes of memory to use for -** the lookaside memory. +** The cnt parameter is the number of slots. If pBuf is NULL the +** space for the lookaside memory is obtained from sqlite3_malloc() +** or similar. If pBuf is not NULL then it is sz*cnt bytes of memory +** to use for the lookaside memory. */ -static int setupLookaside(sqlite3 *db, void *pBuf, int sz, int cnt){ +static int setupLookaside( + sqlite3 *db, /* Database connection being configured */ + void *pBuf, /* Memory to use for lookaside. May be NULL */ + int sz, /* Desired size of each lookaside memory slot */ + int cnt /* Number of slots to allocate */ +){ #ifndef SQLITE_OMIT_LOOKASIDE - void *pStart; - sqlite3_int64 szAlloc = sz*(sqlite3_int64)cnt; - int nBig; /* Number of full-size slots */ - int nSm; /* Number smaller LOOKASIDE_SMALL-byte slots */ + void *pStart; /* Start of the lookaside buffer */ + sqlite3_int64 szAlloc; /* Total space set aside for lookaside memory */ + int nBig; /* Number of full-size slots */ + int nSm; /* Number smaller LOOKASIDE_SMALL-byte slots */ if( sqlite3LookasideUsed(db,0)>0 ){ return SQLITE_BUSY; @@ -179014,17 +185642,22 @@ static int setupLookaside(sqlite3 *db, void *pBuf, int sz, int cnt){ sqlite3_free(db->lookaside.pStart); } /* The size of a lookaside slot after ROUNDDOWN8 needs to be larger - ** than a pointer to be useful. + ** than a pointer and small enough to fit in a u16. */ - sz = ROUNDDOWN8(sz); /* IMP: R-33038-09382 */ + sz = ROUNDDOWN8(sz); if( sz<=(int)sizeof(LookasideSlot*) ) sz = 0; - if( cnt<0 ) cnt = 0; - if( sz==0 || cnt==0 ){ + if( sz>65528 ) sz = 65528; + /* Count must be at least 1 to be useful, but not so large as to use + ** more than 0x7fff0000 total bytes for lookaside. */ + if( cnt<1 ) cnt = 0; + if( sz>0 && cnt>(0x7fff0000/sz) ) cnt = 0x7fff0000/sz; + szAlloc = (i64)sz*(i64)cnt; + if( szAlloc==0 ){ sz = 0; pStart = 0; }else if( pBuf==0 ){ sqlite3BeginBenignMalloc(); - pStart = sqlite3Malloc( szAlloc ); /* IMP: R-61949-35727 */ + pStart = sqlite3Malloc( szAlloc ); sqlite3EndBenignMalloc(); if( pStart ) szAlloc = sqlite3MallocSize(pStart); }else{ @@ -179033,10 +185666,10 @@ static int setupLookaside(sqlite3 *db, void *pBuf, int sz, int cnt){ #ifndef SQLITE_OMIT_TWOSIZE_LOOKASIDE if( sz>=LOOKASIDE_SMALL*3 ){ nBig = szAlloc/(3*LOOKASIDE_SMALL+sz); - nSm = (szAlloc - sz*nBig)/LOOKASIDE_SMALL; + nSm = (szAlloc - (i64)sz*(i64)nBig)/LOOKASIDE_SMALL; }else if( sz>=LOOKASIDE_SMALL*2 ){ nBig = szAlloc/(LOOKASIDE_SMALL+sz); - nSm = (szAlloc - sz*nBig)/LOOKASIDE_SMALL; + nSm = (szAlloc - (i64)sz*(i64)nBig)/LOOKASIDE_SMALL; }else #endif /* SQLITE_OMIT_TWOSIZE_LOOKASIDE */ if( sz>0 ){ @@ -179191,7 +185824,7 @@ SQLITE_API int sqlite3_db_config(sqlite3 *db, int op, ...){ default: { static const struct { int op; /* The opcode */ - u32 mask; /* Mask of the bit in sqlite3.flags to set/clear */ + u64 mask; /* Mask of the bit in sqlite3.flags to set/clear */ } aFlagOp[] = { { SQLITE_DBCONFIG_ENABLE_FKEY, SQLITE_ForeignKeys }, { SQLITE_DBCONFIG_ENABLE_TRIGGER, SQLITE_EnableTrigger }, @@ -179212,6 +185845,9 @@ SQLITE_API int sqlite3_db_config(sqlite3 *db, int op, ...){ { SQLITE_DBCONFIG_TRUSTED_SCHEMA, SQLITE_TrustedSchema }, { SQLITE_DBCONFIG_STMT_SCANSTATUS, SQLITE_StmtScanStatus }, { SQLITE_DBCONFIG_REVERSE_SCANORDER, SQLITE_ReverseOrder }, + { SQLITE_DBCONFIG_ENABLE_ATTACH_CREATE, SQLITE_AttachCreate }, + { SQLITE_DBCONFIG_ENABLE_ATTACH_WRITE, SQLITE_AttachWrite }, + { SQLITE_DBCONFIG_ENABLE_COMMENTS, SQLITE_Comments }, }; unsigned int i; rc = SQLITE_ERROR; /* IMP: R-42790-23372 */ @@ -179608,6 +186244,7 @@ SQLITE_PRIVATE void sqlite3LeaveMutexAndCloseZombie(sqlite3 *db){ /* Clear the TEMP schema separately and last */ if( db->aDb[1].pSchema ){ sqlite3SchemaClear(db->aDb[1].pSchema); + assert( db->aDb[1].pSchema->trigHash.count==0 ); } sqlite3VtabUnlockList(db); @@ -179655,10 +186292,6 @@ SQLITE_PRIVATE void sqlite3LeaveMutexAndCloseZombie(sqlite3 *db){ sqlite3Error(db, SQLITE_OK); /* Deallocates any cached error strings. */ sqlite3ValueFree(db->pErr); sqlite3CloseExtensions(db); -#if SQLITE_USER_AUTHENTICATION - sqlite3_free(db->auth.zAuthUser); - sqlite3_free(db->auth.zAuthPW); -#endif db->eOpenState = SQLITE_STATE_ERROR; @@ -179747,6 +186380,9 @@ SQLITE_PRIVATE const char *sqlite3ErrName(int rc){ case SQLITE_OK: zName = "SQLITE_OK"; break; case SQLITE_ERROR: zName = "SQLITE_ERROR"; break; case SQLITE_ERROR_SNAPSHOT: zName = "SQLITE_ERROR_SNAPSHOT"; break; + case SQLITE_ERROR_RETRY: zName = "SQLITE_ERROR_RETRY"; break; + case SQLITE_ERROR_MISSING_COLLSEQ: + zName = "SQLITE_ERROR_MISSING_COLLSEQ"; break; case SQLITE_INTERNAL: zName = "SQLITE_INTERNAL"; break; case SQLITE_PERM: zName = "SQLITE_PERM"; break; case SQLITE_ABORT: zName = "SQLITE_ABORT"; break; @@ -180002,6 +186638,9 @@ SQLITE_API int sqlite3_busy_handler( db->busyHandler.pBusyArg = pArg; db->busyHandler.nBusy = 0; db->busyTimeout = 0; +#ifdef SQLITE_ENABLE_SETLK_TIMEOUT + db->setlkTimeout = 0; +#endif sqlite3_mutex_leave(db->mutex); return SQLITE_OK; } @@ -180051,12 +186690,49 @@ SQLITE_API int sqlite3_busy_timeout(sqlite3 *db, int ms){ sqlite3_busy_handler(db, (int(*)(void*,int))sqliteDefaultBusyCallback, (void*)db); db->busyTimeout = ms; +#ifdef SQLITE_ENABLE_SETLK_TIMEOUT + db->setlkTimeout = ms; +#endif }else{ sqlite3_busy_handler(db, 0, 0); } return SQLITE_OK; } +/* +** Set the setlk timeout value. +*/ +SQLITE_API int sqlite3_setlk_timeout(sqlite3 *db, int ms, int flags){ +#ifdef SQLITE_ENABLE_SETLK_TIMEOUT + int iDb; + int bBOC = ((flags & SQLITE_SETLK_BLOCK_ON_CONNECT) ? 1 : 0); +#endif +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT; +#endif + if( ms<-1 ) return SQLITE_RANGE; +#ifdef SQLITE_ENABLE_SETLK_TIMEOUT + sqlite3_mutex_enter(db->mutex); + db->setlkTimeout = ms; + db->setlkFlags = flags; + sqlite3BtreeEnterAll(db); + for(iDb=0; iDbnDb; iDb++){ + Btree *pBt = db->aDb[iDb].pBt; + if( pBt ){ + sqlite3_file *fd = sqlite3PagerFile(sqlite3BtreePager(pBt)); + sqlite3OsFileControlHint(fd, SQLITE_FCNTL_BLOCK_ON_CONNECT, (void*)&bBOC); + } + } + sqlite3BtreeLeaveAll(db); + sqlite3_mutex_leave(db->mutex); +#endif +#if !defined(SQLITE_ENABLE_API_ARMOR) && !defined(SQLITE_ENABLE_SETLK_TIMEOUT) + UNUSED_PARAMETER(db); + UNUSED_PARAMETER(flags); +#endif + return SQLITE_OK; +} + /* ** Cause any pending operation to stop at its earliest opportunity. */ @@ -180125,7 +186801,8 @@ SQLITE_PRIVATE int sqlite3CreateFunc( assert( SQLITE_FUNC_CONSTANT==SQLITE_DETERMINISTIC ); assert( SQLITE_FUNC_DIRECT==SQLITE_DIRECTONLY ); extraFlags = enc & (SQLITE_DETERMINISTIC|SQLITE_DIRECTONLY| - SQLITE_SUBTYPE|SQLITE_INNOCUOUS|SQLITE_RESULT_SUBTYPE); + SQLITE_SUBTYPE|SQLITE_INNOCUOUS| + SQLITE_RESULT_SUBTYPE|SQLITE_SELFORDER1); enc &= (SQLITE_FUNC_ENCMASK|SQLITE_ANY); /* The SQLITE_INNOCUOUS flag is the same bit as SQLITE_FUNC_UNSAFE. But @@ -180887,6 +187564,29 @@ SQLITE_API const char *sqlite3_errmsg(sqlite3 *db){ return z; } +/* +** Set the error code and error message associated with the database handle. +** +** This routine is intended to be called by outside extensions (ex: the +** Session extension). Internal logic should invoke sqlite3Error() or +** sqlite3ErrorWithMsg() directly. +*/ +SQLITE_API int sqlite3_set_errmsg(sqlite3 *db, int errcode, const char *zMsg){ + int rc = SQLITE_OK; + if( !sqlite3SafetyCheckOk(db) ){ + return SQLITE_MISUSE_BKPT; + } + sqlite3_mutex_enter(db->mutex); + if( zMsg ){ + sqlite3ErrorWithMsg(db, errcode, "%s", zMsg); + }else{ + sqlite3Error(db, errcode); + } + rc = sqlite3ApiExit(db, rc); + sqlite3_mutex_leave(db->mutex); + return rc; +} + /* ** Return the byte offset of the most recent error */ @@ -181092,8 +187792,8 @@ static const int aHardLimit[] = { #if SQLITE_MAX_VDBE_OP<40 # error SQLITE_MAX_VDBE_OP must be at least 40 #endif -#if SQLITE_MAX_FUNCTION_ARG<0 || SQLITE_MAX_FUNCTION_ARG>127 -# error SQLITE_MAX_FUNCTION_ARG must be between 0 and 127 +#if SQLITE_MAX_FUNCTION_ARG<0 || SQLITE_MAX_FUNCTION_ARG>32767 +# error SQLITE_MAX_FUNCTION_ARG must be between 0 and 32767 #endif #if SQLITE_MAX_ATTACHED<0 || SQLITE_MAX_ATTACHED>125 # error SQLITE_MAX_ATTACHED must be between 0 and 125 @@ -181160,8 +187860,8 @@ SQLITE_API int sqlite3_limit(sqlite3 *db, int limitId, int newLimit){ if( newLimit>=0 ){ /* IMP: R-52476-28732 */ if( newLimit>aHardLimit[limitId] ){ newLimit = aHardLimit[limitId]; /* IMP: R-51463-25634 */ - }else if( newLimit<1 && limitId==SQLITE_LIMIT_LENGTH ){ - newLimit = 1; + }else if( newLimitaLimit[limitId] = newLimit; } @@ -181556,6 +188256,9 @@ static int openDatabase( | SQLITE_EnableTrigger | SQLITE_EnableView | SQLITE_CacheSpill + | SQLITE_AttachCreate + | SQLITE_AttachWrite + | SQLITE_Comments #if !defined(SQLITE_TRUSTED_SCHEMA) || SQLITE_TRUSTED_SCHEMA+0!=0 | SQLITE_TrustedSchema #endif @@ -181680,6 +188383,7 @@ static int openDatabase( if( ((1<<(flags&7)) & 0x46)==0 ){ rc = SQLITE_MISUSE_BKPT; /* IMP: R-18321-05872 */ }else{ + if( zFilename==0 ) zFilename = ":memory:"; rc = sqlite3ParseUri(zVfs, zFilename, &flags, &db->pVfs, &zOpen, &zErrMsg); } if( rc!=SQLITE_OK ){ @@ -182017,7 +188721,7 @@ SQLITE_API int sqlite3_set_clientdata( return SQLITE_OK; }else{ size_t n = strlen(zName); - p = sqlite3_malloc64( sizeof(DbClientData)+n+1 ); + p = sqlite3_malloc64( SZ_DBCLIENTDATA(n+1) ); if( p==0 ){ if( xDestructor ) xDestructor(pData); sqlite3_mutex_leave(db->mutex); @@ -182171,13 +188875,10 @@ SQLITE_API int sqlite3_table_column_metadata( if( zColumnName==0 ){ /* Query for existence of table only */ }else{ - for(iCol=0; iColnCol; iCol++){ + iCol = sqlite3ColumnIndex(pTab, zColumnName); + if( iCol>=0 ){ pCol = &pTab->aCol[iCol]; - if( 0==sqlite3StrICmp(pCol->zCnName, zColumnName) ){ - break; - } - } - if( iCol==pTab->nCol ){ + }else{ if( HasRowid(pTab) && sqlite3IsRowid(zColumnName) ){ iCol = pTab->iPKey; pCol = iCol>=0 ? &pTab->aCol[iCol] : 0; @@ -182386,8 +189087,8 @@ SQLITE_API int sqlite3_test_control(int op, ...){ /* sqlite3_test_control(SQLITE_TESTCTRL_FK_NO_ACTION, sqlite3 *db, int b); ** ** If b is true, then activate the SQLITE_FkNoAction setting. If b is - ** false then clearn that setting. If the SQLITE_FkNoAction setting is - ** abled, all foreign key ON DELETE and ON UPDATE actions behave as if + ** false then clear that setting. If the SQLITE_FkNoAction setting is + ** enabled, all foreign key ON DELETE and ON UPDATE actions behave as if ** they were NO ACTION, regardless of how they are defined. ** ** NB: One must usually run "PRAGMA writable_schema=RESET" after @@ -182504,7 +189205,6 @@ SQLITE_API int sqlite3_test_control(int op, ...){ /* Invoke these debugging routines so that the compiler does not ** issue "defined but not used" warnings. */ if( x==9999 ){ - sqlite3ShowExpr(0); sqlite3ShowExpr(0); sqlite3ShowExprList(0); sqlite3ShowIdList(0); @@ -182592,6 +189292,18 @@ SQLITE_API int sqlite3_test_control(int op, ...){ break; } + /* sqlite3_test_control(SQLITE_TESTCTRL_GETOPT, sqlite3 *db, int *N) + ** + ** Write the current optimization settings into *N. A zero bit means that + ** the optimization is on, and a 1 bit means that the optimization is off. + */ + case SQLITE_TESTCTRL_GETOPT: { + sqlite3 *db = va_arg(ap, sqlite3*); + int *pN = va_arg(ap, int*); + *pN = db->dbOptFlags; + break; + } + /* sqlite3_test_control(SQLITE_TESTCTRL_LOCALTIME_FAULT, onoff, xAlt); ** ** If parameter onoff is 1, subsequent calls to localtime() fail. @@ -182699,13 +189411,15 @@ SQLITE_API int sqlite3_test_control(int op, ...){ break; } - /* sqlite3_test_control(SQLITE_TESTCTRL_IMPOSTER, db, dbName, onOff, tnum); + /* sqlite3_test_control(SQLITE_TESTCTRL_IMPOSTER, db, dbName, mode, tnum); ** ** This test control is used to create imposter tables. "db" is a pointer ** to the database connection. dbName is the database name (ex: "main" or - ** "temp") which will receive the imposter. "onOff" turns imposter mode on - ** or off. "tnum" is the root page of the b-tree to which the imposter - ** table should connect. + ** "temp") which will receive the imposter. "mode" turns imposter mode on + ** or off. mode==0 means imposter mode is off. mode==1 means imposter mode + ** is on. mode==2 means imposter mode is on but results in an imposter + ** table that is read-only unless writable_schema is on. "tnum" is the + ** root page of the b-tree to which the imposter table should connect. ** ** Enable imposter mode only when the schema has already been parsed. Then ** run a single CREATE TABLE statement to construct the imposter table in @@ -182823,24 +189537,6 @@ SQLITE_API int sqlite3_test_control(int op, ...){ break; } -#if !defined(SQLITE_OMIT_WSD) - /* sqlite3_test_control(SQLITE_TESTCTRL_USELONGDOUBLE, int X); - ** - ** X<0 Make no changes to the bUseLongDouble. Just report value. - ** X==0 Disable bUseLongDouble - ** X==1 Enable bUseLongDouble - ** X>=2 Set bUseLongDouble to its default value for this platform - */ - case SQLITE_TESTCTRL_USELONGDOUBLE: { - int b = va_arg(ap, int); - if( b>=2 ) b = hasHighPrecisionDouble(b); - if( b>=0 ) sqlite3Config.bUseLongDouble = b>0; - rc = sqlite3Config.bUseLongDouble!=0; - break; - } -#endif - - #if defined(SQLITE_DEBUG) && !defined(SQLITE_OMIT_WSD) /* sqlite3_test_control(SQLITE_TESTCTRL_TUNE, id, *piValue) ** @@ -183148,7 +189844,11 @@ SQLITE_API int sqlite3_snapshot_get( if( iDb==0 || iDb>1 ){ Btree *pBt = db->aDb[iDb].pBt; if( SQLITE_TXN_WRITE!=sqlite3BtreeTxnState(pBt) ){ + Pager *pPager = sqlite3BtreePager(pBt); + i64 dummy = 0; + sqlite3PagerSnapshotOpen(pPager, (sqlite3_snapshot*)&dummy); rc = sqlite3BtreeBeginTrans(pBt, 0, 0); + sqlite3PagerSnapshotOpen(pPager, 0); if( rc==SQLITE_OK ){ rc = sqlite3PagerSnapshotGet(sqlite3BtreePager(pBt), ppSnapshot); } @@ -183737,7 +190437,7 @@ SQLITE_PRIVATE void sqlite3ConnectionClosed(sqlite3 *db){ ** Here, array { X } means zero or more occurrences of X, adjacent in ** memory. A "position" is an index of a token in the token stream ** generated by the tokenizer. Note that POS_END and POS_COLUMN occur -** in the same logical place as the position element, and act as sentinals +** in the same logical place as the position element, and act as sentinels ** ending a position list array. POS_END is 0. POS_COLUMN is 1. ** The positions numbers are not stored literally but rather as two more ** than the difference from the prior position, or the just the position plus @@ -183956,10 +190656,20 @@ SQLITE_PRIVATE void sqlite3ConnectionClosed(sqlite3 *db){ #ifndef _FTSINT_H #define _FTSINT_H +/* +** Activate assert() only if SQLITE_TEST is enabled. +*/ #if !defined(NDEBUG) && !defined(SQLITE_DEBUG) # define NDEBUG 1 #endif +/* #include */ +/* #include */ +/* #include */ +/* #include */ +/* #include */ +/* #include */ + /* FTS3/FTS4 require virtual tables */ #ifdef SQLITE_OMIT_VIRTUALTABLE # undef SQLITE_ENABLE_FTS3 @@ -184402,13 +191112,6 @@ typedef sqlite3_int64 i64; /* 8-byte signed integer */ */ #define UNUSED_PARAMETER(x) (void)(x) -/* -** Activate assert() only if SQLITE_TEST is enabled. -*/ -#if !defined(NDEBUG) && !defined(SQLITE_DEBUG) -# define NDEBUG 1 -#endif - /* ** The TESTONLY macro is used to enclose variable declarations or ** other bits of code that are needed to support the arguments @@ -184425,6 +191128,19 @@ typedef sqlite3_int64 i64; /* 8-byte signed integer */ #define deliberate_fall_through +/* +** Macros needed to provide flexible arrays in a portable way +*/ +#ifndef offsetof +# define offsetof(ST,M) ((size_t)((char*)&((ST*)0)->M - (char*)0)) +#endif +#if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) +# define FLEXARRAY +#else +# define FLEXARRAY 1 +#endif + + #endif /* SQLITE_AMALGAMATION */ #ifdef SQLITE_DEBUG @@ -184529,7 +191245,7 @@ struct Fts3Table { #endif #if defined(SQLITE_DEBUG) || defined(SQLITE_TEST) - /* True to disable the incremental doclist optimization. This is controled + /* True to disable the incremental doclist optimization. This is controlled ** by special insert command 'test-no-incr-doclist'. */ int bNoIncrDoclist; @@ -184581,7 +191297,7 @@ struct Fts3Cursor { /* ** The Fts3Cursor.eSearch member is always set to one of the following. -** Actualy, Fts3Cursor.eSearch can be greater than or equal to +** Actually, Fts3Cursor.eSearch can be greater than or equal to ** FTS3_FULLTEXT_SEARCH. If so, then Fts3Cursor.eSearch - 2 is the index ** of the column to be searched. For example, in ** @@ -184654,9 +191370,13 @@ struct Fts3Phrase { */ int nToken; /* Number of tokens in the phrase */ int iColumn; /* Index of column this phrase must match */ - Fts3PhraseToken aToken[1]; /* One entry for each token in the phrase */ + Fts3PhraseToken aToken[FLEXARRAY]; /* One for each token in the phrase */ }; +/* Size (in bytes) of an Fts3Phrase object large enough to hold N tokens */ +#define SZ_FTS3PHRASE(N) \ + (offsetof(Fts3Phrase,aToken)+(N)*sizeof(Fts3PhraseToken)) + /* ** A tree of these objects forms the RHS of a MATCH operator. ** @@ -184863,6 +191583,7 @@ SQLITE_PRIVATE int sqlite3Fts3MsrIncrNext( SQLITE_PRIVATE int sqlite3Fts3EvalPhrasePoslist(Fts3Cursor *, Fts3Expr *, int iCol, char **); SQLITE_PRIVATE int sqlite3Fts3MsrOvfl(Fts3Cursor *, Fts3MultiSegReader *, int *); SQLITE_PRIVATE int sqlite3Fts3MsrIncrRestart(Fts3MultiSegReader *pCsr); +SQLITE_PRIVATE int sqlite3Fts3MsrCancel(Fts3Cursor*, Fts3Expr*); /* fts3_tokenize_vtab.c */ SQLITE_PRIVATE int sqlite3Fts3InitTok(sqlite3*, Fts3Hash *, void(*xDestroy)(void*)); @@ -184889,12 +191610,6 @@ SQLITE_PRIVATE int sqlite3Fts3IntegrityCheck(Fts3Table *p, int *pbOk); # define SQLITE_CORE 1 #endif -/* #include */ -/* #include */ -/* #include */ -/* #include */ -/* #include */ -/* #include */ /* #include "fts3.h" */ #ifndef SQLITE_CORE @@ -186938,10 +193653,15 @@ static int fts3PoslistPhraseMerge( if( *p1==POS_COLUMN ){ p1++; p1 += fts3GetVarint32(p1, &iCol1); + /* iCol1==0 indicates corruption. Column 0 does not have a POS_COLUMN + ** entry, so this is actually end-of-doclist. */ + if( iCol1==0 ) return 0; } if( *p2==POS_COLUMN ){ p2++; p2 += fts3GetVarint32(p2, &iCol2); + /* As above, iCol2==0 indicates corruption. */ + if( iCol2==0 ) return 0; } while( 1 ){ @@ -187228,7 +193948,7 @@ static int fts3DoclistOrMerge( ** sizes of the two inputs, plus enough space for exactly one of the input ** docids to grow. ** - ** A symetric argument may be made if the doclists are in descending + ** A symmetric argument may be made if the doclists are in descending ** order. */ aOut = sqlite3_malloc64((i64)n1+n2+FTS3_VARINT_MAX-1+FTS3_BUFFER_PADDING); @@ -188608,22 +195328,24 @@ static int fts3IntegrityMethod( char **pzErr /* Write error message here */ ){ Fts3Table *p = (Fts3Table*)pVtab; - int rc; + int rc = SQLITE_OK; int bOk = 0; UNUSED_PARAMETER(isQuick); rc = sqlite3Fts3IntegrityCheck(p, &bOk); - assert( rc!=SQLITE_CORRUPT_VTAB || bOk==0 ); - if( rc!=SQLITE_OK && rc!=SQLITE_CORRUPT_VTAB ){ + assert( rc!=SQLITE_CORRUPT_VTAB ); + if( rc==SQLITE_ERROR || (rc&0xFF)==SQLITE_CORRUPT ){ *pzErr = sqlite3_mprintf("unable to validate the inverted index for" " FTS%d table %s.%s: %s", p->bFts4 ? 4 : 3, zSchema, zTabname, sqlite3_errstr(rc)); - }else if( bOk==0 ){ + if( *pzErr ) rc = SQLITE_OK; + }else if( rc==SQLITE_OK && bOk==0 ){ *pzErr = sqlite3_mprintf("malformed inverted index for FTS%d table %s.%s", p->bFts4 ? 4 : 3, zSchema, zTabname); + if( *pzErr==0 ) rc = SQLITE_NOMEM; } sqlite3Fts3SegmentsClose(p); - return SQLITE_OK; + return rc; } @@ -189025,7 +195747,7 @@ static int fts3EvalDeferredPhrase(Fts3Cursor *pCsr, Fts3Phrase *pPhrase){ nDistance = iPrev - nMaxUndeferred; } - aOut = (char *)sqlite3Fts3MallocZero(nPoslist+FTS3_BUFFER_PADDING); + aOut = (char *)sqlite3Fts3MallocZero(((i64)nPoslist)+FTS3_BUFFER_PADDING); if( !aOut ){ sqlite3_free(aPoslist); return SQLITE_NOMEM; @@ -189324,7 +196046,7 @@ static int incrPhraseTokenNext( ** ** * does not contain any deferred tokens. ** -** Advance it to the next matching documnent in the database and populate +** Advance it to the next matching document in the database and populate ** the Fts3Doclist.pList and nList fields. ** ** If there is no "next" entry and no error occurs, then *pbEof is set to @@ -190110,7 +196832,7 @@ static int fts3EvalNearTest(Fts3Expr *pExpr, int *pRc){ nTmp += p->pRight->pPhrase->doclist.nList; } nTmp += p->pPhrase->doclist.nList; - aTmp = sqlite3_malloc64(nTmp*2); + aTmp = sqlite3_malloc64(nTmp*2 + FTS3_VARINT_MAX); if( !aTmp ){ *pRc = SQLITE_NOMEM; res = 0; @@ -190331,7 +197053,7 @@ static int fts3EvalNext(Fts3Cursor *pCsr){ } /* -** Restart interation for expression pExpr so that the next call to +** Restart iteration for expression pExpr so that the next call to ** fts3EvalNext() visits the first row. Do not allow incremental ** loading or merging of phrase doclists for this iteration. ** @@ -190374,6 +197096,24 @@ static void fts3EvalRestart( } } +/* +** Expression node pExpr is an MSR phrase. This function restarts pExpr +** so that it is a regular phrase query, not an MSR. SQLITE_OK is returned +** if successful, or an SQLite error code otherwise. +*/ +SQLITE_PRIVATE int sqlite3Fts3MsrCancel(Fts3Cursor *pCsr, Fts3Expr *pExpr){ + int rc = SQLITE_OK; + if( pExpr->bEof==0 ){ + i64 iDocid = pExpr->iDocid; + fts3EvalRestart(pCsr, pExpr, &rc); + while( rc==SQLITE_OK && pExpr->iDocid!=iDocid ){ + fts3EvalNextRow(pCsr, pExpr, &rc); + if( pExpr->bEof ) rc = FTS_CORRUPT_VTAB; + } + } + return rc; +} + /* ** After allocating the Fts3Expr.aMI[] array for each phrase in the ** expression rooted at pExpr, the cursor iterates through all rows matched @@ -190761,7 +197501,7 @@ SQLITE_PRIVATE int sqlite3Fts3Corrupt(){ } #endif -#if !SQLITE_CORE +#if !defined(SQLITE_CORE) /* ** Initialize API pointer table, if required. */ @@ -191505,6 +198245,23 @@ SQLITE_PRIVATE int sqlite3Fts3OpenTokenizer( */ static int fts3ExprParse(ParseContext *, const char *, int, Fts3Expr **, int *); +/* +** Search buffer z[], size n, for a '"' character. Or, if enable_parenthesis +** is defined, search for '(' and ')' as well. Return the index of the first +** such character in the buffer. If there is no such character, return -1. +*/ +static int findBarredChar(const char *z, int n){ + int ii; + for(ii=0; iiiLangid, z, i, &pCursor); + *pnConsumed = n; + rc = sqlite3Fts3OpenTokenizer(pTokenizer, pParse->iLangid, z, n, &pCursor); if( rc==SQLITE_OK ){ const char *zToken; int nToken = 0, iStart = 0, iEnd = 0, iPosition = 0; @@ -191546,7 +198296,18 @@ static int getNextToken( rc = pModule->xNext(pCursor, &zToken, &nToken, &iStart, &iEnd, &iPosition); if( rc==SQLITE_OK ){ - nByte = sizeof(Fts3Expr) + sizeof(Fts3Phrase) + nToken; + /* Check that this tokenization did not gobble up any " characters. Or, + ** if enable_parenthesis is true, that it did not gobble up any + ** open or close parenthesis characters either. If it did, call + ** getNextToken() again, but pass only that part of the input buffer + ** up to the first such character. */ + int iBarred = findBarredChar(z, iEnd); + if( iBarred>=0 ){ + pModule->xClose(pCursor); + return getNextToken(pParse, iCol, z, iBarred, ppExpr, pnConsumed); + } + + nByte = sizeof(Fts3Expr) + SZ_FTS3PHRASE(1) + nToken; pRet = (Fts3Expr *)sqlite3Fts3MallocZero(nByte); if( !pRet ){ rc = SQLITE_NOMEM; @@ -191556,7 +198317,7 @@ static int getNextToken( pRet->pPhrase->nToken = 1; pRet->pPhrase->iColumn = iCol; pRet->pPhrase->aToken[0].n = nToken; - pRet->pPhrase->aToken[0].z = (char *)&pRet->pPhrase[1]; + pRet->pPhrase->aToken[0].z = (char*)&pRet->pPhrase->aToken[1]; memcpy(pRet->pPhrase->aToken[0].z, zToken, nToken); if( iEnd=0 ){ + *pnConsumed = iBarred; + } rc = SQLITE_OK; } @@ -191627,9 +198392,9 @@ static int getNextString( Fts3Expr *p = 0; sqlite3_tokenizer_cursor *pCursor = 0; char *zTemp = 0; - int nTemp = 0; + i64 nTemp = 0; - const int nSpace = sizeof(Fts3Expr) + sizeof(Fts3Phrase); + const int nSpace = sizeof(Fts3Expr) + SZ_FTS3PHRASE(1); int nToken = 0; /* The final Fts3Expr data structure, including the Fts3Phrase, @@ -191663,10 +198428,11 @@ static int getNextString( Fts3PhraseToken *pToken; p = fts3ReallocOrFree(p, nSpace + ii*sizeof(Fts3PhraseToken)); - if( !p ) goto no_mem; - zTemp = fts3ReallocOrFree(zTemp, nTemp + nByte); - if( !zTemp ) goto no_mem; + if( !zTemp || !p ){ + rc = SQLITE_NOMEM; + goto getnextstring_out; + } assert( nToken==ii ); pToken = &((Fts3Phrase *)(&p[1]))->aToken[ii]; @@ -191681,9 +198447,6 @@ static int getNextString( nToken = ii+1; } } - - pModule->xClose(pCursor); - pCursor = 0; } if( rc==SQLITE_DONE ){ @@ -191691,7 +198454,10 @@ static int getNextString( char *zBuf = 0; p = fts3ReallocOrFree(p, nSpace + nToken*sizeof(Fts3PhraseToken) + nTemp); - if( !p ) goto no_mem; + if( !p ){ + rc = SQLITE_NOMEM; + goto getnextstring_out; + } memset(p, 0, (char *)&(((Fts3Phrase *)&p[1])->aToken[0])-(char *)p); p->eType = FTSQUERY_PHRASE; p->pPhrase = (Fts3Phrase *)&p[1]; @@ -191699,11 +198465,9 @@ static int getNextString( p->pPhrase->nToken = nToken; zBuf = (char *)&p->pPhrase->aToken[nToken]; + assert( nTemp==0 || zTemp ); if( zTemp ){ memcpy(zBuf, zTemp, nTemp); - sqlite3_free(zTemp); - }else{ - assert( nTemp==0 ); } for(jj=0; jjpPhrase->nToken; jj++){ @@ -191713,17 +198477,17 @@ static int getNextString( rc = SQLITE_OK; } - *ppExpr = p; - return rc; -no_mem: - + getnextstring_out: if( pCursor ){ pModule->xClose(pCursor); } sqlite3_free(zTemp); - sqlite3_free(p); - *ppExpr = 0; - return SQLITE_NOMEM; + if( rc!=SQLITE_OK ){ + sqlite3_free(p); + p = 0; + } + *ppExpr = p; + return rc; } /* @@ -192002,7 +198766,7 @@ static int fts3ExprParse( /* The isRequirePhrase variable is set to true if a phrase or ** an expression contained in parenthesis is required. If a - ** binary operator (AND, OR, NOT or NEAR) is encounted when + ** binary operator (AND, OR, NOT or NEAR) is encountered when ** isRequirePhrase is set, this is a syntax error. */ if( !isPhrase && isRequirePhrase ){ @@ -192584,7 +199348,6 @@ static void fts3ExprTestCommon( } if( rc!=SQLITE_OK && rc!=SQLITE_NOMEM ){ - sqlite3Fts3ExprFree(pExpr); sqlite3_result_error(context, "Error parsing expression", -1); }else if( rc==SQLITE_NOMEM || !(zBuf = exprToString(pExpr, 0)) ){ sqlite3_result_error_nomem(context); @@ -192827,7 +199590,7 @@ static void fts3HashInsertElement( } -/* Resize the hash table so that it cantains "new_size" buckets. +/* Resize the hash table so that it contains "new_size" buckets. ** "new_size" must be a power of 2. The hash table might fail ** to resize if sqliteMalloc() fails. ** @@ -193282,7 +200045,7 @@ static int star_oh(const char *z){ /* ** If the word ends with zFrom and xCond() is true for the stem -** of the word that preceeds the zFrom ending, then change the +** of the word that precedes the zFrom ending, then change the ** ending to zTo. ** ** The input word *pz and zFrom are both in reverse order. zTo @@ -193917,11 +200680,7 @@ SQLITE_PRIVATE int sqlite3Fts3InitTokenizer( #ifdef SQLITE_TEST -#if defined(INCLUDE_SQLITE_TCL_H) -# include "sqlite_tcl.h" -#else -# include "tcl.h" -#endif +#include "tclsqlite.h" /* #include */ /* @@ -194797,7 +201556,7 @@ static int fts3tokFilterMethod( fts3tokResetCursor(pCsr); if( idxNum==1 ){ const char *zByte = (const char *)sqlite3_value_text(apVal[0]); - int nByte = sqlite3_value_bytes(apVal[0]); + sqlite3_int64 nByte = sqlite3_value_bytes(apVal[0]); pCsr->zInput = sqlite3_malloc64(nByte+1); if( pCsr->zInput==0 ){ rc = SQLITE_NOMEM; @@ -198627,8 +205386,8 @@ struct NodeWriter { ** to an appendable b-tree segment. */ struct IncrmergeWriter { - int nLeafEst; /* Space allocated for leaf blocks */ - int nWork; /* Number of leaf pages flushed */ + i64 nLeafEst; /* Space allocated for leaf blocks */ + i64 nWork; /* Number of leaf pages flushed */ sqlite3_int64 iAbsLevel; /* Absolute level of input segments */ int iIdx; /* Index of *output* segment in iAbsLevel+1 */ sqlite3_int64 iStart; /* Block number of first allocated block */ @@ -198869,7 +205628,7 @@ static int fts3IncrmergePush( ** ** It is assumed that the buffer associated with pNode is already large ** enough to accommodate the new entry. The buffer associated with pPrev -** is extended by this function if requrired. +** is extended by this function if required. ** ** If an error (i.e. OOM condition) occurs, an SQLite error code is ** returned. Otherwise, SQLITE_OK. @@ -199374,7 +206133,7 @@ static int fts3IncrmergeWriter( ){ int rc; /* Return Code */ int i; /* Iterator variable */ - int nLeafEst = 0; /* Blocks allocated for leaf nodes */ + i64 nLeafEst = 0; /* Blocks allocated for leaf nodes */ sqlite3_stmt *pLeafEst = 0; /* SQL used to determine nLeafEst */ sqlite3_stmt *pFirstBlock = 0; /* SQL used to determine first block */ @@ -199384,7 +206143,7 @@ static int fts3IncrmergeWriter( sqlite3_bind_int64(pLeafEst, 1, iAbsLevel); sqlite3_bind_int64(pLeafEst, 2, pCsr->nSegment); if( SQLITE_ROW==sqlite3_step(pLeafEst) ){ - nLeafEst = sqlite3_column_int(pLeafEst, 0); + nLeafEst = sqlite3_column_int64(pLeafEst, 0); } rc = sqlite3_reset(pLeafEst); } @@ -200285,7 +207044,12 @@ SQLITE_PRIVATE int sqlite3Fts3IntegrityCheck(Fts3Table *p, int *pbOk){ sqlite3_finalize(pStmt); } - *pbOk = (rc==SQLITE_OK && cksum1==cksum2); + if( rc==SQLITE_CORRUPT_VTAB ){ + rc = SQLITE_OK; + *pbOk = 0; + }else{ + *pbOk = (rc==SQLITE_OK && cksum1==cksum2); + } return rc; } @@ -200527,7 +207291,7 @@ SQLITE_PRIVATE int sqlite3Fts3DeferToken( /* ** SQLite value pRowid contains the rowid of a row that may or may not be ** present in the FTS3 table. If it is, delete it and adjust the contents -** of subsiduary data structures accordingly. +** of subsidiary data structures accordingly. */ static int fts3DeleteByRowid( Fts3Table *p, @@ -200762,10 +207526,6 @@ SQLITE_PRIVATE int sqlite3Fts3Optimize(Fts3Table *p){ /* #include */ /* #include */ -#ifndef SQLITE_AMALGAMATION -typedef sqlite3_int64 i64; -#endif - /* ** Characters that may appear in the second argument to matchinfo(). */ @@ -200853,9 +207613,13 @@ struct MatchinfoBuffer { int nElem; int bGlobal; /* Set if global data is loaded */ char *zMatchinfo; - u32 aMatchinfo[1]; + u32 aMI[FLEXARRAY]; }; +/* Size (in bytes) of a MatchinfoBuffer sufficient for N elements */ +#define SZ_MATCHINFOBUFFER(N) \ + (offsetof(MatchinfoBuffer,aMI)+(((N)+1)/2)*sizeof(u64)) + /* ** The snippet() and offsets() functions both return text values. An instance @@ -200880,13 +207644,13 @@ struct StrBuffer { static MatchinfoBuffer *fts3MIBufferNew(size_t nElem, const char *zMatchinfo){ MatchinfoBuffer *pRet; sqlite3_int64 nByte = sizeof(u32) * (2*(sqlite3_int64)nElem + 1) - + sizeof(MatchinfoBuffer); + + SZ_MATCHINFOBUFFER(1); sqlite3_int64 nStr = strlen(zMatchinfo); pRet = sqlite3Fts3MallocZero(nByte + nStr+1); if( pRet ){ - pRet->aMatchinfo[0] = (u8*)(&pRet->aMatchinfo[1]) - (u8*)pRet; - pRet->aMatchinfo[1+nElem] = pRet->aMatchinfo[0] + pRet->aMI[0] = (u8*)(&pRet->aMI[1]) - (u8*)pRet; + pRet->aMI[1+nElem] = pRet->aMI[0] + sizeof(u32)*((int)nElem+1); pRet->nElem = (int)nElem; pRet->zMatchinfo = ((char*)pRet) + nByte; @@ -200900,10 +207664,10 @@ static MatchinfoBuffer *fts3MIBufferNew(size_t nElem, const char *zMatchinfo){ static void fts3MIBufferFree(void *p){ MatchinfoBuffer *pBuf = (MatchinfoBuffer*)((u8*)p - ((u32*)p)[-1]); - assert( (u32*)p==&pBuf->aMatchinfo[1] - || (u32*)p==&pBuf->aMatchinfo[pBuf->nElem+2] + assert( (u32*)p==&pBuf->aMI[1] + || (u32*)p==&pBuf->aMI[pBuf->nElem+2] ); - if( (u32*)p==&pBuf->aMatchinfo[1] ){ + if( (u32*)p==&pBuf->aMI[1] ){ pBuf->aRef[1] = 0; }else{ pBuf->aRef[2] = 0; @@ -200920,18 +207684,18 @@ static void (*fts3MIBufferAlloc(MatchinfoBuffer *p, u32 **paOut))(void*){ if( p->aRef[1]==0 ){ p->aRef[1] = 1; - aOut = &p->aMatchinfo[1]; + aOut = &p->aMI[1]; xRet = fts3MIBufferFree; } else if( p->aRef[2]==0 ){ p->aRef[2] = 1; - aOut = &p->aMatchinfo[p->nElem+2]; + aOut = &p->aMI[p->nElem+2]; xRet = fts3MIBufferFree; }else{ aOut = (u32*)sqlite3_malloc64(p->nElem * sizeof(u32)); if( aOut ){ xRet = sqlite3_free; - if( p->bGlobal ) memcpy(aOut, &p->aMatchinfo[1], p->nElem*sizeof(u32)); + if( p->bGlobal ) memcpy(aOut, &p->aMI[1], p->nElem*sizeof(u32)); } } @@ -200941,7 +207705,7 @@ static void (*fts3MIBufferAlloc(MatchinfoBuffer *p, u32 **paOut))(void*){ static void fts3MIBufferSetGlobal(MatchinfoBuffer *p){ p->bGlobal = 1; - memcpy(&p->aMatchinfo[2+p->nElem], &p->aMatchinfo[1], p->nElem*sizeof(u32)); + memcpy(&p->aMI[2+p->nElem], &p->aMI[1], p->nElem*sizeof(u32)); } /* @@ -201143,6 +207907,7 @@ static int fts3SnippetNextCandidate(SnippetIter *pIter){ return 1; } + assert( pIter->nSnippet>=0 ); pIter->iCurrent = iStart = iEnd - pIter->nSnippet + 1; for(i=0; inPhrase; i++){ SnippetPhrase *pPhrase = &pIter->aPhrase[i]; @@ -201191,7 +207956,7 @@ static void fts3SnippetDetails( } mCover |= mPhrase; - for(j=0; jnToken; j++){ + for(j=0; jnToken && jnSnippet; j++){ mHighlight |= (mPos>>j); } @@ -201355,7 +208120,7 @@ static int fts3StringAppend( } /* If there is insufficient space allocated at StrBuffer.z, use realloc() - ** to grow the buffer until so that it is big enough to accomadate the + ** to grow the buffer until so that it is big enough to accommodate the ** appended data. */ if( pStr->n+nAppend+1>=pStr->nAlloc ){ @@ -201767,16 +208532,16 @@ static size_t fts3MatchinfoSize(MatchInfo *pInfo, char cArg){ break; case FTS3_MATCHINFO_LHITS: - nVal = pInfo->nCol * pInfo->nPhrase; + nVal = (size_t)pInfo->nCol * pInfo->nPhrase; break; case FTS3_MATCHINFO_LHITS_BM: - nVal = pInfo->nPhrase * ((pInfo->nCol + 31) / 32); + nVal = (size_t)pInfo->nPhrase * ((pInfo->nCol + 31) / 32); break; default: assert( cArg==FTS3_MATCHINFO_HITS ); - nVal = pInfo->nCol * pInfo->nPhrase * 3; + nVal = (size_t)pInfo->nCol * pInfo->nPhrase * 3; break; } @@ -202330,6 +209095,22 @@ static int fts3ExprTermOffsetInit(Fts3Expr *pExpr, int iPhrase, void *ctx){ return rc; } +/* +** If expression pExpr is a phrase expression that uses an MSR query, +** restart it as a regular, non-incremental query. Return SQLITE_OK +** if successful, or an SQLite error code otherwise. +*/ +static int fts3ExprRestartIfCb(Fts3Expr *pExpr, int iPhrase, void *ctx){ + TermOffsetCtx *p = (TermOffsetCtx*)ctx; + int rc = SQLITE_OK; + UNUSED_PARAMETER(iPhrase); + if( pExpr->pPhrase && pExpr->pPhrase->bIncr ){ + rc = sqlite3Fts3MsrCancel(p->pCsr, pExpr); + pExpr->pPhrase->bIncr = 0; + } + return rc; +} + /* ** Implementation of offsets() function. */ @@ -202366,6 +209147,12 @@ SQLITE_PRIVATE void sqlite3Fts3Offsets( sCtx.iDocid = pCsr->iPrevId; sCtx.pCsr = pCsr; + /* If a query restart will be required, do it here, rather than later of + ** after pointers to poslist buffers that may be invalidated by a restart + ** have been saved. */ + rc = sqlite3Fts3ExprIterate(pCsr->pExpr, fts3ExprRestartIfCb, (void*)&sCtx); + if( rc!=SQLITE_OK ) goto offsets_out; + /* Loop through the table columns, appending offset information to ** string-buffer res for each column. */ @@ -203312,8 +210099,8 @@ SQLITE_PRIVATE int sqlite3FtsUnicodeFold(int c, int eRemoveDiacritic){ ** Beginning with version 3.45.0 (circa 2024-01-01), these routines also ** accept BLOB values that have JSON encoded using a binary representation ** called "JSONB". The name JSONB comes from PostgreSQL, however the on-disk -** format SQLite JSONB is completely different and incompatible with -** PostgreSQL JSONB. +** format for SQLite-JSONB is completely different and incompatible with +** PostgreSQL-JSONB. ** ** Decoding and interpreting JSONB is still O(N) where N is the size of ** the input, the same as text JSON. However, the constant of proportionality @@ -203370,7 +210157,7 @@ SQLITE_PRIVATE int sqlite3FtsUnicodeFold(int c, int eRemoveDiacritic){ ** ** The payload size need not be expressed in its minimal form. For example, ** if the payload size is 10, the size can be expressed in any of 5 different -** ways: (1) (X>>4)==10, (2) (X>>4)==12 following by on 0x0a byte, +** ways: (1) (X>>4)==10, (2) (X>>4)==12 following by one 0x0a byte, ** (3) (X>>4)==13 followed by 0x00 and 0x0a, (4) (X>>4)==14 followed by ** 0x00 0x00 0x00 0x0a, or (5) (X>>4)==15 followed by 7 bytes of 0x00 and ** a single byte of 0x0a. The shorter forms are preferred, of course, but @@ -203380,7 +210167,7 @@ SQLITE_PRIVATE int sqlite3FtsUnicodeFold(int c, int eRemoveDiacritic){ ** the size when it becomes known, resulting in a non-minimal encoding. ** ** The value (X>>4)==15 is not actually used in the current implementation -** (as SQLite is currently unable handle BLOBs larger than about 2GB) +** (as SQLite is currently unable to handle BLOBs larger than about 2GB) ** but is included in the design to allow for future enhancements. ** ** The payload follows the header. NULL, TRUE, and FALSE have no payload and @@ -203440,23 +210227,47 @@ static const char * const jsonbType[] = { ** increase for the text-JSON parser. (Ubuntu14.10 gcc 4.8.4 x64 with -Os). */ static const char jsonIsSpace[] = { - 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 1, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +#ifdef SQLITE_ASCII +/*0 1 2 3 4 5 6 7 8 9 a b c d e f */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 1, 0, 0, /* 0 */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 1 */ + 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 2 */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 3 */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 4 */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 5 */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 6 */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 7 */ + + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 8 */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 9 */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* a */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* b */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* c */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* d */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* e */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* f */ +#endif +#ifdef SQLITE_EBCDIC +/*0 1 2 3 4 5 6 7 8 9 a b c d e f */ + 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, /* 0 */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 1 */ + 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 2 */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 3 */ + 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 4 */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 5 */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 6 */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 7 */ + + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 8 */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 9 */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* a */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* b */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* c */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* d */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* e */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* f */ +#endif + }; #define jsonIsspace(x) (jsonIsSpace[(unsigned char)x]) @@ -203464,7 +210275,13 @@ static const char jsonIsSpace[] = { ** The set of all space characters recognized by jsonIsspace(). ** Useful as the second argument to strspn(). */ +#ifdef SQLITE_ASCII static const char jsonSpaces[] = "\011\012\015\040"; +#endif +#ifdef SQLITE_EBCDIC +static const char jsonSpaces[] = "\005\045\015\100"; +#endif + /* ** Characters that are special to JSON. Control characters, @@ -203473,23 +210290,46 @@ static const char jsonSpaces[] = "\011\012\015\040"; ** it in the set of special characters. */ static const char jsonIsOk[256] = { - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 1, 1, 0, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 +#ifdef SQLITE_ASCII +/*0 1 2 3 4 5 6 7 8 9 a b c d e f */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 0 */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 1 */ + 1, 1, 0, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, /* 2 */ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 3 */ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 4 */ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, /* 5 */ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 6 */ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 7 */ + + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 8 */ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 9 */ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* a */ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* b */ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* c */ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* d */ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* e */ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 /* f */ +#endif +#ifdef SQLITE_EBCDIC +/*0 1 2 3 4 5 6 7 8 9 a b c d e f */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 0 */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 1 */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 2 */ + 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, /* 3 */ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 4 */ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 5 */ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 6 */ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 0, /* 7 */ + + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 8 */ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 9 */ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* a */ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* b */ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* c */ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* d */ + 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* e */ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 /* f */ +#endif }; /* Objects */ @@ -203634,7 +210474,7 @@ struct JsonParse { ** Forward references **************************************************************************/ static void jsonReturnStringAsBlob(JsonString*); -static int jsonFuncArgMightBeBinary(sqlite3_value *pJson); +static int jsonArgIsJsonb(sqlite3_value *pJson, JsonParse *p); static u32 jsonTranslateBlobToText(const JsonParse*,u32,JsonString*); static void jsonReturnParse(sqlite3_context*,JsonParse*); static JsonParse *jsonParseFuncArg(sqlite3_context*,sqlite3_value*,u32); @@ -203708,7 +210548,7 @@ static int jsonCacheInsert( ** most-recently used entry if it isn't so already. ** ** The JsonParse object returned still belongs to the Cache and might -** be deleted at any moment. If the caller whants the JsonParse to +** be deleted at any moment. If the caller wants the JsonParse to ** linger, it needs to increment the nPJRef reference counter. */ static JsonParse *jsonCacheSearch( @@ -203852,7 +210692,6 @@ static void jsonAppendRawNZ(JsonString *p, const char *zIn, u32 N){ } } - /* Append formatted text (not to exceed N bytes) to the JsonString. */ static void jsonPrintf(int N, JsonString *p, const char *zFormat, ...){ @@ -203910,6 +210749,40 @@ static void jsonAppendSeparator(JsonString *p){ jsonAppendChar(p, ','); } +/* c is a control character. Append the canonical JSON representation +** of that control character to p. +** +** This routine assumes that the output buffer has already been enlarged +** sufficiently to hold the worst-case encoding plus a nul terminator. +*/ +static void jsonAppendControlChar(JsonString *p, u8 c){ + static const char aSpecial[] = { + 0, 0, 0, 0, 0, 0, 0, 0, 'b', 't', 'n', 0, 'f', 'r', 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 + }; + assert( sizeof(aSpecial)==32 ); + assert( aSpecial['\b']=='b' ); + assert( aSpecial['\f']=='f' ); + assert( aSpecial['\n']=='n' ); + assert( aSpecial['\r']=='r' ); + assert( aSpecial['\t']=='t' ); + assert( c>=0 && cnUsed+7 <= p->nAlloc ); + if( aSpecial[c] ){ + p->zBuf[p->nUsed] = '\\'; + p->zBuf[p->nUsed+1] = aSpecial[c]; + p->nUsed += 2; + }else{ + p->zBuf[p->nUsed] = '\\'; + p->zBuf[p->nUsed+1] = 'u'; + p->zBuf[p->nUsed+2] = '0'; + p->zBuf[p->nUsed+3] = '0'; + p->zBuf[p->nUsed+4] = "0123456789abcdef"[c>>4]; + p->zBuf[p->nUsed+5] = "0123456789abcdef"[c&0xf]; + p->nUsed += 6; + } +} + /* Append the N-byte string in zIn to the end of the JsonString string ** under construction. Enclose the string in double-quotes ("...") and ** escape any double-quotes or backslash characters contained within the @@ -203969,35 +210842,14 @@ static void jsonAppendString(JsonString *p, const char *zIn, u32 N){ } c = z[0]; if( c=='"' || c=='\\' ){ - json_simple_escape: if( (p->nUsed+N+3 > p->nAlloc) && jsonStringGrow(p,N+3)!=0 ) return; p->zBuf[p->nUsed++] = '\\'; p->zBuf[p->nUsed++] = c; }else if( c=='\'' ){ p->zBuf[p->nUsed++] = c; }else{ - static const char aSpecial[] = { - 0, 0, 0, 0, 0, 0, 0, 0, 'b', 't', 'n', 0, 'f', 'r', 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 - }; - assert( sizeof(aSpecial)==32 ); - assert( aSpecial['\b']=='b' ); - assert( aSpecial['\f']=='f' ); - assert( aSpecial['\n']=='n' ); - assert( aSpecial['\r']=='r' ); - assert( aSpecial['\t']=='t' ); - assert( c>=0 && cnUsed+N+7 > p->nAlloc) && jsonStringGrow(p,N+7)!=0 ) return; - p->zBuf[p->nUsed++] = '\\'; - p->zBuf[p->nUsed++] = 'u'; - p->zBuf[p->nUsed++] = '0'; - p->zBuf[p->nUsed++] = '0'; - p->zBuf[p->nUsed++] = "0123456789abcdef"[c>>4]; - p->zBuf[p->nUsed++] = "0123456789abcdef"[c&0xf]; + jsonAppendControlChar(p, c); } z++; N--; @@ -204040,11 +210892,9 @@ static void jsonAppendSqlValue( break; } default: { - if( jsonFuncArgMightBeBinary(pValue) ){ - JsonParse px; - memset(&px, 0, sizeof(px)); - px.aBlob = (u8*)sqlite3_value_blob(pValue); - px.nBlob = sqlite3_value_bytes(pValue); + JsonParse px; + memset(&px, 0, sizeof(px)); + if( jsonArgIsJsonb(pValue, &px) ){ jsonTranslateBlobToText(&px, 0, p); }else if( p->eErr==0 ){ sqlite3_result_error(p->pCtx, "JSON cannot hold BLOB values", -1); @@ -204363,7 +211213,7 @@ static void jsonWrongNumArgs( */ static int jsonBlobExpand(JsonParse *pParse, u32 N){ u8 *aNew; - u32 t; + u64 t; assert( N>pParse->nBlobAlloc ); if( pParse->nBlobAlloc==0 ){ t = 100; @@ -204373,8 +211223,9 @@ static int jsonBlobExpand(JsonParse *pParse, u32 N){ if( tdb, pParse->aBlob, t); if( aNew==0 ){ pParse->oom = 1; return 1; } + assert( t<0x7fffffff ); pParse->aBlob = aNew; - pParse->nBlobAlloc = t; + pParse->nBlobAlloc = (u32)t; return 0; } @@ -204441,7 +211292,7 @@ static SQLITE_NOINLINE void jsonBlobExpandAndAppendNode( } -/* Append an node type byte together with the payload size and +/* Append a node type byte together with the payload size and ** possibly also the payload. ** ** If aPayload is not NULL, then it is a pointer to the payload which @@ -204510,8 +211361,10 @@ static int jsonBlobChangePayloadSize( nExtra = 1; }else if( szType==13 ){ nExtra = 2; - }else{ + }else if( szType==14 ){ nExtra = 4; + }else{ + nExtra = 8; } if( szPayload<=11 ){ nNeeded = 0; @@ -204698,7 +211551,10 @@ static u32 jsonbValidityCheck( if( !jsonIsOk[z[j]] && z[j]!='\'' ){ if( z[j]=='"' ){ if( x==JSONB_TEXTJ ) return j+1; - }else if( z[j]!='\\' || j+1>=k ){ + }else if( z[j]<=0x1f ){ + /* Control characters in JSON5 string literals are ok */ + if( x==JSONB_TEXTJ ) return j+1; + }else if( NEVER(z[j]!='\\') || j+1>=k ){ return j+1; }else if( strchr("\"\\/bfnrt",z[j+1])!=0 ){ j++; @@ -204978,7 +211834,12 @@ static int jsonTranslateTextToBlob(JsonParse *pParse, u32 i){ || c=='n' || c=='r' || c=='t' || (c=='u' && jsonIs4Hex(&z[j+1])) ){ if( opcode==JSONB_TEXT ) opcode = JSONB_TEXTJ; - }else if( c=='\'' || c=='0' || c=='v' || c=='\n' + }else if( c=='\'' || c=='v' || c=='\n' +#ifdef SQLITE_BUG_COMPATIBLE_20250510 + || (c=='0') /* Legacy bug compatible */ +#else + || (c=='0' && !sqlite3Isdigit(z[j+1])) /* Correct implementation */ +#endif || (0xe2==(u8)c && 0x80==(u8)z[j+1] && (0xa8==(u8)z[j+2] || 0xa9==(u8)z[j+2])) || (c=='x' && jsonIs2Hex(&z[j+1])) ){ @@ -204993,9 +211854,14 @@ static int jsonTranslateTextToBlob(JsonParse *pParse, u32 i){ return -1; } }else if( c<=0x1f ){ - /* Control characters are not allowed in strings */ - pParse->iErr = j; - return -1; + if( c==0 ){ + pParse->iErr = j; + return -1; + } + /* Control characters are not allowed in canonical JSON string + ** literals, but are allowed in JSON5 string literals. */ + opcode = JSONB_TEXT5; + pParse->hasNonstd = 1; }else if( c=='"' ){ opcode = JSONB_TEXT5; } @@ -205211,6 +212077,7 @@ static int jsonTranslateTextToBlob(JsonParse *pParse, u32 i){ return i+4; } /* fall-through into the default case that checks for NaN */ + /* no break */ deliberate_fall_through } default: { u32 k; @@ -205322,10 +212189,7 @@ static u32 jsonbPayloadSize(const JsonParse *pParse, u32 i, u32 *pSz){ u8 x; u32 sz; u32 n; - if( NEVER(i>pParse->nBlob) ){ - *pSz = 0; - return 0; - } + assert( i<=pParse->nBlob ); x = pParse->aBlob[i]>>4; if( x<=11 ){ sz = x; @@ -205362,15 +212226,15 @@ static u32 jsonbPayloadSize(const JsonParse *pParse, u32 i, u32 *pSz){ *pSz = 0; return 0; } - sz = (pParse->aBlob[i+5]<<24) + (pParse->aBlob[i+6]<<16) + + sz = ((u32)pParse->aBlob[i+5]<<24) + (pParse->aBlob[i+6]<<16) + (pParse->aBlob[i+7]<<8) + pParse->aBlob[i+8]; n = 9; } if( (i64)i+sz+n > pParse->nBlob && (i64)i+sz+n > pParse->nBlob-pParse->delta ){ - sz = 0; - n = 0; + *pSz = 0; + return 0; } *pSz = sz; return n; @@ -205467,9 +212331,12 @@ static u32 jsonTranslateBlobToText( } case JSONB_TEXT: case JSONB_TEXTJ: { - jsonAppendChar(pOut, '"'); - jsonAppendRaw(pOut, (const char*)&pParse->aBlob[i+n], sz); - jsonAppendChar(pOut, '"'); + if( pOut->nUsed+sz+2<=pOut->nAlloc || jsonStringGrow(pOut, sz+2)==0 ){ + pOut->zBuf[pOut->nUsed] = '"'; + memcpy(pOut->zBuf+pOut->nUsed+1,(const char*)&pParse->aBlob[i+n],sz); + pOut->zBuf[pOut->nUsed+sz+1] = '"'; + pOut->nUsed += sz+2; + } break; } case JSONB_TEXT5: { @@ -205479,7 +212346,7 @@ static u32 jsonTranslateBlobToText( zIn = (const char*)&pParse->aBlob[i+n]; jsonAppendChar(pOut, '"'); while( sz2>0 ){ - for(k=0; k0 ){ jsonAppendRawNZ(pOut, zIn, k); if( k>=sz2 ){ @@ -205494,6 +212361,13 @@ static u32 jsonTranslateBlobToText( sz2--; continue; } + if( zIn[0]<=0x1f ){ + if( pOut->nUsed+7>pOut->nAlloc && jsonStringGrow(pOut,7) ) break; + jsonAppendControlChar(pOut, zIn[0]); + zIn++; + sz2--; + continue; + } assert( zIn[0]=='\\' ); assert( sz2>=1 ); if( sz2<2 ){ @@ -205505,7 +212379,7 @@ static u32 jsonTranslateBlobToText( jsonAppendChar(pOut, '\''); break; case 'v': - jsonAppendRawNZ(pOut, "\\u0009", 6); + jsonAppendRawNZ(pOut, "\\u000b", 6); break; case 'x': if( sz2<4 ){ @@ -205596,30 +212470,109 @@ static u32 jsonTranslateBlobToText( return i+n+sz; } -/* Return true if the input pJson +/* Context for recursion of json_pretty() +*/ +typedef struct JsonPretty JsonPretty; +struct JsonPretty { + JsonParse *pParse; /* The BLOB being rendered */ + JsonString *pOut; /* Generate pretty output into this string */ + const char *zIndent; /* Use this text for indentation */ + u32 szIndent; /* Bytes in zIndent[] */ + u32 nIndent; /* Current level of indentation */ +}; + +/* Append indentation to the pretty JSON under construction */ +static void jsonPrettyIndent(JsonPretty *pPretty){ + u32 jj; + for(jj=0; jjnIndent; jj++){ + jsonAppendRaw(pPretty->pOut, pPretty->zIndent, pPretty->szIndent); + } +} + +/* +** Translate the binary JSONB representation of JSON beginning at +** pParse->aBlob[i] into a JSON text string. Append the JSON +** text onto the end of pOut. Return the index in pParse->aBlob[] +** of the first byte past the end of the element that is translated. +** +** This is a variant of jsonTranslateBlobToText() that "pretty-prints" +** the output. Extra whitespace is inserted to make the JSON easier +** for humans to read. ** -** For performance reasons, this routine does not do a detailed check of the -** input BLOB to ensure that it is well-formed. Hence, false positives are -** possible. False negatives should never occur, however. +** If an error is detected in the BLOB input, the pOut->eErr flag +** might get set to JSTRING_MALFORMED. But not all BLOB input errors +** are detected. So a malformed JSONB input might either result +** in an error, or in incorrect JSON. +** +** The pOut->eErr JSTRING_OOM flag is set on a OOM. */ -static int jsonFuncArgMightBeBinary(sqlite3_value *pJson){ - u32 sz, n; - const u8 *aBlob; - int nBlob; - JsonParse s; - if( sqlite3_value_type(pJson)!=SQLITE_BLOB ) return 0; - aBlob = sqlite3_value_blob(pJson); - nBlob = sqlite3_value_bytes(pJson); - if( nBlob<1 ) return 0; - if( NEVER(aBlob==0) || (aBlob[0] & 0x0f)>JSONB_OBJECT ) return 0; - memset(&s, 0, sizeof(s)); - s.aBlob = (u8*)aBlob; - s.nBlob = nBlob; - n = jsonbPayloadSize(&s, 0, &sz); - if( n==0 ) return 0; - if( sz+n!=(u32)nBlob ) return 0; - if( (aBlob[0] & 0x0f)<=JSONB_FALSE && sz>0 ) return 0; - return sz+n==(u32)nBlob; +static u32 jsonTranslateBlobToPrettyText( + JsonPretty *pPretty, /* Pretty-printing context */ + u32 i /* Start rendering at this index */ +){ + u32 sz, n, j, iEnd; + const JsonParse *pParse = pPretty->pParse; + JsonString *pOut = pPretty->pOut; + n = jsonbPayloadSize(pParse, i, &sz); + if( n==0 ){ + pOut->eErr |= JSTRING_MALFORMED; + return pParse->nBlob+1; + } + switch( pParse->aBlob[i] & 0x0f ){ + case JSONB_ARRAY: { + j = i+n; + iEnd = j+sz; + jsonAppendChar(pOut, '['); + if( jnIndent++; + while( pOut->eErr==0 ){ + jsonPrettyIndent(pPretty); + j = jsonTranslateBlobToPrettyText(pPretty, j); + if( j>=iEnd ) break; + jsonAppendRawNZ(pOut, ",\n", 2); + } + jsonAppendChar(pOut, '\n'); + pPretty->nIndent--; + jsonPrettyIndent(pPretty); + } + jsonAppendChar(pOut, ']'); + i = iEnd; + break; + } + case JSONB_OBJECT: { + j = i+n; + iEnd = j+sz; + jsonAppendChar(pOut, '{'); + if( jnIndent++; + while( pOut->eErr==0 ){ + jsonPrettyIndent(pPretty); + j = jsonTranslateBlobToText(pParse, j, pOut); + if( j>iEnd ){ + pOut->eErr |= JSTRING_MALFORMED; + break; + } + jsonAppendRawNZ(pOut, ": ", 2); + j = jsonTranslateBlobToPrettyText(pPretty, j); + if( j>=iEnd ) break; + jsonAppendRawNZ(pOut, ",\n", 2); + } + jsonAppendChar(pOut, '\n'); + pPretty->nIndent--; + jsonPrettyIndent(pPretty); + } + jsonAppendChar(pOut, '}'); + i = iEnd; + break; + } + default: { + i = jsonTranslateBlobToText(pParse, i, pOut); + break; + } + } + return i; } /* @@ -205654,6 +212607,82 @@ static void jsonAfterEditSizeAdjust(JsonParse *pParse, u32 iRoot){ pParse->delta += jsonBlobChangePayloadSize(pParse, iRoot, sz); } +/* +** If the JSONB at aIns[0..nIns-1] can be expanded (by denormalizing the +** size field) by d bytes, then write the expansion into aOut[] and +** return true. In this way, an overwrite happens without changing the +** size of the JSONB, which reduces memcpy() operations and also make it +** faster and easier to update the B-Tree entry that contains the JSONB +** in the database. +** +** If the expansion of aIns[] by d bytes cannot be (easily) accomplished +** then return false. +** +** The d parameter is guaranteed to be between 1 and 8. +** +** This routine is an optimization. A correct answer is obtained if it +** always leaves the output unchanged and returns false. +*/ +static int jsonBlobOverwrite( + u8 *aOut, /* Overwrite here */ + const u8 *aIns, /* New content */ + u32 nIns, /* Bytes of new content */ + u32 d /* Need to expand new content by this much */ +){ + u32 szPayload; /* Bytes of payload */ + u32 i; /* New header size, after expansion & a loop counter */ + u8 szHdr; /* Size of header before expansion */ + + /* Lookup table for finding the upper 4 bits of the first byte of the + ** expanded aIns[], based on the size of the expanded aIns[] header: + ** + ** 2 3 4 5 6 7 8 9 */ + static const u8 aType[] = { 0xc0, 0xd0, 0, 0xe0, 0, 0, 0, 0xf0 }; + + if( (aIns[0]&0x0f)<=2 ) return 0; /* Cannot enlarge NULL, true, false */ + switch( aIns[0]>>4 ){ + default: { /* aIns[] header size 1 */ + if( ((1<=2 && i<=9 && aType[i-2]!=0 ); + aOut[0] = (aIns[0] & 0x0f) | aType[i-2]; + memcpy(&aOut[i], &aIns[szHdr], nIns-szHdr); + szPayload = nIns - szHdr; + while( 1/*edit-by-break*/ ){ + i--; + aOut[i] = szPayload & 0xff; + if( i==1 ) break; + szPayload >>= 8; + } + assert( (szPayload>>8)==0 ); + return 1; +} + /* ** Modify the JSONB blob at pParse->aBlob by removing nDel bytes of ** content beginning at iDel, and replacing them with nIns bytes of @@ -205675,6 +212704,11 @@ static void jsonBlobEdit( u32 nIns /* Bytes of content to insert */ ){ i64 d = (i64)nIns - (i64)nDel; + if( d<0 && d>=(-8) && aIns!=0 + && jsonBlobOverwrite(&pParse->aBlob[iDel], aIns, nIns, (int)-d) + ){ + return; + } if( d!=0 ){ if( pParse->nBlob + d > pParse->nBlobAlloc ){ jsonBlobExpand(pParse, pParse->nBlob+d); @@ -205686,7 +212720,9 @@ static void jsonBlobEdit( pParse->nBlob += d; pParse->delta += d; } - if( nIns && aIns ) memcpy(&pParse->aBlob[iDel], aIns, nIns); + if( nIns && aIns ){ + memcpy(&pParse->aBlob[iDel], aIns, nIns); + } } /* @@ -205771,7 +212807,21 @@ static u32 jsonUnescapeOneChar(const char *z, u32 n, u32 *piOut){ case 'r': { *piOut = '\r'; return 2; } case 't': { *piOut = '\t'; return 2; } case 'v': { *piOut = '\v'; return 2; } - case '0': { *piOut = 0; return 2; } + case '0': { + /* JSON5 requires that the \0 escape not be followed by a digit. + ** But SQLite did not enforce this restriction in versions 3.42.0 + ** through 3.49.2. That was a bug. But some applications might have + ** come to depend on that bug. Use the SQLITE_BUG_COMPATIBLE_20250510 + ** option to restore the old buggy behavior. */ +#ifdef SQLITE_BUG_COMPATIBLE_20250510 + /* Legacy bug-compatible behavior */ + *piOut = 0; +#else + /* Correct behavior */ + *piOut = (n>2 && sqlite3Isdigit(z[2])) ? JSON_INVALID_CHAR : 0; +#endif + return 2; + } case '\'': case '"': case '/': @@ -206002,7 +213052,9 @@ static u32 jsonLookupStep( zPath++; if( zPath[0]=='"' ){ zKey = zPath + 1; - for(i=1; zPath[i] && zPath[i]!='"'; i++){} + for(i=1; zPath[i] && zPath[i]!='"'; i++){ + if( zPath[i]=='\\' && zPath[i+1]!=0 ) i++; + } nKey = i-1; if( zPath[i] ){ i++; @@ -206177,19 +213229,27 @@ static void jsonReturnTextJsonFromBlob( ** ** If the value is a primitive, return it as an SQL value. ** If the value is an array or object, return it as either -** JSON text or the BLOB encoding, depending on the JSON_B flag -** on the userdata. +** JSON text or the BLOB encoding, depending on the eMode flag +** as follows: +** +** eMode==0 JSONB if the JSON_B flag is set in userdata or +** text if the JSON_B flag is omitted from userdata. +** +** eMode==1 Text +** +** eMode==2 JSONB */ static void jsonReturnFromBlob( JsonParse *pParse, /* Complete JSON parse tree */ u32 i, /* Index of the node */ sqlite3_context *pCtx, /* Return value for this function */ - int textOnly /* return text JSON. Disregard user-data */ + int eMode /* Format of return: text of JSONB */ ){ u32 n, sz; int rc; sqlite3 *db = sqlite3_context_db_handle(pCtx); + assert( eMode>=0 && eMode<=2 ); n = jsonbPayloadSize(pParse, i, &sz); if( n==0 ){ sqlite3_result_error(pCtx, "malformed JSON", -1); @@ -206230,7 +213290,19 @@ static void jsonReturnFromBlob( rc = sqlite3DecOrHexToI64(z, &iRes); sqlite3DbFree(db, z); if( rc==0 ){ - sqlite3_result_int64(pCtx, bNeg ? -iRes : iRes); + if( iRes<0 ){ + /* A hexadecimal literal with 16 significant digits and with the + ** high-order bit set is a negative integer in SQLite (and hence + ** iRes comes back as negative) but should be interpreted as a + ** positive value if it occurs within JSON. The value is too + ** large to appear as an SQLite integer so it must be converted + ** into floating point. */ + double r; + r = (double)*(sqlite3_uint64*)&iRes; + sqlite3_result_double(pCtx, bNeg ? -r : r); + }else{ + sqlite3_result_int64(pCtx, bNeg ? -iRes : iRes); + } }else if( rc==3 && bNeg ){ sqlite3_result_int64(pCtx, SMALLEST_INT64); }else if( rc==1 ){ @@ -206269,7 +213341,7 @@ static void jsonReturnFromBlob( char *zOut; u32 nOut = sz; z = (const char*)&pParse->aBlob[i+n]; - zOut = sqlite3DbMallocRaw(db, nOut+1); + zOut = sqlite3DbMallocRaw(db, ((u64)nOut)+1); if( zOut==0 ) goto returnfromblob_oom; for(iIn=iOut=0; iInaBlob[i], sz+n, SQLITE_TRANSIENT); }else{ jsonReturnTextJsonFromBlob(pCtx, &pParse->aBlob[i], sz+n); @@ -206364,10 +213442,7 @@ static int jsonFunctionArgToBlob( return 0; } case SQLITE_BLOB: { - if( jsonFuncArgMightBeBinary(pArg) ){ - pParse->aBlob = (u8*)sqlite3_value_blob(pArg); - pParse->nBlob = sqlite3_value_bytes(pArg); - }else{ + if( !jsonArgIsJsonb(pArg, pParse) ){ sqlite3_result_error(ctx, "JSON cannot hold BLOB values", -1); return 1; } @@ -206447,7 +213522,7 @@ static char *jsonBadPathError( } /* argv[0] is a BLOB that seems likely to be a JSONB. Subsequent -** arguments come in parse where each pair contains a JSON path and +** arguments come in pairs where each pair contains a JSON path and ** content to insert or set at that patch. Do the updates ** and return the result. ** @@ -206518,27 +213593,46 @@ static void jsonInsertIntoBlob( /* ** If pArg is a blob that seems like a JSONB blob, then initialize ** p to point to that JSONB and return TRUE. If pArg does not seem like -** a JSONB blob, then return FALSE; -** -** This routine is only called if it is already known that pArg is a -** blob. The only open question is whether or not the blob appears -** to be a JSONB blob. +** a JSONB blob, then return FALSE. +** +** For small BLOBs (having no more than 7 bytes of payload) a full +** validity check is done. So for small BLOBs this routine only returns +** true if the value is guaranteed to be a valid JSONB. For larger BLOBs +** (8 byte or more of payload) only the size of the outermost element is +** checked to verify that the BLOB is superficially valid JSONB. +** +** A full JSONB validation is done on smaller BLOBs because those BLOBs might +** also be text JSON that has been incorrectly cast into a BLOB. +** (See tag-20240123-a and https://sqlite.org/forum/forumpost/012136abd5) +** If the BLOB is 9 bytes are larger, then it is not possible for the +** superficial size check done here to pass if the input is really text +** JSON so we do not need to look deeper in that case. +** +** Why we only need to do full JSONB validation for smaller BLOBs: +** +** The first byte of valid JSON text must be one of: '{', '[', '"', ' ', '\n', +** '\r', '\t', '-', or a digit '0' through '9'. Of these, only a subset +** can also be the first byte of JSONB: '{', '[', and digits '3' +** through '9'. In every one of those cases, the payload size is 7 bytes +** or less. So if we do full JSONB validation for every BLOB where the +** payload is less than 7 bytes, we will never get a false positive for +** JSONB on an input that is really text JSON. */ static int jsonArgIsJsonb(sqlite3_value *pArg, JsonParse *p){ u32 n, sz = 0; + u8 c; + if( sqlite3_value_type(pArg)!=SQLITE_BLOB ) return 0; p->aBlob = (u8*)sqlite3_value_blob(pArg); p->nBlob = (u32)sqlite3_value_bytes(pArg); - if( p->nBlob==0 ){ - p->aBlob = 0; - return 0; - } - if( NEVER(p->aBlob==0) ){ - return 0; - } - if( (p->aBlob[0] & 0x0f)<=JSONB_OBJECT + if( p->nBlob>0 + && ALWAYS(p->aBlob!=0) + && ((c = p->aBlob[0]) & 0x0f)<=JSONB_OBJECT && (n = jsonbPayloadSize(p, 0, &sz))>0 && sz+n==p->nBlob - && ((p->aBlob[0] & 0x0f)>JSONB_FALSE || sz==0) + && ((c & 0x0f)>JSONB_FALSE || sz==0) + && (sz>7 + || (c!=0x7b && c!=0x5b && !sqlite3Isdigit(c)) + || jsonbValidityCheck(p, 0, p->nBlob, 1)==0) ){ return 1; } @@ -206616,7 +213710,7 @@ static JsonParse *jsonParseFuncArg( ** JSON functions were suppose to work. From the beginning, blob was ** reserved for expansion and a blob value should have raised an error. ** But it did not, due to a bug. And many applications came to depend - ** upon this buggy behavior, espeically when using the CLI and reading + ** upon this buggy behavior, especially when using the CLI and reading ** JSON text using readfile(), which returns a blob. For this reason ** we will continue to support the bug moving forward. ** See for example https://sqlite.org/forum/forumpost/012136abd5292b8d @@ -206846,11 +213940,12 @@ static void jsonParseFunc( if( p==0 ) return; if( argc==1 ){ jsonDebugPrintBlob(p, 0, p->nBlob, 0, &out); - sqlite3_result_text64(ctx, out.zText, out.nChar, SQLITE_DYNAMIC, SQLITE_UTF8); + sqlite3_result_text64(ctx,out.zText,out.nChar,SQLITE_TRANSIENT,SQLITE_UTF8); }else{ jsonShowParse(p); } jsonParseFree(p); + sqlite3_str_reset(&out); } #endif /* SQLITE_DEBUG */ @@ -206949,13 +214044,6 @@ static void jsonArrayLengthFunc( jsonParseFree(p); } -/* True if the string is all digits */ -static int jsonAllDigits(const char *z, int n){ - int i; - for(i=0; i $[NUMBER] // PG compatible ** LABEL ==> $.LABEL // PG compatible ** [NUMBER] ==> $[NUMBER] // Not PG. Purely for convenience + ** + ** Updated 2024-05-27: If the NUMBER is negative, then PG counts from + ** the right of the array. Hence for negative NUMBER: + ** + ** NUMBER ==> $[#NUMBER] // PG compatible */ jsonStringInit(&jx, ctx); - if( jsonAllDigits(zPath, nPath) ){ + if( sqlite3_value_type(argv[i])==SQLITE_INTEGER ){ jsonAppendRawNZ(&jx, "[", 1); + if( zPath[0]=='-' ) jsonAppendRawNZ(&jx,"#",1); jsonAppendRaw(&jx, zPath, nPath); jsonAppendRawNZ(&jx, "]", 2); }else if( jsonAllAlphanum(zPath, nPath) ){ @@ -207514,6 +214608,40 @@ static void jsonTypeFunc( jsonParseFree(p); } +/* +** json_pretty(JSON) +** json_pretty(JSON, INDENT) +** +** Return text that is a pretty-printed rendering of the input JSON. +** If the argument is not valid JSON, return NULL. +** +** The INDENT argument is text that is used for indentation. If omitted, +** it defaults to four spaces (the same as PostgreSQL). +*/ +static void jsonPrettyFunc( + sqlite3_context *ctx, + int argc, + sqlite3_value **argv +){ + JsonString s; /* The output string */ + JsonPretty x; /* Pretty printing context */ + + memset(&x, 0, sizeof(x)); + x.pParse = jsonParseFuncArg(ctx, argv[0], 0); + if( x.pParse==0 ) return; + x.pOut = &s; + jsonStringInit(&s, ctx); + if( argc==1 || (x.zIndent = (const char*)sqlite3_value_text(argv[1]))==0 ){ + x.zIndent = " "; + x.szIndent = 4; + }else{ + x.szIndent = (u32)strlen(x.zIndent); + } + jsonTranslateBlobToPrettyText(&x, 0); + jsonReturnString(&s, 0, 0); + jsonParseFree(x.pParse); +} + /* ** json_valid(JSON) ** json_valid(JSON, FLAGS) @@ -207597,21 +214725,17 @@ static void jsonValidFunc( return; } case SQLITE_BLOB: { - if( jsonFuncArgMightBeBinary(argv[0]) ){ + JsonParse py; + memset(&py, 0, sizeof(py)); + if( jsonArgIsJsonb(argv[0], &py) ){ if( flags & 0x04 ){ /* Superficial checking only - accomplished by the - ** jsonFuncArgMightBeBinary() call above. */ + ** jsonArgIsJsonb() call above. */ res = 1; }else if( flags & 0x08 ){ /* Strict checking. Check by translating BLOB->TEXT->BLOB. If ** no errors occur, call that a "strict check". */ - JsonParse px; - u32 iErr; - memset(&px, 0, sizeof(px)); - px.aBlob = (u8*)sqlite3_value_blob(argv[0]); - px.nBlob = sqlite3_value_bytes(argv[0]); - iErr = jsonbValidityCheck(&px, 0, px.nBlob, 1); - res = iErr==0; + res = 0==jsonbValidityCheck(&py, 0, py.nBlob, 1); } break; } @@ -207669,9 +214793,7 @@ static void jsonErrorFunc( UNUSED_PARAMETER(argc); memset(&s, 0, sizeof(s)); s.db = sqlite3_context_db_handle(ctx); - if( jsonFuncArgMightBeBinary(argv[0]) ){ - s.aBlob = (u8*)sqlite3_value_blob(argv[0]); - s.nBlob = sqlite3_value_bytes(argv[0]); + if( jsonArgIsJsonb(argv[0], &s) ){ iErrPos = (i64)jsonbValidityCheck(&s, 0, s.nBlob, 1); }else{ s.zJson = (char*)sqlite3_value_text(argv[0]); @@ -207832,18 +214954,20 @@ static void jsonObjectStep( UNUSED_PARAMETER(argc); pStr = (JsonString*)sqlite3_aggregate_context(ctx, sizeof(*pStr)); if( pStr ){ + z = (const char*)sqlite3_value_text(argv[0]); + n = sqlite3Strlen30(z); if( pStr->zBuf==0 ){ jsonStringInit(pStr, ctx); jsonAppendChar(pStr, '{'); - }else if( pStr->nUsed>1 ){ + }else if( pStr->nUsed>1 && z!=0 ){ jsonAppendChar(pStr, ','); } pStr->pCtx = ctx; - z = (const char*)sqlite3_value_text(argv[0]); - n = sqlite3Strlen30(z); - jsonAppendString(pStr, z, n); - jsonAppendChar(pStr, ':'); - jsonAppendSqlValue(pStr, argv[1]); + if( z!=0 ){ + jsonAppendString(pStr, z, n); + jsonAppendChar(pStr, ':'); + jsonAppendSqlValue(pStr, argv[1]); + } } } static void jsonObjectCompute(sqlite3_context *ctx, int isFinal){ @@ -207910,6 +215034,7 @@ struct JsonEachCursor { u32 nRoot; /* Size of the root path in bytes */ u8 eType; /* Type of the container for element i */ u8 bRecursive; /* True for json_tree(). False for json_each() */ + u8 eMode; /* 1 for json_each(). 2 for jsonb_each() */ u32 nParent; /* Current nesting depth */ u32 nParentAlloc; /* Space allocated for aParent[] */ JsonParent *aParent; /* Parent elements of i */ @@ -207921,6 +215046,8 @@ typedef struct JsonEachConnection JsonEachConnection; struct JsonEachConnection { sqlite3_vtab base; /* Base class - must be first */ sqlite3 *db; /* Database connection */ + u8 eMode; /* 1 for json_each(). 2 for jsonb_each() */ + u8 bRecursive; /* True for json_tree(). False for json_each() */ }; @@ -207963,6 +215090,8 @@ static int jsonEachConnect( if( pNew==0 ) return SQLITE_NOMEM; sqlite3_vtab_config(db, SQLITE_VTAB_INNOCUOUS); pNew->db = db; + pNew->eMode = argv[0][4]=='b' ? 2 : 1; + pNew->bRecursive = argv[0][4+pNew->eMode]=='t'; } return rc; } @@ -207974,8 +215103,8 @@ static int jsonEachDisconnect(sqlite3_vtab *pVtab){ return SQLITE_OK; } -/* constructor for a JsonEachCursor object for json_each(). */ -static int jsonEachOpenEach(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCursor){ +/* constructor for a JsonEachCursor object for json_each()/json_tree(). */ +static int jsonEachOpen(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCursor){ JsonEachConnection *pVtab = (JsonEachConnection*)p; JsonEachCursor *pCur; @@ -207983,21 +215112,13 @@ static int jsonEachOpenEach(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCursor){ pCur = sqlite3DbMallocZero(pVtab->db, sizeof(*pCur)); if( pCur==0 ) return SQLITE_NOMEM; pCur->db = pVtab->db; + pCur->eMode = pVtab->eMode; + pCur->bRecursive = pVtab->bRecursive; jsonStringZero(&pCur->path); *ppCursor = &pCur->base; return SQLITE_OK; } -/* constructor for a JsonEachCursor object for json_tree(). */ -static int jsonEachOpenTree(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCursor){ - int rc = jsonEachOpenEach(p, ppCursor); - if( rc==SQLITE_OK ){ - JsonEachCursor *pCur = (JsonEachCursor*)*ppCursor; - pCur->bRecursive = 1; - } - return rc; -} - /* Reset a JsonEachCursor back to its original state. Free any memory ** held. */ static void jsonEachCursorReset(JsonEachCursor *p){ @@ -208202,7 +215323,7 @@ static int jsonEachColumn( } case JEACH_VALUE: { u32 i = jsonSkipLabel(p); - jsonReturnFromBlob(&p->sParse, i, ctx, 1); + jsonReturnFromBlob(&p->sParse, i, ctx, p->eMode); if( (p->sParse.aBlob[i] & 0x0f)>=JSONB_ARRAY ){ sqlite3_result_subtype(ctx, JSON_SUBTYPE); } @@ -208356,9 +215477,8 @@ static int jsonEachFilter( memset(&p->sParse, 0, sizeof(p->sParse)); p->sParse.nJPRef = 1; p->sParse.db = p->db; - if( jsonFuncArgMightBeBinary(argv[0]) ){ - p->sParse.nBlob = sqlite3_value_bytes(argv[0]); - p->sParse.aBlob = (u8*)sqlite3_value_blob(argv[0]); + if( jsonArgIsJsonb(argv[0], &p->sParse) ){ + /* We have JSONB */ }else{ p->sParse.zJson = (char*)sqlite3_value_text(argv[0]); p->sParse.nJson = sqlite3_value_bytes(argv[0]); @@ -208447,36 +215567,7 @@ static sqlite3_module jsonEachModule = { jsonEachBestIndex, /* xBestIndex */ jsonEachDisconnect, /* xDisconnect */ 0, /* xDestroy */ - jsonEachOpenEach, /* xOpen - open a cursor */ - jsonEachClose, /* xClose - close a cursor */ - jsonEachFilter, /* xFilter - configure scan constraints */ - jsonEachNext, /* xNext - advance a cursor */ - jsonEachEof, /* xEof - check for end of scan */ - jsonEachColumn, /* xColumn - read data */ - jsonEachRowid, /* xRowid - read data */ - 0, /* xUpdate */ - 0, /* xBegin */ - 0, /* xSync */ - 0, /* xCommit */ - 0, /* xRollback */ - 0, /* xFindMethod */ - 0, /* xRename */ - 0, /* xSavepoint */ - 0, /* xRelease */ - 0, /* xRollbackTo */ - 0, /* xShadowName */ - 0 /* xIntegrity */ -}; - -/* The methods of the json_tree virtual table. */ -static sqlite3_module jsonTreeModule = { - 0, /* iVersion */ - 0, /* xCreate */ - jsonEachConnect, /* xConnect */ - jsonEachBestIndex, /* xBestIndex */ - jsonEachDisconnect, /* xDisconnect */ - 0, /* xDestroy */ - jsonEachOpenTree, /* xOpen - open a cursor */ + jsonEachOpen, /* xOpen - open a cursor */ jsonEachClose, /* xClose - close a cursor */ jsonEachFilter, /* xFilter - configure scan constraints */ jsonEachNext, /* xNext - advance a cursor */ @@ -208528,6 +215619,8 @@ SQLITE_PRIVATE void sqlite3RegisterJsonFunctions(void){ JFUNCTION(jsonb_object, -1,0,1, 1,1,0, jsonObjectFunc), JFUNCTION(json_patch, 2,1,1, 0,0,0, jsonPatchFunc), JFUNCTION(jsonb_patch, 2,1,0, 0,1,0, jsonPatchFunc), + JFUNCTION(json_pretty, 1,1,0, 0,0,0, jsonPrettyFunc), + JFUNCTION(json_pretty, 2,1,0, 0,0,0, jsonPrettyFunc), JFUNCTION(json_quote, 1,0,1, 1,0,0, jsonQuoteFunc), JFUNCTION(json_remove, -1,1,1, 0,0,0, jsonRemoveFunc), JFUNCTION(jsonb_remove, -1,1,0, 0,1,0, jsonRemoveFunc), @@ -208563,22 +215656,21 @@ SQLITE_PRIVATE void sqlite3RegisterJsonFunctions(void){ #if !defined(SQLITE_OMIT_VIRTUALTABLE) && !defined(SQLITE_OMIT_JSON) /* -** Register the JSON table-valued functions +** Register the JSON table-valued function named zName and return a +** pointer to its Module object. Return NULL if something goes wrong. */ -SQLITE_PRIVATE int sqlite3JsonTableFunctions(sqlite3 *db){ - int rc = SQLITE_OK; - static const struct { - const char *zName; - sqlite3_module *pModule; - } aMod[] = { - { "json_each", &jsonEachModule }, - { "json_tree", &jsonTreeModule }, - }; +SQLITE_PRIVATE Module *sqlite3JsonVtabRegister(sqlite3 *db, const char *zName){ unsigned int i; - for(i=0; iaModule, zName)==0 ); + for(i=0; i */ /* ** If building separately, we will need some setup that is normally @@ -208680,6 +215774,14 @@ typedef unsigned int u32; # define ALWAYS(X) (X) # define NEVER(X) (X) #endif +#ifndef offsetof +# define offsetof(ST,M) ((size_t)((char*)&((ST*)0)->M - (char*)0)) +#endif +#if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) +# define FLEXARRAY +#else +# define FLEXARRAY 1 +#endif #endif /* !defined(SQLITE_AMALGAMATION) */ /* Macro to check for 4-byte alignment. Only used inside of assert() */ @@ -209000,9 +216102,13 @@ struct RtreeMatchArg { RtreeGeomCallback cb; /* Info about the callback functions */ int nParam; /* Number of parameters to the SQL function */ sqlite3_value **apSqlParam; /* Original SQL parameter values */ - RtreeDValue aParam[1]; /* Values for parameters to the SQL function */ + RtreeDValue aParam[FLEXARRAY]; /* Values for parameters to the SQL function */ }; +/* Size of an RtreeMatchArg object with N parameters */ +#define SZ_RTREEMATCHARG(N) \ + (offsetof(RtreeMatchArg,aParam)+(N)*sizeof(RtreeDValue)) + #ifndef MAX # define MAX(x,y) ((x) < (y) ? (y) : (x)) #endif @@ -209707,6 +216813,12 @@ static void resetCursor(RtreeCursor *pCsr){ pCsr->base.pVtab = (sqlite3_vtab*)pRtree; pCsr->pReadAux = pStmt; + /* The following will only fail if the previous sqlite3_step() call failed, + ** in which case the error has already been caught. This statement never + ** encounters an error within an sqlite3_column_xxx() function, as it + ** calls sqlite3_column_value(), which does not use malloc(). So it is safe + ** to ignore the error code here. */ + sqlite3_reset(pStmt); } /* @@ -210427,6 +217539,8 @@ static int deserializeGeometry(sqlite3_value *pValue, RtreeConstraint *pCons){ return SQLITE_OK; } +SQLITE_PRIVATE int sqlite3IntFloatCompare(i64,double); + /* ** Rtree virtual table module xFilter method. */ @@ -210456,7 +217570,8 @@ static int rtreeFilter( i64 iNode = 0; int eType = sqlite3_value_numeric_type(argv[0]); if( eType==SQLITE_INTEGER - || (eType==SQLITE_FLOAT && sqlite3_value_double(argv[0])==iRowid) + || (eType==SQLITE_FLOAT + && 0==sqlite3IntFloatCompare(iRowid,sqlite3_value_double(argv[0]))) ){ rc = findLeafNode(pRtree, iRowid, &pLeaf, &iNode); }else{ @@ -210688,7 +217803,7 @@ static int rtreeBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){ } /* -** Return the N-dimensional volumn of the cell stored in *p. +** Return the N-dimensional volume of the cell stored in *p. */ static RtreeDValue cellArea(Rtree *pRtree, RtreeCell *p){ RtreeDValue area = (RtreeDValue)1; @@ -211811,6 +218926,7 @@ static int rtreeUpdate( */ static int rtreeBeginTransaction(sqlite3_vtab *pVtab){ Rtree *pRtree = (Rtree *)pVtab; + assert( pRtree->inWrTrans==0 ); pRtree->inWrTrans = 1; return SQLITE_OK; } @@ -212357,8 +219473,8 @@ static void rtreenode(sqlite3_context *ctx, int nArg, sqlite3_value **apArg){ sqlite3_str_append(pOut, "}", 1); } errCode = sqlite3_str_errcode(pOut); - sqlite3_result_text(ctx, sqlite3_str_finish(pOut), -1, sqlite3_free); sqlite3_result_error_code(ctx, errCode); + sqlite3_result_text(ctx, sqlite3_str_finish(pOut), -1, sqlite3_free); } /* This routine implements an SQL function that returns the "depth" parameter @@ -212453,7 +219569,7 @@ static sqlite3_stmt *rtreeCheckPrepare( /* ** The second and subsequent arguments to this function are a printf() ** style format string and arguments. This function formats the string and -** appends it to the report being accumuated in pCheck. +** appends it to the report being accumulated in pCheck. */ static void rtreeCheckAppendMsg(RtreeCheck *pCheck, const char *zFmt, ...){ va_list ap; @@ -213641,7 +220757,7 @@ static void geopolyBBoxFinal( ** Determine if point (x0,y0) is beneath line segment (x1,y1)->(x2,y2). ** Returns: ** -** +2 x0,y0 is on the line segement +** +2 x0,y0 is on the line segment ** ** +1 x0,y0 is beneath line segment ** @@ -213747,7 +220863,7 @@ static void geopolyWithinFunc( sqlite3_free(p2); } -/* Objects used by the overlap algorihm. */ +/* Objects used by the overlap algorithm. */ typedef struct GeoEvent GeoEvent; typedef struct GeoSegment GeoSegment; typedef struct GeoOverlap GeoOverlap; @@ -214794,8 +221910,7 @@ static void geomCallback(sqlite3_context *ctx, int nArg, sqlite3_value **aArg){ sqlite3_int64 nBlob; int memErr = 0; - nBlob = sizeof(RtreeMatchArg) + (nArg-1)*sizeof(RtreeDValue) - + nArg*sizeof(sqlite3_value*); + nBlob = SZ_RTREEMATCHARG(nArg) + nArg*sizeof(sqlite3_value*); pBlob = (RtreeMatchArg *)sqlite3_malloc64(nBlob); if( !pBlob ){ sqlite3_result_error_nomem(ctx); @@ -214874,7 +221989,7 @@ SQLITE_API int sqlite3_rtree_query_callback( ); } -#if !SQLITE_CORE +#ifndef SQLITE_CORE #ifdef _WIN32 __declspec(dllexport) #endif @@ -215365,7 +222480,7 @@ static void icuLoadCollation( UCollator *pUCollator; /* ICU library collation object */ int rc; /* Return code from sqlite3_create_collation_x() */ - assert(nArg==2); + assert(nArg==2 || nArg==3); (void)nArg; /* Unused parameter */ zLocale = (const char *)sqlite3_value_text(apArg[0]); zName = (const char *)sqlite3_value_text(apArg[1]); @@ -215380,7 +222495,39 @@ static void icuLoadCollation( return; } assert(p); - + if(nArg==3){ + const char *zOption = (const char*)sqlite3_value_text(apArg[2]); + static const struct { + const char *zName; + UColAttributeValue val; + } aStrength[] = { + { "PRIMARY", UCOL_PRIMARY }, + { "SECONDARY", UCOL_SECONDARY }, + { "TERTIARY", UCOL_TERTIARY }, + { "DEFAULT", UCOL_DEFAULT_STRENGTH }, + { "QUARTERNARY", UCOL_QUATERNARY }, + { "IDENTICAL", UCOL_IDENTICAL }, + }; + unsigned int i; + for(i=0; i=sizeof(aStrength)/sizeof(aStrength[0]) ){ + sqlite3_str *pStr = sqlite3_str_new(sqlite3_context_db_handle(p)); + sqlite3_str_appendf(pStr, + "unknown collation strength \"%s\" - should be one of:", + zOption); + for(i=0; ipTblIter, &p->zErrmsg); pIter->zTbl = 0; + pIter->zDataTbl = 0; }else{ pIter->zTbl = (const char*)sqlite3_column_text(pIter->pTblIter, 0); pIter->zDataTbl = (const char*)sqlite3_column_text(pIter->pTblIter,1); @@ -218620,13 +225792,13 @@ static char *rbuObjIterGetIndexWhere(sqlite3rbu *p, RbuObjIter *pIter){ else if( c==')' ){ nParen--; if( nParen==0 ){ - int nSpan = &zSql[i] - pIter->aIdxCol[iIdxCol].zSpan; + int nSpan = (int)(&zSql[i] - pIter->aIdxCol[iIdxCol].zSpan); pIter->aIdxCol[iIdxCol++].nSpan = nSpan; i++; break; } }else if( c==',' && nParen==1 ){ - int nSpan = &zSql[i] - pIter->aIdxCol[iIdxCol].zSpan; + int nSpan = (int)(&zSql[i] - pIter->aIdxCol[iIdxCol].zSpan); pIter->aIdxCol[iIdxCol++].nSpan = nSpan; pIter->aIdxCol[iIdxCol].zSpan = &zSql[i+1]; }else if( c=='"' || c=='\'' || c=='`' ){ @@ -219316,6 +226488,8 @@ static void rbuFileSuffix3(const char *zBase, char *z){ for(i=sz-1; i>0 && z[i]!='/' && z[i]!='.'; i--){} if( z[i]=='.' && sz>i+4 ) memmove(&z[i+1], &z[sz-3], 4); } +#else + UNUSED_PARAMETER2(zBase,z); #endif } @@ -219333,7 +226507,7 @@ static i64 rbuShmChecksum(sqlite3rbu *p){ u32 volatile *ptr; p->rc = pDb->pMethods->xShmMap(pDb, 0, 32*1024, 0, (void volatile**)&ptr); if( p->rc==SQLITE_OK ){ - iRet = ((i64)ptr[10] << 32) + ptr[11]; + iRet = (i64)(((u64)ptr[10] << 32) + ptr[11]); } } return iRet; @@ -219900,7 +227074,7 @@ static void rbuSaveState(sqlite3rbu *p, int eStage){ "(%d, %Q), " "(%d, %Q), " "(%d, %d), " - "(%d, %d), " + "(%d, %lld), " "(%d, %lld), " "(%d, %lld), " "(%d, %lld), " @@ -220258,6 +227432,7 @@ static void rbuIndexCntFunc( sqlite3 *db = (rbuIsVacuum(p) ? p->dbRbu : p->dbMain); assert( nVal==1 ); + UNUSED_PARAMETER(nVal); rc = prepareFreeAndCollectError(db, &pStmt, &zErrmsg, sqlite3_mprintf("SELECT count(*) FROM sqlite_schema " @@ -220533,7 +227708,7 @@ SQLITE_API sqlite3rbu *sqlite3rbu_vacuum( ){ if( zTarget==0 ){ return rbuMisuseError(); } if( zState ){ - int n = strlen(zState); + size_t n = strlen(zState); if( n>=7 && 0==memcmp("-vactmp", &zState[n-7], 7) ){ return rbuMisuseError(); } @@ -220750,6 +227925,7 @@ SQLITE_API int sqlite3rbu_savestate(sqlite3rbu *p){ */ static int xDefaultRename(void *pArg, const char *zOld, const char *zNew){ int rc = SQLITE_OK; + UNUSED_PARAMETER(pArg); #if defined(_WIN32_WCE) { LPWSTR zWideOld; @@ -221148,7 +228324,7 @@ static int rbuVfsFileSize(sqlite3_file *pFile, sqlite_int64 *pSize){ /* If this is an RBU vacuum operation and this is the target database, ** pretend that it has at least one page. Otherwise, SQLite will not - ** check for the existance of a *-wal file. rbuVfsRead() contains + ** check for the existence of a *-wal file. rbuVfsRead() contains ** similar logic. */ if( rc==SQLITE_OK && *pSize==0 && p->pRbu && rbuIsVacuum(p->pRbu) @@ -221654,6 +228830,9 @@ static int rbuVfsCurrentTime(sqlite3_vfs *pVfs, double *pTimeOut){ ** No-op. */ static int rbuVfsGetLastError(sqlite3_vfs *pVfs, int a, char *b){ + UNUSED_PARAMETER(pVfs); + UNUSED_PARAMETER(a); + UNUSED_PARAMETER(b); return 0; } @@ -222052,6 +229231,7 @@ static int statBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){ pIdxInfo->orderByConsumed = 1; pIdxInfo->idxNum |= 0x08; } + pIdxInfo->idxFlags |= SQLITE_INDEX_SCAN_HEX; return SQLITE_OK; } @@ -222709,7 +229889,13 @@ SQLITE_PRIVATE int sqlite3DbstatRegister(sqlite3 *db){ return SQLITE_OK; } ** ** The data field of sqlite_dbpage table can be updated. The new ** value must be a BLOB which is the correct page size, otherwise the -** update fails. Rows may not be deleted or inserted. +** update fails. INSERT operations also work, and operate as if they +** where REPLACE. The size of the database can be extended by INSERT-ing +** new pages on the end. +** +** Rows may not be deleted. However, doing an INSERT to page number N +** with NULL page data causes the N-th page and all subsequent pages to be +** deleted and the database to be truncated. */ /* #include "sqliteInt.h" ** Requires access to internal data structures ** */ @@ -222721,8 +229907,8 @@ typedef struct DbpageCursor DbpageCursor; struct DbpageCursor { sqlite3_vtab_cursor base; /* Base class. Must be first */ - int pgno; /* Current page number */ - int mxPgno; /* Last page to visit on this scan */ + Pgno pgno; /* Current page number */ + Pgno mxPgno; /* Last page to visit on this scan */ Pager *pPager; /* Pager being read/written */ DbPage *pPage1; /* Page 1 of the database */ int iDb; /* Index of database to analyze */ @@ -222732,6 +229918,8 @@ struct DbpageCursor { struct DbpageTable { sqlite3_vtab base; /* Base class. Must be first */ sqlite3 *db; /* The database */ + int iDbTrunc; /* Database to truncate */ + Pgno pgnoTrunc; /* Size to truncate to */ }; /* Columns */ @@ -222740,7 +229928,6 @@ struct DbpageTable { #define DBPAGE_COLUMN_SCHEMA 2 - /* ** Connect to or create a dbpagevfs virtual table. */ @@ -222858,7 +230045,7 @@ static int dbpageOpen(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCursor){ }else{ memset(pCsr, 0, sizeof(DbpageCursor)); pCsr->base.pVtab = pVTab; - pCsr->pgno = -1; + pCsr->pgno = 0; } *ppCursor = (sqlite3_vtab_cursor *)pCsr; @@ -222911,7 +230098,8 @@ static int dbpageFilter( sqlite3 *db = pTab->db; Btree *pBt; - (void)idxStr; + UNUSED_PARAMETER(idxStr); + UNUSED_PARAMETER(argc); /* Default setting is no rows of result */ pCsr->pgno = 1; @@ -222957,12 +230145,12 @@ static int dbpageColumn( int rc = SQLITE_OK; switch( i ){ case 0: { /* pgno */ - sqlite3_result_int(ctx, pCsr->pgno); + sqlite3_result_int64(ctx, (sqlite3_int64)pCsr->pgno); break; } case 1: { /* data */ DbPage *pDbPage = 0; - if( pCsr->pgno==((PENDING_BYTE/pCsr->szPage)+1) ){ + if( pCsr->pgno==(Pgno)((PENDING_BYTE/pCsr->szPage)+1) ){ /* The pending byte page. Assume it is zeroed out. Attempting to ** request this page from the page is an SQLITE_CORRUPT error. */ sqlite3_result_zeroblob(ctx, pCsr->szPage); @@ -222991,6 +230179,24 @@ static int dbpageRowid(sqlite3_vtab_cursor *pCursor, sqlite_int64 *pRowid){ return SQLITE_OK; } +/* +** Open write transactions. Since we do not know in advance which database +** files will be written by the sqlite_dbpage virtual table, start a write +** transaction on them all. +** +** Return SQLITE_OK if successful, or an SQLite error code otherwise. +*/ +static int dbpageBeginTrans(DbpageTable *pTab){ + sqlite3 *db = pTab->db; + int rc = SQLITE_OK; + int i; + for(i=0; rc==SQLITE_OK && inDb; i++){ + Btree *pBt = db->aDb[i].pBt; + if( pBt ) rc = sqlite3BtreeBeginTrans(pBt, 1, 0); + } + return rc; +} + static int dbpageUpdate( sqlite3_vtab *pVtab, int argc, @@ -223002,11 +230208,11 @@ static int dbpageUpdate( DbPage *pDbPage = 0; int rc = SQLITE_OK; char *zErr = 0; - const char *zSchema; int iDb; Btree *pBt; Pager *pPager; int szPage; + int isInsert; (void)pRowid; if( pTab->db->flags & SQLITE_Defensive ){ @@ -223017,21 +230223,29 @@ static int dbpageUpdate( zErr = "cannot delete"; goto update_fail; } - pgno = sqlite3_value_int(argv[0]); - if( sqlite3_value_type(argv[0])==SQLITE_NULL - || (Pgno)sqlite3_value_int(argv[1])!=pgno - ){ - zErr = "cannot insert"; - goto update_fail; + if( sqlite3_value_type(argv[0])==SQLITE_NULL ){ + pgno = (Pgno)sqlite3_value_int64(argv[2]); + isInsert = 1; + }else{ + pgno = (Pgno)sqlite3_value_int64(argv[0]); + if( (Pgno)sqlite3_value_int(argv[1])!=pgno ){ + zErr = "cannot insert"; + goto update_fail; + } + isInsert = 0; } - zSchema = (const char*)sqlite3_value_text(argv[4]); - iDb = ALWAYS(zSchema) ? sqlite3FindDbName(pTab->db, zSchema) : -1; - if( NEVER(iDb<0) ){ - zErr = "no such schema"; - goto update_fail; + if( sqlite3_value_type(argv[4])==SQLITE_NULL ){ + iDb = 0; + }else{ + const char *zSchema = (const char*)sqlite3_value_text(argv[4]); + iDb = sqlite3FindDbName(pTab->db, zSchema); + if( iDb<0 ){ + zErr = "no such schema"; + goto update_fail; + } } pBt = pTab->db->aDb[iDb].pBt; - if( NEVER(pgno<1) || NEVER(pBt==0) || NEVER(pgno>sqlite3BtreeLastPage(pBt)) ){ + if( pgno<1 || NEVER(pBt==0) ){ zErr = "bad page number"; goto update_fail; } @@ -223039,51 +230253,84 @@ static int dbpageUpdate( if( sqlite3_value_type(argv[3])!=SQLITE_BLOB || sqlite3_value_bytes(argv[3])!=szPage ){ - zErr = "bad page value"; + if( sqlite3_value_type(argv[3])==SQLITE_NULL && isInsert && pgno>1 ){ + /* "INSERT INTO dbpage($PGNO,NULL)" causes page number $PGNO and + ** all subsequent pages to be deleted. */ + pTab->iDbTrunc = iDb; + pTab->pgnoTrunc = pgno-1; + pgno = 1; + }else{ + zErr = "bad page value"; + goto update_fail; + } + } + + if( dbpageBeginTrans(pTab)!=SQLITE_OK ){ + zErr = "failed to open transaction"; goto update_fail; } + pPager = sqlite3BtreePager(pBt); rc = sqlite3PagerGet(pPager, pgno, (DbPage**)&pDbPage, 0); if( rc==SQLITE_OK ){ const void *pData = sqlite3_value_blob(argv[3]); - assert( pData!=0 || pTab->db->mallocFailed ); - if( pData - && (rc = sqlite3PagerWrite(pDbPage))==SQLITE_OK - ){ - memcpy(sqlite3PagerGetData(pDbPage), pData, szPage); + if( (rc = sqlite3PagerWrite(pDbPage))==SQLITE_OK && pData ){ + unsigned char *aPage = sqlite3PagerGetData(pDbPage); + memcpy(aPage, pData, szPage); + pTab->pgnoTrunc = 0; } } + if( rc!=SQLITE_OK ){ + pTab->pgnoTrunc = 0; + } sqlite3PagerUnref(pDbPage); return rc; update_fail: + pTab->pgnoTrunc = 0; sqlite3_free(pVtab->zErrMsg); pVtab->zErrMsg = sqlite3_mprintf("%s", zErr); return SQLITE_ERROR; } -/* Since we do not know in advance which database files will be -** written by the sqlite_dbpage virtual table, start a write transaction -** on them all. -*/ static int dbpageBegin(sqlite3_vtab *pVtab){ DbpageTable *pTab = (DbpageTable *)pVtab; - sqlite3 *db = pTab->db; - int i; - for(i=0; inDb; i++){ - Btree *pBt = db->aDb[i].pBt; - if( pBt ) (void)sqlite3BtreeBeginTrans(pBt, 1, 0); + pTab->pgnoTrunc = 0; + return SQLITE_OK; +} + +/* Invoke sqlite3PagerTruncate() as necessary, just prior to COMMIT +*/ +static int dbpageSync(sqlite3_vtab *pVtab){ + DbpageTable *pTab = (DbpageTable *)pVtab; + if( pTab->pgnoTrunc>0 ){ + Btree *pBt = pTab->db->aDb[pTab->iDbTrunc].pBt; + Pager *pPager = sqlite3BtreePager(pBt); + sqlite3BtreeEnter(pBt); + if( pTab->pgnoTruncpgnoTrunc); + } + sqlite3BtreeLeave(pBt); } + pTab->pgnoTrunc = 0; return SQLITE_OK; } +/* Cancel any pending truncate. +*/ +static int dbpageRollbackTo(sqlite3_vtab *pVtab, int notUsed1){ + DbpageTable *pTab = (DbpageTable *)pVtab; + pTab->pgnoTrunc = 0; + (void)notUsed1; + return SQLITE_OK; +} /* ** Invoke this routine to register the "dbpage" virtual table module */ SQLITE_PRIVATE int sqlite3DbpageRegister(sqlite3 *db){ static sqlite3_module dbpage_module = { - 0, /* iVersion */ + 2, /* iVersion */ dbpageConnect, /* xCreate */ dbpageConnect, /* xConnect */ dbpageBestIndex, /* xBestIndex */ @@ -223098,14 +230345,14 @@ SQLITE_PRIVATE int sqlite3DbpageRegister(sqlite3 *db){ dbpageRowid, /* xRowid - read data */ dbpageUpdate, /* xUpdate */ dbpageBegin, /* xBegin */ - 0, /* xSync */ + dbpageSync, /* xSync */ 0, /* xCommit */ 0, /* xRollback */ 0, /* xFindMethod */ 0, /* xRename */ 0, /* xSavepoint */ 0, /* xRelease */ - 0, /* xRollbackTo */ + dbpageRollbackTo, /* xRollbackTo */ 0, /* xShadowName */ 0 /* xIntegrity */ }; @@ -223116,6 +230363,536 @@ SQLITE_PRIVATE int sqlite3DbpageRegister(sqlite3 *db){ return SQLITE_OK; } #endif /* SQLITE_ENABLE_DBSTAT_VTAB */ /************** End of dbpage.c **********************************************/ +/************** Begin file carray.c ******************************************/ +/* +** 2016-06-29 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This file implements a table-valued-function that +** returns the values in a C-language array. +** Examples: +** +** SELECT * FROM carray($ptr,5) +** +** The query above returns 5 integers contained in a C-language array +** at the address $ptr. $ptr is a pointer to the array of integers. +** The pointer value must be assigned to $ptr using the +** sqlite3_bind_pointer() interface with a pointer type of "carray". +** For example: +** +** static int aX[] = { 53, 9, 17, 2231, 4, 99 }; +** int i = sqlite3_bind_parameter_index(pStmt, "$ptr"); +** sqlite3_bind_pointer(pStmt, i, aX, "carray", 0); +** +** There is an optional third parameter to determine the datatype of +** the C-language array. Allowed values of the third parameter are +** 'int32', 'int64', 'double', 'char*', 'struct iovec'. Example: +** +** SELECT * FROM carray($ptr,10,'char*'); +** +** The default value of the third parameter is 'int32'. +** +** HOW IT WORKS +** +** The carray "function" is really a virtual table with the +** following schema: +** +** CREATE TABLE carray( +** value, +** pointer HIDDEN, +** count HIDDEN, +** ctype TEXT HIDDEN +** ); +** +** If the hidden columns "pointer" and "count" are unconstrained, then +** the virtual table has no rows. Otherwise, the virtual table interprets +** the integer value of "pointer" as a pointer to the array and "count" +** as the number of elements in the array. The virtual table steps through +** the array, element by element. +*/ +#if !defined(SQLITE_OMIT_VIRTUALTABLE) && defined(SQLITE_ENABLE_CARRAY) +/* #include "sqliteInt.h" */ +#if defined(_WIN32) || defined(__RTP__) || defined(_WRS_KERNEL) + struct iovec { + void *iov_base; + size_t iov_len; + }; +#else +# include +#endif + +/* +** Names of allowed datatypes +*/ +static const char *azCarrayType[] = { + "int32", "int64", "double", "char*", "struct iovec" +}; + +/* +** Structure used to hold the sqlite3_carray_bind() information +*/ +typedef struct carray_bind carray_bind; +struct carray_bind { + void *aData; /* The data */ + int nData; /* Number of elements */ + int mFlags; /* Control flags */ + void (*xDel)(void*); /* Destructor for aData */ +}; + + +/* carray_cursor is a subclass of sqlite3_vtab_cursor which will +** serve as the underlying representation of a cursor that scans +** over rows of the result +*/ +typedef struct carray_cursor carray_cursor; +struct carray_cursor { + sqlite3_vtab_cursor base; /* Base class - must be first */ + sqlite3_int64 iRowid; /* The rowid */ + void *pPtr; /* Pointer to the array of values */ + sqlite3_int64 iCnt; /* Number of integers in the array */ + unsigned char eType; /* One of the CARRAY_type values */ +}; + +/* +** The carrayConnect() method is invoked to create a new +** carray_vtab that describes the carray virtual table. +** +** Think of this routine as the constructor for carray_vtab objects. +** +** All this routine needs to do is: +** +** (1) Allocate the carray_vtab object and initialize all fields. +** +** (2) Tell SQLite (via the sqlite3_declare_vtab() interface) what the +** result set of queries against carray will look like. +*/ +static int carrayConnect( + sqlite3 *db, + void *pAux, + int argc, const char *const*argv, + sqlite3_vtab **ppVtab, + char **pzErr +){ + sqlite3_vtab *pNew; + int rc; + +/* Column numbers */ +#define CARRAY_COLUMN_VALUE 0 +#define CARRAY_COLUMN_POINTER 1 +#define CARRAY_COLUMN_COUNT 2 +#define CARRAY_COLUMN_CTYPE 3 + + rc = sqlite3_declare_vtab(db, + "CREATE TABLE x(value,pointer hidden,count hidden,ctype hidden)"); + if( rc==SQLITE_OK ){ + pNew = *ppVtab = sqlite3_malloc( sizeof(*pNew) ); + if( pNew==0 ) return SQLITE_NOMEM; + memset(pNew, 0, sizeof(*pNew)); + } + return rc; +} + +/* +** This method is the destructor for carray_cursor objects. +*/ +static int carrayDisconnect(sqlite3_vtab *pVtab){ + sqlite3_free(pVtab); + return SQLITE_OK; +} + +/* +** Constructor for a new carray_cursor object. +*/ +static int carrayOpen(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCursor){ + carray_cursor *pCur; + pCur = sqlite3_malloc( sizeof(*pCur) ); + if( pCur==0 ) return SQLITE_NOMEM; + memset(pCur, 0, sizeof(*pCur)); + *ppCursor = &pCur->base; + return SQLITE_OK; +} + +/* +** Destructor for a carray_cursor. +*/ +static int carrayClose(sqlite3_vtab_cursor *cur){ + sqlite3_free(cur); + return SQLITE_OK; +} + + +/* +** Advance a carray_cursor to its next row of output. +*/ +static int carrayNext(sqlite3_vtab_cursor *cur){ + carray_cursor *pCur = (carray_cursor*)cur; + pCur->iRowid++; + return SQLITE_OK; +} + +/* +** Return values of columns for the row at which the carray_cursor +** is currently pointing. +*/ +static int carrayColumn( + sqlite3_vtab_cursor *cur, /* The cursor */ + sqlite3_context *ctx, /* First argument to sqlite3_result_...() */ + int i /* Which column to return */ +){ + carray_cursor *pCur = (carray_cursor*)cur; + sqlite3_int64 x = 0; + switch( i ){ + case CARRAY_COLUMN_POINTER: return SQLITE_OK; + case CARRAY_COLUMN_COUNT: x = pCur->iCnt; break; + case CARRAY_COLUMN_CTYPE: { + sqlite3_result_text(ctx, azCarrayType[pCur->eType], -1, SQLITE_STATIC); + return SQLITE_OK; + } + default: { + switch( pCur->eType ){ + case CARRAY_INT32: { + int *p = (int*)pCur->pPtr; + sqlite3_result_int(ctx, p[pCur->iRowid-1]); + return SQLITE_OK; + } + case CARRAY_INT64: { + sqlite3_int64 *p = (sqlite3_int64*)pCur->pPtr; + sqlite3_result_int64(ctx, p[pCur->iRowid-1]); + return SQLITE_OK; + } + case CARRAY_DOUBLE: { + double *p = (double*)pCur->pPtr; + sqlite3_result_double(ctx, p[pCur->iRowid-1]); + return SQLITE_OK; + } + case CARRAY_TEXT: { + const char **p = (const char**)pCur->pPtr; + sqlite3_result_text(ctx, p[pCur->iRowid-1], -1, SQLITE_TRANSIENT); + return SQLITE_OK; + } + default: { + const struct iovec *p = (struct iovec*)pCur->pPtr; + assert( pCur->eType==CARRAY_BLOB ); + sqlite3_result_blob(ctx, p[pCur->iRowid-1].iov_base, + (int)p[pCur->iRowid-1].iov_len, SQLITE_TRANSIENT); + return SQLITE_OK; + } + } + } + } + sqlite3_result_int64(ctx, x); + return SQLITE_OK; +} + +/* +** Return the rowid for the current row. In this implementation, the +** rowid is the same as the output value. +*/ +static int carrayRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){ + carray_cursor *pCur = (carray_cursor*)cur; + *pRowid = pCur->iRowid; + return SQLITE_OK; +} + +/* +** Return TRUE if the cursor has been moved off of the last +** row of output. +*/ +static int carrayEof(sqlite3_vtab_cursor *cur){ + carray_cursor *pCur = (carray_cursor*)cur; + return pCur->iRowid>pCur->iCnt; +} + +/* +** This method is called to "rewind" the carray_cursor object back +** to the first row of output. +*/ +static int carrayFilter( + sqlite3_vtab_cursor *pVtabCursor, + int idxNum, const char *idxStr, + int argc, sqlite3_value **argv +){ + carray_cursor *pCur = (carray_cursor *)pVtabCursor; + pCur->pPtr = 0; + pCur->iCnt = 0; + switch( idxNum ){ + case 1: { + carray_bind *pBind = sqlite3_value_pointer(argv[0], "carray-bind"); + if( pBind==0 ) break; + pCur->pPtr = pBind->aData; + pCur->iCnt = pBind->nData; + pCur->eType = pBind->mFlags & 0x07; + break; + } + case 2: + case 3: { + pCur->pPtr = sqlite3_value_pointer(argv[0], "carray"); + pCur->iCnt = pCur->pPtr ? sqlite3_value_int64(argv[1]) : 0; + if( idxNum<3 ){ + pCur->eType = CARRAY_INT32; + }else{ + unsigned char i; + const char *zType = (const char*)sqlite3_value_text(argv[2]); + for(i=0; i=sizeof(azCarrayType)/sizeof(azCarrayType[0]) ){ + pVtabCursor->pVtab->zErrMsg = sqlite3_mprintf( + "unknown datatype: %Q", zType); + return SQLITE_ERROR; + }else{ + pCur->eType = i; + } + } + break; + } + } + pCur->iRowid = 1; + return SQLITE_OK; +} + +/* +** SQLite will invoke this method one or more times while planning a query +** that uses the carray virtual table. This routine needs to create +** a query plan for each invocation and compute an estimated cost for that +** plan. +** +** In this implementation idxNum is used to represent the +** query plan. idxStr is unused. +** +** idxNum is: +** +** 1 If only the pointer= constraint exists. In this case, the +** parameter must be bound using sqlite3_carray_bind(). +** +** 2 if the pointer= and count= constraints exist. +** +** 3 if the ctype= constraint also exists. +** +** idxNum is 0 otherwise and carray becomes an empty table. +*/ +static int carrayBestIndex( + sqlite3_vtab *tab, + sqlite3_index_info *pIdxInfo +){ + int i; /* Loop over constraints */ + int ptrIdx = -1; /* Index of the pointer= constraint, or -1 if none */ + int cntIdx = -1; /* Index of the count= constraint, or -1 if none */ + int ctypeIdx = -1; /* Index of the ctype= constraint, or -1 if none */ + unsigned seen = 0; /* Bitmask of == constrainted columns */ + + const struct sqlite3_index_constraint *pConstraint; + pConstraint = pIdxInfo->aConstraint; + for(i=0; inConstraint; i++, pConstraint++){ + if( pConstraint->op!=SQLITE_INDEX_CONSTRAINT_EQ ) continue; + if( pConstraint->iColumn>=0 ) seen |= 1 << pConstraint->iColumn; + if( pConstraint->usable==0 ) continue; + switch( pConstraint->iColumn ){ + case CARRAY_COLUMN_POINTER: + ptrIdx = i; + break; + case CARRAY_COLUMN_COUNT: + cntIdx = i; + break; + case CARRAY_COLUMN_CTYPE: + ctypeIdx = i; + break; + } + } + if( ptrIdx>=0 ){ + pIdxInfo->aConstraintUsage[ptrIdx].argvIndex = 1; + pIdxInfo->aConstraintUsage[ptrIdx].omit = 1; + pIdxInfo->estimatedCost = (double)1; + pIdxInfo->estimatedRows = 100; + pIdxInfo->idxNum = 1; + if( cntIdx>=0 ){ + pIdxInfo->aConstraintUsage[cntIdx].argvIndex = 2; + pIdxInfo->aConstraintUsage[cntIdx].omit = 1; + pIdxInfo->idxNum = 2; + if( ctypeIdx>=0 ){ + pIdxInfo->aConstraintUsage[ctypeIdx].argvIndex = 3; + pIdxInfo->aConstraintUsage[ctypeIdx].omit = 1; + pIdxInfo->idxNum = 3; + }else if( seen & (1<estimatedCost = (double)2147483647; + pIdxInfo->estimatedRows = 2147483647; + pIdxInfo->idxNum = 0; + } + return SQLITE_OK; +} + +/* +** This following structure defines all the methods for the +** carray virtual table. +*/ +static sqlite3_module carrayModule = { + 0, /* iVersion */ + 0, /* xCreate */ + carrayConnect, /* xConnect */ + carrayBestIndex, /* xBestIndex */ + carrayDisconnect, /* xDisconnect */ + 0, /* xDestroy */ + carrayOpen, /* xOpen - open a cursor */ + carrayClose, /* xClose - close a cursor */ + carrayFilter, /* xFilter - configure scan constraints */ + carrayNext, /* xNext - advance a cursor */ + carrayEof, /* xEof - check for end of scan */ + carrayColumn, /* xColumn - read data */ + carrayRowid, /* xRowid - read data */ + 0, /* xUpdate */ + 0, /* xBegin */ + 0, /* xSync */ + 0, /* xCommit */ + 0, /* xRollback */ + 0, /* xFindMethod */ + 0, /* xRename */ + 0, /* xSavepoint */ + 0, /* xRelease */ + 0, /* xRollbackTo */ + 0, /* xShadow */ + 0 /* xIntegrity */ +}; + +/* +** Destructor for the carray_bind object +*/ +static void carrayBindDel(void *pPtr){ + carray_bind *p = (carray_bind*)pPtr; + if( p->xDel!=SQLITE_STATIC ){ + p->xDel(p->aData); + } + sqlite3_free(p); +} + +/* +** Invoke this interface in order to bind to the single-argument +** version of CARRAY(). +*/ +SQLITE_API int sqlite3_carray_bind( + sqlite3_stmt *pStmt, + int idx, + void *aData, + int nData, + int mFlags, + void (*xDestroy)(void*) +){ + carray_bind *pNew = 0; + int i; + int rc = SQLITE_OK; + + /* Ensure that the mFlags value is acceptable. */ + assert( CARRAY_INT32==0 && CARRAY_INT64==1 && CARRAY_DOUBLE==2 ); + assert( CARRAY_TEXT==3 && CARRAY_BLOB==4 ); + if( mFlagsCARRAY_BLOB ){ + rc = SQLITE_ERROR; + goto carray_bind_error; + } + + pNew = sqlite3_malloc64(sizeof(*pNew)); + if( pNew==0 ){ + rc = SQLITE_NOMEM; + goto carray_bind_error; + } + + pNew->nData = nData; + pNew->mFlags = mFlags; + if( xDestroy==SQLITE_TRANSIENT ){ + sqlite3_int64 sz = nData; + switch( mFlags ){ + case CARRAY_INT32: sz *= 4; break; + case CARRAY_INT64: sz *= 8; break; + case CARRAY_DOUBLE: sz *= 8; break; + case CARRAY_TEXT: sz *= sizeof(char*); break; + default: sz *= sizeof(struct iovec); break; + } + if( mFlags==CARRAY_TEXT ){ + for(i=0; iaData = sqlite3_malloc64( sz ); + if( pNew->aData==0 ){ + rc = SQLITE_NOMEM; + goto carray_bind_error; + } + + if( mFlags==CARRAY_TEXT ){ + char **az = (char**)pNew->aData; + char *z = (char*)&az[nData]; + for(i=0; iaData; + unsigned char *z = (unsigned char*)&p[nData]; + for(i=0; iaData, aData, sz); + } + pNew->xDel = sqlite3_free; + }else{ + pNew->aData = aData; + pNew->xDel = xDestroy; + } + return sqlite3_bind_pointer(pStmt, idx, pNew, "carray-bind", carrayBindDel); + + carray_bind_error: + if( xDestroy!=SQLITE_STATIC && xDestroy!=SQLITE_TRANSIENT ){ + xDestroy(aData); + } + sqlite3_free(pNew); + return rc; +} + +/* +** Invoke this routine to register the carray() function. +*/ +SQLITE_PRIVATE Module *sqlite3CarrayRegister(sqlite3 *db){ + return sqlite3VtabCreateModule(db, "carray", &carrayModule, 0, 0); +} + +#endif /* !defined(SQLITE_OMIT_VIRTUALTABLE) && defined(SQLITE_ENABLE_CARRAY) */ + +/************** End of carray.c **********************************************/ /************** Begin file sqlite3session.c **********************************/ #if defined(SQLITE_ENABLE_SESSION) && defined(SQLITE_ENABLE_PREUPDATE_HOOK) @@ -223193,6 +230970,10 @@ struct SessionBuffer { ** input data. Input data may be supplied either as a single large buffer ** (e.g. sqlite3changeset_start()) or using a stream function (e.g. ** sqlite3changeset_start_strm()). +** +** bNoDiscard: +** If true, then the only time data is discarded is as a result of explicit +** sessionDiscardData() calls. Not within every sessionInputBuffer() call. */ struct SessionInput { int bNoDiscard; /* If true, do not discard in InputBuffer() */ @@ -223254,11 +231035,13 @@ struct sqlite3_changeset_iter { struct SessionTable { SessionTable *pNext; char *zName; /* Local name of table */ - int nCol; /* Number of columns in table zName */ + int nCol; /* Number of non-hidden columns */ + int nTotalCol; /* Number of columns including hidden */ int bStat1; /* True if this is sqlite_stat1 */ int bRowid; /* True if this table uses rowid for PK */ const char **azCol; /* Column names */ const char **azDflt; /* Default value expressions */ + int *aiIdx; /* Index to pass to xNew/xOld */ u8 *abPK; /* Array of primary key flags */ int nEntry; /* Total number of entries in hash table */ int nChange; /* Size of apChange[] array */ @@ -223661,22 +231444,22 @@ static int sessionPreupdateHash( unsigned int h = 0; /* Hash value to return */ int i; /* Used to iterate through columns */ + assert( pTab->nTotalCol==pSession->hook.xCount(pSession->hook.pCtx) ); if( pTab->bRowid ){ - assert( pTab->nCol-1==pSession->hook.xCount(pSession->hook.pCtx) ); h = sessionHashAppendI64(h, iRowid); }else{ assert( *pbNullPK==0 ); - assert( pTab->nCol==pSession->hook.xCount(pSession->hook.pCtx) ); for(i=0; inCol; i++){ if( pTab->abPK[i] ){ int rc; int eType; sqlite3_value *pVal; + int iIdx = pTab->aiIdx[i]; if( bNew ){ - rc = pSession->hook.xNew(pSession->hook.pCtx, i, &pVal); + rc = pSession->hook.xNew(pSession->hook.pCtx, iIdx, &pVal); }else{ - rc = pSession->hook.xOld(pSession->hook.pCtx, i, &pVal); + rc = pSession->hook.xOld(pSession->hook.pCtx, iIdx, &pVal); } if( rc!=SQLITE_OK ) return rc; @@ -224013,6 +231796,7 @@ static int sessionPreupdateEqual( sqlite3_value *pVal; /* Value returned by preupdate_new/old */ int rc; /* Error code from preupdate_new/old */ int eType = *a++; /* Type of value from change record */ + int iIdx = pTab->aiIdx[iCol]; /* The following calls to preupdate_new() and preupdate_old() can not ** fail. This is because they cache their return values, and by the @@ -224021,10 +231805,10 @@ static int sessionPreupdateEqual( ** this (that the method has already been called). */ if( op==SQLITE_INSERT ){ /* assert( db->pPreUpdate->pNewUnpacked || db->pPreUpdate->aNew ); */ - rc = pSession->hook.xNew(pSession->hook.pCtx, iCol, &pVal); + rc = pSession->hook.xNew(pSession->hook.pCtx, iIdx, &pVal); }else{ /* assert( db->pPreUpdate->pUnpacked ); */ - rc = pSession->hook.xOld(pSession->hook.pCtx, iCol, &pVal); + rc = pSession->hook.xOld(pSession->hook.pCtx, iIdx, &pVal); } assert( rc==SQLITE_OK ); (void)rc; /* Suppress warning about unused variable */ @@ -224149,9 +231933,11 @@ static int sessionTableInfo( const char *zDb, /* Name of attached database (e.g. "main") */ const char *zThis, /* Table name */ int *pnCol, /* OUT: number of columns */ + int *pnTotalCol, /* OUT: number of hidden columns */ const char **pzTab, /* OUT: Copy of zThis */ const char ***pazCol, /* OUT: Array of column names for table */ const char ***pazDflt, /* OUT: Array of default value expressions */ + int **paiIdx, /* OUT: Array of xNew/xOld indexes */ u8 **pabPK, /* OUT: Array of booleans - true for PK col */ int *pbRowid /* OUT: True if only PK is a rowid */ ){ @@ -224166,6 +231952,7 @@ static int sessionTableInfo( char **azCol = 0; char **azDflt = 0; u8 *abPK = 0; + int *aiIdx = 0; int bRowid = 0; /* Set to true to use rowid as PK */ assert( pazCol && pabPK ); @@ -224173,6 +231960,8 @@ static int sessionTableInfo( *pazCol = 0; *pabPK = 0; *pnCol = 0; + if( pnTotalCol ) *pnTotalCol = 0; + if( paiIdx ) *paiIdx = 0; if( pzTab ) *pzTab = 0; if( pazDflt ) *pazDflt = 0; @@ -224182,9 +231971,9 @@ static int sessionTableInfo( if( rc==SQLITE_OK ){ /* For sqlite_stat1, pretend that (tbl,idx) is the PRIMARY KEY. */ zPragma = sqlite3_mprintf( - "SELECT 0, 'tbl', '', 0, '', 1 UNION ALL " - "SELECT 1, 'idx', '', 0, '', 2 UNION ALL " - "SELECT 2, 'stat', '', 0, '', 0" + "SELECT 0, 'tbl', '', 0, '', 1, 0 UNION ALL " + "SELECT 1, 'idx', '', 0, '', 2, 0 UNION ALL " + "SELECT 2, 'stat', '', 0, '', 0, 0" ); }else if( rc==SQLITE_ERROR ){ zPragma = sqlite3_mprintf(""); @@ -224192,7 +231981,7 @@ static int sessionTableInfo( return rc; } }else{ - zPragma = sqlite3_mprintf("PRAGMA '%q'.table_info('%q')", zDb, zThis); + zPragma = sqlite3_mprintf("PRAGMA '%q'.table_xinfo('%q')", zDb, zThis); } if( !zPragma ){ return SQLITE_NOMEM; @@ -224209,7 +231998,9 @@ static int sessionTableInfo( while( SQLITE_ROW==sqlite3_step(pStmt) ){ nByte += sqlite3_column_bytes(pStmt, 1); /* name */ nByte += sqlite3_column_bytes(pStmt, 4); /* dflt_value */ - nDbCol++; + if( sqlite3_column_int(pStmt, 6)==0 ){ /* !hidden */ + nDbCol++; + } if( sqlite3_column_int(pStmt, 5) ) bRowid = 0; /* pk */ } if( nDbCol==0 ) bRowid = 0; @@ -224218,7 +232009,7 @@ static int sessionTableInfo( rc = sqlite3_reset(pStmt); if( rc==SQLITE_OK ){ - nByte += nDbCol * (sizeof(const char *)*2 + sizeof(u8) + 1 + 1); + nByte += nDbCol * (sizeof(const char *)*2 +sizeof(int)+sizeof(u8) + 1 + 1); pAlloc = sessionMalloc64(pSession, nByte); if( pAlloc==0 ){ rc = SQLITE_NOMEM; @@ -224229,8 +232020,8 @@ static int sessionTableInfo( if( rc==SQLITE_OK ){ azCol = (char **)pAlloc; azDflt = (char**)&azCol[nDbCol]; - pAlloc = (u8 *)&azDflt[nDbCol]; - abPK = (u8 *)pAlloc; + aiIdx = (int*)&azDflt[nDbCol]; + abPK = (u8 *)&aiIdx[nDbCol]; pAlloc = &abPK[nDbCol]; if( pzTab ){ memcpy(pAlloc, zThis, nThis+1); @@ -224245,27 +232036,32 @@ static int sessionTableInfo( azCol[i] = (char*)pAlloc; pAlloc += nName+1; abPK[i] = 1; + aiIdx[i] = -1; i++; } while( SQLITE_ROW==sqlite3_step(pStmt) ){ - int nName = sqlite3_column_bytes(pStmt, 1); - int nDflt = sqlite3_column_bytes(pStmt, 4); - const unsigned char *zName = sqlite3_column_text(pStmt, 1); - const unsigned char *zDflt = sqlite3_column_text(pStmt, 4); - - if( zName==0 ) break; - memcpy(pAlloc, zName, nName+1); - azCol[i] = (char *)pAlloc; - pAlloc += nName+1; - if( zDflt ){ - memcpy(pAlloc, zDflt, nDflt+1); - azDflt[i] = (char *)pAlloc; - pAlloc += nDflt+1; - }else{ - azDflt[i] = 0; + if( sqlite3_column_int(pStmt, 6)==0 ){ /* !hidden */ + int nName = sqlite3_column_bytes(pStmt, 1); + int nDflt = sqlite3_column_bytes(pStmt, 4); + const unsigned char *zName = sqlite3_column_text(pStmt, 1); + const unsigned char *zDflt = sqlite3_column_text(pStmt, 4); + + if( zName==0 ) break; + memcpy(pAlloc, zName, nName+1); + azCol[i] = (char *)pAlloc; + pAlloc += nName+1; + if( zDflt ){ + memcpy(pAlloc, zDflt, nDflt+1); + azDflt[i] = (char *)pAlloc; + pAlloc += nDflt+1; + }else{ + azDflt[i] = 0; + } + abPK[i] = sqlite3_column_int(pStmt, 5); + aiIdx[i] = sqlite3_column_int(pStmt, 0); + i++; } - abPK[i] = sqlite3_column_int(pStmt, 5); - i++; + if( pnTotalCol ) (*pnTotalCol)++; } rc = sqlite3_reset(pStmt); } @@ -224278,6 +232074,7 @@ static int sessionTableInfo( if( pazDflt ) *pazDflt = (const char**)azDflt; *pabPK = abPK; *pnCol = nDbCol; + if( paiIdx ) *paiIdx = aiIdx; }else{ sessionFree(pSession, azCol); } @@ -224289,7 +232086,7 @@ static int sessionTableInfo( /* ** This function is called to initialize the SessionTable.nCol, azCol[] ** abPK[] and azDflt[] members of SessionTable object pTab. If these -** fields are already initilialized, this function is a no-op. +** fields are already initialized, this function is a no-op. ** ** If an error occurs, an error code is stored in sqlite3_session.rc and ** non-zero returned. Or, if no error occurs but the table has no primary @@ -224308,8 +232105,11 @@ static int sessionInitTable( if( pTab->nCol==0 ){ u8 *abPK; assert( pTab->azCol==0 || pTab->abPK==0 ); + sqlite3_free(pTab->azCol); + pTab->abPK = 0; rc = sessionTableInfo(pSession, db, zDb, - pTab->zName, &pTab->nCol, 0, &pTab->azCol, &pTab->azDflt, &abPK, + pTab->zName, &pTab->nCol, &pTab->nTotalCol, 0, &pTab->azCol, + &pTab->azDflt, &pTab->aiIdx, &abPK, ((pSession==0 || pSession->bImplicitPK) ? &pTab->bRowid : 0) ); if( rc==SQLITE_OK ){ @@ -224344,15 +232144,17 @@ static int sessionInitTable( */ static int sessionReinitTable(sqlite3_session *pSession, SessionTable *pTab){ int nCol = 0; + int nTotalCol = 0; const char **azCol = 0; const char **azDflt = 0; + int *aiIdx = 0; u8 *abPK = 0; int bRowid = 0; assert( pSession->rc==SQLITE_OK ); pSession->rc = sessionTableInfo(pSession, pSession->db, pSession->zDb, - pTab->zName, &nCol, 0, &azCol, &azDflt, &abPK, + pTab->zName, &nCol, &nTotalCol, 0, &azCol, &azDflt, &aiIdx, &abPK, (pSession->bImplicitPK ? &bRowid : 0) ); if( pSession->rc==SQLITE_OK ){ @@ -224375,8 +232177,10 @@ static int sessionReinitTable(sqlite3_session *pSession, SessionTable *pTab){ const char **a = pTab->azCol; pTab->azCol = azCol; pTab->nCol = nCol; + pTab->nTotalCol = nTotalCol; pTab->azDflt = azDflt; pTab->abPK = abPK; + pTab->aiIdx = aiIdx; azCol = a; } if( pSession->bEnableSize ){ @@ -224694,7 +232498,7 @@ static int sessionUpdateMaxSize( int ii; for(ii=0; iinCol; ii++){ sqlite3_value *p = 0; - pSession->hook.xNew(pSession->hook.pCtx, ii, &p); + pSession->hook.xNew(pSession->hook.pCtx, pTab->aiIdx[ii], &p); sessionSerializeValue(0, p, &nNew); } } @@ -224714,8 +232518,9 @@ static int sessionUpdateMaxSize( int bChanged = 1; int nOld = 0; int eType; + int iIdx = pTab->aiIdx[ii]; sqlite3_value *p = 0; - pSession->hook.xNew(pSession->hook.pCtx, ii-pTab->bRowid, &p); + pSession->hook.xNew(pSession->hook.pCtx, iIdx, &p); if( p==0 ){ return SQLITE_NOMEM; } @@ -224812,11 +232617,11 @@ static void sessionPreupdateOneChange( /* Check the number of columns in this xPreUpdate call matches the ** number of columns in the table. */ nExpect = pSession->hook.xCount(pSession->hook.pCtx); - if( (pTab->nCol-pTab->bRowid)nTotalColnCol-pTab->bRowid)!=nExpect ){ + if( pTab->nTotalCol!=nExpect ){ pSession->rc = SQLITE_SCHEMA; return; } @@ -224873,19 +232678,23 @@ static void sessionPreupdateOneChange( /* Figure out how large an allocation is required */ nByte = sizeof(SessionChange); - for(i=0; i<(pTab->nCol-pTab->bRowid); i++){ + for(i=pTab->bRowid; inCol; i++){ + int iIdx = pTab->aiIdx[i]; sqlite3_value *p = 0; if( op!=SQLITE_INSERT ){ - TESTONLY(int trc = ) pSession->hook.xOld(pSession->hook.pCtx, i, &p); - assert( trc==SQLITE_OK ); + /* This may fail if the column has a non-NULL default and was added + ** using ALTER TABLE ADD COLUMN after this record was created. */ + rc = pSession->hook.xOld(pSession->hook.pCtx, iIdx, &p); }else if( pTab->abPK[i] ){ - TESTONLY(int trc = ) pSession->hook.xNew(pSession->hook.pCtx, i, &p); + TESTONLY(int trc = ) pSession->hook.xNew(pSession->hook.pCtx,iIdx,&p); assert( trc==SQLITE_OK ); } - /* This may fail if SQLite value p contains a utf-16 string that must - ** be converted to utf-8 and an OOM error occurs while doing so. */ - rc = sessionSerializeValue(0, p, &nByte); + if( rc==SQLITE_OK ){ + /* This may fail if SQLite value p contains a utf-16 string that must + ** be converted to utf-8 and an OOM error occurs while doing so. */ + rc = sessionSerializeValue(0, p, &nByte); + } if( rc!=SQLITE_OK ) goto error_out; } if( pTab->bRowid ){ @@ -224912,12 +232721,13 @@ static void sessionPreupdateOneChange( sessionPutI64(&pC->aRecord[1], iRowid); nByte = 9; } - for(i=0; i<(pTab->nCol-pTab->bRowid); i++){ + for(i=pTab->bRowid; inCol; i++){ sqlite3_value *p = 0; + int iIdx = pTab->aiIdx[i]; if( op!=SQLITE_INSERT ){ - pSession->hook.xOld(pSession->hook.pCtx, i, &p); + pSession->hook.xOld(pSession->hook.pCtx, iIdx, &p); }else if( pTab->abPK[i] ){ - pSession->hook.xNew(pSession->hook.pCtx, i, &p); + pSession->hook.xNew(pSession->hook.pCtx, iIdx, &p); } sessionSerializeValue(&pC->aRecord[nByte], p, &nByte); } @@ -225304,7 +233114,9 @@ SQLITE_API int sqlite3session_diff( SessionTable *pTo; /* Table zTbl */ /* Locate and if necessary initialize the target table object */ + pSession->bAutoAttach++; rc = sessionFindTable(pSession, zTbl, &pTo); + pSession->bAutoAttach--; if( pTo==0 ) goto diff_out; if( sessionInitTable(pSession, pTo, pSession->db, pSession->zDb) ){ rc = pSession->rc; @@ -225315,16 +233127,43 @@ SQLITE_API int sqlite3session_diff( if( rc==SQLITE_OK ){ int bHasPk = 0; int bMismatch = 0; - int nCol; /* Columns in zFrom.zTbl */ + int nCol = 0; /* Columns in zFrom.zTbl */ int bRowid = 0; - u8 *abPK; + u8 *abPK = 0; const char **azCol = 0; - rc = sessionTableInfo(0, db, zFrom, zTbl, &nCol, 0, &azCol, 0, &abPK, - pSession->bImplicitPK ? &bRowid : 0 - ); + char *zDbExists = 0; + + /* Check that database zFrom is attached. */ + zDbExists = sqlite3_mprintf("SELECT * FROM %Q.sqlite_schema", zFrom); + if( zDbExists==0 ){ + rc = SQLITE_NOMEM; + }else{ + sqlite3_stmt *pDbExists = 0; + rc = sqlite3_prepare_v2(db, zDbExists, -1, &pDbExists, 0); + if( rc==SQLITE_ERROR ){ + rc = SQLITE_OK; + nCol = -1; + } + sqlite3_finalize(pDbExists); + sqlite3_free(zDbExists); + } + + if( rc==SQLITE_OK && nCol==0 ){ + rc = sessionTableInfo(0, db, zFrom, zTbl, + &nCol, 0, 0, &azCol, 0, 0, &abPK, + pSession->bImplicitPK ? &bRowid : 0 + ); + } if( rc==SQLITE_OK ){ if( pTo->nCol!=nCol ){ - bMismatch = 1; + if( nCol<=0 ){ + rc = SQLITE_SCHEMA; + if( pzErrMsg ){ + *pzErrMsg = sqlite3_mprintf("no such table: %s.%s", zFrom, zTbl); + } + }else{ + bMismatch = 1; + } }else{ int i; for(i=0; iaBuf[p->nBuf]; const char *zIn = zStr; *zOut++ = '"'; - while( *zIn ){ - if( *zIn=='"' ) *zOut++ = '"'; - *zOut++ = *(zIn++); + if( zIn!=0 ){ + while( *zIn ){ + if( *zIn=='"' ) *zOut++ = '"'; + *zOut++ = *(zIn++); + } } *zOut++ = '"'; p->nBuf = (int)((u8 *)zOut - p->aBuf); @@ -225870,6 +233711,19 @@ static int sessionAppendDelete( return rc; } +static int sessionPrepare( + sqlite3 *db, + sqlite3_stmt **pp, + char **pzErrmsg, + const char *zSql +){ + int rc = sqlite3_prepare_v2(db, zSql, -1, pp, 0); + if( pzErrmsg && rc!=SQLITE_OK ){ + *pzErrmsg = sqlite3_mprintf("%s", sqlite3_errmsg(db)); + } + return rc; +} + /* ** Formulate and prepare a SELECT statement to retrieve a row from table ** zTab in database zDb based on its primary key. i.e. @@ -225891,15 +233745,15 @@ static int sessionSelectStmt( int nCol, /* Number of columns in table */ const char **azCol, /* Names of table columns */ u8 *abPK, /* PRIMARY KEY array */ - sqlite3_stmt **ppStmt /* OUT: Prepared SELECT statement */ + sqlite3_stmt **ppStmt, /* OUT: Prepared SELECT statement */ + char **pzErrmsg /* OUT: Error message */ ){ int rc = SQLITE_OK; char *zSql = 0; const char *zSep = ""; - const char *zCols = bRowid ? SESSIONS_ROWID ", *" : "*"; - int nSql = -1; int i; + SessionBuffer cols = {0, 0, 0}; SessionBuffer nooptest = {0, 0, 0}; SessionBuffer pkfield = {0, 0, 0}; SessionBuffer pkvar = {0, 0, 0}; @@ -225912,9 +233766,16 @@ static int sessionSelectStmt( sessionAppendStr(&pkvar, "?1, (CASE WHEN ?2=X'' THEN NULL ELSE ?2 END)", &rc ); - zCols = "tbl, ?2, stat"; + sessionAppendStr(&cols, "tbl, ?2, stat", &rc); }else{ + #if 0 + if( bRowid ){ + sessionAppendStr(&cols, SESSIONS_ROWID, &rc); + } + #endif for(i=0; idb; /* Source database handle */ SessionTable *pTab; /* Used to iterate through attached tables */ - SessionBuffer buf = {0,0,0}; /* Buffer in which to accumlate changeset */ + SessionBuffer buf = {0,0,0}; /* Buffer in which to accumulate changeset */ int rc; /* Return code */ assert( xOutput==0 || (pnChangeset==0 && ppChangeset==0) ); @@ -226132,7 +233994,7 @@ static int sessionGenerateChangeset( /* Build and compile a statement to execute: */ if( rc==SQLITE_OK ){ rc = sessionSelectStmt(db, 0, pSession->zDb, - zName, pTab->bRowid, pTab->nCol, pTab->azCol, pTab->abPK, &pSel + zName, pTab->bRowid, pTab->nCol, pTab->azCol, pTab->abPK, &pSel, 0 ); } @@ -226443,14 +234305,15 @@ SQLITE_API int sqlite3changeset_start_v2_strm( ** object and the buffer is full, discard some data to free up space. */ static void sessionDiscardData(SessionInput *pIn){ - if( pIn->xInput && pIn->iNext>=sessions_strm_chunk_size ){ - int nMove = pIn->buf.nBuf - pIn->iNext; + if( pIn->xInput && pIn->iCurrent>=sessions_strm_chunk_size ){ + int nMove = pIn->buf.nBuf - pIn->iCurrent; assert( nMove>=0 ); if( nMove>0 ){ - memmove(pIn->buf.aBuf, &pIn->buf.aBuf[pIn->iNext], nMove); + memmove(pIn->buf.aBuf, &pIn->buf.aBuf[pIn->iCurrent], nMove); } - pIn->buf.nBuf -= pIn->iNext; - pIn->iNext = 0; + pIn->buf.nBuf -= pIn->iCurrent; + pIn->iNext -= pIn->iCurrent; + pIn->iCurrent = 0; pIn->nData = pIn->buf.nBuf; } } @@ -226804,14 +234667,14 @@ static int sessionChangesetNextOne( p->rc = sessionInputBuffer(&p->in, 2); if( p->rc!=SQLITE_OK ) return p->rc; + p->in.iCurrent = p->in.iNext; + sessionDiscardData(&p->in); + /* If the iterator is already at the end of the changeset, return DONE. */ if( p->in.iNext>=p->in.nData ){ return SQLITE_DONE; } - sessionDiscardData(&p->in); - p->in.iCurrent = p->in.iNext; - op = p->in.aData[p->in.iNext++]; while( op=='T' || op=='P' ){ if( pbNew ) *pbNew = 1; @@ -227340,6 +235203,7 @@ struct SessionApplyCtx { u8 bRebase; /* True to collect rebase information */ u8 bIgnoreNoop; /* True to ignore no-op conflicts */ int bRowid; + char *zErr; /* Error message, if any */ }; /* Number of prepared UPDATE statements to cache. */ @@ -227565,7 +235429,7 @@ static int sessionDeleteRow( } if( rc==SQLITE_OK ){ - rc = sqlite3_prepare_v2(db, (char *)buf.aBuf, buf.nBuf, &p->pDelete, 0); + rc = sessionPrepare(db, &p->pDelete, &p->zErr, (char*)buf.aBuf); } sqlite3_free(buf.aBuf); @@ -227592,7 +235456,7 @@ static int sessionSelectRow( ){ /* TODO */ return sessionSelectStmt(db, p->bIgnoreNoop, - "main", zTab, p->bRowid, p->nCol, p->azCol, p->abPK, &p->pSelect + "main", zTab, p->bRowid, p->nCol, p->azCol, p->abPK, &p->pSelect, &p->zErr ); } @@ -227629,16 +235493,12 @@ static int sessionInsertRow( sessionAppendStr(&buf, ")", &rc); if( rc==SQLITE_OK ){ - rc = sqlite3_prepare_v2(db, (char *)buf.aBuf, buf.nBuf, &p->pInsert, 0); + rc = sessionPrepare(db, &p->pInsert, &p->zErr, (char*)buf.aBuf); } sqlite3_free(buf.aBuf); return rc; } -static int sessionPrepare(sqlite3 *db, sqlite3_stmt **pp, const char *zSql){ - return sqlite3_prepare_v2(db, zSql, -1, pp, 0); -} - /* ** Prepare statements for applying changes to the sqlite_stat1 table. ** These are similar to those created by sessionSelectRow(), @@ -227648,14 +235508,14 @@ static int sessionPrepare(sqlite3 *db, sqlite3_stmt **pp, const char *zSql){ static int sessionStat1Sql(sqlite3 *db, SessionApplyCtx *p){ int rc = sessionSelectRow(db, "sqlite_stat1", p); if( rc==SQLITE_OK ){ - rc = sessionPrepare(db, &p->pInsert, + rc = sessionPrepare(db, &p->pInsert, 0, "INSERT INTO main.sqlite_stat1 VALUES(?1, " "CASE WHEN length(?2)=0 AND typeof(?2)='blob' THEN NULL ELSE ?2 END, " "?3)" ); } if( rc==SQLITE_OK ){ - rc = sessionPrepare(db, &p->pDelete, + rc = sessionPrepare(db, &p->pDelete, 0, "DELETE FROM main.sqlite_stat1 WHERE tbl=?1 AND idx IS " "CASE WHEN length(?2)=0 AND typeof(?2)='blob' THEN NULL ELSE ?2 END " "AND (?4 OR stat IS ?3)" @@ -227879,7 +235739,7 @@ static int sessionConflictHandler( void *pCtx, /* First argument for conflict handler */ int *pbReplace /* OUT: Set to true if PK row is found */ ){ - int res = 0; /* Value returned by conflict handler */ + int res = SQLITE_CHANGESET_OMIT;/* Value returned by conflict handler */ int rc; int nCol; int op; @@ -227900,11 +235760,9 @@ static int sessionConflictHandler( if( rc==SQLITE_ROW ){ /* There exists another row with the new.* primary key. */ - if( p->bIgnoreNoop - && sqlite3_column_int(p->pSelect, sqlite3_column_count(p->pSelect)-1) + if( 0==p->bIgnoreNoop + || 0==sqlite3_column_int(p->pSelect, sqlite3_column_count(p->pSelect)-1) ){ - res = SQLITE_CHANGESET_OMIT; - }else{ pIter->pConflict = p->pSelect; res = xConflict(pCtx, eType, pIter); pIter->pConflict = 0; @@ -227918,7 +235776,9 @@ static int sessionConflictHandler( int nBlob = pIter->in.iNext - pIter->in.iCurrent; sessionAppendBlob(&p->constraints, aBlob, nBlob, &rc); return SQLITE_OK; - }else{ + }else if( p->bIgnoreNoop==0 || op!=SQLITE_DELETE + || eType==SQLITE_CHANGESET_CONFLICT + ){ /* No other row with the new.* primary key. */ res = xConflict(pCtx, eType+1, pIter); if( res==SQLITE_CHANGESET_REPLACE ) rc = SQLITE_MISUSE; @@ -228016,7 +235876,7 @@ static int sessionApplyOneOp( sqlite3_step(p->pDelete); rc = sqlite3_reset(p->pDelete); - if( rc==SQLITE_OK && sqlite3_changes(p->db)==0 && p->bIgnoreNoop==0 ){ + if( rc==SQLITE_OK && sqlite3_changes(p->db)==0 ){ rc = sessionConflictHandler( SQLITE_CHANGESET_DATA, p, pIter, xConflict, pCtx, pbRetry ); @@ -228228,6 +236088,10 @@ static int sessionChangesetApply( void *pCtx, /* Copy of sixth arg to _apply() */ const char *zTab /* Table name */ ), + int(*xFilterIter)( + void *pCtx, /* Copy of sixth arg to _apply() */ + sqlite3_changeset_iter *p + ), int(*xConflict)( void *pCtx, /* Copy of fifth arg to _apply() */ int eConflict, /* DATA, MISSING, CONFLICT, CONSTRAINT */ @@ -228243,15 +236107,21 @@ static int sessionChangesetApply( int nTab = 0; /* Result of sqlite3Strlen30(zTab) */ SessionApplyCtx sApply; /* changeset_apply() context object */ int bPatchset; + u64 savedFlag = db->flags & SQLITE_FkNoAction; assert( xConflict!=0 ); + sqlite3_mutex_enter(sqlite3_db_mutex(db)); + if( flags & SQLITE_CHANGESETAPPLY_FKNOACTION ){ + db->flags |= ((u64)SQLITE_FkNoAction); + db->aDb[0].pSchema->schema_cookie -= 32; + } + pIter->in.bNoDiscard = 1; memset(&sApply, 0, sizeof(sApply)); sApply.bRebase = (ppRebase && pnRebase); sApply.bInvertConstraints = !!(flags & SQLITE_CHANGESETAPPLY_INVERT); sApply.bIgnoreNoop = !!(flags & SQLITE_CHANGESETAPPLY_IGNORENOOP); - sqlite3_mutex_enter(sqlite3_db_mutex(db)); if( (flags & SQLITE_CHANGESETAPPLY_NOSAVEPOINT)==0 ){ rc = sqlite3_exec(db, "SAVEPOINT changeset_apply", 0, 0, 0); } @@ -228309,7 +236179,8 @@ static int sessionChangesetApply( sqlite3changeset_pk(pIter, &abPK, 0); rc = sessionTableInfo(0, db, "main", zNew, - &sApply.nCol, &zTab, &sApply.azCol, 0, &sApply.abPK, &sApply.bRowid + &sApply.nCol, 0, &zTab, &sApply.azCol, 0, 0, + &sApply.abPK, &sApply.bRowid ); if( rc!=SQLITE_OK ) break; for(i=0; iflags & SQLITE_FkNoAction ); + db->flags &= ~((u64)SQLITE_FkNoAction); + db->aDb[0].pSchema->schema_cookie -= 32; + } + + assert( rc!=SQLITE_OK || sApply.zErr==0 ); + sqlite3_set_errmsg(db, rc, sApply.zErr); + sqlite3_free(sApply.zErr); + sqlite3_mutex_leave(sqlite3_db_mutex(db)); return rc; } +/* +** This function is called by all six sqlite3changeset_apply() variants: +** +** + sqlite3changeset_apply() +** + sqlite3changeset_apply_v2() +** + sqlite3changeset_apply_v3() +** + sqlite3changeset_apply_strm() +** + sqlite3changeset_apply_strm_v2() +** + sqlite3changeset_apply_strm_v3() +** +** Arguments passed to this function are as follows: +** +** db: +** Database handle to apply changeset to main database of. +** +** nChangeset/pChangeset: +** These are both passed zero for the streaming variants. For the normal +** apply() functions, these are passed the size of and the buffer containing +** the changeset, respectively. +** +** xInput/pIn: +** These are both passed zero for the normal variants. For the streaming +** apply() functions, these are passed the input callback and context +** pointer, respectively. +** +** xFilter: +** The filter function as passed to apply() or apply_v2() (to filter by +** table name), if any. This is always NULL for apply_v3() calls. +** +** xFilterIter: +** The filter function as passed to apply_v3(), if any. +** +** xConflict: +** The conflict handler callback (must not be NULL). +** +** pCtx: +** The context pointer passed to the xFilter and xConflict handler callbacks. +** +** ppRebase, pnRebase: +** Zero for apply(). The rebase changeset output pointers, if any, for +** apply_v2() and apply_v3(). +** +** flags: +** Zero for apply(). The flags parameter for apply_v2() and apply_v3(). +*/ +static int sessionChangesetApplyV23( + sqlite3 *db, /* Apply change to "main" db of this handle */ + int nChangeset, /* Size of changeset in bytes */ + void *pChangeset, /* Changeset blob */ + int (*xInput)(void *pIn, void *pData, int *pnData), /* Input function */ + void *pIn, /* First arg for xInput */ + int(*xFilter)( + void *pCtx, /* Copy of sixth arg to _apply() */ + const char *zTab /* Table name */ + ), + int(*xFilterIter)( + void *pCtx, /* Copy of sixth arg to _apply() */ + sqlite3_changeset_iter *p /* Handle describing current change */ + ), + int(*xConflict)( + void *pCtx, /* Copy of sixth arg to _apply() */ + int eConflict, /* DATA, MISSING, CONFLICT, CONSTRAINT */ + sqlite3_changeset_iter *p /* Handle describing change and conflict */ + ), + void *pCtx, /* First argument passed to xConflict */ + void **ppRebase, int *pnRebase, + int flags +){ + sqlite3_changeset_iter *pIter; /* Iterator to skip through changeset */ + int bInverse = !!(flags & SQLITE_CHANGESETAPPLY_INVERT); + int rc = sessionChangesetStart( + &pIter, xInput, pIn, nChangeset, pChangeset, bInverse, 1 + ); + if( rc==SQLITE_OK ){ + rc = sessionChangesetApply(db, pIter, + xFilter, xFilterIter, xConflict, pCtx, ppRebase, pnRebase, flags + ); + } + return rc; +} + /* ** Apply the changeset passed via pChangeset/nChangeset to the main ** database attached to handle "db". @@ -228438,28 +236410,39 @@ SQLITE_API int sqlite3changeset_apply_v2( void **ppRebase, int *pnRebase, int flags ){ - sqlite3_changeset_iter *pIter; /* Iterator to skip through changeset */ - int bInv = !!(flags & SQLITE_CHANGESETAPPLY_INVERT); - int rc = sessionChangesetStart(&pIter, 0, 0, nChangeset, pChangeset, bInv, 1); - u64 savedFlag = db->flags & SQLITE_FkNoAction; - - if( flags & SQLITE_CHANGESETAPPLY_FKNOACTION ){ - db->flags |= ((u64)SQLITE_FkNoAction); - db->aDb[0].pSchema->schema_cookie -= 32; - } - - if( rc==SQLITE_OK ){ - rc = sessionChangesetApply( - db, pIter, xFilter, xConflict, pCtx, ppRebase, pnRebase, flags - ); - } + return sessionChangesetApplyV23(db, + nChangeset, pChangeset, 0, 0, + xFilter, 0, xConflict, pCtx, + ppRebase, pnRebase, flags + ); +} - if( (flags & SQLITE_CHANGESETAPPLY_FKNOACTION) && savedFlag==0 ){ - assert( db->flags & SQLITE_FkNoAction ); - db->flags &= ~((u64)SQLITE_FkNoAction); - db->aDb[0].pSchema->schema_cookie -= 32; - } - return rc; +/* +** Apply the changeset passed via pChangeset/nChangeset to the main +** database attached to handle "db". +*/ +SQLITE_API int sqlite3changeset_apply_v3( + sqlite3 *db, /* Apply change to "main" db of this handle */ + int nChangeset, /* Size of changeset in bytes */ + void *pChangeset, /* Changeset blob */ + int(*xFilter)( + void *pCtx, /* Copy of sixth arg to _apply() */ + sqlite3_changeset_iter *p /* Handle describing current change */ + ), + int(*xConflict)( + void *pCtx, /* Copy of sixth arg to _apply() */ + int eConflict, /* DATA, MISSING, CONFLICT, CONSTRAINT */ + sqlite3_changeset_iter *p /* Handle describing change and conflict */ + ), + void *pCtx, /* First argument passed to xConflict */ + void **ppRebase, int *pnRebase, + int flags +){ + return sessionChangesetApplyV23(db, + nChangeset, pChangeset, 0, 0, + 0, xFilter, xConflict, pCtx, + ppRebase, pnRebase, flags + ); } /* @@ -228482,8 +236465,10 @@ SQLITE_API int sqlite3changeset_apply( ), void *pCtx /* First argument passed to xConflict */ ){ - return sqlite3changeset_apply_v2( - db, nChangeset, pChangeset, xFilter, xConflict, pCtx, 0, 0, 0 + return sessionChangesetApplyV23(db, + nChangeset, pChangeset, 0, 0, + xFilter, 0, xConflict, pCtx, + 0, 0, 0 ); } @@ -228492,6 +236477,29 @@ SQLITE_API int sqlite3changeset_apply( ** attached to handle "db". Invoke the supplied conflict handler callback ** to resolve any conflicts encountered while applying the change. */ +SQLITE_API int sqlite3changeset_apply_v3_strm( + sqlite3 *db, /* Apply change to "main" db of this handle */ + int (*xInput)(void *pIn, void *pData, int *pnData), /* Input function */ + void *pIn, /* First arg for xInput */ + int(*xFilter)( + void *pCtx, /* Copy of sixth arg to _apply() */ + sqlite3_changeset_iter *p + ), + int(*xConflict)( + void *pCtx, /* Copy of sixth arg to _apply() */ + int eConflict, /* DATA, MISSING, CONFLICT, CONSTRAINT */ + sqlite3_changeset_iter *p /* Handle describing change and conflict */ + ), + void *pCtx, /* First argument passed to xConflict */ + void **ppRebase, int *pnRebase, + int flags +){ + return sessionChangesetApplyV23(db, + 0, 0, xInput, pIn, + 0, xFilter, xConflict, pCtx, + ppRebase, pnRebase, flags + ); +} SQLITE_API int sqlite3changeset_apply_v2_strm( sqlite3 *db, /* Apply change to "main" db of this handle */ int (*xInput)(void *pIn, void *pData, int *pnData), /* Input function */ @@ -228509,15 +236517,11 @@ SQLITE_API int sqlite3changeset_apply_v2_strm( void **ppRebase, int *pnRebase, int flags ){ - sqlite3_changeset_iter *pIter; /* Iterator to skip through changeset */ - int bInverse = !!(flags & SQLITE_CHANGESETAPPLY_INVERT); - int rc = sessionChangesetStart(&pIter, xInput, pIn, 0, 0, bInverse, 1); - if( rc==SQLITE_OK ){ - rc = sessionChangesetApply( - db, pIter, xFilter, xConflict, pCtx, ppRebase, pnRebase, flags - ); - } - return rc; + return sessionChangesetApplyV23(db, + 0, 0, xInput, pIn, + xFilter, 0, xConflict, pCtx, + ppRebase, pnRebase, flags + ); } SQLITE_API int sqlite3changeset_apply_strm( sqlite3 *db, /* Apply change to "main" db of this handle */ @@ -228534,8 +236538,10 @@ SQLITE_API int sqlite3changeset_apply_strm( ), void *pCtx /* First argument passed to xConflict */ ){ - return sqlite3changeset_apply_v2_strm( - db, xInput, pIn, xFilter, xConflict, pCtx, 0, 0, 0 + return sessionChangesetApplyV23(db, + 0, 0, xInput, pIn, + xFilter, 0, xConflict, pCtx, + 0, 0, 0 ); } @@ -228546,6 +236552,7 @@ struct sqlite3_changegroup { int rc; /* Error code */ int bPatch; /* True to accumulate patchsets */ SessionTable *pList; /* List of tables in current patch */ + SessionBuffer rec; sqlite3 *db; /* Configured by changegroup_schema() */ char *zDb; /* Configured by changegroup_schema() */ @@ -228778,6 +236785,9 @@ static int sessionChangesetExtendRecord( sessionAppendBlob(pOut, aRec, nRec, &rc); if( rc==SQLITE_OK && pTab->pDfltStmt==0 ){ rc = sessionPrepareDfltStmt(pGrp->db, pTab, &pTab->pDfltStmt); + if( rc==SQLITE_OK && SQLITE_ROW!=sqlite3_step(pTab->pDfltStmt) ){ + rc = sqlite3_errcode(pGrp->db); + } } for(ii=nCol; rc==SQLITE_OK && iinCol; ii++){ int eType = sqlite3_column_type(pTab->pDfltStmt, ii); @@ -228794,6 +236804,7 @@ static int sessionChangesetExtendRecord( } if( SQLITE_OK==sessionBufferGrow(pOut, 8, &rc) ){ sessionPutI64(&pOut->aBuf[pOut->nBuf], iVal); + pOut->nBuf += 8; } break; } @@ -228844,108 +236855,130 @@ static int sessionChangesetExtendRecord( } /* -** Add all changes in the changeset traversed by the iterator passed as -** the first argument to the changegroup hash tables. +** Locate or create a SessionTable object that may be used to add the +** change currently pointed to by iterator pIter to changegroup pGrp. +** If successful, set output variable (*ppTab) to point to the table +** object and return SQLITE_OK. Otherwise, if some error occurs, return +** an SQLite error code and leave (*ppTab) set to NULL. */ -static int sessionChangesetToHash( - sqlite3_changeset_iter *pIter, /* Iterator to read from */ - sqlite3_changegroup *pGrp, /* Changegroup object to add changeset to */ - int bRebase /* True if hash table is for rebasing */ +static int sessionChangesetFindTable( + sqlite3_changegroup *pGrp, + const char *zTab, + sqlite3_changeset_iter *pIter, + SessionTable **ppTab ){ - u8 *aRec; - int nRec; int rc = SQLITE_OK; SessionTable *pTab = 0; - SessionBuffer rec = {0, 0, 0}; - - while( SQLITE_ROW==sessionChangesetNext(pIter, &aRec, &nRec, 0) ){ - const char *zNew; - int nCol; - int op; - int iHash; - int bIndirect; - SessionChange *pChange; - SessionChange *pExist = 0; - SessionChange **pp; - - /* Ensure that only changesets, or only patchsets, but not a mixture - ** of both, are being combined. It is an error to try to combine a - ** changeset and a patchset. */ - if( pGrp->pList==0 ){ - pGrp->bPatch = pIter->bPatchset; - }else if( pIter->bPatchset!=pGrp->bPatch ){ - rc = SQLITE_ERROR; - break; - } + int nTab = (int)strlen(zTab); + u8 *abPK = 0; + int nCol = 0; - sqlite3changeset_op(pIter, &zNew, &nCol, &op, &bIndirect); - if( !pTab || sqlite3_stricmp(zNew, pTab->zName) ){ - /* Search the list for a matching table */ - int nNew = (int)strlen(zNew); - u8 *abPK; + *ppTab = 0; + sqlite3changeset_pk(pIter, &abPK, &nCol); - sqlite3changeset_pk(pIter, &abPK, 0); - for(pTab = pGrp->pList; pTab; pTab=pTab->pNext){ - if( 0==sqlite3_strnicmp(pTab->zName, zNew, nNew+1) ) break; - } - if( !pTab ){ - SessionTable **ppTab; + /* Search the list for an existing table */ + for(pTab = pGrp->pList; pTab; pTab=pTab->pNext){ + if( 0==sqlite3_strnicmp(pTab->zName, zTab, nTab+1) ) break; + } - pTab = sqlite3_malloc64(sizeof(SessionTable) + nCol + nNew+1); - if( !pTab ){ - rc = SQLITE_NOMEM; - break; - } - memset(pTab, 0, sizeof(SessionTable)); - pTab->nCol = nCol; - pTab->abPK = (u8*)&pTab[1]; - memcpy(pTab->abPK, abPK, nCol); - pTab->zName = (char*)&pTab->abPK[nCol]; - memcpy(pTab->zName, zNew, nNew+1); - - if( pGrp->db ){ - pTab->nCol = 0; - rc = sessionInitTable(0, pTab, pGrp->db, pGrp->zDb); - if( rc ){ - assert( pTab->azCol==0 ); - sqlite3_free(pTab); - break; - } - } + /* If one was not found above, create a new table now */ + if( !pTab ){ + SessionTable **ppNew; - /* The new object must be linked on to the end of the list, not - ** simply added to the start of it. This is to ensure that the - ** tables within the output of sqlite3changegroup_output() are in - ** the right order. */ - for(ppTab=&pGrp->pList; *ppTab; ppTab=&(*ppTab)->pNext); - *ppTab = pTab; - } + pTab = sqlite3_malloc64(sizeof(SessionTable) + nCol + nTab+1); + if( !pTab ){ + return SQLITE_NOMEM; + } + memset(pTab, 0, sizeof(SessionTable)); + pTab->nCol = nCol; + pTab->abPK = (u8*)&pTab[1]; + memcpy(pTab->abPK, abPK, nCol); + pTab->zName = (char*)&pTab->abPK[nCol]; + memcpy(pTab->zName, zTab, nTab+1); - if( !sessionChangesetCheckCompat(pTab, nCol, abPK) ){ - rc = SQLITE_SCHEMA; - break; + if( pGrp->db ){ + pTab->nCol = 0; + rc = sessionInitTable(0, pTab, pGrp->db, pGrp->zDb); + if( rc ){ + assert( pTab->azCol==0 ); + sqlite3_free(pTab); + return rc; } } - if( nColnCol ){ - assert( pGrp->db ); - rc = sessionChangesetExtendRecord(pGrp, pTab, nCol, op, aRec, nRec, &rec); - if( rc ) break; - aRec = rec.aBuf; - nRec = rec.nBuf; - } + /* The new object must be linked on to the end of the list, not + ** simply added to the start of it. This is to ensure that the + ** tables within the output of sqlite3changegroup_output() are in + ** the right order. */ + for(ppNew=&pGrp->pList; *ppNew; ppNew=&(*ppNew)->pNext); + *ppNew = pTab; + } - if( sessionGrowHash(0, pIter->bPatchset, pTab) ){ - rc = SQLITE_NOMEM; - break; - } + /* Check that the table is compatible. */ + if( !sessionChangesetCheckCompat(pTab, nCol, abPK) ){ + rc = SQLITE_SCHEMA; + } + + *ppTab = pTab; + return rc; +} + +/* +** Add the change currently indicated by iterator pIter to the hash table +** belonging to changegroup pGrp. +*/ +static int sessionOneChangeToHash( + sqlite3_changegroup *pGrp, + sqlite3_changeset_iter *pIter, + int bRebase +){ + int rc = SQLITE_OK; + int nCol = 0; + int op = 0; + int iHash = 0; + int bIndirect = 0; + SessionChange *pChange = 0; + SessionChange *pExist = 0; + SessionChange **pp = 0; + SessionTable *pTab = 0; + u8 *aRec = &pIter->in.aData[pIter->in.iCurrent + 2]; + int nRec = (pIter->in.iNext - pIter->in.iCurrent) - 2; + + assert( nRec>0 ); + + /* Ensure that only changesets, or only patchsets, but not a mixture + ** of both, are being combined. It is an error to try to combine a + ** changeset and a patchset. */ + if( pGrp->pList==0 ){ + pGrp->bPatch = pIter->bPatchset; + }else if( pIter->bPatchset!=pGrp->bPatch ){ + rc = SQLITE_ERROR; + } + + if( rc==SQLITE_OK ){ + const char *zTab = 0; + sqlite3changeset_op(pIter, &zTab, &nCol, &op, &bIndirect); + rc = sessionChangesetFindTable(pGrp, zTab, pIter, &pTab); + } + + if( rc==SQLITE_OK && nColnCol ){ + SessionBuffer *pBuf = &pGrp->rec; + rc = sessionChangesetExtendRecord(pGrp, pTab, nCol, op, aRec, nRec, pBuf); + aRec = pBuf->aBuf; + nRec = pBuf->nBuf; + assert( pGrp->db ); + } + + if( rc==SQLITE_OK && sessionGrowHash(0, pIter->bPatchset, pTab) ){ + rc = SQLITE_NOMEM; + } + + if( rc==SQLITE_OK ){ + /* Search for existing entry. If found, remove it from the hash table. + ** Code below may link it back in. */ iHash = sessionChangeHash( pTab, (pIter->bPatchset && op==SQLITE_DELETE), aRec, pTab->nChange ); - - /* Search for existing entry. If found, remove it from the hash table. - ** Code below may link it back in. - */ for(pp=&pTab->apChange[iHash]; *pp; pp=&(*pp)->pNext){ int bPkOnly1 = 0; int bPkOnly2 = 0; @@ -228960,19 +236993,42 @@ static int sessionChangesetToHash( break; } } + } + if( rc==SQLITE_OK ){ rc = sessionChangeMerge(pTab, bRebase, pIter->bPatchset, pExist, op, bIndirect, aRec, nRec, &pChange ); - if( rc ) break; - if( pChange ){ - pChange->pNext = pTab->apChange[iHash]; - pTab->apChange[iHash] = pChange; - pTab->nEntry++; - } + } + if( rc==SQLITE_OK && pChange ){ + pChange->pNext = pTab->apChange[iHash]; + pTab->apChange[iHash] = pChange; + pTab->nEntry++; + } + + if( rc==SQLITE_OK ) rc = pIter->rc; + return rc; +} + +/* +** Add all changes in the changeset traversed by the iterator passed as +** the first argument to the changegroup hash tables. +*/ +static int sessionChangesetToHash( + sqlite3_changeset_iter *pIter, /* Iterator to read from */ + sqlite3_changegroup *pGrp, /* Changegroup object to add changeset to */ + int bRebase /* True if hash table is for rebasing */ +){ + u8 *aRec; + int nRec; + int rc = SQLITE_OK; + + pIter->in.bNoDiscard = 1; + while( SQLITE_ROW==(sessionChangesetNext(pIter, &aRec, &nRec, 0)) ){ + rc = sessionOneChangeToHash(pGrp, pIter, bRebase); + if( rc!=SQLITE_OK ) break; } - sqlite3_free(rec.aBuf); if( rc==SQLITE_OK ) rc = pIter->rc; return rc; } @@ -229100,6 +237156,28 @@ SQLITE_API int sqlite3changegroup_add(sqlite3_changegroup *pGrp, int nData, void return rc; } +/* +** Add a single change to a changeset-group. +*/ +SQLITE_API int sqlite3changegroup_add_change( + sqlite3_changegroup *pGrp, + sqlite3_changeset_iter *pIter +){ + int rc = SQLITE_OK; + + if( pIter->in.iCurrent==pIter->in.iNext + || pIter->rc!=SQLITE_OK + || pIter->bInvert + ){ + /* Iterator does not point to any valid entry or is an INVERT iterator. */ + rc = SQLITE_ERROR; + }else{ + pIter->in.bNoDiscard = 1; + rc = sessionOneChangeToHash(pGrp, pIter, 0); + } + return rc; +} + /* ** Obtain a buffer containing a changeset representing the concatenation ** of all changesets added to the group so far. @@ -229149,6 +237227,7 @@ SQLITE_API void sqlite3changegroup_delete(sqlite3_changegroup *pGrp){ if( pGrp ){ sqlite3_free(pGrp->zDb); sessionDeleteTable(0, pGrp->pList); + sqlite3_free(pGrp->rec.aBuf); sqlite3_free(pGrp); } } @@ -229550,6 +237629,7 @@ SQLITE_API int sqlite3rebaser_rebase_strm( SQLITE_API void sqlite3rebaser_delete(sqlite3_rebaser *p){ if( p ){ sessionDeleteTable(0, p->grp.pList); + sqlite3_free(p->grp.rec.aBuf); sqlite3_free(p); } } @@ -229580,7 +237660,27 @@ SQLITE_API int sqlite3session_config(int op, void *pArg){ /************** End of sqlite3session.c **************************************/ /************** Begin file fts5.c ********************************************/ - +/* +** This, the "fts5.c" source file, is a composite file that is itself +** assembled from the following files: +** +** fts5.h +** fts5Int.h +** fts5parse.h <--- Generated from fts5parse.y by Lemon +** fts5parse.c <--- Generated from fts5parse.y by Lemon +** fts5_aux.c +** fts5_buffer.c +** fts5_config.c +** fts5_expr.c +** fts5_hash.c +** fts5_index.c +** fts5_main.c +** fts5_storage.c +** fts5_tokenize.c +** fts5_unicode2.c +** fts5_varint.c +** fts5_vocab.c +*/ #if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS5) #if !defined(NDEBUG) && !defined(SQLITE_DEBUG) @@ -229590,6 +237690,12 @@ SQLITE_API int sqlite3session_config(int op, void *pArg){ # undef NDEBUG #endif +#ifdef HAVE_STDINT_H +/* #include */ +#endif +#ifdef HAVE_INTTYPES_H +/* #include */ +#endif /* ** 2014 May 31 ** @@ -229647,8 +237753,8 @@ struct Fts5PhraseIter { ** EXTENSION API FUNCTIONS ** ** xUserData(pFts): -** Return a copy of the context pointer the extension function was -** registered with. +** Return a copy of the pUserData pointer passed to the xCreateFunction() +** API when the extension function was registered. ** ** xColumnTotalSize(pFts, iCol, pnToken): ** If parameter iCol is less than zero, set output variable *pnToken @@ -229830,6 +237936,10 @@ struct Fts5PhraseIter { ** (i.e. if it is a contentless table), then this API always iterates ** through an empty set (all calls to xPhraseFirst() set iCol to -1). ** +** In all cases, matches are visited in (column ASC, offset ASC) order. +** i.e. all those in column 0, sorted by offset, followed by those in +** column 1, etc. +** ** xPhraseNext() ** See xPhraseFirst above. ** @@ -229886,19 +237996,57 @@ struct Fts5PhraseIter { ** value returned by xInstCount(), SQLITE_RANGE is returned. Otherwise, ** output variable (*ppToken) is set to point to a buffer containing the ** matching document token, and (*pnToken) to the size of that buffer in -** bytes. This API is not available if the specified token matches a -** prefix query term. In that case both output variables are always set -** to 0. +** bytes. ** ** The output text is not a copy of the document text that was tokenized. ** It is the output of the tokenizer module. For tokendata=1 tables, this ** includes any embedded 0x00 and trailing data. ** +** This API may be slow in some cases if the token identified by parameters +** iIdx and iToken matched a prefix token in the query. In most cases, the +** first call to this API for each prefix token in the query is forced +** to scan the portion of the full-text index that matches the prefix +** token to collect the extra data required by this API. If the prefix +** token matches a large number of token instances in the document set, +** this may be a performance problem. +** +** If the user knows in advance that a query may use this API for a +** prefix token, FTS5 may be configured to collect all required data as part +** of the initial querying of the full-text index, avoiding the second scan +** entirely. This also causes prefix queries that do not use this API to +** run more slowly and use more memory. FTS5 may be configured in this way +** either on a per-table basis using the [FTS5 insttoken | 'insttoken'] +** option, or on a per-query basis using the +** [fts5_insttoken | fts5_insttoken()] user function. +** ** This API can be quite slow if used with an FTS5 table created with the ** "detail=none" or "detail=column" option. +** +** xColumnLocale(pFts5, iIdx, pzLocale, pnLocale) +** If parameter iCol is less than zero, or greater than or equal to the +** number of columns in the table, SQLITE_RANGE is returned. +** +** Otherwise, this function attempts to retrieve the locale associated +** with column iCol of the current row. Usually, there is no associated +** locale, and output parameters (*pzLocale) and (*pnLocale) are set +** to NULL and 0, respectively. However, if the fts5_locale() function +** was used to associate a locale with the value when it was inserted +** into the fts5 table, then (*pzLocale) is set to point to a nul-terminated +** buffer containing the name of the locale in utf-8 encoding. (*pnLocale) +** is set to the size in bytes of the buffer, not including the +** nul-terminator. +** +** If successful, SQLITE_OK is returned. Or, if an error occurs, an +** SQLite error code is returned. The final value of the output parameters +** is undefined in this case. +** +** xTokenize_v2: +** Tokenize text using the tokenizer belonging to the FTS5 table. This +** API is the same as the xTokenize() API, except that it allows a tokenizer +** locale to be specified. */ struct Fts5ExtensionApi { - int iVersion; /* Currently always set to 3 */ + int iVersion; /* Currently always set to 4 */ void *(*xUserData)(Fts5Context*); @@ -229940,6 +238088,15 @@ struct Fts5ExtensionApi { const char **ppToken, int *pnToken ); int (*xInstToken)(Fts5Context*, int iIdx, int iToken, const char**, int*); + + /* Below this point are iVersion>=4 only */ + int (*xColumnLocale)(Fts5Context*, int iCol, const char **pz, int *pn); + int (*xTokenize_v2)(Fts5Context*, + const char *pText, int nText, /* Text to tokenize */ + const char *pLocale, int nLocale, /* Locale to pass to tokenizer */ + void *pCtx, /* Context passed to xToken() */ + int (*xToken)(void*, int, const char*, int, int, int) /* Callback */ + ); }; /* @@ -229960,7 +238117,7 @@ struct Fts5ExtensionApi { ** A tokenizer instance is required to actually tokenize text. ** ** The first argument passed to this function is a copy of the (void*) -** pointer provided by the application when the fts5_tokenizer object +** pointer provided by the application when the fts5_tokenizer_v2 object ** was registered with FTS5 (the third argument to xCreateTokenizer()). ** The second and third arguments are an array of nul-terminated strings ** containing the tokenizer arguments, if any, specified following the @@ -229984,7 +238141,7 @@ struct Fts5ExtensionApi { ** argument passed to this function is a pointer to an Fts5Tokenizer object ** returned by an earlier call to xCreate(). ** -** The second argument indicates the reason that FTS5 is requesting +** The third argument indicates the reason that FTS5 is requesting ** tokenization of the supplied text. This is always one of the following ** four values: ** @@ -230008,6 +238165,13 @@ struct Fts5ExtensionApi { ** on a columnsize=0 database. ** ** +** The sixth and seventh arguments passed to xTokenize() - pLocale and +** nLocale - are a pointer to a buffer containing the locale to use for +** tokenization (e.g. "en_US") and its size in bytes, respectively. The +** pLocale buffer is not nul-terminated. pLocale may be passed NULL (in +** which case nLocale is always 0) to indicate that the tokenizer should +** use its default locale. +** ** For each token in the input string, the supplied callback xToken() must ** be invoked. The first argument to it should be a copy of the pointer ** passed as the second argument to xTokenize(). The third and fourth @@ -230031,6 +238195,30 @@ struct Fts5ExtensionApi { ** may abandon the tokenization and return any error code other than ** SQLITE_OK or SQLITE_DONE. ** +** If the tokenizer is registered using an fts5_tokenizer_v2 object, +** then the xTokenize() method has two additional arguments - pLocale +** and nLocale. These specify the locale that the tokenizer should use +** for the current request. If pLocale and nLocale are both 0, then the +** tokenizer should use its default locale. Otherwise, pLocale points to +** an nLocale byte buffer containing the name of the locale to use as utf-8 +** text. pLocale is not nul-terminated. +** +** FTS5_TOKENIZER +** +** There is also an fts5_tokenizer object. This is an older, deprecated, +** version of fts5_tokenizer_v2. It is similar except that: +** +**
            +**
          • There is no "iVersion" field, and +**
          • The xTokenize() method does not take a locale argument. +**
          +** +** Legacy fts5_tokenizer tokenizers must be registered using the +** legacy xCreateTokenizer() function, instead of xCreateTokenizer_v2(). +** +** Tokenizer implementations registered using either API may be retrieved +** using both xFindTokenizer() and xFindTokenizer_v2(). +** ** SYNONYM SUPPORT ** ** Custom tokenizers may also support synonyms. Consider a case in which a @@ -230139,6 +238327,33 @@ struct Fts5ExtensionApi { ** inefficient. */ typedef struct Fts5Tokenizer Fts5Tokenizer; +typedef struct fts5_tokenizer_v2 fts5_tokenizer_v2; +struct fts5_tokenizer_v2 { + int iVersion; /* Currently always 2 */ + + int (*xCreate)(void*, const char **azArg, int nArg, Fts5Tokenizer **ppOut); + void (*xDelete)(Fts5Tokenizer*); + int (*xTokenize)(Fts5Tokenizer*, + void *pCtx, + int flags, /* Mask of FTS5_TOKENIZE_* flags */ + const char *pText, int nText, + const char *pLocale, int nLocale, + int (*xToken)( + void *pCtx, /* Copy of 2nd argument to xTokenize() */ + int tflags, /* Mask of FTS5_TOKEN_* flags */ + const char *pToken, /* Pointer to buffer containing token */ + int nToken, /* Size of token in bytes */ + int iStart, /* Byte offset of token within input text */ + int iEnd /* Byte offset of end of token within input text */ + ) + ); +}; + +/* +** New code should use the fts5_tokenizer_v2 type to define tokenizer +** implementations. The following type is included for legacy applications +** that still use it. +*/ typedef struct fts5_tokenizer fts5_tokenizer; struct fts5_tokenizer { int (*xCreate)(void*, const char **azArg, int nArg, Fts5Tokenizer **ppOut); @@ -230158,6 +238373,7 @@ struct fts5_tokenizer { ); }; + /* Flags that may be passed as the third argument to xTokenize() */ #define FTS5_TOKENIZE_QUERY 0x0001 #define FTS5_TOKENIZE_PREFIX 0x0002 @@ -230177,7 +238393,7 @@ struct fts5_tokenizer { */ typedef struct fts5_api fts5_api; struct fts5_api { - int iVersion; /* Currently always set to 2 */ + int iVersion; /* Currently always set to 3 */ /* Create a new tokenizer */ int (*xCreateTokenizer)( @@ -230204,6 +238420,25 @@ struct fts5_api { fts5_extension_function xFunction, void (*xDestroy)(void*) ); + + /* APIs below this point are only available if iVersion>=3 */ + + /* Create a new tokenizer */ + int (*xCreateTokenizer_v2)( + fts5_api *pApi, + const char *zName, + void *pUserData, + fts5_tokenizer_v2 *pTokenizer, + void (*xDestroy)(void*) + ); + + /* Find an existing tokenizer */ + int (*xFindTokenizer_v2)( + fts5_api *pApi, + const char *zName, + void **ppUserData, + fts5_tokenizer_v2 **ppTokenizer + ); }; /* @@ -230238,6 +238473,7 @@ SQLITE_EXTENSION_INIT1 /* #include */ /* #include */ +/* #include */ #ifndef SQLITE_AMALGAMATION @@ -230277,6 +238513,27 @@ typedef sqlite3_uint64 u64; # define LARGEST_INT64 (0xffffffff|(((i64)0x7fffffff)<<32)) # define SMALLEST_INT64 (((i64)-1) - LARGEST_INT64) +/* +** This macro is used in a single assert() within fts5 to check that an +** allocation is aligned to an 8-byte boundary. But it is a complicated +** macro to get right for multiple platforms without generating warnings. +** So instead of reproducing the entire definition from sqliteInt.h, we +** just do without this assert() for the rare non-amalgamation builds. +*/ +#define EIGHT_BYTE_ALIGNMENT(x) 1 + +/* +** Macros needed to provide flexible arrays in a portable way +*/ +#ifndef offsetof +# define offsetof(ST,M) ((size_t)((char*)&((ST*)0)->M - (char*)0)) +#endif +#if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) +# define FLEXARRAY +#else +# define FLEXARRAY 1 +#endif + #endif /* Truncate very long tokens to this many bytes. Hard limit is @@ -230349,10 +238606,11 @@ typedef struct Fts5Colset Fts5Colset; */ struct Fts5Colset { int nCol; - int aiCol[1]; + int aiCol[FLEXARRAY]; }; - +/* Size (int bytes) of a complete Fts5Colset object with N columns. */ +#define SZ_FTS5COLSET(N) (sizeof(i64)*((N+2)/2)) /************************************************************************** ** Interface to code in fts5_config.c. fts5_config.c contains contains code @@ -230360,6 +238618,18 @@ struct Fts5Colset { */ typedef struct Fts5Config Fts5Config; +typedef struct Fts5TokenizerConfig Fts5TokenizerConfig; + +struct Fts5TokenizerConfig { + Fts5Tokenizer *pTok; + fts5_tokenizer_v2 *pApi2; + fts5_tokenizer *pApi1; + const char **azArg; + int nArg; + int ePattern; /* FTS_PATTERN_XXX constant */ + const char *pLocale; /* Current locale to use */ + int nLocale; /* Size of pLocale in bytes */ +}; /* ** An instance of the following structure encodes all information that can @@ -230399,9 +238669,12 @@ typedef struct Fts5Config Fts5Config; ** ** INSERT INTO tbl(tbl, rank) VALUES('prefix-index', $bPrefixIndex); ** +** bLocale: +** Set to true if locale=1 was specified when the table was created. */ struct Fts5Config { sqlite3 *db; /* Database handle */ + Fts5Global *pGlobal; /* Global fts5 object for handle db */ char *zDb; /* Database holding FTS index (e.g. "main") */ char *zName; /* Name of FTS index */ int nCol; /* Number of columns */ @@ -230411,16 +238684,17 @@ struct Fts5Config { int *aPrefix; /* Sizes in bytes of nPrefix prefix indexes */ int eContent; /* An FTS5_CONTENT value */ int bContentlessDelete; /* "contentless_delete=" option (dflt==0) */ + int bContentlessUnindexed; /* "contentless_unindexed=" option (dflt=0) */ char *zContent; /* content table */ char *zContentRowid; /* "content_rowid=" option value */ int bColumnsize; /* "columnsize=" option value (dflt==1) */ int bTokendata; /* "tokendata=" option value (dflt==0) */ + int bLocale; /* "locale=" option value (dflt==0) */ int eDetail; /* FTS5_DETAIL_XXX value */ char *zContentExprlist; - Fts5Tokenizer *pTok; - fts5_tokenizer *pTokApi; + Fts5TokenizerConfig t; int bLock; /* True when table is preparing statement */ - int ePattern; /* FTS_PATTERN_XXX constant */ + /* Values loaded from the %_config table */ int iVersion; /* fts5 file format 'version' */ @@ -230433,7 +238707,8 @@ struct Fts5Config { char *zRank; /* Name of rank function */ char *zRankArgs; /* Arguments to rank function */ int bSecureDelete; /* 'secure-delete' */ - int nDeleteMerge; /* 'deletemerge' */ + int nDeleteMerge; /* 'deletemerge' */ + int bPrefixInsttoken; /* 'prefix-insttoken' */ /* If non-NULL, points to sqlite3_vtab.base.zErrmsg. Often NULL. */ char **pzErrmsg; @@ -230449,9 +238724,10 @@ struct Fts5Config { #define FTS5_CURRENT_VERSION 4 #define FTS5_CURRENT_VERSION_SECUREDELETE 5 -#define FTS5_CONTENT_NORMAL 0 -#define FTS5_CONTENT_NONE 1 -#define FTS5_CONTENT_EXTERNAL 2 +#define FTS5_CONTENT_NORMAL 0 +#define FTS5_CONTENT_NONE 1 +#define FTS5_CONTENT_EXTERNAL 2 +#define FTS5_CONTENT_UNINDEXED 3 #define FTS5_DETAIL_FULL 0 #define FTS5_DETAIL_NONE 1 @@ -230486,6 +238762,8 @@ static int sqlite3Fts5ConfigSetValue(Fts5Config*, const char*, sqlite3_value*, i static int sqlite3Fts5ConfigParseRank(const char*, char**, char**); +static void sqlite3Fts5ConfigErrmsg(Fts5Config *pConfig, const char *zFmt, ...); + /* ** End of interface to code in fts5_config.c. **************************************************************************/ @@ -230530,7 +238808,7 @@ static char *sqlite3Fts5Mprintf(int *pRc, const char *zFmt, ...); static void sqlite3Fts5Put32(u8*, int); static int sqlite3Fts5Get32(const u8*); -#define FTS5_POS2COLUMN(iPos) (int)(iPos >> 32) +#define FTS5_POS2COLUMN(iPos) (int)((iPos >> 32) & 0x7FFFFFFF) #define FTS5_POS2OFFSET(iPos) (int)(iPos & 0x7FFFFFFF) typedef struct Fts5PoslistReader Fts5PoslistReader; @@ -230687,7 +238965,14 @@ static int sqlite3Fts5StructureTest(Fts5Index*, void*); /* ** Used by xInstToken(): */ -static int sqlite3Fts5IterToken(Fts5IndexIter*, i64, int, int, const char**, int*); +static int sqlite3Fts5IterToken( + Fts5IndexIter *pIndexIter, + const char *pToken, int nToken, + i64 iRowid, + int iCol, + int iOff, + const char **ppOut, int *pnOut +); /* ** Insert or remove data to or from the index. Each time a document is @@ -230815,18 +239100,20 @@ struct Fts5Table { Fts5Index *pIndex; /* Full-text index */ }; -static int sqlite3Fts5GetTokenizer( - Fts5Global*, - const char **azArg, - int nArg, - Fts5Config*, - char **pzErr -); +static int sqlite3Fts5LoadTokenizer(Fts5Config *pConfig); static Fts5Table *sqlite3Fts5TableFromCsrid(Fts5Global*, i64); static int sqlite3Fts5FlushToDisk(Fts5Table*); +static void sqlite3Fts5ClearLocale(Fts5Config *pConfig); +static void sqlite3Fts5SetLocale(Fts5Config *pConfig, const char *pLoc, int nLoc); + +static int sqlite3Fts5IsLocaleValue(Fts5Config *pConfig, sqlite3_value *pVal); +static int sqlite3Fts5DecodeLocaleValue(sqlite3_value *pVal, + const char **ppText, int *pnText, const char **ppLoc, int *pnLoc +); + /* ** End of interface to code in fts5.c. **************************************************************************/ @@ -230906,8 +239193,8 @@ static int sqlite3Fts5StorageRename(Fts5Storage*, const char *zName); static int sqlite3Fts5DropAll(Fts5Config*); static int sqlite3Fts5CreateTable(Fts5Config*, const char*, const char*, int, char **); -static int sqlite3Fts5StorageDelete(Fts5Storage *p, i64, sqlite3_value**); -static int sqlite3Fts5StorageContentInsert(Fts5Storage *p, sqlite3_value**, i64*); +static int sqlite3Fts5StorageDelete(Fts5Storage *p, i64, sqlite3_value**, int); +static int sqlite3Fts5StorageContentInsert(Fts5Storage *p, int, sqlite3_value**, i64*); static int sqlite3Fts5StorageIndexInsert(Fts5Storage *p, sqlite3_value**, i64); static int sqlite3Fts5StorageIntegrity(Fts5Storage *p, int iArg); @@ -230932,6 +239219,9 @@ static int sqlite3Fts5StorageOptimize(Fts5Storage *p); static int sqlite3Fts5StorageMerge(Fts5Storage *p, int nMerge); static int sqlite3Fts5StorageReset(Fts5Storage *p); +static void sqlite3Fts5StorageReleaseDeleteRow(Fts5Storage*); +static int sqlite3Fts5StorageFindDeleteRow(Fts5Storage *p, i64 iDel); + /* ** End of interface to code in fts5_storage.c. **************************************************************************/ @@ -230978,7 +239268,7 @@ static int sqlite3Fts5ExprPattern( ** i64 iRowid = sqlite3Fts5ExprRowid(pExpr); ** } */ -static int sqlite3Fts5ExprFirst(Fts5Expr*, Fts5Index *pIdx, i64 iMin, int bDesc); +static int sqlite3Fts5ExprFirst(Fts5Expr*, Fts5Index *pIdx, i64 iMin, i64, int bDesc); static int sqlite3Fts5ExprNext(Fts5Expr*, i64 iMax); static int sqlite3Fts5ExprEof(Fts5Expr*); static i64 sqlite3Fts5ExprRowid(Fts5Expr*); @@ -231084,6 +239374,7 @@ static int sqlite3Fts5TokenizerPattern( int (*xCreate)(void*, const char**, int, Fts5Tokenizer**), Fts5Tokenizer *pTok ); +static int sqlite3Fts5TokenizerPreload(Fts5TokenizerConfig*); /* ** End of interface to code in fts5_tokenizer.c. **************************************************************************/ @@ -231148,7 +239439,7 @@ static void sqlite3Fts5UnicodeAscii(u8*, u8*); ** ** The "lemon" program processes an LALR(1) input grammar file, then uses ** this template to construct a parser. The "lemon" program inserts text -** at each "%%" line. Also, any "P-a-r-s-e" identifer prefix (without the +** at each "%%" line. Also, any "P-a-r-s-e" identifier prefix (without the ** interstitial "-" characters) contained in this template is changed into ** the value of the %name directive from the grammar. Otherwise, the content ** of this template is copied straight through into the generate parser @@ -231244,6 +239535,9 @@ static void sqlite3Fts5UnicodeAscii(u8*, u8*); ** sqlite3Fts5ParserARG_STORE Code to store %extra_argument into fts5yypParser ** sqlite3Fts5ParserARG_FETCH Code to extract %extra_argument from fts5yypParser ** sqlite3Fts5ParserCTX_* As sqlite3Fts5ParserARG_ except for %extra_context +** fts5YYREALLOC Name of the realloc() function to use +** fts5YYFREE Name of the free() function to use +** fts5YYDYNSTACK True if stack space should be extended on heap ** fts5YYERRORSYMBOL is the code number of the error symbol. If not ** defined, then do no error processing. ** fts5YYNSTATE the combined number of states. @@ -231257,6 +239551,8 @@ static void sqlite3Fts5UnicodeAscii(u8*, u8*); ** fts5YY_NO_ACTION The fts5yy_action[] code for no-op ** fts5YY_MIN_REDUCE Minimum value for reduce actions ** fts5YY_MAX_REDUCE Maximum value for reduce actions +** fts5YY_MIN_DSTRCTR Minimum symbol value that has a destructor +** fts5YY_MAX_DSTRCTR Maximum symbol value that has a destructor */ #ifndef INTERFACE # define INTERFACE 1 @@ -231283,6 +239579,9 @@ typedef union { #define sqlite3Fts5ParserARG_PARAM ,pParse #define sqlite3Fts5ParserARG_FETCH Fts5Parse *pParse=fts5yypParser->pParse; #define sqlite3Fts5ParserARG_STORE fts5yypParser->pParse=pParse; +#define fts5YYREALLOC realloc +#define fts5YYFREE free +#define fts5YYDYNSTACK 0 #define sqlite3Fts5ParserCTX_SDECL #define sqlite3Fts5ParserCTX_PDECL #define sqlite3Fts5ParserCTX_PARAM @@ -231300,6 +239599,8 @@ typedef union { #define fts5YY_NO_ACTION 82 #define fts5YY_MIN_REDUCE 83 #define fts5YY_MAX_REDUCE 110 +#define fts5YY_MIN_DSTRCTR 16 +#define fts5YY_MAX_DSTRCTR 24 /************* End control #defines *******************************************/ #define fts5YY_NLOOKAHEAD ((int)(sizeof(fts5yy_lookahead)/sizeof(fts5yy_lookahead[0]))) @@ -231315,6 +239616,22 @@ typedef union { # define fts5yytestcase(X) #endif +/* Macro to determine if stack space has the ability to grow using +** heap memory. +*/ +#if fts5YYSTACKDEPTH<=0 || fts5YYDYNSTACK +# define fts5YYGROWABLESTACK 1 +#else +# define fts5YYGROWABLESTACK 0 +#endif + +/* Guarantee a minimum number of initial stack slots. +*/ +#if fts5YYSTACKDEPTH<=0 +# undef fts5YYSTACKDEPTH +# define fts5YYSTACKDEPTH 2 /* Need a minimum stack size */ +#endif + /* Next are the tables used to determine what action to take based on the ** current state and lookahead token. These tables are used to implement @@ -231475,14 +239792,9 @@ struct fts5yyParser { #endif sqlite3Fts5ParserARG_SDECL /* A place to hold %extra_argument */ sqlite3Fts5ParserCTX_SDECL /* A place to hold %extra_context */ -#if fts5YYSTACKDEPTH<=0 - int fts5yystksz; /* Current side of the stack */ - fts5yyStackEntry *fts5yystack; /* The parser's stack */ - fts5yyStackEntry fts5yystk0; /* First stack entry */ -#else - fts5yyStackEntry fts5yystack[fts5YYSTACKDEPTH]; /* The parser's stack */ - fts5yyStackEntry *fts5yystackEnd; /* Last entry in the stack */ -#endif + fts5yyStackEntry *fts5yystackEnd; /* Last entry in the stack */ + fts5yyStackEntry *fts5yystack; /* The parser stack */ + fts5yyStackEntry fts5yystk0[fts5YYSTACKDEPTH]; /* Initial stack space */ }; typedef struct fts5yyParser fts5yyParser; @@ -231589,37 +239901,45 @@ static const char *const fts5yyRuleName[] = { #endif /* NDEBUG */ -#if fts5YYSTACKDEPTH<=0 +#if fts5YYGROWABLESTACK /* ** Try to increase the size of the parser stack. Return the number ** of errors. Return 0 on success. */ static int fts5yyGrowStack(fts5yyParser *p){ + int oldSize = 1 + (int)(p->fts5yystackEnd - p->fts5yystack); int newSize; int idx; fts5yyStackEntry *pNew; - newSize = p->fts5yystksz*2 + 100; - idx = p->fts5yytos ? (int)(p->fts5yytos - p->fts5yystack) : 0; - if( p->fts5yystack==&p->fts5yystk0 ){ - pNew = malloc(newSize*sizeof(pNew[0])); - if( pNew ) pNew[0] = p->fts5yystk0; + newSize = oldSize*2 + 100; + idx = (int)(p->fts5yytos - p->fts5yystack); + if( p->fts5yystack==p->fts5yystk0 ){ + pNew = fts5YYREALLOC(0, newSize*sizeof(pNew[0])); + if( pNew==0 ) return 1; + memcpy(pNew, p->fts5yystack, oldSize*sizeof(pNew[0])); }else{ - pNew = realloc(p->fts5yystack, newSize*sizeof(pNew[0])); + pNew = fts5YYREALLOC(p->fts5yystack, newSize*sizeof(pNew[0])); + if( pNew==0 ) return 1; } - if( pNew ){ - p->fts5yystack = pNew; - p->fts5yytos = &p->fts5yystack[idx]; + p->fts5yystack = pNew; + p->fts5yytos = &p->fts5yystack[idx]; #ifndef NDEBUG - if( fts5yyTraceFILE ){ - fprintf(fts5yyTraceFILE,"%sStack grows from %d to %d entries.\n", - fts5yyTracePrompt, p->fts5yystksz, newSize); - } -#endif - p->fts5yystksz = newSize; + if( fts5yyTraceFILE ){ + fprintf(fts5yyTraceFILE,"%sStack grows from %d to %d entries.\n", + fts5yyTracePrompt, oldSize, newSize); } - return pNew==0; +#endif + p->fts5yystackEnd = &p->fts5yystack[newSize-1]; + return 0; } +#endif /* fts5YYGROWABLESTACK */ + +#if !fts5YYGROWABLESTACK +/* For builds that do no have a growable stack, fts5yyGrowStack always +** returns an error. +*/ +# define fts5yyGrowStack(X) 1 #endif /* Datatype of the argument to the memory allocated passed as the @@ -231639,24 +239959,14 @@ static void sqlite3Fts5ParserInit(void *fts5yypRawParser sqlite3Fts5ParserCTX_PD #ifdef fts5YYTRACKMAXSTACKDEPTH fts5yypParser->fts5yyhwm = 0; #endif -#if fts5YYSTACKDEPTH<=0 - fts5yypParser->fts5yytos = NULL; - fts5yypParser->fts5yystack = NULL; - fts5yypParser->fts5yystksz = 0; - if( fts5yyGrowStack(fts5yypParser) ){ - fts5yypParser->fts5yystack = &fts5yypParser->fts5yystk0; - fts5yypParser->fts5yystksz = 1; - } -#endif + fts5yypParser->fts5yystack = fts5yypParser->fts5yystk0; + fts5yypParser->fts5yystackEnd = &fts5yypParser->fts5yystack[fts5YYSTACKDEPTH-1]; #ifndef fts5YYNOERRORRECOVERY fts5yypParser->fts5yyerrcnt = -1; #endif fts5yypParser->fts5yytos = fts5yypParser->fts5yystack; fts5yypParser->fts5yystack[0].stateno = 0; fts5yypParser->fts5yystack[0].major = 0; -#if fts5YYSTACKDEPTH>0 - fts5yypParser->fts5yystackEnd = &fts5yypParser->fts5yystack[fts5YYSTACKDEPTH-1]; -#endif } #ifndef sqlite3Fts5Parser_ENGINEALWAYSONSTACK @@ -231770,9 +240080,26 @@ static void fts5yy_pop_parser_stack(fts5yyParser *pParser){ */ static void sqlite3Fts5ParserFinalize(void *p){ fts5yyParser *pParser = (fts5yyParser*)p; - while( pParser->fts5yytos>pParser->fts5yystack ) fts5yy_pop_parser_stack(pParser); -#if fts5YYSTACKDEPTH<=0 - if( pParser->fts5yystack!=&pParser->fts5yystk0 ) free(pParser->fts5yystack); + + /* In-lined version of calling fts5yy_pop_parser_stack() for each + ** element left in the stack */ + fts5yyStackEntry *fts5yytos = pParser->fts5yytos; + while( fts5yytos>pParser->fts5yystack ){ +#ifndef NDEBUG + if( fts5yyTraceFILE ){ + fprintf(fts5yyTraceFILE,"%sPopping %s\n", + fts5yyTracePrompt, + fts5yyTokenName[fts5yytos->major]); + } +#endif + if( fts5yytos->major>=fts5YY_MIN_DSTRCTR ){ + fts5yy_destructor(pParser, fts5yytos->major, &fts5yytos->minor); + } + fts5yytos--; + } + +#if fts5YYGROWABLESTACK + if( pParser->fts5yystack!=pParser->fts5yystk0 ) fts5YYFREE(pParser->fts5yystack); #endif } @@ -231999,25 +240326,19 @@ static void fts5yy_shift( assert( fts5yypParser->fts5yyhwm == (int)(fts5yypParser->fts5yytos - fts5yypParser->fts5yystack) ); } #endif -#if fts5YYSTACKDEPTH>0 - if( fts5yypParser->fts5yytos>fts5yypParser->fts5yystackEnd ){ - fts5yypParser->fts5yytos--; - fts5yyStackOverflow(fts5yypParser); - return; - } -#else - if( fts5yypParser->fts5yytos>=&fts5yypParser->fts5yystack[fts5yypParser->fts5yystksz] ){ + fts5yytos = fts5yypParser->fts5yytos; + if( fts5yytos>fts5yypParser->fts5yystackEnd ){ if( fts5yyGrowStack(fts5yypParser) ){ fts5yypParser->fts5yytos--; fts5yyStackOverflow(fts5yypParser); return; } + fts5yytos = fts5yypParser->fts5yytos; + assert( fts5yytos <= fts5yypParser->fts5yystackEnd ); } -#endif if( fts5yyNewState > fts5YY_MAX_SHIFT ){ fts5yyNewState += fts5YY_MIN_REDUCE - fts5YY_MIN_SHIFTREDUCE; } - fts5yytos = fts5yypParser->fts5yytos; fts5yytos->stateno = fts5yyNewState; fts5yytos->major = fts5yyMajor; fts5yytos->minor.fts5yy0 = fts5yyMinor; @@ -232454,19 +240775,12 @@ static void sqlite3Fts5Parser( (int)(fts5yypParser->fts5yytos - fts5yypParser->fts5yystack)); } #endif -#if fts5YYSTACKDEPTH>0 if( fts5yypParser->fts5yytos>=fts5yypParser->fts5yystackEnd ){ - fts5yyStackOverflow(fts5yypParser); - break; - } -#else - if( fts5yypParser->fts5yytos>=&fts5yypParser->fts5yystack[fts5yypParser->fts5yystksz-1] ){ if( fts5yyGrowStack(fts5yypParser) ){ fts5yyStackOverflow(fts5yypParser); break; } } -#endif } fts5yyact = fts5yy_reduce(fts5yypParser,fts5yyruleno,fts5yymajor,fts5yyminor sqlite3Fts5ParserCTX_PARAM); }else if( fts5yyact <= fts5YY_MAX_SHIFTREDUCE ){ @@ -232838,6 +241152,7 @@ static int fts5HighlightCb( return rc; } + /* ** Implementation of highlight() function. */ @@ -232868,12 +241183,19 @@ static void fts5HighlightFunction( sqlite3_result_text(pCtx, "", -1, SQLITE_STATIC); rc = SQLITE_OK; }else if( ctx.zIn ){ + const char *pLoc = 0; /* Locale of column iCol */ + int nLoc = 0; /* Size of pLoc in bytes */ if( rc==SQLITE_OK ){ rc = fts5CInstIterInit(pApi, pFts, iCol, &ctx.iter); } if( rc==SQLITE_OK ){ - rc = pApi->xTokenize(pFts, ctx.zIn, ctx.nIn, (void*)&ctx,fts5HighlightCb); + rc = pApi->xColumnLocale(pFts, iCol, &pLoc, &nLoc); + } + if( rc==SQLITE_OK ){ + rc = pApi->xTokenize_v2( + pFts, ctx.zIn, ctx.nIn, pLoc, nLoc, (void*)&ctx, fts5HighlightCb + ); } if( ctx.bOpen ){ fts5HighlightAppend(&rc, &ctx, ctx.zClose, -1); @@ -233070,6 +241392,8 @@ static void fts5SnippetFunction( memset(&sFinder, 0, sizeof(Fts5SFinder)); for(i=0; ixColumnText(pFts, i, &sFinder.zDoc, &nDoc); if( rc!=SQLITE_OK ) break; - rc = pApi->xTokenize(pFts, - sFinder.zDoc, nDoc, (void*)&sFinder,fts5SentenceFinderCb + rc = pApi->xColumnLocale(pFts, i, &pLoc, &nLoc); + if( rc!=SQLITE_OK ) break; + rc = pApi->xTokenize_v2(pFts, + sFinder.zDoc, nDoc, pLoc, nLoc, (void*)&sFinder, fts5SentenceFinderCb ); if( rc!=SQLITE_OK ) break; rc = pApi->xColumnSize(pFts, i, &nDocsize); @@ -233136,6 +241462,9 @@ static void fts5SnippetFunction( rc = pApi->xColumnSize(pFts, iBestCol, &nColSize); } if( ctx.zIn ){ + const char *pLoc = 0; /* Locale of column iBestCol */ + int nLoc = 0; /* Bytes in pLoc */ + if( rc==SQLITE_OK ){ rc = fts5CInstIterInit(pApi, pFts, iBestCol, &ctx.iter); } @@ -233154,7 +241483,12 @@ static void fts5SnippetFunction( } if( rc==SQLITE_OK ){ - rc = pApi->xTokenize(pFts, ctx.zIn, ctx.nIn, (void*)&ctx,fts5HighlightCb); + rc = pApi->xColumnLocale(pFts, iBestCol, &pLoc, &nLoc); + } + if( rc==SQLITE_OK ){ + rc = pApi->xTokenize_v2( + pFts, ctx.zIn, ctx.nIn, pLoc, nLoc, (void*)&ctx,fts5HighlightCb + ); } if( ctx.bOpen ){ fts5HighlightAppend(&rc, &ctx, ctx.zClose, -1); @@ -233259,7 +241593,7 @@ static int fts5Bm25GetData( ** under consideration. ** ** The problem with this is that if (N < 2*nHit), the IDF is - ** negative. Which is undesirable. So the mimimum allowable IDF is + ** negative. Which is undesirable. So the minimum allowable IDF is ** (1e-6) - roughly the same as a term that appears in just over ** half of set of 5,000,000 documents. */ double idf = log( (nRow - nHit + 0.5) / (nHit + 0.5) ); @@ -233338,6 +241672,53 @@ static void fts5Bm25Function( } } +/* +** Implementation of fts5_get_locale() function. +*/ +static void fts5GetLocaleFunction( + const Fts5ExtensionApi *pApi, /* API offered by current FTS version */ + Fts5Context *pFts, /* First arg to pass to pApi functions */ + sqlite3_context *pCtx, /* Context for returning result/error */ + int nVal, /* Number of values in apVal[] array */ + sqlite3_value **apVal /* Array of trailing arguments */ +){ + int iCol = 0; + int eType = 0; + int rc = SQLITE_OK; + const char *zLocale = 0; + int nLocale = 0; + + /* xColumnLocale() must be available */ + assert( pApi->iVersion>=4 ); + + if( nVal!=1 ){ + const char *z = "wrong number of arguments to function fts5_get_locale()"; + sqlite3_result_error(pCtx, z, -1); + return; + } + + eType = sqlite3_value_numeric_type(apVal[0]); + if( eType!=SQLITE_INTEGER ){ + const char *z = "non-integer argument passed to function fts5_get_locale()"; + sqlite3_result_error(pCtx, z, -1); + return; + } + + iCol = sqlite3_value_int(apVal[0]); + if( iCol<0 || iCol>=pApi->xColumnCount(pFts) ){ + sqlite3_result_error_code(pCtx, SQLITE_RANGE); + return; + } + + rc = pApi->xColumnLocale(pFts, iCol, &zLocale, &nLocale); + if( rc!=SQLITE_OK ){ + sqlite3_result_error_code(pCtx, rc); + return; + } + + sqlite3_result_text(pCtx, zLocale, nLocale, SQLITE_TRANSIENT); +} + static int sqlite3Fts5AuxInit(fts5_api *pApi){ struct Builtin { const char *zFunc; /* Function name (nul-terminated) */ @@ -233345,9 +241726,10 @@ static int sqlite3Fts5AuxInit(fts5_api *pApi){ fts5_extension_function xFunc;/* Callback function */ void (*xDestroy)(void*); /* Destructor function */ } aBuiltin [] = { - { "snippet", 0, fts5SnippetFunction, 0 }, - { "highlight", 0, fts5HighlightFunction, 0 }, - { "bm25", 0, fts5Bm25Function, 0 }, + { "snippet", 0, fts5SnippetFunction, 0 }, + { "highlight", 0, fts5HighlightFunction, 0 }, + { "bm25", 0, fts5Bm25Function, 0 }, + { "fts5_get_locale", 0, fts5GetLocaleFunction, 0 }, }; int rc = SQLITE_OK; /* Return code */ int i; /* To iterate through builtin functions */ @@ -233674,7 +242056,7 @@ static char *sqlite3Fts5Strndup(int *pRc, const char *pIn, int nIn){ ** * The 52 upper and lower case ASCII characters, and ** * The 10 integer ASCII characters. ** * The underscore character "_" (0x5F). -** * The unicode "subsitute" character (0x1A). +** * The unicode "substitute" character (0x1A). */ static int sqlite3Fts5IsBareword(char t){ u8 aBareword[128] = { @@ -234012,7 +242394,6 @@ static int fts5ConfigSetEnum( ** eventually free any such error message using sqlite3_free(). */ static int fts5ConfigParseSpecial( - Fts5Global *pGlobal, Fts5Config *pConfig, /* Configuration object to update */ const char *zCmd, /* Special command to parse */ const char *zArg, /* Argument to parse */ @@ -234020,6 +242401,7 @@ static int fts5ConfigParseSpecial( ){ int rc = SQLITE_OK; int nCmd = (int)strlen(zCmd); + if( sqlite3_strnicmp("prefix", zCmd, nCmd)==0 ){ const int nByte = sizeof(int) * FTS5_MAX_PREFIX_INDEXES; const char *p; @@ -234076,12 +242458,11 @@ static int fts5ConfigParseSpecial( if( sqlite3_strnicmp("tokenize", zCmd, nCmd)==0 ){ const char *p = (const char*)zArg; sqlite3_int64 nArg = strlen(zArg) + 1; - char **azArg = sqlite3Fts5MallocZero(&rc, sizeof(char*) * nArg); - char *pDel = sqlite3Fts5MallocZero(&rc, nArg * 2); - char *pSpace = pDel; + char **azArg = sqlite3Fts5MallocZero(&rc, (sizeof(char*) + 2) * nArg); - if( azArg && pSpace ){ - if( pConfig->pTok ){ + if( azArg ){ + char *pSpace = (char*)&azArg[nArg]; + if( pConfig->t.azArg ){ *pzErr = sqlite3_mprintf("multiple tokenize=... directives"); rc = SQLITE_ERROR; }else{ @@ -234104,16 +242485,14 @@ static int fts5ConfigParseSpecial( *pzErr = sqlite3_mprintf("parse error in tokenize directive"); rc = SQLITE_ERROR; }else{ - rc = sqlite3Fts5GetTokenizer(pGlobal, - (const char**)azArg, (int)nArg, pConfig, - pzErr - ); + pConfig->t.azArg = (const char**)azArg; + pConfig->t.nArg = nArg; + azArg = 0; } } } - sqlite3_free(azArg); - sqlite3_free(pDel); + return rc; } @@ -234142,6 +242521,16 @@ static int fts5ConfigParseSpecial( return rc; } + if( sqlite3_strnicmp("contentless_unindexed", zCmd, nCmd)==0 ){ + if( (zArg[0]!='0' && zArg[0]!='1') || zArg[1]!='\0' ){ + *pzErr = sqlite3_mprintf("malformed contentless_delete=... directive"); + rc = SQLITE_ERROR; + }else{ + pConfig->bContentlessUnindexed = (zArg[0]=='1'); + } + return rc; + } + if( sqlite3_strnicmp("content_rowid", zCmd, nCmd)==0 ){ if( pConfig->zContentRowid ){ *pzErr = sqlite3_mprintf("multiple content_rowid=... directives"); @@ -234162,6 +242551,16 @@ static int fts5ConfigParseSpecial( return rc; } + if( sqlite3_strnicmp("locale", zCmd, nCmd)==0 ){ + if( (zArg[0]!='0' && zArg[0]!='1') || zArg[1]!='\0' ){ + *pzErr = sqlite3_mprintf("malformed locale=... directive"); + rc = SQLITE_ERROR; + }else{ + pConfig->bLocale = (zArg[0]=='1'); + } + return rc; + } + if( sqlite3_strnicmp("detail", zCmd, nCmd)==0 ){ const Fts5Enum aDetail[] = { { "none", FTS5_DETAIL_NONE }, @@ -234190,16 +242589,6 @@ static int fts5ConfigParseSpecial( return SQLITE_ERROR; } -/* -** Allocate an instance of the default tokenizer ("simple") at -** Fts5Config.pTokenizer. Return SQLITE_OK if successful, or an SQLite error -** code if an error occurs. -*/ -static int fts5ConfigDefaultTokenizer(Fts5Global *pGlobal, Fts5Config *pConfig){ - assert( pConfig->pTok==0 && pConfig->pTokApi==0 ); - return sqlite3Fts5GetTokenizer(pGlobal, 0, 0, pConfig, 0); -} - /* ** Gobble up the first bareword or quoted word from the input buffer zIn. ** Return a pointer to the character immediately following the last in @@ -234259,7 +242648,8 @@ static int fts5ConfigParseColumn( Fts5Config *p, char *zCol, char *zArg, - char **pzErr + char **pzErr, + int *pbUnindexed ){ int rc = SQLITE_OK; if( 0==sqlite3_stricmp(zCol, FTS5_RANK_NAME) @@ -234270,6 +242660,7 @@ static int fts5ConfigParseColumn( }else if( zArg ){ if( 0==sqlite3_stricmp(zArg, "unindexed") ){ p->abUnindexed[p->nCol] = 1; + *pbUnindexed = 1; }else{ *pzErr = sqlite3_mprintf("unrecognized column option: %s", zArg); rc = SQLITE_ERROR; @@ -234290,11 +242681,26 @@ static int fts5ConfigMakeExprlist(Fts5Config *p){ sqlite3Fts5BufferAppendPrintf(&rc, &buf, "T.%Q", p->zContentRowid); if( p->eContent!=FTS5_CONTENT_NONE ){ + assert( p->eContent==FTS5_CONTENT_EXTERNAL + || p->eContent==FTS5_CONTENT_NORMAL + || p->eContent==FTS5_CONTENT_UNINDEXED + ); for(i=0; inCol; i++){ if( p->eContent==FTS5_CONTENT_EXTERNAL ){ sqlite3Fts5BufferAppendPrintf(&rc, &buf, ", T.%Q", p->azCol[i]); - }else{ + }else if( p->eContent==FTS5_CONTENT_NORMAL || p->abUnindexed[i] ){ sqlite3Fts5BufferAppendPrintf(&rc, &buf, ", T.c%d", i); + }else{ + sqlite3Fts5BufferAppendPrintf(&rc, &buf, ", NULL"); + } + } + } + if( p->eContent==FTS5_CONTENT_NORMAL && p->bLocale ){ + for(i=0; inCol; i++){ + if( p->abUnindexed[i]==0 ){ + sqlite3Fts5BufferAppendPrintf(&rc, &buf, ", T.l%d", i); + }else{ + sqlite3Fts5BufferAppendPrintf(&rc, &buf, ", NULL"); } } } @@ -234328,10 +242734,12 @@ static int sqlite3Fts5ConfigParse( Fts5Config *pRet; /* New object to return */ int i; sqlite3_int64 nByte; + int bUnindexed = 0; /* True if there are one or more UNINDEXED */ *ppOut = pRet = (Fts5Config*)sqlite3_malloc(sizeof(Fts5Config)); if( pRet==0 ) return SQLITE_NOMEM; memset(pRet, 0, sizeof(Fts5Config)); + pRet->pGlobal = pGlobal; pRet->db = db; pRet->iCookie = -1; @@ -234380,13 +242788,13 @@ static int sqlite3Fts5ConfigParse( rc = SQLITE_ERROR; }else{ if( bOption ){ - rc = fts5ConfigParseSpecial(pGlobal, pRet, + rc = fts5ConfigParseSpecial(pRet, ALWAYS(zOne)?zOne:"", zTwo?zTwo:"", pzErr ); }else{ - rc = fts5ConfigParseColumn(pRet, zOne, zTwo, pzErr); + rc = fts5ConfigParseColumn(pRet, zOne, zTwo, pzErr, &bUnindexed); zOne = 0; } } @@ -234418,11 +242826,17 @@ static int sqlite3Fts5ConfigParse( rc = SQLITE_ERROR; } - /* If a tokenizer= option was successfully parsed, the tokenizer has - ** already been allocated. Otherwise, allocate an instance of the default - ** tokenizer (unicode61) now. */ - if( rc==SQLITE_OK && pRet->pTok==0 ){ - rc = fts5ConfigDefaultTokenizer(pGlobal, pRet); + /* We only allow contentless_unindexed=1 if the table is actually a + ** contentless one. + */ + if( rc==SQLITE_OK + && pRet->bContentlessUnindexed + && pRet->eContent!=FTS5_CONTENT_NONE + ){ + *pzErr = sqlite3_mprintf( + "contentless_unindexed=1 requires a contentless table" + ); + rc = SQLITE_ERROR; } /* If no zContent option was specified, fill in the default values. */ @@ -234433,6 +242847,9 @@ static int sqlite3Fts5ConfigParse( ); if( pRet->eContent==FTS5_CONTENT_NORMAL ){ zTail = "content"; + }else if( bUnindexed && pRet->bContentlessUnindexed ){ + pRet->eContent = FTS5_CONTENT_UNINDEXED; + zTail = "content"; }else if( pRet->bColumnsize ){ zTail = "docsize"; } @@ -234466,9 +242883,14 @@ static int sqlite3Fts5ConfigParse( static void sqlite3Fts5ConfigFree(Fts5Config *pConfig){ if( pConfig ){ int i; - if( pConfig->pTok ){ - pConfig->pTokApi->xDelete(pConfig->pTok); + if( pConfig->t.pTok ){ + if( pConfig->t.pApi1 ){ + pConfig->t.pApi1->xDelete(pConfig->t.pTok); + }else{ + pConfig->t.pApi2->xDelete(pConfig->t.pTok); + } } + sqlite3_free((char*)pConfig->t.azArg); sqlite3_free(pConfig->zDb); sqlite3_free(pConfig->zName); for(i=0; inCol; i++){ @@ -234543,10 +242965,24 @@ static int sqlite3Fts5Tokenize( void *pCtx, /* Context passed to xToken() */ int (*xToken)(void*, int, const char*, int, int, int) /* Callback */ ){ - if( pText==0 ) return SQLITE_OK; - return pConfig->pTokApi->xTokenize( - pConfig->pTok, pCtx, flags, pText, nText, xToken - ); + int rc = SQLITE_OK; + if( pText ){ + if( pConfig->t.pTok==0 ){ + rc = sqlite3Fts5LoadTokenizer(pConfig); + } + if( rc==SQLITE_OK ){ + if( pConfig->t.pApi1 ){ + rc = pConfig->t.pApi1->xTokenize( + pConfig->t.pTok, pCtx, flags, pText, nText, xToken + ); + }else{ + rc = pConfig->t.pApi2->xTokenize(pConfig->t.pTok, pCtx, flags, + pText, nText, pConfig->t.pLocale, pConfig->t.nLocale, xToken + ); + } + } + } + return rc; } /* @@ -234750,6 +243186,19 @@ static int sqlite3Fts5ConfigSetValue( }else{ pConfig->bSecureDelete = (bVal ? 1 : 0); } + } + + else if( 0==sqlite3_stricmp(zKey, "insttoken") ){ + int bVal = -1; + if( SQLITE_INTEGER==sqlite3_value_numeric_type(pVal) ){ + bVal = sqlite3_value_int(pVal); + } + if( bVal<0 ){ + *pbBadkey = 1; + }else{ + pConfig->bPrefixInsttoken = (bVal ? 1 : 0); + } + }else{ *pbBadkey = 1; } @@ -234800,13 +243249,10 @@ static int sqlite3Fts5ConfigLoad(Fts5Config *pConfig, int iCookie){ && iVersion!=FTS5_CURRENT_VERSION_SECUREDELETE ){ rc = SQLITE_ERROR; - if( pConfig->pzErrmsg ){ - assert( 0==*pConfig->pzErrmsg ); - *pConfig->pzErrmsg = sqlite3_mprintf("invalid fts5 file format " - "(found %d, expected %d or %d) - run 'rebuild'", - iVersion, FTS5_CURRENT_VERSION, FTS5_CURRENT_VERSION_SECUREDELETE - ); - } + sqlite3Fts5ConfigErrmsg(pConfig, "invalid fts5 file format " + "(found %d, expected %d or %d) - run 'rebuild'", + iVersion, FTS5_CURRENT_VERSION, FTS5_CURRENT_VERSION_SECUREDELETE + ); }else{ pConfig->iVersion = iVersion; } @@ -234817,6 +243263,29 @@ static int sqlite3Fts5ConfigLoad(Fts5Config *pConfig, int iCookie){ return rc; } +/* +** Set (*pConfig->pzErrmsg) to point to an sqlite3_malloc()ed buffer +** containing the error message created using printf() style formatting +** string zFmt and its trailing arguments. +*/ +static void sqlite3Fts5ConfigErrmsg(Fts5Config *pConfig, const char *zFmt, ...){ + va_list ap; /* ... printf arguments */ + char *zMsg = 0; + + va_start(ap, zFmt); + zMsg = sqlite3_vmprintf(zFmt, ap); + if( pConfig->pzErrmsg ){ + assert( *pConfig->pzErrmsg==0 ); + *pConfig->pzErrmsg = zMsg; + }else{ + sqlite3_free(zMsg); + } + + va_end(ap); +} + + + /* ** 2014 May 31 ** @@ -234873,7 +243342,7 @@ struct Fts5Expr { /* ** eType: -** Expression node type. Always one of: +** Expression node type. Usually one of: ** ** FTS5_AND (nChild, apChild valid) ** FTS5_OR (nChild, apChild valid) @@ -234881,6 +243350,10 @@ struct Fts5Expr { ** FTS5_STRING (pNear valid) ** FTS5_TERM (pNear valid) ** +** An expression node with eType==0 may also exist. It always matches zero +** rows. This is created when a phrase containing no tokens is parsed. +** e.g. "". +** ** iHeight: ** Distance from this node to furthest leaf. This is always 0 for nodes ** of type FTS5_STRING and FTS5_TERM. For all other nodes it is one @@ -234901,9 +243374,13 @@ struct Fts5ExprNode { /* Child nodes. For a NOT node, this array always contains 2 entries. For ** AND or OR nodes, it contains 2 or more entries. */ int nChild; /* Number of child nodes */ - Fts5ExprNode *apChild[1]; /* Array of child nodes */ + Fts5ExprNode *apChild[FLEXARRAY]; /* Array of child nodes */ }; +/* Size (in bytes) of an Fts5ExprNode object that holds up to N children */ +#define SZ_FTS5EXPRNODE(N) \ + (offsetof(Fts5ExprNode,apChild) + (N)*sizeof(Fts5ExprNode*)) + #define Fts5NodeIsString(p) ((p)->eType==FTS5_TERM || (p)->eType==FTS5_STRING) /* @@ -234934,9 +243411,13 @@ struct Fts5ExprPhrase { Fts5ExprNode *pNode; /* FTS5_STRING node this phrase is part of */ Fts5Buffer poslist; /* Current position list */ int nTerm; /* Number of entries in aTerm[] */ - Fts5ExprTerm aTerm[1]; /* Terms that make up this phrase */ + Fts5ExprTerm aTerm[FLEXARRAY]; /* Terms that make up this phrase */ }; +/* Size (in bytes) of an Fts5ExprPhrase object that holds up to N terms */ +#define SZ_FTS5EXPRPHRASE(N) \ + (offsetof(Fts5ExprPhrase,aTerm) + (N)*sizeof(Fts5ExprTerm)) + /* ** One or more phrases that must appear within a certain token distance of ** each other within each matching document. @@ -234945,9 +243426,12 @@ struct Fts5ExprNearset { int nNear; /* NEAR parameter */ Fts5Colset *pColset; /* Columns to search (NULL -> all columns) */ int nPhrase; /* Number of entries in aPhrase[] array */ - Fts5ExprPhrase *apPhrase[1]; /* Array of phrase pointers */ + Fts5ExprPhrase *apPhrase[FLEXARRAY]; /* Array of phrase pointers */ }; +/* Size (in bytes) of an Fts5ExprNearset object covering up to N phrases */ +#define SZ_FTS5EXPRNEARSET(N) \ + (offsetof(Fts5ExprNearset,apPhrase)+(N)*sizeof(Fts5ExprPhrase*)) /* ** Parse context. @@ -235101,12 +243585,13 @@ static int sqlite3Fts5ExprNew( }while( sParse.rc==SQLITE_OK && t!=FTS5_EOF ); sqlite3Fts5ParserFree(pEngine, fts5ParseFree); + assert( sParse.pExpr || sParse.rc!=SQLITE_OK ); assert_expr_depth_ok(sParse.rc, sParse.pExpr); /* If the LHS of the MATCH expression was a user column, apply the ** implicit column-filter. */ - if( iColnCol && sParse.pExpr && sParse.rc==SQLITE_OK ){ - int n = sizeof(Fts5Colset); + if( sParse.rc==SQLITE_OK && iColnCol ){ + int n = SZ_FTS5COLSET(1); Fts5Colset *pColset = (Fts5Colset*)sqlite3Fts5MallocZero(&sParse.rc, n); if( pColset ){ pColset->nCol = 1; @@ -235122,15 +243607,7 @@ static int sqlite3Fts5ExprNew( sParse.rc = SQLITE_NOMEM; sqlite3Fts5ParseNodeFree(sParse.pExpr); }else{ - if( !sParse.pExpr ){ - const int nByte = sizeof(Fts5ExprNode); - pNew->pRoot = (Fts5ExprNode*)sqlite3Fts5MallocZero(&sParse.rc, nByte); - if( pNew->pRoot ){ - pNew->pRoot->bEof = 1; - } - }else{ - pNew->pRoot = sParse.pExpr; - } + pNew->pRoot = sParse.pExpr; pNew->pIndex = 0; pNew->pConfig = pConfig; pNew->apExprPhrase = sParse.apPhrase; @@ -235143,7 +243620,11 @@ static int sqlite3Fts5ExprNew( } sqlite3_free(sParse.apPhrase); - *pzErr = sParse.zErr; + if( 0==*pzErr ){ + *pzErr = sParse.zErr; + }else{ + sqlite3_free(sParse.zErr); + } return sParse.rc; } @@ -235944,7 +244425,7 @@ static int fts5ExprNodeTest_STRING( } }else{ Fts5IndexIter *pIter = pPhrase->aTerm[j].pIter; - if( pIter->iRowid==iLast || pIter->bEof ) continue; + if( pIter->iRowid==iLast ) continue; bMatch = 0; if( fts5ExprAdvanceto(pIter, bDesc, &iLast, &rc, &pNode->bEof) ){ return rc; @@ -236356,7 +244837,13 @@ static int fts5ExprNodeFirst(Fts5Expr *pExpr, Fts5ExprNode *pNode){ ** Return SQLITE_OK if successful, or an SQLite error code otherwise. It ** is not considered an error if the query does not match any documents. */ -static int sqlite3Fts5ExprFirst(Fts5Expr *p, Fts5Index *pIdx, i64 iFirst, int bDesc){ +static int sqlite3Fts5ExprFirst( + Fts5Expr *p, + Fts5Index *pIdx, + i64 iFirst, + i64 iLast, + int bDesc +){ Fts5ExprNode *pRoot = p->pRoot; int rc; /* Return code */ @@ -236378,6 +244865,9 @@ static int sqlite3Fts5ExprFirst(Fts5Expr *p, Fts5Index *pIdx, i64 iFirst, int bD assert( pRoot->bEof==0 ); rc = fts5ExprNodeNext(p, pRoot, 0, 0); } + if( fts5RowidCmp(p, pRoot->iRowid, iLast)>0 ){ + pRoot->bEof = 1; + } return rc; } @@ -236466,12 +244956,9 @@ static Fts5ExprNearset *sqlite3Fts5ParseNearset( Fts5ExprNearset *pRet = 0; if( pParse->rc==SQLITE_OK ){ - if( pPhrase==0 ){ - return pNear; - } if( pNear==0 ){ sqlite3_int64 nByte; - nByte = sizeof(Fts5ExprNearset) + SZALLOC * sizeof(Fts5ExprPhrase*); + nByte = SZ_FTS5EXPRNEARSET(SZALLOC+1); pRet = sqlite3_malloc64(nByte); if( pRet==0 ){ pParse->rc = SQLITE_NOMEM; @@ -236482,7 +244969,7 @@ static Fts5ExprNearset *sqlite3Fts5ParseNearset( int nNew = pNear->nPhrase + SZALLOC; sqlite3_int64 nByte; - nByte = sizeof(Fts5ExprNearset) + nNew * sizeof(Fts5ExprPhrase*); + nByte = SZ_FTS5EXPRNEARSET(nNew+1); pRet = (Fts5ExprNearset*)sqlite3_realloc64(pNear, nByte); if( pRet==0 ){ pParse->rc = SQLITE_NOMEM; @@ -236573,12 +245060,12 @@ static int fts5ParseTokenize( int nNew = SZALLOC + (pPhrase ? pPhrase->nTerm : 0); pNew = (Fts5ExprPhrase*)sqlite3_realloc64(pPhrase, - sizeof(Fts5ExprPhrase) + sizeof(Fts5ExprTerm) * nNew + SZ_FTS5EXPRPHRASE(nNew+1) ); if( pNew==0 ){ rc = SQLITE_NOMEM; }else{ - if( pPhrase==0 ) memset(pNew, 0, sizeof(Fts5ExprPhrase)); + if( pPhrase==0 ) memset(pNew, 0, SZ_FTS5EXPRPHRASE(1)); pCtx->pPhrase = pPhrase = pNew; pNew->nTerm = nNew - SZALLOC; } @@ -236686,10 +245173,11 @@ static Fts5ExprPhrase *sqlite3Fts5ParseTerm( if( sCtx.pPhrase==0 ){ /* This happens when parsing a token or quoted phrase that contains ** no token characters at all. (e.g ... MATCH '""'). */ - sCtx.pPhrase = sqlite3Fts5MallocZero(&pParse->rc, sizeof(Fts5ExprPhrase)); + sCtx.pPhrase = sqlite3Fts5MallocZero(&pParse->rc, SZ_FTS5EXPRPHRASE(1)); }else if( sCtx.pPhrase->nTerm ){ sCtx.pPhrase->aTerm[sCtx.pPhrase->nTerm-1].bPrefix = (u8)bPrefix; } + assert( pParse->apPhrase!=0 ); pParse->apPhrase[pParse->nPhrase-1] = sCtx.pPhrase; } @@ -236709,7 +245197,7 @@ static int sqlite3Fts5ExprClonePhrase( Fts5ExprPhrase *pOrig = 0; /* The phrase extracted from pExpr */ Fts5Expr *pNew = 0; /* Expression to return via *ppNew */ TokenCtx sCtx = {0,0,0}; /* Context object for fts5ParseTokenize */ - if( iPhrase<0 || iPhrase>=pExpr->nPhrase ){ + if( !pExpr || iPhrase<0 || iPhrase>=pExpr->nPhrase ){ rc = SQLITE_RANGE; }else{ pOrig = pExpr->apExprPhrase[iPhrase]; @@ -236720,19 +245208,18 @@ static int sqlite3Fts5ExprClonePhrase( sizeof(Fts5ExprPhrase*)); } if( rc==SQLITE_OK ){ - pNew->pRoot = (Fts5ExprNode*)sqlite3Fts5MallocZero(&rc, - sizeof(Fts5ExprNode)); + pNew->pRoot = (Fts5ExprNode*)sqlite3Fts5MallocZero(&rc, SZ_FTS5EXPRNODE(1)); } if( rc==SQLITE_OK ){ pNew->pRoot->pNear = (Fts5ExprNearset*)sqlite3Fts5MallocZero(&rc, - sizeof(Fts5ExprNearset) + sizeof(Fts5ExprPhrase*)); + SZ_FTS5EXPRNEARSET(2)); } if( rc==SQLITE_OK && ALWAYS(pOrig!=0) ){ Fts5Colset *pColsetOrig = pOrig->pNode->pNear->pColset; if( pColsetOrig ){ sqlite3_int64 nByte; Fts5Colset *pColset; - nByte = sizeof(Fts5Colset) + (pColsetOrig->nCol-1) * sizeof(int); + nByte = SZ_FTS5COLSET(pColsetOrig->nCol); pColset = (Fts5Colset*)sqlite3Fts5MallocZero(&rc, nByte); if( pColset ){ memcpy(pColset, pColsetOrig, (size_t)nByte); @@ -236760,7 +245247,7 @@ static int sqlite3Fts5ExprClonePhrase( }else{ /* This happens when parsing a token or quoted phrase that contains ** no token characters at all. (e.g ... MATCH '""'). */ - sCtx.pPhrase = sqlite3Fts5MallocZero(&rc, sizeof(Fts5ExprPhrase)); + sCtx.pPhrase = sqlite3Fts5MallocZero(&rc, SZ_FTS5EXPRPHRASE(1)); } } @@ -236825,7 +245312,8 @@ static void sqlite3Fts5ParseSetDistance( ); return; } - nNear = nNear * 10 + (p->p[i] - '0'); + if( nNear<214748363 ) nNear = nNear * 10 + (p->p[i] - '0'); + /* ^^^^^^^^^^^^^^^--- Prevent integer overflow */ } }else{ nNear = FTS5_DEFAULT_NEARDIST; @@ -236854,7 +245342,7 @@ static Fts5Colset *fts5ParseColset( assert( pParse->rc==SQLITE_OK ); assert( iCol>=0 && iColpConfig->nCol ); - pNew = sqlite3_realloc64(p, sizeof(Fts5Colset) + sizeof(int)*nCol); + pNew = sqlite3_realloc64(p, SZ_FTS5COLSET(nCol+1)); if( pNew==0 ){ pParse->rc = SQLITE_NOMEM; }else{ @@ -236889,7 +245377,7 @@ static Fts5Colset *sqlite3Fts5ParseColsetInvert(Fts5Parse *pParse, Fts5Colset *p int nCol = pParse->pConfig->nCol; pRet = (Fts5Colset*)sqlite3Fts5MallocZero(&pParse->rc, - sizeof(Fts5Colset) + sizeof(int)*nCol + SZ_FTS5COLSET(nCol+1) ); if( pRet ){ int i; @@ -236950,7 +245438,7 @@ static Fts5Colset *sqlite3Fts5ParseColset( static Fts5Colset *fts5CloneColset(int *pRc, Fts5Colset *pOrig){ Fts5Colset *pRet; if( pOrig ){ - sqlite3_int64 nByte = sizeof(Fts5Colset) + (pOrig->nCol-1) * sizeof(int); + sqlite3_int64 nByte = SZ_FTS5COLSET(pOrig->nCol); pRet = (Fts5Colset*)sqlite3Fts5MallocZero(pRc, nByte); if( pRet ){ memcpy(pRet, pOrig, (size_t)nByte); @@ -237077,6 +245565,9 @@ static void fts5ExprAssignXNext(Fts5ExprNode *pNode){ } } +/* +** Add pSub as a child of p. +*/ static void fts5ExprAddChildren(Fts5ExprNode *p, Fts5ExprNode *pSub){ int ii = p->nChild; if( p->eType!=FTS5_NOT && pSub->eType==p->eType ){ @@ -237115,7 +245606,7 @@ static Fts5ExprNode *fts5ParsePhraseToAnd( assert( pNear->nPhrase==1 ); assert( pParse->bPhraseToAnd ); - nByte = sizeof(Fts5ExprNode) + nTerm*sizeof(Fts5ExprNode*); + nByte = SZ_FTS5EXPRNODE(nTerm+1); pRet = (Fts5ExprNode*)sqlite3Fts5MallocZero(&pParse->rc, nByte); if( pRet ){ pRet->eType = FTS5_AND; @@ -237125,7 +245616,7 @@ static Fts5ExprNode *fts5ParsePhraseToAnd( pParse->nPhrase--; for(ii=0; iirc, sizeof(Fts5ExprPhrase) + &pParse->rc, SZ_FTS5EXPRPHRASE(1) ); if( pPhrase ){ if( parseGrowPhraseArray(pParse) ){ @@ -237194,7 +245685,7 @@ static Fts5ExprNode *sqlite3Fts5ParseNode( if( pRight->eType==eType ) nChild += pRight->nChild-1; } - nByte = sizeof(Fts5ExprNode) + sizeof(Fts5ExprNode*)*(nChild-1); + nByte = SZ_FTS5EXPRNODE(nChild); pRet = (Fts5ExprNode*)sqlite3Fts5MallocZero(&pParse->rc, nByte); if( pRet ){ @@ -237221,19 +245712,23 @@ static Fts5ExprNode *sqlite3Fts5ParseNode( "fts5: %s queries are not supported (detail!=full)", pNear->nPhrase==1 ? "phrase": "NEAR" ); - sqlite3_free(pRet); + sqlite3Fts5ParseNodeFree(pRet); pRet = 0; + pNear = 0; + assert( pLeft==0 && pRight==0 ); } } }else{ + assert( pNear==0 ); fts5ExprAddChildren(pRet, pLeft); fts5ExprAddChildren(pRet, pRight); + pLeft = pRight = 0; if( pRet->iHeight>SQLITE_FTS5_MAX_EXPR_DEPTH ){ sqlite3Fts5ParseError(pParse, "fts5 expression tree is too large (maximum depth %d)", SQLITE_FTS5_MAX_EXPR_DEPTH ); - sqlite3_free(pRet); + sqlite3Fts5ParseNodeFree(pRet); pRet = 0; } } @@ -237271,6 +245766,7 @@ static Fts5ExprNode *sqlite3Fts5ParseImplicitAnd( assert( pRight->eType==FTS5_STRING || pRight->eType==FTS5_TERM || pRight->eType==FTS5_EOF + || (pRight->eType==FTS5_AND && pParse->bPhraseToAnd) ); if( pLeft->eType==FTS5_AND ){ @@ -237284,6 +245780,8 @@ static Fts5ExprNode *sqlite3Fts5ParseImplicitAnd( ); if( pRight->eType==FTS5_EOF ){ + assert( pParse->apPhrase!=0 ); + assert( pParse->nPhrase>0 ); assert( pParse->apPhrase[pParse->nPhrase-1]==pRight->pNear->apPhrase[0] ); sqlite3Fts5ParseNodeFree(pRight); pRet = pLeft; @@ -237856,7 +246354,7 @@ static int fts5ExprPopulatePoslistsCb( int rc = sqlite3Fts5PoslistWriterAppend( &pExpr->apExprPhrase[i]->poslist, &p->aPopulator[i].writer, p->iOff ); - if( rc==SQLITE_OK && pExpr->pConfig->bTokendata && !pT->bPrefix ){ + if( rc==SQLITE_OK && (pExpr->pConfig->bTokendata || pT->bPrefix) ){ int iCol = p->iOff>>32; int iTokOff = p->iOff & 0x7FFFFFFF; rc = sqlite3Fts5IndexIterWriteTokendata( @@ -237916,6 +246414,7 @@ static int fts5ExprCheckPoslists(Fts5ExprNode *pNode, i64 iRowid){ pNode->iRowid = iRowid; pNode->bEof = 0; switch( pNode->eType ){ + case 0: case FTS5_TERM: case FTS5_STRING: return (pNode->pNear->apPhrase[0]->poslist.n>0); @@ -238048,21 +246547,20 @@ static int sqlite3Fts5ExprInstToken( return SQLITE_RANGE; } pTerm = &pPhrase->aTerm[iToken]; - if( pTerm->bPrefix==0 ){ - if( pExpr->pConfig->bTokendata ){ - rc = sqlite3Fts5IterToken( - pTerm->pIter, iRowid, iCol, iOff+iToken, ppOut, pnOut - ); - }else{ - *ppOut = pTerm->pTerm; - *pnOut = pTerm->nFullTerm; - } + if( pExpr->pConfig->bTokendata || pTerm->bPrefix ){ + rc = sqlite3Fts5IterToken( + pTerm->pIter, pTerm->pTerm, pTerm->nQueryTerm, + iRowid, iCol, iOff+iToken, ppOut, pnOut + ); + }else{ + *ppOut = pTerm->pTerm; + *pnOut = pTerm->nFullTerm; } return rc; } /* -** Clear the token mappings for all Fts5IndexIter objects mannaged by +** Clear the token mappings for all Fts5IndexIter objects managed by ** the expression passed as the only argument. */ static void sqlite3Fts5ExprClearTokens(Fts5Expr *pExpr){ @@ -238097,7 +246595,7 @@ typedef struct Fts5HashEntry Fts5HashEntry; /* ** This file contains the implementation of an in-memory hash table used -** to accumuluate "term -> doclist" content before it is flused to a level-0 +** to accumulate "term -> doclist" content before it is flushed to a level-0 ** segment. */ @@ -238154,7 +246652,7 @@ struct Fts5HashEntry { }; /* -** Eqivalent to: +** Equivalent to: ** ** char *fts5EntryKey(Fts5HashEntry *pEntry){ return zKey; } */ @@ -239090,9 +247588,13 @@ struct Fts5Structure { u64 nOriginCntr; /* Origin value for next top-level segment */ int nSegment; /* Total segments in this structure */ int nLevel; /* Number of levels in this index */ - Fts5StructureLevel aLevel[1]; /* Array of nLevel level objects */ + Fts5StructureLevel aLevel[FLEXARRAY]; /* Array of nLevel level objects */ }; +/* Size (in bytes) of an Fts5Structure object holding up to N levels */ +#define SZ_FTS5STRUCTURE(N) \ + (offsetof(Fts5Structure,aLevel) + (N)*sizeof(Fts5StructureLevel)) + /* ** An object of type Fts5SegWriter is used to write to segments. */ @@ -239218,15 +247720,49 @@ struct Fts5SegIter { u8 bDel; /* True if the delete flag is set */ }; +static int fts5IndexCorruptRowid(Fts5Index *pIdx, i64 iRowid){ + pIdx->rc = FTS5_CORRUPT; + sqlite3Fts5ConfigErrmsg(pIdx->pConfig, + "fts5: corruption found reading blob %lld from table \"%s\"", + iRowid, pIdx->pConfig->zName + ); + return SQLITE_CORRUPT_VTAB; +} +#define FTS5_CORRUPT_ROWID(pIdx, iRowid) fts5IndexCorruptRowid(pIdx, iRowid) + +static int fts5IndexCorruptIter(Fts5Index *pIdx, Fts5SegIter *pIter){ + pIdx->rc = FTS5_CORRUPT; + sqlite3Fts5ConfigErrmsg(pIdx->pConfig, + "fts5: corruption on page %d, segment %d, table \"%s\"", + pIter->iLeafPgno, pIter->pSeg->iSegid, pIdx->pConfig->zName + ); + return SQLITE_CORRUPT_VTAB; +} +#define FTS5_CORRUPT_ITER(pIdx, pIter) fts5IndexCorruptIter(pIdx, pIter) + +static int fts5IndexCorruptIdx(Fts5Index *pIdx){ + pIdx->rc = FTS5_CORRUPT; + sqlite3Fts5ConfigErrmsg(pIdx->pConfig, + "fts5: corruption in table \"%s\"", pIdx->pConfig->zName + ); + return SQLITE_CORRUPT_VTAB; +} +#define FTS5_CORRUPT_IDX(pIdx) fts5IndexCorruptIdx(pIdx) + + /* ** Array of tombstone pages. Reference counted. */ struct Fts5TombstoneArray { - int nRef; /* Number of pointers to this object */ + int nRef; /* Number of pointers to this object */ int nTombstone; - Fts5Data *apTombstone[1]; /* Array of tombstone pages */ + Fts5Data *apTombstone[FLEXARRAY]; /* Array of tombstone pages */ }; +/* Size (in bytes) of an Fts5TombstoneArray holding up to N tombstones */ +#define SZ_FTS5TOMBSTONEARRAY(N) \ + (offsetof(Fts5TombstoneArray,apTombstone)+(N)*sizeof(Fts5Data*)) + /* ** Argument is a pointer to an Fts5Data structure that contains a ** leaf page. @@ -239295,9 +247831,12 @@ struct Fts5Iter { i64 iSwitchRowid; /* Firstest rowid of other than aFirst[1] */ Fts5CResult *aFirst; /* Current merge state (see above) */ - Fts5SegIter aSeg[1]; /* Array of segment iterators */ + Fts5SegIter aSeg[FLEXARRAY]; /* Array of segment iterators */ }; +/* Size (in bytes) of an Fts5Iter object holding up to N segment iterators */ +#define SZ_FTS5ITER(N) (offsetof(Fts5Iter,aSeg)+(N)*sizeof(Fts5SegIter)) + /* ** An instance of the following type is used to iterate through the contents ** of a doclist-index record. @@ -239324,9 +247863,13 @@ struct Fts5DlidxLvl { struct Fts5DlidxIter { int nLvl; int iSegid; - Fts5DlidxLvl aLvl[1]; + Fts5DlidxLvl aLvl[FLEXARRAY]; }; +/* Size (in bytes) of an Fts5DlidxIter object with up to N levels */ +#define SZ_FTS5DLIDXITER(N) \ + (offsetof(Fts5DlidxIter,aLvl)+(N)*sizeof(Fts5DlidxLvl)) + static void fts5PutU16(u8 *aOut, u16 iVal){ aOut[0] = (iVal>>8); aOut[1] = (iVal&0xFF); @@ -239446,11 +247989,13 @@ static int fts5LeafFirstTermOff(Fts5Data *pLeaf){ /* ** Close the read-only blob handle, if it is open. */ -static void sqlite3Fts5IndexCloseReader(Fts5Index *p){ +static void fts5IndexCloseReader(Fts5Index *p){ if( p->pReader ){ + int rc; sqlite3_blob *pReader = p->pReader; p->pReader = 0; - sqlite3_blob_close(pReader); + rc = sqlite3_blob_close(pReader); + if( p->rc==SQLITE_OK ) p->rc = rc; } } @@ -239475,7 +248020,7 @@ static Fts5Data *fts5DataRead(Fts5Index *p, i64 iRowid){ assert( p->pReader==0 ); p->pReader = pBlob; if( rc!=SQLITE_OK ){ - sqlite3Fts5IndexCloseReader(p); + fts5IndexCloseReader(p); } if( rc==SQLITE_ABORT ) rc = SQLITE_OK; } @@ -239494,16 +248039,17 @@ static Fts5Data *fts5DataRead(Fts5Index *p, i64 iRowid){ ** All the reasons those functions might return SQLITE_ERROR - missing ** table, missing row, non-blob/text in block column - indicate ** backing store corruption. */ - if( rc==SQLITE_ERROR ) rc = FTS5_CORRUPT; + if( rc==SQLITE_ERROR ) rc = FTS5_CORRUPT_ROWID(p, iRowid); if( rc==SQLITE_OK ){ u8 *aOut = 0; /* Read blob data into this buffer */ - int nByte = sqlite3_blob_bytes(p->pReader); - sqlite3_int64 nAlloc = sizeof(Fts5Data) + nByte + FTS5_DATA_PADDING; + i64 nByte = sqlite3_blob_bytes(p->pReader); + i64 szData = (sizeof(Fts5Data) + 7) & ~7; + i64 nAlloc = szData + nByte + FTS5_DATA_PADDING; pRet = (Fts5Data*)sqlite3_malloc64(nAlloc); if( pRet ){ pRet->nn = nByte; - aOut = pRet->p = (u8*)&pRet[1]; + aOut = pRet->p = (u8*)pRet + szData; }else{ rc = SQLITE_NOMEM; } @@ -239526,6 +248072,7 @@ static Fts5Data *fts5DataRead(Fts5Index *p, i64 iRowid){ } assert( (pRet==0)==(p->rc!=SQLITE_OK) ); + assert( pRet==0 || EIGHT_BYTE_ALIGNMENT( pRet->p ) ); return pRet; } @@ -239542,7 +248089,7 @@ static Fts5Data *fts5LeafRead(Fts5Index *p, i64 iRowid){ Fts5Data *pRet = fts5DataRead(p, iRowid); if( pRet ){ if( pRet->nn<4 || pRet->szLeaf>pRet->nn ){ - p->rc = FTS5_CORRUPT; + FTS5_CORRUPT_ROWID(p, iRowid); fts5DataRelease(pRet); pRet = 0; } @@ -239557,9 +248104,13 @@ static int fts5IndexPrepareStmt( ){ if( p->rc==SQLITE_OK ){ if( zSql ){ - p->rc = sqlite3_prepare_v3(p->pConfig->db, zSql, -1, + int rc = sqlite3_prepare_v3(p->pConfig->db, zSql, -1, SQLITE_PREPARE_PERSISTENT|SQLITE_PREPARE_NO_VTAB, ppStmt, 0); + /* If this prepare() call fails with SQLITE_ERROR, then one of the + ** %_idx or %_data tables has been removed or modified. Call this + ** corruption. */ + p->rc = (rc==SQLITE_ERROR ? SQLITE_CORRUPT : rc); }else{ p->rc = SQLITE_NOMEM; } @@ -239686,7 +248237,7 @@ static int sqlite3Fts5StructureTest(Fts5Index *p, void *pStruct){ static void fts5StructureMakeWritable(int *pRc, Fts5Structure **pp){ Fts5Structure *p = *pp; if( *pRc==SQLITE_OK && p->nRef>1 ){ - i64 nByte = sizeof(Fts5Structure)+(p->nLevel-1)*sizeof(Fts5StructureLevel); + i64 nByte = SZ_FTS5STRUCTURE(p->nLevel); Fts5Structure *pNew; pNew = (Fts5Structure*)sqlite3Fts5MallocZero(pRc, nByte); if( pNew ){ @@ -239760,10 +248311,7 @@ static int fts5StructureDecode( ){ return FTS5_CORRUPT; } - nByte = ( - sizeof(Fts5Structure) + /* Main structure */ - sizeof(Fts5StructureLevel) * (nLevel-1) /* aLevel[] array */ - ); + nByte = SZ_FTS5STRUCTURE(nLevel); pRet = (Fts5Structure*)sqlite3Fts5MallocZero(&rc, nByte); if( pRet ){ @@ -239843,10 +248391,7 @@ static void fts5StructureAddLevel(int *pRc, Fts5Structure **ppStruct){ if( *pRc==SQLITE_OK ){ Fts5Structure *pStruct = *ppStruct; int nLevel = pStruct->nLevel; - sqlite3_int64 nByte = ( - sizeof(Fts5Structure) + /* Main structure */ - sizeof(Fts5StructureLevel) * (nLevel+1) /* aLevel[] array */ - ); + sqlite3_int64 nByte = SZ_FTS5STRUCTURE(nLevel+2); pStruct = sqlite3_realloc64(pStruct, nByte); if( pStruct ){ @@ -239903,8 +248448,14 @@ static Fts5Structure *fts5StructureReadUncached(Fts5Index *p){ /* TODO: Do we need this if the leaf-index is appended? Probably... */ memset(&pData->p[pData->nn], 0, FTS5_DATA_PADDING); p->rc = fts5StructureDecode(pData->p, pData->nn, &iCookie, &pRet); - if( p->rc==SQLITE_OK && (pConfig->pgsz==0 || pConfig->iCookie!=iCookie) ){ - p->rc = sqlite3Fts5ConfigLoad(pConfig, iCookie); + if( p->rc==SQLITE_OK ){ + if( (pConfig->pgsz==0 || pConfig->iCookie!=iCookie) ){ + p->rc = sqlite3Fts5ConfigLoad(pConfig, iCookie); + } + }else if( p->rc==SQLITE_CORRUPT_VTAB ){ + sqlite3Fts5ConfigErrmsg(p->pConfig, + "fts5: corrupt structure record for table \"%s\"", p->pConfig->zName + ); } fts5DataRelease(pData); if( p->rc!=SQLITE_OK ){ @@ -240385,7 +248936,7 @@ static Fts5DlidxIter *fts5DlidxIterInit( int bDone = 0; for(i=0; p->rc==SQLITE_OK && bDone==0; i++){ - sqlite3_int64 nByte = sizeof(Fts5DlidxIter) + i * sizeof(Fts5DlidxLvl); + sqlite3_int64 nByte = SZ_FTS5DLIDXITER(i+1); Fts5DlidxIter *pNew; pNew = (Fts5DlidxIter*)sqlite3_realloc64(pIter, nByte); @@ -240527,7 +249078,7 @@ static void fts5SegIterLoadRowid(Fts5Index *p, Fts5SegIter *pIter){ while( iOff>=pIter->pLeaf->szLeaf ){ fts5SegIterNextPage(p, pIter); if( pIter->pLeaf==0 ){ - if( p->rc==SQLITE_OK ) p->rc = FTS5_CORRUPT; + if( p->rc==SQLITE_OK ) FTS5_CORRUPT_ITER(p, pIter); return; } iOff = 4; @@ -240559,7 +249110,7 @@ static void fts5SegIterLoadTerm(Fts5Index *p, Fts5SegIter *pIter, int nKeep){ iOff += fts5GetVarint32(&a[iOff], nNew); if( iOff+nNew>pIter->pLeaf->szLeaf || nKeep>pIter->term.n || nNew==0 ){ - p->rc = FTS5_CORRUPT; + FTS5_CORRUPT_ITER(p, pIter); return; } pIter->term.n = nKeep; @@ -240601,9 +249152,9 @@ static void fts5SegIterSetNext(Fts5Index *p, Fts5SegIter *pIter){ ** leave an error in the Fts5Index object. */ static void fts5SegIterAllocTombstone(Fts5Index *p, Fts5SegIter *pIter){ - const int nTomb = pIter->pSeg->nPgTombstone; + const i64 nTomb = (i64)pIter->pSeg->nPgTombstone; if( nTomb>0 ){ - int nByte = nTomb * sizeof(Fts5Data*) + sizeof(Fts5TombstoneArray); + i64 nByte = SZ_FTS5TOMBSTONEARRAY(nTomb+1); Fts5TombstoneArray *pNew; pNew = (Fts5TombstoneArray*)sqlite3Fts5MallocZero(&p->rc, nByte); if( pNew ){ @@ -240689,6 +249240,7 @@ static void fts5SegIterReverseInitPage(Fts5Index *p, Fts5SegIter *pIter){ while( 1 ){ u64 iDelta = 0; + if( i>=n ) break; if( eDetail==FTS5_DETAIL_NONE ){ /* todo */ if( i=pNew->szLeaf ){ - p->rc = FTS5_CORRUPT; + FTS5_CORRUPT_ITER(p, pIter); }else{ pIter->pLeaf = pNew; pIter->iLeafOffset = iRowidOff; @@ -240851,7 +249403,7 @@ static void fts5SegIterNext_None( if( iOffiEndofDoclist ){ /* Next entry is on the current page */ - i64 iDelta; + u64 iDelta; iOff += sqlite3Fts5GetVarint(&pIter->pLeaf->p[iOff], (u64*)&iDelta); pIter->iLeafOffset = iOff; pIter->iRowid += iDelta; @@ -240988,7 +249540,7 @@ static void fts5SegIterNext( } assert_nc( iOffszLeaf ); if( iOff>pLeaf->szLeaf ){ - p->rc = FTS5_CORRUPT; + FTS5_CORRUPT_ITER(p, pIter); return; } } @@ -241096,18 +249648,20 @@ static void fts5SegIterReverse(Fts5Index *p, Fts5SegIter *pIter){ fts5DataRelease(pIter->pLeaf); pIter->pLeaf = pLast; pIter->iLeafPgno = pgnoLast; - iOff = fts5LeafFirstRowidOff(pLast); - if( iOff>pLast->szLeaf ){ - p->rc = FTS5_CORRUPT; - return; - } - iOff += fts5GetVarint(&pLast->p[iOff], (u64*)&pIter->iRowid); - pIter->iLeafOffset = iOff; + if( p->rc==SQLITE_OK ){ + iOff = fts5LeafFirstRowidOff(pLast); + if( iOff>pLast->szLeaf ){ + FTS5_CORRUPT_ITER(p, pIter); + return; + } + iOff += fts5GetVarint(&pLast->p[iOff], (u64*)&pIter->iRowid); + pIter->iLeafOffset = iOff; - if( fts5LeafIsTermless(pLast) ){ - pIter->iEndofDoclist = pLast->nn+1; - }else{ - pIter->iEndofDoclist = fts5LeafFirstTermOff(pLast); + if( fts5LeafIsTermless(pLast) ){ + pIter->iEndofDoclist = pLast->nn+1; + }else{ + pIter->iEndofDoclist = fts5LeafFirstTermOff(pLast); + } } } @@ -241177,7 +249731,7 @@ static void fts5LeafSeek( iPgidx += fts5GetVarint32(&a[iPgidx], iTermOff); iOff = iTermOff; if( iOff>n ){ - p->rc = FTS5_CORRUPT; + FTS5_CORRUPT_ITER(p, pIter); return; } @@ -241220,7 +249774,7 @@ static void fts5LeafSeek( iOff = iTermOff; if( iOff>=n ){ - p->rc = FTS5_CORRUPT; + FTS5_CORRUPT_ITER(p, pIter); return; } @@ -241242,7 +249796,7 @@ static void fts5LeafSeek( iPgidx = (u32)pIter->pLeaf->szLeaf; iPgidx += fts5GetVarint32(&pIter->pLeaf->p[iPgidx], iOff); if( iOff<4 || (i64)iOff>=pIter->pLeaf->szLeaf ){ - p->rc = FTS5_CORRUPT; + FTS5_CORRUPT_ITER(p, pIter); return; }else{ nKeep = 0; @@ -241257,7 +249811,7 @@ static void fts5LeafSeek( search_success: if( (i64)iOff+nNew>n || nNew<1 ){ - p->rc = FTS5_CORRUPT; + FTS5_CORRUPT_ITER(p, pIter); return; } pIter->iLeafOffset = iOff + nNew; @@ -241722,7 +250276,7 @@ static void fts5SegIterGotoPage( assert( iLeafPgno>pIter->iLeafPgno ); if( iLeafPgno>pIter->pSeg->pgnoLast ){ - p->rc = FTS5_CORRUPT; + FTS5_CORRUPT_IDX(p); }else{ fts5DataRelease(pIter->pNextLeaf); pIter->pNextLeaf = 0; @@ -241737,7 +250291,7 @@ static void fts5SegIterGotoPage( u8 *a = pIter->pLeaf->p; int n = pIter->pLeaf->szLeaf; if( iOff<4 || iOff>=n ){ - p->rc = FTS5_CORRUPT; + FTS5_CORRUPT_IDX(p); }else{ iOff += fts5GetVarint(&a[iOff], (u64*)&pIter->iRowid); pIter->iLeafOffset = iOff; @@ -242064,8 +250618,7 @@ static Fts5Iter *fts5MultiIterAlloc( for(nSlot=2; nSlotaSeg[] */ + SZ_FTS5ITER(nSlot) + /* pNew + pNew->aSeg[] */ sizeof(Fts5CResult) * nSlot /* pNew->aFirst[] */ ); if( pNew ){ @@ -242217,7 +250770,7 @@ static void fts5ChunkIterate( if( nRem<=0 ){ break; }else if( pSeg->pSeg==0 ){ - p->rc = FTS5_CORRUPT; + FTS5_CORRUPT_IDX(p); return; }else{ pgno++; @@ -243320,7 +251873,7 @@ static void fts5TrimSegments(Fts5Index *p, Fts5Iter *pIter){ ** a single page has been assigned to more than one segment. In ** this case a prior iteration of this loop may have corrupted the ** segment currently being trimmed. */ - p->rc = FTS5_CORRUPT; + FTS5_CORRUPT_ROWID(p, iLeafRowid); }else{ fts5BufferZero(&buf); fts5BufferGrow(&p->rc, &buf, pData->nn); @@ -243555,6 +252108,11 @@ static int fts5IndexFindDeleteMerge(Fts5Index *p, Fts5Structure *pStruct){ nBest = nPercent; } } + + /* If pLvl is already the input level to an ongoing merge, look no + ** further for a merge candidate. The caller should be allowed to + ** continue merging from pLvl first. */ + if( pLvl->nMerge ) break; } } return iRet; @@ -243666,6 +252224,14 @@ static int fts5IndexReturn(Fts5Index *p){ return rc; } +/* +** Close the read-only blob handle, if it is open. +*/ +static void sqlite3Fts5IndexCloseReader(Fts5Index *p){ + fts5IndexCloseReader(p); + fts5IndexReturn(p); +} + typedef struct Fts5FlushCtx Fts5FlushCtx; struct Fts5FlushCtx { Fts5Index *pIdx; @@ -243774,7 +252340,7 @@ static void fts5SecureDeleteOverflow( }else if( bDetailNone ){ break; }else if( iNext>=pLeaf->szLeaf || pLeaf->nnszLeaf || iNext<4 ){ - p->rc = FTS5_CORRUPT; + FTS5_CORRUPT_ROWID(p, iRowid); break; }else{ int nShift = iNext - 4; @@ -243794,7 +252360,7 @@ static void fts5SecureDeleteOverflow( i1 += fts5GetVarint32(&aPg[i1], iFirst); if( iFirstrc = FTS5_CORRUPT; + FTS5_CORRUPT_ROWID(p, iRowid); break; } aIdx = sqlite3Fts5MallocZero(&p->rc, (pLeaf->nn-pLeaf->szLeaf)+2); @@ -243853,7 +252419,7 @@ static void fts5DoSecureDelete( int iDelKeyOff = 0; /* Offset of deleted key, if any */ nIdx = nPg-iPgIdx; - aIdx = sqlite3Fts5MallocZero(&p->rc, nIdx+16); + aIdx = sqlite3Fts5MallocZero(&p->rc, ((i64)nIdx)+16); if( p->rc ) return; memcpy(aIdx, &aPg[iPgIdx], nIdx); @@ -244017,14 +252583,14 @@ static void fts5DoSecureDelete( nSuffix = (nPrefix2 + nSuffix2) - nPrefix; if( (iKeyOff+nSuffix)>iPgIdx || (iNextOff+nSuffix2)>iPgIdx ){ - p->rc = FTS5_CORRUPT; + FTS5_CORRUPT_IDX(p); }else{ if( iKey!=1 ){ iOff += sqlite3Fts5PutVarint(&aPg[iOff], nPrefix); } iOff += sqlite3Fts5PutVarint(&aPg[iOff], nSuffix); if( nPrefix2>pSeg->term.n ){ - p->rc = FTS5_CORRUPT; + FTS5_CORRUPT_IDX(p); }else if( nPrefix2>nPrefix ){ memcpy(&aPg[iOff], &pSeg->term.p[nPrefix], nPrefix2-nPrefix); iOff += (nPrefix2-nPrefix); @@ -244123,8 +252689,11 @@ static void fts5DoSecureDelete( ** This is called as part of flushing a delete to disk in 'secure-delete' ** mode. It edits the segments within the database described by argument ** pStruct to remove the entries for term zTerm, rowid iRowid. +** +** Return SQLITE_OK if successful, or an SQLite error code if an error +** has occurred. Any error code is also stored in the Fts5Index handle. */ -static void fts5FlushSecureDelete( +static int fts5FlushSecureDelete( Fts5Index *p, Fts5Structure *pStruct, const char *zTerm, @@ -244134,6 +252703,24 @@ static void fts5FlushSecureDelete( const int f = FTS5INDEX_QUERY_SKIPHASH; Fts5Iter *pIter = 0; /* Used to find term instance */ + /* If the version number has not been set to SECUREDELETE, do so now. */ + if( p->pConfig->iVersion!=FTS5_CURRENT_VERSION_SECUREDELETE ){ + Fts5Config *pConfig = p->pConfig; + sqlite3_stmt *pStmt = 0; + fts5IndexPrepareStmt(p, &pStmt, sqlite3_mprintf( + "REPLACE INTO %Q.'%q_config' VALUES ('version', %d)", + pConfig->zDb, pConfig->zName, FTS5_CURRENT_VERSION_SECUREDELETE + )); + if( p->rc==SQLITE_OK ){ + int rc; + sqlite3_step(pStmt); + rc = sqlite3_finalize(pStmt); + if( p->rc==SQLITE_OK ) p->rc = rc; + pConfig->iCookie++; + pConfig->iVersion = FTS5_CURRENT_VERSION_SECUREDELETE; + } + } + fts5MultiIterNew(p, pStruct, f, 0, (const u8*)zTerm, nTerm, -1, 0, &pIter); if( fts5MultiIterEof(p, pIter)==0 ){ i64 iThis = fts5MultiIterRowid(pIter); @@ -244151,6 +252738,7 @@ static void fts5FlushSecureDelete( } fts5MultiIterFree(pIter); + return p->rc; } @@ -244234,8 +252822,9 @@ static void fts5FlushOneHash(Fts5Index *p){ ** using fts5FlushSecureDelete(). */ if( bSecureDelete ){ if( eDetail==FTS5_DETAIL_NONE ){ - if( iOffrc!=SQLITE_OK || pDoclist[iOff]==0x01 ){ iOff++; continue; @@ -244394,7 +252984,7 @@ static Fts5Structure *fts5IndexOptimizeStruct( Fts5Structure *pStruct ){ Fts5Structure *pNew = 0; - sqlite3_int64 nByte = sizeof(Fts5Structure); + sqlite3_int64 nByte = SZ_FTS5STRUCTURE(1); int nSeg = pStruct->nSegment; int i; @@ -244423,7 +253013,8 @@ static Fts5Structure *fts5IndexOptimizeStruct( assert( pStruct->aLevel[i].nMerge<=nThis ); } - nByte += (pStruct->nLevel+1) * sizeof(Fts5StructureLevel); + nByte += (((i64)pStruct->nLevel)+1) * sizeof(Fts5StructureLevel); + assert( nByte==(i64)SZ_FTS5STRUCTURE(pStruct->nLevel+2) ); pNew = (Fts5Structure*)sqlite3Fts5MallocZero(&p->rc, nByte); if( pNew ){ @@ -244792,7 +253383,7 @@ static void fts5MergePrefixLists( } if( pHead==0 || pHead->pNext==0 ){ - p->rc = FTS5_CORRUPT; + FTS5_CORRUPT_IDX(p); break; } @@ -244829,7 +253420,7 @@ static void fts5MergePrefixLists( assert_nc( tmp.n+nTail<=nTmp ); assert( tmp.n+nTail<=nTmp+nMerge*10 ); if( tmp.n+nTail>nTmp-FTS5_DATA_ZERO_PADDING ){ - if( p->rc==SQLITE_OK ) p->rc = FTS5_CORRUPT; + if( p->rc==SQLITE_OK ) FTS5_CORRUPT_IDX(p); break; } fts5BufferSafeAppendVarint(&out, (tmp.n+nTail) * 2); @@ -244864,6 +253455,387 @@ static void fts5MergePrefixLists( *p1 = out; } + +/* +** Iterate through a range of entries in the FTS index, invoking the xVisit +** callback for each of them. +** +** Parameter pToken points to an nToken buffer containing an FTS index term +** (i.e. a document term with the preceding 1 byte index identifier - +** FTS5_MAIN_PREFIX or similar). If bPrefix is true, then the call visits +** all entries for terms that have pToken/nToken as a prefix. If bPrefix +** is false, then only entries with pToken/nToken as the entire key are +** visited. +** +** If the current table is a tokendata=1 table, then if bPrefix is true then +** each index term is treated separately. However, if bPrefix is false, then +** all index terms corresponding to pToken/nToken are collapsed into a single +** term before the callback is invoked. +** +** The callback invoked for each entry visited is specified by paramter xVisit. +** Each time it is invoked, it is passed a pointer to the Fts5Index object, +** a copy of the 7th paramter to this function (pCtx) and a pointer to the +** iterator that indicates the current entry. If the current entry is the +** first with a new term (i.e. different from that of the previous entry, +** including the very first term), then the final two parameters are passed +** a pointer to the term and its size in bytes, respectively. If the current +** entry is not the first associated with its term, these two parameters +** are passed 0. +** +** If parameter pColset is not NULL, then it is used to filter entries before +** the callback is invoked. +*/ +static int fts5VisitEntries( + Fts5Index *p, /* Fts5 index object */ + Fts5Colset *pColset, /* Columns filter to apply, or NULL */ + u8 *pToken, /* Buffer containing token */ + int nToken, /* Size of buffer pToken in bytes */ + int bPrefix, /* True for a prefix scan */ + void (*xVisit)(Fts5Index*, void *pCtx, Fts5Iter *pIter, const u8*, int), + void *pCtx /* Passed as second argument to xVisit() */ +){ + const int flags = (bPrefix ? FTS5INDEX_QUERY_SCAN : 0) + | FTS5INDEX_QUERY_SKIPEMPTY + | FTS5INDEX_QUERY_NOOUTPUT; + Fts5Iter *p1 = 0; /* Iterator used to gather data from index */ + int bNewTerm = 1; + Fts5Structure *pStruct = fts5StructureRead(p); + + fts5MultiIterNew(p, pStruct, flags, pColset, pToken, nToken, -1, 0, &p1); + fts5IterSetOutputCb(&p->rc, p1); + for( /* no-op */ ; + fts5MultiIterEof(p, p1)==0; + fts5MultiIterNext2(p, p1, &bNewTerm) + ){ + Fts5SegIter *pSeg = &p1->aSeg[ p1->aFirst[1].iFirst ]; + int nNew = 0; + const u8 *pNew = 0; + + p1->xSetOutputs(p1, pSeg); + if( p->rc ) break; + + if( bNewTerm ){ + nNew = pSeg->term.n; + pNew = pSeg->term.p; + if( nNewrc; +} + + +/* +** Usually, a tokendata=1 iterator (struct Fts5TokenDataIter) accumulates an +** array of these for each row it visits (so all iRowid fields are the same). +** Or, for an iterator used by an "ORDER BY rank" query, it accumulates an +** array of these for the entire query (in which case iRowid fields may take +** a variety of values). +** +** Each instance in the array indicates the iterator (and therefore term) +** associated with position iPos of rowid iRowid. This is used by the +** xInstToken() API. +** +** iRowid: +** Rowid for the current entry. +** +** iPos: +** Position of current entry within row. In the usual ((iCol<<32)+iOff) +** format (e.g. see macros FTS5_POS2COLUMN() and FTS5_POS2OFFSET()). +** +** iIter: +** If the Fts5TokenDataIter iterator that the entry is part of is +** actually an iterator (i.e. with nIter>0, not just a container for +** Fts5TokenDataMap structures), then this variable is an index into +** the apIter[] array. The corresponding term is that which the iterator +** at apIter[iIter] currently points to. +** +** Or, if the Fts5TokenDataIter iterator is just a container object +** (nIter==0), then iIter is an index into the term.p[] buffer where +** the term is stored. +** +** nByte: +** In the case where iIter is an index into term.p[], this variable +** is the size of the term in bytes. If iIter is an index into apIter[], +** this variable is unused. +*/ +struct Fts5TokenDataMap { + i64 iRowid; /* Row this token is located in */ + i64 iPos; /* Position of token */ + int iIter; /* Iterator token was read from */ + int nByte; /* Length of token in bytes (or 0) */ +}; + +/* +** An object used to supplement Fts5Iter for tokendata=1 iterators. +** +** This object serves two purposes. The first is as a container for an array +** of Fts5TokenDataMap structures, which are used to find the token required +** when the xInstToken() API is used. This is done by the nMapAlloc, nMap and +** aMap[] variables. +*/ +struct Fts5TokenDataIter { + int nMapAlloc; /* Allocated size of aMap[] in entries */ + int nMap; /* Number of valid entries in aMap[] */ + Fts5TokenDataMap *aMap; /* Array of (rowid+pos -> token) mappings */ + + /* The following are used for prefix-queries only. */ + Fts5Buffer terms; + + /* The following are used for other full-token tokendata queries only. */ + int nIter; + int nIterAlloc; + Fts5PoslistReader *aPoslistReader; + int *aPoslistToIter; + Fts5Iter *apIter[FLEXARRAY]; +}; + +/* Size in bytes of an Fts5TokenDataIter object holding up to N iterators */ +#define SZ_FTS5TOKENDATAITER(N) \ + (offsetof(Fts5TokenDataIter,apIter) + (N)*sizeof(Fts5Iter)) + +/* +** The two input arrays - a1[] and a2[] - are in sorted order. This function +** merges the two arrays together and writes the result to output array +** aOut[]. aOut[] is guaranteed to be large enough to hold the result. +** +** Duplicate entries are copied into the output. So the size of the output +** array is always (n1+n2) entries. +*/ +static void fts5TokendataMerge( + Fts5TokenDataMap *a1, int n1, /* Input array 1 */ + Fts5TokenDataMap *a2, int n2, /* Input array 2 */ + Fts5TokenDataMap *aOut /* Output array */ +){ + int i1 = 0; + int i2 = 0; + + assert( n1>=0 && n2>=0 ); + while( i1=n2 || (i1rc==SQLITE_OK ){ + if( pT->nMap==pT->nMapAlloc ){ + int nNew = pT->nMapAlloc ? pT->nMapAlloc*2 : 64; + int nAlloc = nNew * sizeof(Fts5TokenDataMap); + Fts5TokenDataMap *aNew; + + aNew = (Fts5TokenDataMap*)sqlite3_realloc(pT->aMap, nAlloc); + if( aNew==0 ){ + p->rc = SQLITE_NOMEM; + return; + } + + pT->aMap = aNew; + pT->nMapAlloc = nNew; + } + + pT->aMap[pT->nMap].iRowid = iRowid; + pT->aMap[pT->nMap].iPos = iPos; + pT->aMap[pT->nMap].iIter = iIter; + pT->aMap[pT->nMap].nByte = nByte; + pT->nMap++; + } +} + +/* +** Sort the contents of the pT->aMap[] array. +** +** The sorting algorithm requires a malloc(). If this fails, an error code +** is left in Fts5Index.rc before returning. +*/ +static void fts5TokendataIterSortMap(Fts5Index *p, Fts5TokenDataIter *pT){ + Fts5TokenDataMap *aTmp = 0; + int nByte = pT->nMap * sizeof(Fts5TokenDataMap); + + aTmp = (Fts5TokenDataMap*)sqlite3Fts5MallocZero(&p->rc, nByte); + if( aTmp ){ + Fts5TokenDataMap *a1 = pT->aMap; + Fts5TokenDataMap *a2 = aTmp; + i64 nHalf; + + for(nHalf=1; nHalfnMap; nHalf=nHalf*2){ + int i1; + for(i1=0; i1nMap; i1+=(nHalf*2)){ + int n1 = MIN(nHalf, pT->nMap-i1); + int n2 = MIN(nHalf, pT->nMap-i1-n1); + fts5TokendataMerge(&a1[i1], n1, &a1[i1+n1], n2, &a2[i1]); + } + SWAPVAL(Fts5TokenDataMap*, a1, a2); + } + + if( a1!=pT->aMap ){ + memcpy(pT->aMap, a1, pT->nMap*sizeof(Fts5TokenDataMap)); + } + sqlite3_free(aTmp); + +#ifdef SQLITE_DEBUG + { + int ii; + for(ii=1; iinMap; ii++){ + Fts5TokenDataMap *p1 = &pT->aMap[ii-1]; + Fts5TokenDataMap *p2 = &pT->aMap[ii]; + assert( p1->iRowidiRowid + || (p1->iRowid==p2->iRowid && p1->iPos<=p2->iPos) + ); + } + } +#endif + } +} + +/* +** Delete an Fts5TokenDataIter structure and its contents. +*/ +static void fts5TokendataIterDelete(Fts5TokenDataIter *pSet){ + if( pSet ){ + int ii; + for(ii=0; iinIter; ii++){ + fts5MultiIterFree(pSet->apIter[ii]); + } + fts5BufferFree(&pSet->terms); + sqlite3_free(pSet->aPoslistReader); + sqlite3_free(pSet->aMap); + sqlite3_free(pSet); + } +} + + +/* +** fts5VisitEntries() context object used by fts5SetupPrefixIterTokendata() +** to pass data to prefixIterSetupTokendataCb(). +*/ +typedef struct TokendataSetupCtx TokendataSetupCtx; +struct TokendataSetupCtx { + Fts5TokenDataIter *pT; /* Object being populated with mappings */ + int iTermOff; /* Offset of current term in terms.p[] */ + int nTermByte; /* Size of current term in bytes */ +}; + +/* +** fts5VisitEntries() callback used by fts5SetupPrefixIterTokendata(). This +** callback adds an entry to the Fts5TokenDataIter.aMap[] array for each +** position in the current position-list. It doesn't matter that some of +** these may be out of order - they will be sorted later. +*/ +static void prefixIterSetupTokendataCb( + Fts5Index *p, + void *pCtx, + Fts5Iter *p1, + const u8 *pNew, + int nNew +){ + TokendataSetupCtx *pSetup = (TokendataSetupCtx*)pCtx; + int iPosOff = 0; + i64 iPos = 0; + + if( pNew ){ + pSetup->nTermByte = nNew-1; + pSetup->iTermOff = pSetup->pT->terms.n; + fts5BufferAppendBlob(&p->rc, &pSetup->pT->terms, nNew-1, pNew+1); + } + + while( 0==sqlite3Fts5PoslistNext64( + p1->base.pData, p1->base.nData, &iPosOff, &iPos + ) ){ + fts5TokendataIterAppendMap(p, + pSetup->pT, pSetup->iTermOff, pSetup->nTermByte, p1->base.iRowid, iPos + ); + } +} + + +/* +** Context object passed by fts5SetupPrefixIter() to fts5VisitEntries(). +*/ +typedef struct PrefixSetupCtx PrefixSetupCtx; +struct PrefixSetupCtx { + void (*xMerge)(Fts5Index*, Fts5Buffer*, int, Fts5Buffer*); + void (*xAppend)(Fts5Index*, u64, Fts5Iter*, Fts5Buffer*); + i64 iLastRowid; + int nMerge; + Fts5Buffer *aBuf; + int nBuf; + Fts5Buffer doclist; + TokendataSetupCtx *pTokendata; +}; + +/* +** fts5VisitEntries() callback used by fts5SetupPrefixIter() +*/ +static void prefixIterSetupCb( + Fts5Index *p, + void *pCtx, + Fts5Iter *p1, + const u8 *pNew, + int nNew +){ + PrefixSetupCtx *pSetup = (PrefixSetupCtx*)pCtx; + const int nMerge = pSetup->nMerge; + + if( p1->base.nData>0 ){ + if( p1->base.iRowid<=pSetup->iLastRowid && pSetup->doclist.n>0 ){ + int i; + for(i=0; p->rc==SQLITE_OK && pSetup->doclist.n; i++){ + int i1 = i*nMerge; + int iStore; + assert( i1+nMerge<=pSetup->nBuf ); + for(iStore=i1; iStoreaBuf[iStore].n==0 ){ + fts5BufferSwap(&pSetup->doclist, &pSetup->aBuf[iStore]); + fts5BufferZero(&pSetup->doclist); + break; + } + } + if( iStore==i1+nMerge ){ + pSetup->xMerge(p, &pSetup->doclist, nMerge, &pSetup->aBuf[i1]); + for(iStore=i1; iStoreaBuf[iStore]); + } + } + } + pSetup->iLastRowid = 0; + } + + pSetup->xAppend( + p, (u64)p1->base.iRowid-(u64)pSetup->iLastRowid, p1, &pSetup->doclist + ); + pSetup->iLastRowid = p1->base.iRowid; + } + + if( pSetup->pTokendata ){ + prefixIterSetupTokendataCb(p, (void*)pSetup->pTokendata, p1, pNew, nNew); + } +} + static void fts5SetupPrefixIter( Fts5Index *p, /* Index to read from */ int bDesc, /* True for "ORDER BY rowid DESC" */ @@ -244874,38 +253846,41 @@ static void fts5SetupPrefixIter( Fts5Iter **ppIter /* OUT: New iterator */ ){ Fts5Structure *pStruct; - Fts5Buffer *aBuf; - int nBuf = 32; - int nMerge = 1; + PrefixSetupCtx s; + TokendataSetupCtx s2; + + memset(&s, 0, sizeof(s)); + memset(&s2, 0, sizeof(s2)); + + s.nMerge = 1; + s.iLastRowid = 0; + s.nBuf = 32; + if( iIdx==0 + && p->pConfig->eDetail==FTS5_DETAIL_FULL + && p->pConfig->bPrefixInsttoken + ){ + s.pTokendata = &s2; + s2.pT = (Fts5TokenDataIter*)fts5IdxMalloc(p, SZ_FTS5TOKENDATAITER(1)); + } - void (*xMerge)(Fts5Index*, Fts5Buffer*, int, Fts5Buffer*); - void (*xAppend)(Fts5Index*, u64, Fts5Iter*, Fts5Buffer*); if( p->pConfig->eDetail==FTS5_DETAIL_NONE ){ - xMerge = fts5MergeRowidLists; - xAppend = fts5AppendRowid; + s.xMerge = fts5MergeRowidLists; + s.xAppend = fts5AppendRowid; }else{ - nMerge = FTS5_MERGE_NLIST-1; - nBuf = nMerge*8; /* Sufficient to merge (16^8)==(2^32) lists */ - xMerge = fts5MergePrefixLists; - xAppend = fts5AppendPoslist; + s.nMerge = FTS5_MERGE_NLIST-1; + s.nBuf = s.nMerge*8; /* Sufficient to merge (16^8)==(2^32) lists */ + s.xMerge = fts5MergePrefixLists; + s.xAppend = fts5AppendPoslist; } - aBuf = (Fts5Buffer*)fts5IdxMalloc(p, sizeof(Fts5Buffer)*nBuf); + s.aBuf = (Fts5Buffer*)fts5IdxMalloc(p, sizeof(Fts5Buffer)*s.nBuf); pStruct = fts5StructureRead(p); - assert( p->rc!=SQLITE_OK || (aBuf && pStruct) ); + assert( p->rc!=SQLITE_OK || (s.aBuf && pStruct) ); if( p->rc==SQLITE_OK ){ - const int flags = FTS5INDEX_QUERY_SCAN - | FTS5INDEX_QUERY_SKIPEMPTY - | FTS5INDEX_QUERY_NOOUTPUT; + void *pCtx = (void*)&s; int i; - i64 iLastRowid = 0; - Fts5Iter *p1 = 0; /* Iterator used to gather data from index */ Fts5Data *pData; - Fts5Buffer doclist; - int bNewTerm = 1; - - memset(&doclist, 0, sizeof(doclist)); /* If iIdx is non-zero, then it is the number of a prefix-index for ** prefixes 1 character longer than the prefix being queried for. That @@ -244913,94 +253888,46 @@ static void fts5SetupPrefixIter( ** corresponding to the prefix itself. That one is extracted from the ** main term index here. */ if( iIdx!=0 ){ - int dummy = 0; - const int f2 = FTS5INDEX_QUERY_SKIPEMPTY|FTS5INDEX_QUERY_NOOUTPUT; pToken[0] = FTS5_MAIN_PREFIX; - fts5MultiIterNew(p, pStruct, f2, pColset, pToken, nToken, -1, 0, &p1); - fts5IterSetOutputCb(&p->rc, p1); - for(; - fts5MultiIterEof(p, p1)==0; - fts5MultiIterNext2(p, p1, &dummy) - ){ - Fts5SegIter *pSeg = &p1->aSeg[ p1->aFirst[1].iFirst ]; - p1->xSetOutputs(p1, pSeg); - if( p1->base.nData ){ - xAppend(p, (u64)p1->base.iRowid-(u64)iLastRowid, p1, &doclist); - iLastRowid = p1->base.iRowid; - } - } - fts5MultiIterFree(p1); + fts5VisitEntries(p, pColset, pToken, nToken, 0, prefixIterSetupCb, pCtx); } pToken[0] = FTS5_MAIN_PREFIX + iIdx; - fts5MultiIterNew(p, pStruct, flags, pColset, pToken, nToken, -1, 0, &p1); - fts5IterSetOutputCb(&p->rc, p1); + fts5VisitEntries(p, pColset, pToken, nToken, 1, prefixIterSetupCb, pCtx); - for( /* no-op */ ; - fts5MultiIterEof(p, p1)==0; - fts5MultiIterNext2(p, p1, &bNewTerm) - ){ - Fts5SegIter *pSeg = &p1->aSeg[ p1->aFirst[1].iFirst ]; - int nTerm = pSeg->term.n; - const u8 *pTerm = pSeg->term.p; - p1->xSetOutputs(p1, pSeg); - - assert_nc( memcmp(pToken, pTerm, MIN(nToken, nTerm))<=0 ); - if( bNewTerm ){ - if( nTermbase.nData==0 ) continue; - if( p1->base.iRowid<=iLastRowid && doclist.n>0 ){ - for(i=0; p->rc==SQLITE_OK && doclist.n; i++){ - int i1 = i*nMerge; - int iStore; - assert( i1+nMerge<=nBuf ); - for(iStore=i1; iStorebase.iRowid-(u64)iLastRowid, p1, &doclist); - iLastRowid = p1->base.iRowid; - } - - assert( (nBuf%nMerge)==0 ); - for(i=0; irc==SQLITE_OK ){ - xMerge(p, &doclist, nMerge, &aBuf[i]); + s.xMerge(p, &s.doclist, s.nMerge, &s.aBuf[i]); } - for(iFree=i; iFreerc!=SQLITE_OK ); if( pData ){ pData->p = (u8*)&pData[1]; - pData->nn = pData->szLeaf = doclist.n; - if( doclist.n ) memcpy(pData->p, doclist.p, doclist.n); + pData->nn = pData->szLeaf = s.doclist.n; + if( s.doclist.n ) memcpy(pData->p, s.doclist.p, s.doclist.n); fts5MultiIterNew2(p, pData, bDesc, ppIter); } - fts5BufferFree(&doclist); + + assert( (*ppIter)!=0 || p->rc!=SQLITE_OK ); + if( p->rc==SQLITE_OK && s.pTokendata ){ + fts5TokendataIterSortMap(p, s2.pT); + (*ppIter)->pTokenDataIter = s2.pT; + s2.pT = 0; + } } + fts5TokendataIterDelete(s2.pT); + fts5BufferFree(&s.doclist); fts5StructureRelease(pStruct); - sqlite3_free(aBuf); + sqlite3_free(s.aBuf); } @@ -245038,7 +253965,7 @@ static int sqlite3Fts5IndexBeginWrite(Fts5Index *p, int bDelete, i64 iRowid){ static int sqlite3Fts5IndexSync(Fts5Index *p){ assert( p->rc==SQLITE_OK ); fts5IndexFlush(p); - sqlite3Fts5IndexCloseReader(p); + fts5IndexCloseReader(p); return fts5IndexReturn(p); } @@ -245049,11 +253976,10 @@ static int sqlite3Fts5IndexSync(Fts5Index *p){ ** records must be invalidated. */ static int sqlite3Fts5IndexRollback(Fts5Index *p){ - sqlite3Fts5IndexCloseReader(p); + fts5IndexCloseReader(p); fts5IndexDiscardData(p); fts5StructureInvalidate(p); - /* assert( p->rc==SQLITE_OK ); */ - return SQLITE_OK; + return fts5IndexReturn(p); } /* @@ -245062,15 +253988,20 @@ static int sqlite3Fts5IndexRollback(Fts5Index *p){ ** and the initial version of the "averages" record (a zero-byte blob). */ static int sqlite3Fts5IndexReinit(Fts5Index *p){ - Fts5Structure s; + Fts5Structure *pTmp; + union { + Fts5Structure sFts; + u8 tmpSpace[SZ_FTS5STRUCTURE(1)]; + } uFts; fts5StructureInvalidate(p); fts5IndexDiscardData(p); - memset(&s, 0, sizeof(Fts5Structure)); + pTmp = &uFts.sFts; + memset(uFts.tmpSpace, 0, sizeof(uFts.tmpSpace)); if( p->pConfig->bContentlessDelete ){ - s.nOriginCntr = 1; + pTmp->nOriginCntr = 1; } fts5DataWrite(p, FTS5_AVERAGES_ROWID, (const u8*)"", 0); - fts5StructureWrite(p, &s); + fts5StructureWrite(p, pTmp); return fts5IndexReturn(p); } @@ -245254,37 +254185,15 @@ static void fts5SegIterSetEOF(Fts5SegIter *pSeg){ pSeg->pLeaf = 0; } -/* -** Usually, a tokendata=1 iterator (struct Fts5TokenDataIter) accumulates an -** array of these for each row it visits. Or, for an iterator used by an -** "ORDER BY rank" query, it accumulates an array of these for the entire -** query. -** -** Each instance in the array indicates the iterator (and therefore term) -** associated with position iPos of rowid iRowid. This is used by the -** xInstToken() API. -*/ -struct Fts5TokenDataMap { - i64 iRowid; /* Row this token is located in */ - i64 iPos; /* Position of token */ - int iIter; /* Iterator token was read from */ -}; - -/* -** An object used to supplement Fts5Iter for tokendata=1 iterators. -*/ -struct Fts5TokenDataIter { - int nIter; - int nIterAlloc; - - int nMap; - int nMapAlloc; - Fts5TokenDataMap *aMap; - - Fts5PoslistReader *aPoslistReader; - int *aPoslistToIter; - Fts5Iter *apIter[1]; -}; +static void fts5IterClose(Fts5IndexIter *pIndexIter){ + if( pIndexIter ){ + Fts5Iter *pIter = (Fts5Iter*)pIndexIter; + Fts5Index *pIndex = pIter->pIndex; + fts5TokendataIterDelete(pIter->pTokenDataIter); + fts5MultiIterFree(pIter); + fts5IndexCloseReader(pIndex); + } +} /* ** This function appends iterator pAppend to Fts5TokenDataIter pIn and @@ -245300,7 +254209,7 @@ static Fts5TokenDataIter *fts5AppendTokendataIter( if( p->rc==SQLITE_OK ){ if( pIn==0 || pIn->nIter==pIn->nIterAlloc ){ int nAlloc = pIn ? pIn->nIterAlloc*2 : 16; - int nByte = nAlloc * sizeof(Fts5Iter*) + sizeof(Fts5TokenDataIter); + int nByte = SZ_FTS5TOKENDATAITER(nAlloc+1); Fts5TokenDataIter *pNew = (Fts5TokenDataIter*)sqlite3_realloc(pIn, nByte); if( pNew==0 ){ @@ -245313,7 +254222,7 @@ static Fts5TokenDataIter *fts5AppendTokendataIter( } } if( p->rc ){ - sqlite3Fts5IterClose((Fts5IndexIter*)pAppend); + fts5IterClose((Fts5IndexIter*)pAppend); }else{ pRet->apIter[pRet->nIter++] = pAppend; } @@ -245322,54 +254231,6 @@ static Fts5TokenDataIter *fts5AppendTokendataIter( return pRet; } -/* -** Delete an Fts5TokenDataIter structure and its contents. -*/ -static void fts5TokendataIterDelete(Fts5TokenDataIter *pSet){ - if( pSet ){ - int ii; - for(ii=0; iinIter; ii++){ - fts5MultiIterFree(pSet->apIter[ii]); - } - sqlite3_free(pSet->aPoslistReader); - sqlite3_free(pSet->aMap); - sqlite3_free(pSet); - } -} - -/* -** Append a mapping to the token-map belonging to object pT. -*/ -static void fts5TokendataIterAppendMap( - Fts5Index *p, - Fts5TokenDataIter *pT, - int iIter, - i64 iRowid, - i64 iPos -){ - if( p->rc==SQLITE_OK ){ - if( pT->nMap==pT->nMapAlloc ){ - int nNew = pT->nMapAlloc ? pT->nMapAlloc*2 : 64; - int nByte = nNew * sizeof(Fts5TokenDataMap); - Fts5TokenDataMap *aNew; - - aNew = (Fts5TokenDataMap*)sqlite3_realloc(pT->aMap, nByte); - if( aNew==0 ){ - p->rc = SQLITE_NOMEM; - return; - } - - pT->aMap = aNew; - pT->nMapAlloc = nNew; - } - - pT->aMap[pT->nMap].iRowid = iRowid; - pT->aMap[pT->nMap].iPos = iPos; - pT->aMap[pT->nMap].iIter = iIter; - pT->nMap++; - } -} - /* ** The iterator passed as the only argument must be a tokendata=1 iterator ** (pIter->pTokenDataIter!=0). This function sets the iterator output @@ -245410,7 +254271,7 @@ static void fts5IterSetOutputsTokendata(Fts5Iter *pIter){ pIter->base.iRowid = iRowid; if( nHit==1 && eDetail==FTS5_DETAIL_FULL ){ - fts5TokendataIterAppendMap(pIter->pIndex, pT, iMin, iRowid, -1); + fts5TokendataIterAppendMap(pIter->pIndex, pT, iMin, 0, iRowid, -1); }else if( nHit>1 && eDetail!=FTS5_DETAIL_NONE ){ int nReader = 0; @@ -245574,7 +254435,7 @@ static Fts5Iter *fts5SetupTokendataIter( fts5BufferSet(&p->rc, &bSeek, nToken, pToken); } if( p->rc ){ - sqlite3Fts5IterClose((Fts5IndexIter*)pNew); + fts5IterClose((Fts5IndexIter*)pNew); break; } @@ -245639,7 +254500,7 @@ static Fts5Iter *fts5SetupTokendataIter( ** not point to any terms that match the query. So delete it and break ** out of the loop - all required iterators have been collected. */ if( pSmall==0 ){ - sqlite3Fts5IterClose((Fts5IndexIter*)pNew); + fts5IterClose((Fts5IndexIter*)pNew); break; } @@ -245663,6 +254524,7 @@ static Fts5Iter *fts5SetupTokendataIter( pRet = fts5MultiIterAlloc(p, 0); } if( pRet ){ + pRet->nSeg = 0; pRet->pTokenDataIter = pSet; if( pSet ){ fts5IterSetOutputsTokendata(pRet); @@ -245678,7 +254540,6 @@ static Fts5Iter *fts5SetupTokendataIter( return pRet; } - /* ** Open a new iterator to iterate though all rowid that match the ** specified token or token prefix. @@ -245701,8 +254562,14 @@ static int sqlite3Fts5IndexQuery( int iIdx = 0; /* Index to search */ int iPrefixIdx = 0; /* +1 prefix index */ int bTokendata = pConfig->bTokendata; + assert( buf.p!=0 ); if( nToken>0 ) memcpy(&buf.p[1], pToken, nToken); + /* The NOTOKENDATA flag is set when each token in a tokendata=1 table + ** should be treated individually, instead of merging all those with + ** a common prefix into a single entry. This is used, for example, by + ** queries performed as part of an integrity-check, or by the fts5vocab + ** module. */ if( flags & (FTS5INDEX_QUERY_NOTOKENDATA|FTS5INDEX_QUERY_SCAN) ){ bTokendata = 0; } @@ -245733,7 +254600,7 @@ static int sqlite3Fts5IndexQuery( } if( bTokendata && iIdx==0 ){ - buf.p[0] = '0'; + buf.p[0] = FTS5_MAIN_PREFIX; pRet = fts5SetupTokendataIter(p, buf.p, nToken+1, pColset); }else if( iIdx<=pConfig->nPrefix ){ /* Straight index lookup */ @@ -245746,7 +254613,7 @@ static int sqlite3Fts5IndexQuery( fts5StructureRelease(pStruct); } }else{ - /* Scan multiple terms in the main index */ + /* Scan multiple terms in the main index for a prefix query. */ int bDesc = (flags & FTS5INDEX_QUERY_DESC)!=0; fts5SetupPrefixIter(p, bDesc, iPrefixIdx, buf.p, nToken+1, pColset,&pRet); if( pRet==0 ){ @@ -245762,9 +254629,9 @@ static int sqlite3Fts5IndexQuery( } if( p->rc ){ - sqlite3Fts5IterClose((Fts5IndexIter*)pRet); + fts5IterClose((Fts5IndexIter*)pRet); pRet = 0; - sqlite3Fts5IndexCloseReader(p); + fts5IndexCloseReader(p); } *ppIter = (Fts5IndexIter*)pRet; @@ -245782,7 +254649,8 @@ static int sqlite3Fts5IndexQuery( static int sqlite3Fts5IterNext(Fts5IndexIter *pIndexIter){ Fts5Iter *pIter = (Fts5Iter*)pIndexIter; assert( pIter->pIndex->rc==SQLITE_OK ); - if( pIter->pTokenDataIter ){ + if( pIter->nSeg==0 ){ + assert( pIter->pTokenDataIter ); fts5TokendataIterNext(pIter, 0, 0); }else{ fts5MultiIterNext(pIter->pIndex, pIter, 0, 0); @@ -245819,7 +254687,8 @@ static int sqlite3Fts5IterNextScan(Fts5IndexIter *pIndexIter){ */ static int sqlite3Fts5IterNextFrom(Fts5IndexIter *pIndexIter, i64 iMatch){ Fts5Iter *pIter = (Fts5Iter*)pIndexIter; - if( pIter->pTokenDataIter ){ + if( pIter->nSeg==0 ){ + assert( pIter->pTokenDataIter ); fts5TokendataIterNext(pIter, 1, iMatch); }else{ fts5MultiIterNextFrom(pIter->pIndex, pIter, iMatch); @@ -245838,14 +254707,62 @@ static const char *sqlite3Fts5IterTerm(Fts5IndexIter *pIndexIter, int *pn){ return (z ? &z[1] : 0); } +/* +** pIter is a prefix query. This function populates pIter->pTokenDataIter +** with an Fts5TokenDataIter object containing mappings for all rows +** matched by the query. +*/ +static int fts5SetupPrefixIterTokendata( + Fts5Iter *pIter, + const char *pToken, /* Token prefix to search for */ + int nToken /* Size of pToken in bytes */ +){ + Fts5Index *p = pIter->pIndex; + Fts5Buffer token = {0, 0, 0}; + TokendataSetupCtx ctx; + + memset(&ctx, 0, sizeof(ctx)); + + fts5BufferGrow(&p->rc, &token, nToken+1); + assert( token.p!=0 || p->rc!=SQLITE_OK ); + ctx.pT = (Fts5TokenDataIter*)sqlite3Fts5MallocZero(&p->rc, + SZ_FTS5TOKENDATAITER(1)); + + if( p->rc==SQLITE_OK ){ + + /* Fill in the token prefix to search for */ + token.p[0] = FTS5_MAIN_PREFIX; + memcpy(&token.p[1], pToken, nToken); + token.n = nToken+1; + + fts5VisitEntries( + p, 0, token.p, token.n, 1, prefixIterSetupTokendataCb, (void*)&ctx + ); + + fts5TokendataIterSortMap(p, ctx.pT); + } + + if( p->rc==SQLITE_OK ){ + pIter->pTokenDataIter = ctx.pT; + }else{ + fts5TokendataIterDelete(ctx.pT); + } + fts5BufferFree(&token); + + return fts5IndexReturn(p); +} + /* ** This is used by xInstToken() to access the token at offset iOff, column ** iCol of row iRowid. The token is returned via output variables *ppOut ** and *pnOut. The iterator passed as the first argument must be a tokendata=1 ** iterator (pIter->pTokenDataIter!=0). +** +** pToken/nToken: */ static int sqlite3Fts5IterToken( Fts5IndexIter *pIndexIter, + const char *pToken, int nToken, i64 iRowid, int iCol, int iOff, @@ -245853,13 +254770,22 @@ static int sqlite3Fts5IterToken( ){ Fts5Iter *pIter = (Fts5Iter*)pIndexIter; Fts5TokenDataIter *pT = pIter->pTokenDataIter; - Fts5TokenDataMap *aMap = pT->aMap; i64 iPos = (((i64)iCol)<<32) + iOff; - + Fts5TokenDataMap *aMap = 0; int i1 = 0; - int i2 = pT->nMap; + int i2 = 0; int iTest = 0; + assert( pT || (pToken && pIter->nSeg>0) ); + if( pT==0 ){ + int rc = fts5SetupPrefixIterTokendata(pIter, pToken, nToken); + if( rc!=SQLITE_OK ) return rc; + pT = pIter->pTokenDataIter; + } + + i2 = pT->nMap; + aMap = pT->aMap; + while( i2>i1 ){ iTest = (i1 + i2) / 2; @@ -245882,9 +254808,15 @@ static int sqlite3Fts5IterToken( } if( i2>i1 ){ - Fts5Iter *pMap = pT->apIter[aMap[iTest].iIter]; - *ppOut = (const char*)pMap->aSeg[0].term.p+1; - *pnOut = pMap->aSeg[0].term.n-1; + if( pIter->nSeg==0 ){ + Fts5Iter *pMap = pT->apIter[aMap[iTest].iIter]; + *ppOut = (const char*)pMap->aSeg[0].term.p+1; + *pnOut = pMap->aSeg[0].term.n-1; + }else{ + Fts5TokenDataMap *p = &aMap[iTest]; + *ppOut = (const char*)&pT->terms.p[p->iIter]; + *pnOut = aMap[iTest].nByte; + } } return SQLITE_OK; @@ -245896,7 +254828,9 @@ static int sqlite3Fts5IterToken( */ static void sqlite3Fts5IndexIterClearTokendata(Fts5IndexIter *pIndexIter){ Fts5Iter *pIter = (Fts5Iter*)pIndexIter; - if( pIter && pIter->pTokenDataIter ){ + if( pIter && pIter->pTokenDataIter + && (pIter->nSeg==0 || pIter->pIndex->pConfig->eDetail!=FTS5_DETAIL_FULL) + ){ pIter->pTokenDataIter->nMap = 0; } } @@ -245916,17 +254850,30 @@ static int sqlite3Fts5IndexIterWriteTokendata( Fts5Iter *pIter = (Fts5Iter*)pIndexIter; Fts5TokenDataIter *pT = pIter->pTokenDataIter; Fts5Index *p = pIter->pIndex; - int ii; + i64 iPos = (((i64)iCol)<<32) + iOff; assert( p->pConfig->eDetail!=FTS5_DETAIL_FULL ); - assert( pIter->pTokenDataIter ); - - for(ii=0; iinIter; ii++){ - Fts5Buffer *pTerm = &pT->apIter[ii]->aSeg[0].term; - if( nToken==pTerm->n-1 && memcmp(pToken, pTerm->p+1, nToken)==0 ) break; - } - if( iinIter ){ - fts5TokendataIterAppendMap(p, pT, ii, iRowid, (((i64)iCol)<<32) + iOff); + assert( pIter->pTokenDataIter || pIter->nSeg>0 ); + if( pIter->nSeg>0 ){ + /* This is a prefix term iterator. */ + if( pT==0 ){ + pT = (Fts5TokenDataIter*)sqlite3Fts5MallocZero(&p->rc, + SZ_FTS5TOKENDATAITER(1)); + pIter->pTokenDataIter = pT; + } + if( pT ){ + fts5TokendataIterAppendMap(p, pT, pT->terms.n, nToken, iRowid, iPos); + fts5BufferAppendBlob(&p->rc, &pT->terms, nToken, (const u8*)pToken); + } + }else{ + int ii; + for(ii=0; iinIter; ii++){ + Fts5Buffer *pTerm = &pT->apIter[ii]->aSeg[0].term; + if( nToken==pTerm->n-1 && memcmp(pToken, pTerm->p+1, nToken)==0 ) break; + } + if( iinIter ){ + fts5TokendataIterAppendMap(p, pT, ii, 0, iRowid, iPos); + } } return fts5IndexReturn(p); } @@ -245936,11 +254883,9 @@ static int sqlite3Fts5IndexIterWriteTokendata( */ static void sqlite3Fts5IterClose(Fts5IndexIter *pIndexIter){ if( pIndexIter ){ - Fts5Iter *pIter = (Fts5Iter*)pIndexIter; - Fts5Index *pIndex = pIter->pIndex; - fts5TokendataIterDelete(pIter->pTokenDataIter); - fts5MultiIterFree(pIter); - sqlite3Fts5IndexCloseReader(pIndex); + Fts5Index *pIndex = ((Fts5Iter*)pIndexIter)->pIndex; + fts5IterClose(pIndexIter); + fts5IndexReturn(pIndex); } } @@ -246470,7 +255415,7 @@ static int fts5QueryCksum( rc = sqlite3Fts5IterNext(pIter); } } - sqlite3Fts5IterClose(pIter); + fts5IterClose(pIter); *pCksum = cksum; return rc; @@ -246511,19 +255456,27 @@ static int fts5TestUtf8(const char *z, int n){ /* ** This function is also purely an internal test. It does not contribute to ** FTS functionality, or even the integrity-check, in any way. +** +** This function sets output variable (*pbFail) to true if the test fails. Or +** leaves it unchanged if the test succeeds. */ static void fts5TestTerm( Fts5Index *p, Fts5Buffer *pPrev, /* Previous term */ const char *z, int n, /* Possibly new term to test */ u64 expected, - u64 *pCksum + u64 *pCksum, + int *pbFail ){ int rc = p->rc; if( pPrev->n==0 ){ fts5BufferSet(&rc, pPrev, n, (const u8*)z); }else - if( rc==SQLITE_OK && (pPrev->n!=n || memcmp(pPrev->p, z, n)) ){ + if( *pbFail==0 + && rc==SQLITE_OK + && (pPrev->n!=n || memcmp(pPrev->p, z, n)) + && (p->pHash==0 || p->pHash->nEntry==0) + ){ u64 cksum3 = *pCksum; const char *zTerm = (const char*)&pPrev->p[1]; /* term sans prefix-byte */ int nTerm = pPrev->n-1; /* Size of zTerm in bytes */ @@ -246573,7 +255526,7 @@ static void fts5TestTerm( fts5BufferSet(&rc, pPrev, n, (const u8*)z); if( rc==SQLITE_OK && cksum3!=expected ){ - rc = FTS5_CORRUPT; + *pbFail = 1; } *pCksum = cksum3; } @@ -246582,7 +255535,7 @@ static void fts5TestTerm( #else # define fts5TestDlidxReverse(x,y,z) -# define fts5TestTerm(u,v,w,x,y,z) +# define fts5TestTerm(t,u,v,w,x,y,z) #endif /* @@ -246607,14 +255560,17 @@ static void fts5IndexIntegrityCheckEmpty( for(i=iFirst; p->rc==SQLITE_OK && i<=iLast; i++){ Fts5Data *pLeaf = fts5DataRead(p, FTS5_SEGMENT_ROWID(pSeg->iSegid, i)); if( pLeaf ){ - if( !fts5LeafIsTermless(pLeaf) ) p->rc = FTS5_CORRUPT; - if( i>=iNoRowid && 0!=fts5LeafFirstRowidOff(pLeaf) ) p->rc = FTS5_CORRUPT; + if( !fts5LeafIsTermless(pLeaf) + || (i>=iNoRowid && 0!=fts5LeafFirstRowidOff(pLeaf)) + ){ + FTS5_CORRUPT_ROWID(p, FTS5_SEGMENT_ROWID(pSeg->iSegid, i)); + } } fts5DataRelease(pLeaf); } } -static void fts5IntegrityCheckPgidx(Fts5Index *p, Fts5Data *pLeaf){ +static void fts5IntegrityCheckPgidx(Fts5Index *p, i64 iRowid, Fts5Data *pLeaf){ i64 iTermOff = 0; int ii; @@ -246632,12 +255588,12 @@ static void fts5IntegrityCheckPgidx(Fts5Index *p, Fts5Data *pLeaf){ iOff = iTermOff; if( iOff>=pLeaf->szLeaf ){ - p->rc = FTS5_CORRUPT; + FTS5_CORRUPT_ROWID(p, iRowid); }else if( iTermOff==nIncr ){ int nByte; iOff += fts5GetVarint32(&pLeaf->p[iOff], nByte); if( (iOff+nByte)>pLeaf->szLeaf ){ - p->rc = FTS5_CORRUPT; + FTS5_CORRUPT_ROWID(p, iRowid); }else{ fts5BufferSet(&p->rc, &buf1, nByte, &pLeaf->p[iOff]); } @@ -246646,7 +255602,7 @@ static void fts5IntegrityCheckPgidx(Fts5Index *p, Fts5Data *pLeaf){ iOff += fts5GetVarint32(&pLeaf->p[iOff], nKeep); iOff += fts5GetVarint32(&pLeaf->p[iOff], nByte); if( nKeep>buf1.n || (iOff+nByte)>pLeaf->szLeaf ){ - p->rc = FTS5_CORRUPT; + FTS5_CORRUPT_ROWID(p, iRowid); }else{ buf1.n = nKeep; fts5BufferAppendBlob(&p->rc, &buf1, nByte, &pLeaf->p[iOff]); @@ -246654,7 +255610,7 @@ static void fts5IntegrityCheckPgidx(Fts5Index *p, Fts5Data *pLeaf){ if( p->rc==SQLITE_OK ){ res = fts5BufferCompare(&buf1, &buf2); - if( res<=0 ) p->rc = FTS5_CORRUPT; + if( res<=0 ) FTS5_CORRUPT_ROWID(p, iRowid); } } fts5BufferSet(&p->rc, &buf2, buf1.n, buf1.p); @@ -246715,7 +255671,7 @@ static void fts5IndexIntegrityCheckSegment( ** entry even if all the terms are removed from it by secure-delete ** operations. */ }else{ - p->rc = FTS5_CORRUPT; + FTS5_CORRUPT_ROWID(p, iRow); } }else{ @@ -246727,15 +255683,15 @@ static void fts5IndexIntegrityCheckSegment( iOff = fts5LeafFirstTermOff(pLeaf); iRowidOff = fts5LeafFirstRowidOff(pLeaf); if( iRowidOff>=iOff || iOff>=pLeaf->szLeaf ){ - p->rc = FTS5_CORRUPT; + FTS5_CORRUPT_ROWID(p, iRow); }else{ iOff += fts5GetVarint32(&pLeaf->p[iOff], nTerm); res = fts5Memcmp(&pLeaf->p[iOff], zIdxTerm, MIN(nTerm, nIdxTerm)); if( res==0 ) res = nTerm - nIdxTerm; - if( res<0 ) p->rc = FTS5_CORRUPT; + if( res<0 ) FTS5_CORRUPT_ROWID(p, iRow); } - fts5IntegrityCheckPgidx(p, pLeaf); + fts5IntegrityCheckPgidx(p, iRow, pLeaf); } fts5DataRelease(pLeaf); if( p->rc ) break; @@ -246765,7 +255721,7 @@ static void fts5IndexIntegrityCheckSegment( iKey = FTS5_SEGMENT_ROWID(iSegid, iPg); pLeaf = fts5DataRead(p, iKey); if( pLeaf ){ - if( fts5LeafFirstRowidOff(pLeaf)!=0 ) p->rc = FTS5_CORRUPT; + if( fts5LeafFirstRowidOff(pLeaf)!=0 ) FTS5_CORRUPT_ROWID(p, iKey); fts5DataRelease(pLeaf); } } @@ -246780,12 +255736,12 @@ static void fts5IndexIntegrityCheckSegment( int iRowidOff = fts5LeafFirstRowidOff(pLeaf); ASSERT_SZLEAF_OK(pLeaf); if( iRowidOff>=pLeaf->szLeaf ){ - p->rc = FTS5_CORRUPT; + FTS5_CORRUPT_ROWID(p, iKey); }else if( bSecureDelete==0 || iRowidOff>0 ){ i64 iDlRowid = fts5DlidxIterRowid(pDlidx); fts5GetVarint(&pLeaf->p[iRowidOff], (u64*)&iRowid); if( iRowidrc = FTS5_CORRUPT; + FTS5_CORRUPT_ROWID(p, iKey); } } fts5DataRelease(pLeaf); @@ -246837,6 +255793,7 @@ static int sqlite3Fts5IndexIntegrityCheck(Fts5Index *p, u64 cksum, int bUseCksum /* Used by extra internal tests only run if NDEBUG is not defined */ u64 cksum3 = 0; /* Checksum based on contents of indexes */ Fts5Buffer term = {0,0,0}; /* Buffer used to hold most recent term */ + int bTestFail = 0; #endif const int flags = FTS5INDEX_QUERY_NOOUTPUT; @@ -246879,7 +255836,7 @@ static int sqlite3Fts5IndexIntegrityCheck(Fts5Index *p, u64 cksum, int bUseCksum char *z = (char*)fts5MultiIterTerm(pIter, &n); /* If this is a new term, query for it. Update cksum3 with the results. */ - fts5TestTerm(p, &term, z, n, cksum2, &cksum3); + fts5TestTerm(p, &term, z, n, cksum2, &cksum3, &bTestFail); if( p->rc ) break; if( eDetail==FTS5_DETAIL_NONE ){ @@ -246897,15 +255854,26 @@ static int sqlite3Fts5IndexIntegrityCheck(Fts5Index *p, u64 cksum, int bUseCksum } } } - fts5TestTerm(p, &term, 0, 0, cksum2, &cksum3); + fts5TestTerm(p, &term, 0, 0, cksum2, &cksum3, &bTestFail); fts5MultiIterFree(pIter); - if( p->rc==SQLITE_OK && bUseCksum && cksum!=cksum2 ) p->rc = FTS5_CORRUPT; - - fts5StructureRelease(pStruct); + if( p->rc==SQLITE_OK && bUseCksum && cksum!=cksum2 ){ + p->rc = FTS5_CORRUPT; + sqlite3Fts5ConfigErrmsg(p->pConfig, + "fts5: checksum mismatch for table \"%s\"", p->pConfig->zName + ); + } #ifdef SQLITE_DEBUG + /* In SQLITE_DEBUG builds, expensive extra checks were run as part of + ** the integrity-check above. If no other errors were detected, but one + ** of these tests failed, set the result to SQLITE_CORRUPT_VTAB here. */ + if( p->rc==SQLITE_OK && bTestFail ){ + p->rc = FTS5_CORRUPT; + } fts5BufferFree(&term); #endif + + fts5StructureRelease(pStruct); fts5BufferFree(&poslist); return fts5IndexReturn(p); } @@ -246947,7 +255915,7 @@ static void fts5DecodeRowid( #if defined(SQLITE_TEST) || defined(SQLITE_FTS5_DEBUG) static void fts5DebugRowid(int *pRc, Fts5Buffer *pBuf, i64 iKey){ - int iSegid, iHeight, iPgno, bDlidx, bTomb; /* Rowid compenents */ + int iSegid, iHeight, iPgno, bDlidx, bTomb; /* Rowid components */ fts5DecodeRowid(iKey, &bTomb, &iSegid, &bDlidx, &iHeight, &iPgno); if( iSegid==0 ){ @@ -247193,7 +256161,7 @@ static void fts5DecodeFunction( ** buffer overreads even if the record is corrupt. */ n = sqlite3_value_bytes(apVal[1]); aBlob = sqlite3_value_blob(apVal[1]); - nSpace = n + FTS5_DATA_ZERO_PADDING; + nSpace = ((i64)n) + FTS5_DATA_ZERO_PADDING; a = (u8*)sqlite3Fts5MallocZero(&rc, nSpace); if( a==0 ) goto decode_out; if( n>0 ) memcpy(a, aBlob, n); @@ -247479,7 +256447,7 @@ static int fts5structConnectMethod( /* ** We must have a single struct=? constraint that will be passed through -** into the xFilter method. If there is no valid stmt=? constraint, +** into the xFilter method. If there is no valid struct=? constraint, ** then return an SQLITE_CONSTRAINT error. */ static int fts5structBestIndexMethod( @@ -247821,8 +256789,18 @@ struct Fts5Global { Fts5TokenizerModule *pTok; /* First in list of all tokenizer modules */ Fts5TokenizerModule *pDfltTok; /* Default tokenizer module */ Fts5Cursor *pCsr; /* First in list of all open cursors */ + u32 aLocaleHdr[4]; }; +/* +** Size of header on fts5_locale() values. And macro to access a buffer +** containing a copy of the header from an Fts5Config pointer. +*/ +#define FTS5_LOCALE_HDR_SIZE ((int)sizeof( ((Fts5Global*)0)->aLocaleHdr )) +#define FTS5_LOCALE_HDR(pConfig) ((const u8*)(pConfig->pGlobal->aLocaleHdr)) + +#define FTS5_INSTTOKEN_SUBTYPE 73 + /* ** Each auxiliary function registered with the FTS5 module is represented ** by an object of the following type. All such objects are stored as part @@ -247841,11 +256819,28 @@ struct Fts5Auxiliary { ** Each tokenizer module registered with the FTS5 module is represented ** by an object of the following type. All such objects are stored as part ** of the Fts5Global.pTok list. +** +** bV2Native: +** True if the tokenizer was registered using xCreateTokenizer_v2(), false +** for xCreateTokenizer(). If this variable is true, then x2 is populated +** with the routines as supplied by the caller and x1 contains synthesized +** wrapper routines. In this case the user-data pointer passed to +** x1.xCreate should be a pointer to the Fts5TokenizerModule structure, +** not a copy of pUserData. +** +** Of course, if bV2Native is false, then x1 contains the real routines and +** x2 the synthesized ones. In this case a pointer to the Fts5TokenizerModule +** object should be passed to x2.xCreate. +** +** The synthesized wrapper routines are necessary for xFindTokenizer(_v2) +** calls. */ struct Fts5TokenizerModule { char *zName; /* Name of tokenizer */ void *pUserData; /* User pointer passed to xCreate() */ - fts5_tokenizer x; /* Tokenizer functions */ + int bV2Native; /* True if v2 native tokenizer */ + fts5_tokenizer x1; /* Tokenizer functions */ + fts5_tokenizer_v2 x2; /* V2 tokenizer functions */ void (*xDestroy)(void*); /* Destructor function */ Fts5TokenizerModule *pNext; /* Next registered tokenizer module */ }; @@ -247881,9 +256876,11 @@ struct Fts5Sorter { i64 iRowid; /* Current rowid */ const u8 *aPoslist; /* Position lists for current row */ int nIdx; /* Number of entries in aIdx[] */ - int aIdx[1]; /* Offsets into aPoslist for current row */ + int aIdx[FLEXARRAY]; /* Offsets into aPoslist for current row */ }; +/* Size (int bytes) of an Fts5Sorter object with N indexes */ +#define SZ_FTS5SORTER(N) (offsetof(Fts5Sorter,nIdx)+((N+2)/2)*sizeof(i64)) /* ** Virtual-table cursor object. @@ -247933,7 +256930,7 @@ struct Fts5Cursor { Fts5Auxiliary *pAux; /* Currently executing extension function */ Fts5Auxdata *pAuxdata; /* First in linked list of saved aux-data */ - /* Cache used by auxiliary functions xInst() and xInstCount() */ + /* Cache used by auxiliary API functions xInst() and xInstCount() */ Fts5PoslistReader *aInstIter; /* One for each phrase */ int nInstAlloc; /* Size of aInst[] array (entries / 3) */ int nInstCount; /* Number of phrase instances */ @@ -248044,10 +257041,16 @@ static void fts5CheckTransactionState(Fts5FullTable *p, int op, int iSavepoint){ #endif /* -** Return true if pTab is a contentless table. +** Return true if pTab is a contentless table. If parameter bIncludeUnindexed +** is true, this includes contentless tables that store UNINDEXED columns +** only. */ -static int fts5IsContentless(Fts5FullTable *pTab){ - return pTab->p.pConfig->eContent==FTS5_CONTENT_NONE; +static int fts5IsContentless(Fts5FullTable *pTab, int bIncludeUnindexed){ + int eContent = pTab->p.pConfig->eContent; + return ( + eContent==FTS5_CONTENT_NONE + || (bIncludeUnindexed && eContent==FTS5_CONTENT_UNINDEXED) + ); } /* @@ -248115,8 +257118,12 @@ static int fts5InitVtab( assert( (rc==SQLITE_OK && *pzErr==0) || pConfig==0 ); } if( rc==SQLITE_OK ){ + pConfig->pzErrmsg = pzErr; pTab->p.pConfig = pConfig; pTab->pGlobal = pGlobal; + if( bCreate || sqlite3Fts5TokenizerPreload(&pConfig->t) ){ + rc = sqlite3Fts5LoadTokenizer(pConfig); + } } /* Open the index sub-system */ @@ -248138,11 +257145,7 @@ static int fts5InitVtab( /* Load the initial configuration */ if( rc==SQLITE_OK ){ - assert( pConfig->pzErrmsg==0 ); - pConfig->pzErrmsg = pzErr; - rc = sqlite3Fts5IndexLoadConfig(pTab->p.pIndex); - sqlite3Fts5IndexRollback(pTab->p.pIndex); - pConfig->pzErrmsg = 0; + rc = sqlite3Fts5ConfigLoad(pTab->p.pConfig, pTab->p.pConfig->iCookie-1); } if( rc==SQLITE_OK && pConfig->eContent==FTS5_CONTENT_NORMAL ){ @@ -248152,6 +257155,7 @@ static int fts5InitVtab( rc = sqlite3_vtab_config(db, SQLITE_VTAB_INNOCUOUS); } + if( pConfig ) pConfig->pzErrmsg = 0; if( rc!=SQLITE_OK ){ fts5FreeVtab(pTab); pTab = 0; @@ -248213,16 +257217,27 @@ static void fts5SetUniqueFlag(sqlite3_index_info *pIdxInfo){ #endif } +static void fts5SetEstimatedRows(sqlite3_index_info *pIdxInfo, i64 nRow){ +#if SQLITE_VERSION_NUMBER>=3008002 +#ifndef SQLITE_CORE + if( sqlite3_libversion_number()>=3008002 ) +#endif + { + pIdxInfo->estimatedRows = nRow; + } +#endif +} + static int fts5UsePatternMatch( Fts5Config *pConfig, struct sqlite3_index_constraint *p ){ assert( FTS5_PATTERN_GLOB==SQLITE_INDEX_CONSTRAINT_GLOB ); assert( FTS5_PATTERN_LIKE==SQLITE_INDEX_CONSTRAINT_LIKE ); - if( pConfig->ePattern==FTS5_PATTERN_GLOB && p->op==FTS5_PATTERN_GLOB ){ + if( pConfig->t.ePattern==FTS5_PATTERN_GLOB && p->op==FTS5_PATTERN_GLOB ){ return 1; } - if( pConfig->ePattern==FTS5_PATTERN_LIKE + if( pConfig->t.ePattern==FTS5_PATTERN_LIKE && (p->op==FTS5_PATTERN_LIKE || p->op==FTS5_PATTERN_GLOB) ){ return 1; @@ -248269,10 +257284,10 @@ static int fts5UsePatternMatch( ** This function ensures that there is at most one "r" or "=". And that if ** there exists an "=" then there is no "<" or ">". ** -** Costs are assigned as follows: +** If an unusable MATCH operator is present in the WHERE clause, then +** SQLITE_CONSTRAINT is returned. ** -** a) If an unusable MATCH operator is present in the WHERE clause, the -** cost is unconditionally set to 1e50 (a really big number). +** Costs are assigned as follows: ** ** a) If a MATCH operator is present, the cost depends on the other ** constraints also present. As follows: @@ -248305,7 +257320,7 @@ static int fts5BestIndexMethod(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){ int bSeenEq = 0; int bSeenGt = 0; int bSeenLt = 0; - int bSeenMatch = 0; + int nSeenMatch = 0; int bSeenRank = 0; @@ -248336,21 +257351,19 @@ static int fts5BestIndexMethod(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){ /* A MATCH operator or equivalent */ if( p->usable==0 || iCol<0 ){ /* As there exists an unusable MATCH constraint this is an - ** unusable plan. Set a prohibitively high cost. */ - pInfo->estimatedCost = 1e50; - assert( iIdxStr < pInfo->nConstraint*6 + 1 ); + ** unusable plan. Return SQLITE_CONSTRAINT. */ idxStr[iIdxStr] = 0; - return SQLITE_OK; + return SQLITE_CONSTRAINT; }else{ if( iCol==nCol+1 ){ if( bSeenRank ) continue; idxStr[iIdxStr++] = 'r'; bSeenRank = 1; - }else if( iCol>=0 ){ - bSeenMatch = 1; + }else{ + nSeenMatch++; idxStr[iIdxStr++] = 'M'; sqlite3_snprintf(6, &idxStr[iIdxStr], "%d", iCol); - idxStr += strlen(&idxStr[iIdxStr]); + iIdxStr += (int)strlen(&idxStr[iIdxStr]); assert( idxStr[iIdxStr]=='\0' ); } pInfo->aConstraintUsage[i].argvIndex = ++iCons; @@ -248364,10 +257377,12 @@ static int fts5BestIndexMethod(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){ idxStr += strlen(&idxStr[iIdxStr]); pInfo->aConstraintUsage[i].argvIndex = ++iCons; assert( idxStr[iIdxStr]=='\0' ); + nSeenMatch++; }else if( bSeenEq==0 && p->op==SQLITE_INDEX_CONSTRAINT_EQ && iCol<0 ){ idxStr[iIdxStr++] = '='; bSeenEq = 1; pInfo->aConstraintUsage[i].argvIndex = ++iCons; + pInfo->aConstraintUsage[i].omit = 1; } } } @@ -248400,7 +257415,7 @@ static int fts5BestIndexMethod(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){ */ if( pInfo->nOrderBy==1 ){ int iSort = pInfo->aOrderBy[0].iColumn; - if( iSort==(pConfig->nCol+1) && bSeenMatch ){ + if( iSort==(pConfig->nCol+1) && nSeenMatch>0 ){ idxFlags |= FTS5_BI_ORDER_RANK; }else if( iSort==-1 && (!pInfo->aOrderBy[0].desc || !pConfig->bTokendata) ){ idxFlags |= FTS5_BI_ORDER_ROWID; @@ -248415,14 +257430,21 @@ static int fts5BestIndexMethod(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){ /* Calculate the estimated cost based on the flags set in idxFlags. */ if( bSeenEq ){ - pInfo->estimatedCost = bSeenMatch ? 100.0 : 10.0; - if( bSeenMatch==0 ) fts5SetUniqueFlag(pInfo); - }else if( bSeenLt && bSeenGt ){ - pInfo->estimatedCost = bSeenMatch ? 500.0 : 250000.0; - }else if( bSeenLt || bSeenGt ){ - pInfo->estimatedCost = bSeenMatch ? 750.0 : 750000.0; - }else{ - pInfo->estimatedCost = bSeenMatch ? 1000.0 : 1000000.0; + pInfo->estimatedCost = nSeenMatch ? 1000.0 : 25.0; + fts5SetUniqueFlag(pInfo); + fts5SetEstimatedRows(pInfo, 1); + }else{ + if( bSeenLt && bSeenGt ){ + pInfo->estimatedCost = nSeenMatch ? 5000.0 : 750000.0; + }else if( bSeenLt || bSeenGt ){ + pInfo->estimatedCost = nSeenMatch ? 7500.0 : 2250000.0; + }else{ + pInfo->estimatedCost = nSeenMatch ? 10000.0 : 3000000.0; + } + for(i=1; iestimatedCost *= 0.4; + } + fts5SetEstimatedRows(pInfo, (i64)(pInfo->estimatedCost / 4.0)); } pInfo->idxNum = idxFlags; @@ -248621,7 +257643,9 @@ static int fts5CursorReseek(Fts5Cursor *pCsr, int *pbSkip){ int bDesc = pCsr->bDesc; i64 iRowid = sqlite3Fts5ExprRowid(pCsr->pExpr); - rc = sqlite3Fts5ExprFirst(pCsr->pExpr, pTab->p.pIndex, iRowid, bDesc); + rc = sqlite3Fts5ExprFirst( + pCsr->pExpr, pTab->p.pIndex, iRowid, pCsr->iLastRowid, bDesc + ); if( rc==SQLITE_OK && iRowid!=sqlite3Fts5ExprRowid(pCsr->pExpr) ){ *pbSkip = 1; } @@ -248698,6 +257722,7 @@ static int fts5NextMethod(sqlite3_vtab_cursor *pCursor){ } }else{ rc = SQLITE_OK; + CsrFlagSet(pCsr, FTS5CSR_REQUIRE_DOCSIZE); } break; } @@ -248727,7 +257752,7 @@ static int fts5PrepareStatement( rc = sqlite3_prepare_v3(pConfig->db, zSql, -1, SQLITE_PREPARE_PERSISTENT, &pRet, 0); if( rc!=SQLITE_OK ){ - *pConfig->pzErrmsg = sqlite3_mprintf("%s", sqlite3_errmsg(pConfig->db)); + sqlite3Fts5ConfigErrmsg(pConfig, "%s", sqlite3_errmsg(pConfig->db)); } sqlite3_free(zSql); } @@ -248751,7 +257776,7 @@ static int fts5CursorFirstSorted( const char *zRankArgs = pCsr->zRankArgs; nPhrase = sqlite3Fts5ExprPhraseCount(pCsr->pExpr); - nByte = sizeof(Fts5Sorter) + sizeof(int) * (nPhrase-1); + nByte = SZ_FTS5SORTER(nPhrase); pSorter = (Fts5Sorter*)sqlite3_malloc64(nByte); if( pSorter==0 ) return SQLITE_NOMEM; memset(pSorter, 0, (size_t)nByte); @@ -248792,7 +257817,9 @@ static int fts5CursorFirstSorted( static int fts5CursorFirst(Fts5FullTable *pTab, Fts5Cursor *pCsr, int bDesc){ int rc; Fts5Expr *pExpr = pCsr->pExpr; - rc = sqlite3Fts5ExprFirst(pExpr, pTab->p.pIndex, pCsr->iFirstRowid, bDesc); + rc = sqlite3Fts5ExprFirst( + pExpr, pTab->p.pIndex, pCsr->iFirstRowid, pCsr->iLastRowid, bDesc + ); if( sqlite3Fts5ExprEof(pExpr) ){ CsrFlagSet(pCsr, FTS5CSR_EOF); } @@ -248951,6 +257978,145 @@ static i64 fts5GetRowidLimit(sqlite3_value *pVal, i64 iDefault){ return iDefault; } +/* +** Set the error message on the virtual table passed as the first argument. +*/ +static void fts5SetVtabError(Fts5FullTable *p, const char *zFormat, ...){ + va_list ap; /* ... printf arguments */ + va_start(ap, zFormat); + sqlite3_free(p->p.base.zErrMsg); + p->p.base.zErrMsg = sqlite3_vmprintf(zFormat, ap); + va_end(ap); +} + +/* +** Arrange for subsequent calls to sqlite3Fts5Tokenize() to use the locale +** specified by pLocale/nLocale. The buffer indicated by pLocale must remain +** valid until after the final call to sqlite3Fts5Tokenize() that will use +** the locale. +*/ +static void sqlite3Fts5SetLocale( + Fts5Config *pConfig, + const char *zLocale, + int nLocale +){ + Fts5TokenizerConfig *pT = &pConfig->t; + pT->pLocale = zLocale; + pT->nLocale = nLocale; +} + +/* +** Clear any locale configured by an earlier call to sqlite3Fts5SetLocale(). +*/ +static void sqlite3Fts5ClearLocale(Fts5Config *pConfig){ + sqlite3Fts5SetLocale(pConfig, 0, 0); +} + +/* +** Return true if the value passed as the only argument is an +** fts5_locale() value. +*/ +static int sqlite3Fts5IsLocaleValue(Fts5Config *pConfig, sqlite3_value *pVal){ + int ret = 0; + if( sqlite3_value_type(pVal)==SQLITE_BLOB ){ + /* Call sqlite3_value_bytes() after sqlite3_value_blob() in this case. + ** If the blob was created using zeroblob(), then sqlite3_value_blob() + ** may call malloc(). If this malloc() fails, then the values returned + ** by both value_blob() and value_bytes() will be 0. If value_bytes() were + ** called first, then the NULL pointer returned by value_blob() might + ** be dereferenced. */ + const u8 *pBlob = sqlite3_value_blob(pVal); + int nBlob = sqlite3_value_bytes(pVal); + if( nBlob>FTS5_LOCALE_HDR_SIZE + && 0==memcmp(pBlob, FTS5_LOCALE_HDR(pConfig), FTS5_LOCALE_HDR_SIZE) + ){ + ret = 1; + } + } + return ret; +} + +/* +** Value pVal is guaranteed to be an fts5_locale() value, according to +** sqlite3Fts5IsLocaleValue(). This function extracts the text and locale +** from the value and returns them separately. +** +** If successful, SQLITE_OK is returned and (*ppText) and (*ppLoc) set +** to point to buffers containing the text and locale, as utf-8, +** respectively. In this case output parameters (*pnText) and (*pnLoc) are +** set to the sizes in bytes of these two buffers. +** +** Or, if an error occurs, then an SQLite error code is returned. The final +** value of the four output parameters is undefined in this case. +*/ +static int sqlite3Fts5DecodeLocaleValue( + sqlite3_value *pVal, + const char **ppText, + int *pnText, + const char **ppLoc, + int *pnLoc +){ + const char *p = sqlite3_value_blob(pVal); + int n = sqlite3_value_bytes(pVal); + int nLoc = 0; + + assert( sqlite3_value_type(pVal)==SQLITE_BLOB ); + assert( n>FTS5_LOCALE_HDR_SIZE ); + + for(nLoc=FTS5_LOCALE_HDR_SIZE; p[nLoc]; nLoc++){ + if( nLoc==(n-1) ){ + return SQLITE_MISMATCH; + } + } + *ppLoc = &p[FTS5_LOCALE_HDR_SIZE]; + *pnLoc = nLoc - FTS5_LOCALE_HDR_SIZE; + + *ppText = &p[nLoc+1]; + *pnText = n - nLoc - 1; + return SQLITE_OK; +} + +/* +** Argument pVal is the text of a full-text search expression. It may or +** may not have been wrapped by fts5_locale(). This function extracts +** the text of the expression, and sets output variable (*pzText) to +** point to a nul-terminated buffer containing the expression. +** +** If pVal was an fts5_locale() value, then sqlite3Fts5SetLocale() is called +** to set the tokenizer to use the specified locale. +** +** If output variable (*pbFreeAndReset) is set to true, then the caller +** is required to (a) call sqlite3Fts5ClearLocale() to reset the tokenizer +** locale, and (b) call sqlite3_free() to free (*pzText). +*/ +static int fts5ExtractExprText( + Fts5Config *pConfig, /* Fts5 configuration */ + sqlite3_value *pVal, /* Value to extract expression text from */ + char **pzText, /* OUT: nul-terminated buffer of text */ + int *pbFreeAndReset /* OUT: Free (*pzText) and clear locale */ +){ + int rc = SQLITE_OK; + + if( sqlite3Fts5IsLocaleValue(pConfig, pVal) ){ + const char *pText = 0; + int nText = 0; + const char *pLoc = 0; + int nLoc = 0; + rc = sqlite3Fts5DecodeLocaleValue(pVal, &pText, &nText, &pLoc, &nLoc); + *pzText = sqlite3Fts5Mprintf(&rc, "%.*s", nText, pText); + if( rc==SQLITE_OK ){ + sqlite3Fts5SetLocale(pConfig, pLoc, nLoc); + } + *pbFreeAndReset = 1; + }else{ + *pzText = (char*)sqlite3_value_text(pVal); + *pbFreeAndReset = 0; + } + + return rc; +} + + /* ** This is the xFilter interface for the virtual table. See ** the virtual table xFilter method documentation for additional @@ -248981,17 +258147,12 @@ static int fts5FilterMethod( sqlite3_value *pRowidGe = 0; /* rowid >= ? expression (or NULL) */ int iCol; /* Column on LHS of MATCH operator */ char **pzErrmsg = pConfig->pzErrmsg; + int bPrefixInsttoken = pConfig->bPrefixInsttoken; int i; int iIdxStr = 0; Fts5Expr *pExpr = 0; - if( pConfig->bLock ){ - pTab->p.base.zErrMsg = sqlite3_mprintf( - "recursively defined fts5 content table" - ); - return SQLITE_ERROR; - } - + assert( pConfig->bLock==0 ); if( pCsr->ePlan ){ fts5FreeCursorComponents(pCsr); memset(&pCsr->ePlan, 0, sizeof(Fts5Cursor) - ((u8*)&pCsr->ePlan-(u8*)pCsr)); @@ -249015,8 +258176,17 @@ static int fts5FilterMethod( pRank = apVal[i]; break; case 'M': { - const char *zText = (const char*)sqlite3_value_text(apVal[i]); + char *zText = 0; + int bFreeAndReset = 0; + int bInternal = 0; + + rc = fts5ExtractExprText(pConfig, apVal[i], &zText, &bFreeAndReset); + if( rc!=SQLITE_OK ) goto filter_out; if( zText==0 ) zText = ""; + if( sqlite3_value_subtype(apVal[i])==FTS5_INSTTOKEN_SUBTYPE ){ + pConfig->bPrefixInsttoken = 1; + } + iCol = 0; do{ iCol = iCol*10 + (idxStr[iIdxStr]-'0'); @@ -249028,7 +258198,7 @@ static int fts5FilterMethod( ** indicates that the MATCH expression is not a full text query, ** but a request for an internal parameter. */ rc = fts5SpecialMatch(pTab, pCsr, &zText[1]); - goto filter_out; + bInternal = 1; }else{ char **pzErr = &pTab->p.base.zErrMsg; rc = sqlite3Fts5ExprNew(pConfig, 0, iCol, zText, &pExpr, pzErr); @@ -249036,9 +258206,15 @@ static int fts5FilterMethod( rc = sqlite3Fts5ExprAnd(&pCsr->pExpr, pExpr); pExpr = 0; } - if( rc!=SQLITE_OK ) goto filter_out; } + if( bFreeAndReset ){ + sqlite3_free(zText); + sqlite3Fts5ClearLocale(pConfig); + } + + if( bInternal || rc!=SQLITE_OK ) goto filter_out; + break; } case 'L': @@ -249126,9 +258302,7 @@ static int fts5FilterMethod( } } }else if( pConfig->zContent==0 ){ - *pConfig->pzErrmsg = sqlite3_mprintf( - "%s: table does not support scanning", pConfig->zName - ); + fts5SetVtabError(pTab,"%s: table does not support scanning",pConfig->zName); rc = SQLITE_ERROR; }else{ /* This is either a full-table scan (ePlan==FTS5_PLAN_SCAN) or a lookup @@ -249152,6 +258326,7 @@ static int fts5FilterMethod( filter_out: sqlite3Fts5ExprFree(pExpr); pConfig->pzErrmsg = pzErrmsg; + pConfig->bPrefixInsttoken = bPrefixInsttoken; return rc; } @@ -249171,9 +258346,13 @@ static i64 fts5CursorRowid(Fts5Cursor *pCsr){ assert( pCsr->ePlan==FTS5_PLAN_MATCH || pCsr->ePlan==FTS5_PLAN_SORTED_MATCH || pCsr->ePlan==FTS5_PLAN_SOURCE + || pCsr->ePlan==FTS5_PLAN_SCAN + || pCsr->ePlan==FTS5_PLAN_ROWID ); if( pCsr->pSorter ){ return pCsr->pSorter->iRowid; + }else if( pCsr->ePlan>=FTS5_PLAN_SCAN ){ + return sqlite3_column_int64(pCsr->pStmt, 0); }else{ return sqlite3Fts5ExprRowid(pCsr->pExpr); } @@ -249190,25 +258369,16 @@ static int fts5RowidMethod(sqlite3_vtab_cursor *pCursor, sqlite_int64 *pRowid){ int ePlan = pCsr->ePlan; assert( CsrFlagTest(pCsr, FTS5CSR_EOF)==0 ); - switch( ePlan ){ - case FTS5_PLAN_SPECIAL: - *pRowid = 0; - break; - - case FTS5_PLAN_SOURCE: - case FTS5_PLAN_MATCH: - case FTS5_PLAN_SORTED_MATCH: - *pRowid = fts5CursorRowid(pCsr); - break; - - default: - *pRowid = sqlite3_column_int64(pCsr->pStmt, 0); - break; + if( ePlan==FTS5_PLAN_SPECIAL ){ + *pRowid = 0; + }else{ + *pRowid = fts5CursorRowid(pCsr); } return SQLITE_OK; } + /* ** If the cursor requires seeking (bSeekRequired flag is set), seek it. ** Return SQLITE_OK if no error occurs, or an SQLite error code otherwise. @@ -249245,8 +258415,13 @@ static int fts5SeekCursor(Fts5Cursor *pCsr, int bErrormsg){ rc = sqlite3_reset(pCsr->pStmt); if( rc==SQLITE_OK ){ rc = FTS5_CORRUPT; + fts5SetVtabError((Fts5FullTable*)pTab, + "fts5: missing row %lld from content table %s", + fts5CursorRowid(pCsr), + pTab->pConfig->zContent + ); }else if( pTab->pConfig->pzErrmsg ){ - *pTab->pConfig->pzErrmsg = sqlite3_mprintf( + fts5SetVtabError((Fts5FullTable*)pTab, "%s", sqlite3_errmsg(pTab->pConfig->db) ); } @@ -249255,14 +258430,6 @@ static int fts5SeekCursor(Fts5Cursor *pCsr, int bErrormsg){ return rc; } -static void fts5SetVtabError(Fts5FullTable *p, const char *zFormat, ...){ - va_list ap; /* ... printf arguments */ - va_start(ap, zFormat); - assert( p->p.base.zErrMsg==0 ); - p->p.base.zErrMsg = sqlite3_vmprintf(zFormat, ap); - va_end(ap); -} - /* ** This function is called to handle an FTS INSERT command. In other words, ** an INSERT statement of the form: @@ -249300,7 +258467,7 @@ static int fts5SpecialInsert( } bLoadConfig = 1; }else if( 0==sqlite3_stricmp("rebuild", zCmd) ){ - if( pConfig->eContent==FTS5_CONTENT_NONE ){ + if( fts5IsContentless(pTab, 1) ){ fts5SetVtabError(pTab, "'rebuild' may not be used with a contentless fts5 table" ); @@ -249356,7 +258523,7 @@ static int fts5SpecialDelete( int eType1 = sqlite3_value_type(apVal[1]); if( eType1==SQLITE_INTEGER ){ sqlite3_int64 iDel = sqlite3_value_int64(apVal[1]); - rc = sqlite3Fts5StorageDelete(pTab->pStorage, iDel, &apVal[2]); + rc = sqlite3Fts5StorageDelete(pTab->pStorage, iDel, &apVal[2], 0); } return rc; } @@ -249369,7 +258536,7 @@ static void fts5StorageInsert( ){ int rc = *pRc; if( rc==SQLITE_OK ){ - rc = sqlite3Fts5StorageContentInsert(pTab->pStorage, apVal, piRowid); + rc = sqlite3Fts5StorageContentInsert(pTab->pStorage, 0, apVal, piRowid); } if( rc==SQLITE_OK ){ rc = sqlite3Fts5StorageIndexInsert(pTab->pStorage, apVal, *piRowid); @@ -249377,6 +258544,67 @@ static void fts5StorageInsert( *pRc = rc; } +/* +** +** This function is called when the user attempts an UPDATE on a contentless +** table. Parameter bRowidModified is true if the UPDATE statement modifies +** the rowid value. Parameter apVal[] contains the new values for each user +** defined column of the fts5 table. pConfig is the configuration object of the +** table being updated (guaranteed to be contentless). The contentless_delete=1 +** and contentless_unindexed=1 options may or may not be set. +** +** This function returns SQLITE_OK if the UPDATE can go ahead, or an SQLite +** error code if it cannot. In this case an error message is also loaded into +** pConfig. Output parameter (*pbContent) is set to true if the caller should +** update the %_content table only - not the FTS index or any other shadow +** table. This occurs when an UPDATE modifies only UNINDEXED columns of the +** table. +** +** An UPDATE may proceed if: +** +** * The only columns modified are UNINDEXED columns, or +** +** * The contentless_delete=1 option was specified and all of the indexed +** columns (not a subset) have been modified. +*/ +static int fts5ContentlessUpdate( + Fts5Config *pConfig, + sqlite3_value **apVal, + int bRowidModified, + int *pbContent +){ + int ii; + int bSeenIndex = 0; /* Have seen modified indexed column */ + int bSeenIndexNC = 0; /* Have seen unmodified indexed column */ + int rc = SQLITE_OK; + + for(ii=0; iinCol; ii++){ + if( pConfig->abUnindexed[ii]==0 ){ + if( sqlite3_value_nochange(apVal[ii]) ){ + bSeenIndexNC++; + }else{ + bSeenIndex++; + } + } + } + + if( bSeenIndex==0 && bRowidModified==0 ){ + *pbContent = 1; + }else{ + if( bSeenIndexNC || pConfig->bContentlessDelete==0 ){ + rc = SQLITE_ERROR; + sqlite3Fts5ConfigErrmsg(pConfig, + (pConfig->bContentlessDelete ? + "%s a subset of columns on fts5 contentless-delete table: %s" : + "%s contentless fts5 table: %s") + , "cannot UPDATE", pConfig->zName + ); + } + } + + return rc; +} + /* ** This function is the implementation of the xUpdate callback used by ** FTS3 virtual tables. It is invoked by SQLite each time a row is to be @@ -249401,7 +258629,6 @@ static int fts5UpdateMethod( Fts5Config *pConfig = pTab->p.pConfig; int eType0; /* value_type() of apVal[0] */ int rc = SQLITE_OK; /* Return code */ - int bUpdateOrDelete = 0; /* A transaction must be open when this is called. */ assert( pTab->ts.eState==1 || pTab->ts.eState==2 ); @@ -249413,7 +258640,7 @@ static int fts5UpdateMethod( ); assert( pTab->p.pConfig->pzErrmsg==0 ); if( pConfig->pgsz==0 ){ - rc = sqlite3Fts5IndexLoadConfig(pTab->p.pIndex); + rc = sqlite3Fts5ConfigLoad(pTab->p.pConfig, pTab->p.pConfig->iCookie); if( rc!=SQLITE_OK ) return rc; } @@ -249462,88 +258689,104 @@ static int fts5UpdateMethod( assert( eType0==SQLITE_INTEGER || eType0==SQLITE_NULL ); assert( nArg!=1 || eType0==SQLITE_INTEGER ); - /* Filter out attempts to run UPDATE or DELETE on contentless tables. - ** This is not suported. Except - they are both supported if the CREATE - ** VIRTUAL TABLE statement contained "contentless_delete=1". */ - if( eType0==SQLITE_INTEGER - && pConfig->eContent==FTS5_CONTENT_NONE - && pConfig->bContentlessDelete==0 - ){ - pTab->p.base.zErrMsg = sqlite3_mprintf( - "cannot %s contentless fts5 table: %s", - (nArg>1 ? "UPDATE" : "DELETE from"), pConfig->zName - ); - rc = SQLITE_ERROR; - } - /* DELETE */ - else if( nArg==1 ){ - i64 iDel = sqlite3_value_int64(apVal[0]); /* Rowid to delete */ - rc = sqlite3Fts5StorageDelete(pTab->pStorage, iDel, 0); - bUpdateOrDelete = 1; + if( nArg==1 ){ + /* It is only possible to DELETE from a contentless table if the + ** contentless_delete=1 flag is set. */ + if( fts5IsContentless(pTab, 1) && pConfig->bContentlessDelete==0 ){ + fts5SetVtabError(pTab, + "cannot DELETE from contentless fts5 table: %s", pConfig->zName + ); + rc = SQLITE_ERROR; + }else{ + i64 iDel = sqlite3_value_int64(apVal[0]); /* Rowid to delete */ + rc = sqlite3Fts5StorageDelete(pTab->pStorage, iDel, 0, 0); + } } /* INSERT or UPDATE */ else{ int eType1 = sqlite3_value_numeric_type(apVal[1]); - if( eType1!=SQLITE_INTEGER && eType1!=SQLITE_NULL ){ - rc = SQLITE_MISMATCH; + /* It is an error to write an fts5_locale() value to a table without + ** the locale=1 option. */ + if( pConfig->bLocale==0 ){ + int ii; + for(ii=0; iinCol; ii++){ + sqlite3_value *pVal = apVal[ii+2]; + if( sqlite3Fts5IsLocaleValue(pConfig, pVal) ){ + fts5SetVtabError(pTab, "fts5_locale() requires locale=1"); + rc = SQLITE_MISMATCH; + goto update_out; + } + } } - else if( eType0!=SQLITE_INTEGER ){ + if( eType0!=SQLITE_INTEGER ){ /* An INSERT statement. If the conflict-mode is REPLACE, first remove ** the current entry (if any). */ if( eConflict==SQLITE_REPLACE && eType1==SQLITE_INTEGER ){ i64 iNew = sqlite3_value_int64(apVal[1]); /* Rowid to delete */ - rc = sqlite3Fts5StorageDelete(pTab->pStorage, iNew, 0); - bUpdateOrDelete = 1; + rc = sqlite3Fts5StorageDelete(pTab->pStorage, iNew, 0, 0); } fts5StorageInsert(&rc, pTab, apVal, pRowid); } /* UPDATE */ else{ + Fts5Storage *pStorage = pTab->pStorage; i64 iOld = sqlite3_value_int64(apVal[0]); /* Old rowid */ i64 iNew = sqlite3_value_int64(apVal[1]); /* New rowid */ - if( eType1==SQLITE_INTEGER && iOld!=iNew ){ + int bContent = 0; /* Content only update */ + + /* If this is a contentless table (including contentless_unindexed=1 + ** tables), check if the UPDATE may proceed. */ + if( fts5IsContentless(pTab, 1) ){ + rc = fts5ContentlessUpdate(pConfig, &apVal[2], iOld!=iNew, &bContent); + if( rc!=SQLITE_OK ) goto update_out; + } + + if( eType1!=SQLITE_INTEGER ){ + rc = SQLITE_MISMATCH; + }else if( iOld!=iNew ){ + assert( bContent==0 ); if( eConflict==SQLITE_REPLACE ){ - rc = sqlite3Fts5StorageDelete(pTab->pStorage, iOld, 0); + rc = sqlite3Fts5StorageDelete(pStorage, iOld, 0, 1); if( rc==SQLITE_OK ){ - rc = sqlite3Fts5StorageDelete(pTab->pStorage, iNew, 0); + rc = sqlite3Fts5StorageDelete(pStorage, iNew, 0, 0); } fts5StorageInsert(&rc, pTab, apVal, pRowid); }else{ - rc = sqlite3Fts5StorageContentInsert(pTab->pStorage, apVal, pRowid); + rc = sqlite3Fts5StorageFindDeleteRow(pStorage, iOld); if( rc==SQLITE_OK ){ - rc = sqlite3Fts5StorageDelete(pTab->pStorage, iOld, 0); + rc = sqlite3Fts5StorageContentInsert(pStorage, 0, apVal, pRowid); } if( rc==SQLITE_OK ){ - rc = sqlite3Fts5StorageIndexInsert(pTab->pStorage, apVal,*pRowid); + rc = sqlite3Fts5StorageDelete(pStorage, iOld, 0, 0); + } + if( rc==SQLITE_OK ){ + rc = sqlite3Fts5StorageIndexInsert(pStorage, apVal, *pRowid); } } + }else if( bContent ){ + /* This occurs when an UPDATE on a contentless table affects *only* + ** UNINDEXED columns. This is a no-op for contentless_unindexed=0 + ** tables, or a write to the %_content table only for =1 tables. */ + assert( fts5IsContentless(pTab, 1) ); + rc = sqlite3Fts5StorageFindDeleteRow(pStorage, iOld); + if( rc==SQLITE_OK ){ + rc = sqlite3Fts5StorageContentInsert(pStorage, 1, apVal, pRowid); + } }else{ - rc = sqlite3Fts5StorageDelete(pTab->pStorage, iOld, 0); + rc = sqlite3Fts5StorageDelete(pStorage, iOld, 0, 1); fts5StorageInsert(&rc, pTab, apVal, pRowid); } - bUpdateOrDelete = 1; + sqlite3Fts5StorageReleaseDeleteRow(pStorage); } } } - if( rc==SQLITE_OK - && bUpdateOrDelete - && pConfig->bSecureDelete - && pConfig->iVersion==FTS5_CURRENT_VERSION - ){ - rc = sqlite3Fts5StorageConfigValue( - pTab->pStorage, "version", 0, FTS5_CURRENT_VERSION_SECUREDELETE - ); - if( rc==SQLITE_OK ){ - pConfig->iVersion = FTS5_CURRENT_VERSION_SECUREDELETE; - } - } - + update_out: pTab->p.pConfig->pzErrmsg = 0; return rc; } @@ -249565,9 +258808,11 @@ static int fts5SyncMethod(sqlite3_vtab *pVtab){ ** Implementation of xBegin() method. */ static int fts5BeginMethod(sqlite3_vtab *pVtab){ - fts5CheckTransactionState((Fts5FullTable*)pVtab, FTS5_BEGIN, 0); - fts5NewTransaction((Fts5FullTable*)pVtab); - return SQLITE_OK; + int rc = fts5NewTransaction((Fts5FullTable*)pVtab); + if( rc==SQLITE_OK ){ + fts5CheckTransactionState((Fts5FullTable*)pVtab, FTS5_BEGIN, 0); + } + return rc; } /* @@ -249590,6 +258835,7 @@ static int fts5RollbackMethod(sqlite3_vtab *pVtab){ Fts5FullTable *pTab = (Fts5FullTable*)pVtab; fts5CheckTransactionState(pTab, FTS5_ROLLBACK, 0); rc = sqlite3Fts5StorageRollback(pTab->pStorage); + pTab->p.pConfig->pgsz = 0; return rc; } @@ -249621,17 +258867,40 @@ static int fts5ApiRowCount(Fts5Context *pCtx, i64 *pnRow){ return sqlite3Fts5StorageRowCount(pTab->pStorage, pnRow); } -static int fts5ApiTokenize( +/* +** Implementation of xTokenize_v2() API. +*/ +static int fts5ApiTokenize_v2( Fts5Context *pCtx, const char *pText, int nText, + const char *pLoc, int nLoc, void *pUserData, int (*xToken)(void*, int, const char*, int, int, int) ){ Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; Fts5Table *pTab = (Fts5Table*)(pCsr->base.pVtab); - return sqlite3Fts5Tokenize( - pTab->pConfig, FTS5_TOKENIZE_AUX, pText, nText, pUserData, xToken + int rc = SQLITE_OK; + + sqlite3Fts5SetLocale(pTab->pConfig, pLoc, nLoc); + rc = sqlite3Fts5Tokenize(pTab->pConfig, + FTS5_TOKENIZE_AUX, pText, nText, pUserData, xToken ); + sqlite3Fts5SetLocale(pTab->pConfig, 0, 0); + + return rc; +} + +/* +** Implementation of xTokenize() API. This is just xTokenize_v2() with NULL/0 +** passed as the locale. +*/ +static int fts5ApiTokenize( + Fts5Context *pCtx, + const char *pText, int nText, + void *pUserData, + int (*xToken)(void*, int, const char*, int, int, int) +){ + return fts5ApiTokenize_v2(pCtx, pText, nText, 0, 0, pUserData, xToken); } static int fts5ApiPhraseCount(Fts5Context *pCtx){ @@ -249644,6 +258913,49 @@ static int fts5ApiPhraseSize(Fts5Context *pCtx, int iPhrase){ return sqlite3Fts5ExprPhraseSize(pCsr->pExpr, iPhrase); } +/* +** Argument pStmt is an SQL statement of the type used by Fts5Cursor. This +** function extracts the text value of column iCol of the current row. +** Additionally, if there is an associated locale, it invokes +** sqlite3Fts5SetLocale() to configure the tokenizer. In all cases the caller +** should invoke sqlite3Fts5ClearLocale() to clear the locale at some point +** after this function returns. +** +** If successful, (*ppText) is set to point to a buffer containing the text +** value as utf-8 and SQLITE_OK returned. (*pnText) is set to the size of that +** buffer in bytes. It is not guaranteed to be nul-terminated. If an error +** occurs, an SQLite error code is returned. The final values of the two +** output parameters are undefined in this case. +*/ +static int fts5TextFromStmt( + Fts5Config *pConfig, + sqlite3_stmt *pStmt, + int iCol, + const char **ppText, + int *pnText +){ + sqlite3_value *pVal = sqlite3_column_value(pStmt, iCol+1); + const char *pLoc = 0; + int nLoc = 0; + int rc = SQLITE_OK; + + if( pConfig->bLocale + && pConfig->eContent==FTS5_CONTENT_EXTERNAL + && sqlite3Fts5IsLocaleValue(pConfig, pVal) + ){ + rc = sqlite3Fts5DecodeLocaleValue(pVal, ppText, pnText, &pLoc, &nLoc); + }else{ + *ppText = (const char*)sqlite3_value_text(pVal); + *pnText = sqlite3_value_bytes(pVal); + if( pConfig->bLocale && pConfig->eContent==FTS5_CONTENT_NORMAL ){ + pLoc = (const char*)sqlite3_column_text(pStmt, iCol+1+pConfig->nCol); + nLoc = sqlite3_column_bytes(pStmt, iCol+1+pConfig->nCol); + } + } + sqlite3Fts5SetLocale(pConfig, pLoc, nLoc); + return rc; +} + static int fts5ApiColumnText( Fts5Context *pCtx, int iCol, @@ -249653,28 +258965,35 @@ static int fts5ApiColumnText( int rc = SQLITE_OK; Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; Fts5Table *pTab = (Fts5Table*)(pCsr->base.pVtab); + + assert( pCsr->ePlan!=FTS5_PLAN_SPECIAL ); if( iCol<0 || iCol>=pTab->pConfig->nCol ){ rc = SQLITE_RANGE; - }else if( fts5IsContentless((Fts5FullTable*)(pCsr->base.pVtab)) - || pCsr->ePlan==FTS5_PLAN_SPECIAL - ){ + }else if( fts5IsContentless((Fts5FullTable*)(pCsr->base.pVtab), 0) ){ *pz = 0; *pn = 0; }else{ rc = fts5SeekCursor(pCsr, 0); if( rc==SQLITE_OK ){ - *pz = (const char*)sqlite3_column_text(pCsr->pStmt, iCol+1); - *pn = sqlite3_column_bytes(pCsr->pStmt, iCol+1); + rc = fts5TextFromStmt(pTab->pConfig, pCsr->pStmt, iCol, pz, pn); + sqlite3Fts5ClearLocale(pTab->pConfig); } } return rc; } +/* +** This is called by various API functions - xInst, xPhraseFirst, +** xPhraseFirstColumn etc. - to obtain the position list for phrase iPhrase +** of the current row. This function works for both detail=full tables (in +** which case the position-list was read from the fts index) or for other +** detail= modes if the row content is available. +*/ static int fts5CsrPoslist( - Fts5Cursor *pCsr, - int iPhrase, - const u8 **pa, - int *pn + Fts5Cursor *pCsr, /* Fts5 cursor object */ + int iPhrase, /* Phrase to find position list for */ + const u8 **pa, /* OUT: Pointer to position list buffer */ + int *pn /* OUT: Size of (*pa) in bytes */ ){ Fts5Config *pConfig = ((Fts5Table*)(pCsr->base.pVtab))->pConfig; int rc = SQLITE_OK; @@ -249682,20 +259001,32 @@ static int fts5CsrPoslist( if( iPhrase<0 || iPhrase>=sqlite3Fts5ExprPhraseCount(pCsr->pExpr) ){ rc = SQLITE_RANGE; + }else if( pConfig->eDetail!=FTS5_DETAIL_FULL + && fts5IsContentless((Fts5FullTable*)pCsr->base.pVtab, 1) + ){ + *pa = 0; + *pn = 0; + return SQLITE_OK; }else if( CsrFlagTest(pCsr, FTS5CSR_REQUIRE_POSLIST) ){ if( pConfig->eDetail!=FTS5_DETAIL_FULL ){ Fts5PoslistPopulator *aPopulator; int i; + aPopulator = sqlite3Fts5ExprClearPoslists(pCsr->pExpr, bLive); if( aPopulator==0 ) rc = SQLITE_NOMEM; + if( rc==SQLITE_OK ){ + rc = fts5SeekCursor(pCsr, 0); + } for(i=0; inCol && rc==SQLITE_OK; i++){ - int n; const char *z; - rc = fts5ApiColumnText((Fts5Context*)pCsr, i, &z, &n); + const char *z = 0; + int n = 0; + rc = fts5TextFromStmt(pConfig, pCsr->pStmt, i, &z, &n); if( rc==SQLITE_OK ){ rc = sqlite3Fts5ExprPopulatePoslists( pConfig, pCsr->pExpr, aPopulator, i, z, n ); } + sqlite3Fts5ClearLocale(pConfig); } sqlite3_free(aPopulator); @@ -249720,7 +259051,6 @@ static int fts5CsrPoslist( *pn = 0; } - return rc; } @@ -249789,7 +259119,8 @@ static int fts5CacheInstArray(Fts5Cursor *pCsr){ aInst[0] = iBest; aInst[1] = FTS5_POS2COLUMN(aIter[iBest].iPos); aInst[2] = FTS5_POS2OFFSET(aIter[iBest].iPos); - if( aInst[1]<0 || aInst[1]>=nCol ){ + assert( aInst[1]>=0 ); + if( aInst[1]>=nCol ){ rc = FTS5_CORRUPT; break; } @@ -249867,7 +259198,7 @@ static int fts5ApiColumnSize(Fts5Context *pCtx, int iCol, int *pnToken){ if( pConfig->bColumnsize ){ i64 iRowid = fts5CursorRowid(pCsr); rc = sqlite3Fts5StorageDocsize(pTab->pStorage, iRowid, pCsr->aColumnSize); - }else if( pConfig->zContent==0 ){ + }else if( !pConfig->zContent || pConfig->eContent==FTS5_CONTENT_UNINDEXED ){ int i; for(i=0; inCol; i++){ if( pConfig->abUnindexed[i]==0 ){ @@ -249876,17 +259207,19 @@ static int fts5ApiColumnSize(Fts5Context *pCtx, int iCol, int *pnToken){ } }else{ int i; + rc = fts5SeekCursor(pCsr, 0); for(i=0; rc==SQLITE_OK && inCol; i++){ if( pConfig->abUnindexed[i]==0 ){ - const char *z; int n; - void *p = (void*)(&pCsr->aColumnSize[i]); + const char *z = 0; + int n = 0; pCsr->aColumnSize[i] = 0; - rc = fts5ApiColumnText(pCtx, i, &z, &n); + rc = fts5TextFromStmt(pConfig, pCsr->pStmt, i, &z, &n); if( rc==SQLITE_OK ){ - rc = sqlite3Fts5Tokenize( - pConfig, FTS5_TOKENIZE_AUX, z, n, p, fts5ColumnSizeCb + rc = sqlite3Fts5Tokenize(pConfig, FTS5_TOKENIZE_AUX, + z, n, (void*)&pCsr->aColumnSize[i], fts5ColumnSizeCb ); } + sqlite3Fts5ClearLocale(pConfig); } } } @@ -249966,11 +259299,10 @@ static void *fts5ApiGetAuxdata(Fts5Context *pCtx, int bClear){ } static void fts5ApiPhraseNext( - Fts5Context *pUnused, + Fts5Context *pCtx, Fts5PhraseIter *pIter, int *piCol, int *piOff ){ - UNUSED_PARAM(pUnused); if( pIter->a>=pIter->b ){ *piCol = -1; *piOff = -1; @@ -249978,8 +259310,12 @@ static void fts5ApiPhraseNext( int iVal; pIter->a += fts5GetVarint32(pIter->a, iVal); if( iVal==1 ){ + /* Avoid returning a (*piCol) value that is too large for the table, + ** even if the position-list is corrupt. The caller might not be + ** expecting it. */ + int nCol = ((Fts5Table*)(((Fts5Cursor*)pCtx)->base.pVtab))->pConfig->nCol; pIter->a += fts5GetVarint32(pIter->a, iVal); - *piCol = iVal; + *piCol = (iVal>=nCol ? nCol-1 : iVal); *piOff = 0; pIter->a += fts5GetVarint32(pIter->a, iVal); } @@ -250129,8 +259465,48 @@ static int fts5ApiQueryPhrase(Fts5Context*, int, void*, int(*)(const Fts5ExtensionApi*, Fts5Context*, void*) ); +/* +** The xColumnLocale() API. +*/ +static int fts5ApiColumnLocale( + Fts5Context *pCtx, + int iCol, + const char **pzLocale, + int *pnLocale +){ + int rc = SQLITE_OK; + Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; + Fts5Config *pConfig = ((Fts5Table*)(pCsr->base.pVtab))->pConfig; + + *pzLocale = 0; + *pnLocale = 0; + + assert( pCsr->ePlan!=FTS5_PLAN_SPECIAL ); + if( iCol<0 || iCol>=pConfig->nCol ){ + rc = SQLITE_RANGE; + }else if( + pConfig->abUnindexed[iCol]==0 + && 0==fts5IsContentless((Fts5FullTable*)pCsr->base.pVtab, 1) + && pConfig->bLocale + ){ + rc = fts5SeekCursor(pCsr, 0); + if( rc==SQLITE_OK ){ + const char *zDummy = 0; + int nDummy = 0; + rc = fts5TextFromStmt(pConfig, pCsr->pStmt, iCol, &zDummy, &nDummy); + if( rc==SQLITE_OK ){ + *pzLocale = pConfig->t.pLocale; + *pnLocale = pConfig->t.nLocale; + } + sqlite3Fts5ClearLocale(pConfig); + } + } + + return rc; +} + static const Fts5ExtensionApi sFts5Api = { - 3, /* iVersion */ + 4, /* iVersion */ fts5ApiUserData, fts5ApiColumnCount, fts5ApiRowCount, @@ -250151,7 +259527,9 @@ static const Fts5ExtensionApi sFts5Api = { fts5ApiPhraseFirstColumn, fts5ApiPhraseNextColumn, fts5ApiQueryToken, - fts5ApiInstToken + fts5ApiInstToken, + fts5ApiColumnLocale, + fts5ApiTokenize_v2 }; /* @@ -250202,6 +259580,7 @@ static void fts5ApiInvoke( sqlite3_value **argv ){ assert( pCsr->pAux==0 ); + assert( pCsr->ePlan!=FTS5_PLAN_SPECIAL ); pCsr->pAux = pAux; pAux->xFunc(&sFts5Api, (Fts5Context*)pCsr, context, argc, argv); pCsr->pAux = 0; @@ -250215,6 +259594,21 @@ static Fts5Cursor *fts5CursorFromCsrid(Fts5Global *pGlobal, i64 iCsrId){ return pCsr; } +/* +** Parameter zFmt is a printf() style formatting string. This function +** formats it using the trailing arguments and returns the result as +** an error message to the context passed as the first argument. +*/ +static void fts5ResultError(sqlite3_context *pCtx, const char *zFmt, ...){ + char *zErr = 0; + va_list ap; + va_start(ap, zFmt); + zErr = sqlite3_vmprintf(zFmt, ap); + sqlite3_result_error(pCtx, zErr, -1); + sqlite3_free(zErr); + va_end(ap); +} + static void fts5ApiCallback( sqlite3_context *context, int argc, @@ -250230,12 +259624,13 @@ static void fts5ApiCallback( iCsrId = sqlite3_value_int64(argv[0]); pCsr = fts5CursorFromCsrid(pAux->pGlobal, iCsrId); - if( pCsr==0 || pCsr->ePlan==0 ){ - char *zErr = sqlite3_mprintf("no such cursor: %lld", iCsrId); - sqlite3_result_error(context, zErr, -1); - sqlite3_free(zErr); + if( pCsr==0 || (pCsr->ePlan==0 || pCsr->ePlan==FTS5_PLAN_SPECIAL) ){ + fts5ResultError(context, "no such cursor: %lld", iCsrId); }else{ + sqlite3_vtab *pTab = pCsr->base.pVtab; fts5ApiInvoke(pAux, pCsr, context, argc-1, &argv[1]); + sqlite3_free(pTab->zErrMsg); + pTab->zErrMsg = 0; } } @@ -250353,8 +259748,8 @@ static int fts5ColumnMethod( ** auxiliary function. */ sqlite3_result_int64(pCtx, pCsr->iCsrId); }else if( iCol==pConfig->nCol+1 ){ - /* The value of the "rank" column. */ + if( pCsr->ePlan==FTS5_PLAN_SOURCE ){ fts5PoslistBlob(pCtx, pCsr); }else if( @@ -250365,20 +259760,32 @@ static int fts5ColumnMethod( fts5ApiInvoke(pCsr->pRank, pCsr, pCtx, pCsr->nRankArg, pCsr->apRankArg); } } - }else if( !fts5IsContentless(pTab) ){ - pConfig->pzErrmsg = &pTab->p.base.zErrMsg; - rc = fts5SeekCursor(pCsr, 1); - if( rc==SQLITE_OK ){ - sqlite3_result_value(pCtx, sqlite3_column_value(pCsr->pStmt, iCol+1)); + }else{ + if( !sqlite3_vtab_nochange(pCtx) && pConfig->eContent!=FTS5_CONTENT_NONE ){ + pConfig->pzErrmsg = &pTab->p.base.zErrMsg; + rc = fts5SeekCursor(pCsr, 1); + if( rc==SQLITE_OK ){ + sqlite3_value *pVal = sqlite3_column_value(pCsr->pStmt, iCol+1); + if( pConfig->bLocale + && pConfig->eContent==FTS5_CONTENT_EXTERNAL + && sqlite3Fts5IsLocaleValue(pConfig, pVal) + ){ + const char *z = 0; + int n = 0; + rc = fts5TextFromStmt(pConfig, pCsr->pStmt, iCol, &z, &n); + if( rc==SQLITE_OK ){ + sqlite3_result_text(pCtx, z, n, SQLITE_TRANSIENT); + } + sqlite3Fts5ClearLocale(pConfig); + }else{ + sqlite3_result_value(pCtx, pVal); + } + } + + pConfig->pzErrmsg = 0; } - pConfig->pzErrmsg = 0; - }else if( pConfig->bContentlessDelete && sqlite3_vtab_nochange(pCtx) ){ - char *zErr = sqlite3_mprintf("cannot UPDATE a subset of " - "columns on fts5 contentless-delete table: %s", pConfig->zName - ); - sqlite3_result_error(pCtx, zErr, -1); - sqlite3_free(zErr); } + return rc; } @@ -250518,47 +259925,210 @@ static int fts5CreateAux( } /* -** Register a new tokenizer. This is the implementation of the -** fts5_api.xCreateTokenizer() method. +** This function is used by xCreateTokenizer_v2() and xCreateTokenizer(). +** It allocates and partially populates a new Fts5TokenizerModule object. +** The new object is already linked into the Fts5Global context before +** returning. +** +** If successful, SQLITE_OK is returned and a pointer to the new +** Fts5TokenizerModule object returned via output parameter (*ppNew). All +** that is required is for the caller to fill in the methods in +** Fts5TokenizerModule.x1 and x2, and to set Fts5TokenizerModule.bV2Native +** as appropriate. +** +** If an error occurs, an SQLite error code is returned and the final value +** of (*ppNew) undefined. */ -static int fts5CreateTokenizer( - fts5_api *pApi, /* Global context (one per db handle) */ +static int fts5NewTokenizerModule( + Fts5Global *pGlobal, /* Global context (one per db handle) */ const char *zName, /* Name of new function */ void *pUserData, /* User data for aux. function */ - fts5_tokenizer *pTokenizer, /* Tokenizer implementation */ - void(*xDestroy)(void*) /* Destructor for pUserData */ + void(*xDestroy)(void*), /* Destructor for pUserData */ + Fts5TokenizerModule **ppNew ){ - Fts5Global *pGlobal = (Fts5Global*)pApi; - Fts5TokenizerModule *pNew; - sqlite3_int64 nName; /* Size of zName and its \0 terminator */ - sqlite3_int64 nByte; /* Bytes of space to allocate */ int rc = SQLITE_OK; + Fts5TokenizerModule *pNew; + sqlite3_int64 nName; /* Size of zName and its \0 terminator */ + sqlite3_int64 nByte; /* Bytes of space to allocate */ nName = strlen(zName) + 1; nByte = sizeof(Fts5TokenizerModule) + nName; - pNew = (Fts5TokenizerModule*)sqlite3_malloc64(nByte); + *ppNew = pNew = (Fts5TokenizerModule*)sqlite3Fts5MallocZero(&rc, nByte); if( pNew ){ - memset(pNew, 0, (size_t)nByte); pNew->zName = (char*)&pNew[1]; memcpy(pNew->zName, zName, nName); pNew->pUserData = pUserData; - pNew->x = *pTokenizer; pNew->xDestroy = xDestroy; pNew->pNext = pGlobal->pTok; pGlobal->pTok = pNew; if( pNew->pNext==0 ){ pGlobal->pDfltTok = pNew; } + } + + return rc; +} + +/* +** An instance of this type is used as the Fts5Tokenizer object for +** wrapper tokenizers - those that provide access to a v1 tokenizer via +** the fts5_tokenizer_v2 API, and those that provide access to a v2 tokenizer +** via the fts5_tokenizer API. +*/ +typedef struct Fts5VtoVTokenizer Fts5VtoVTokenizer; +struct Fts5VtoVTokenizer { + int bV2Native; /* True if v2 native tokenizer */ + fts5_tokenizer x1; /* Tokenizer functions */ + fts5_tokenizer_v2 x2; /* V2 tokenizer functions */ + Fts5Tokenizer *pReal; +}; + +/* +** Create a wrapper tokenizer. The context argument pCtx points to the +** Fts5TokenizerModule object. +*/ +static int fts5VtoVCreate( + void *pCtx, + const char **azArg, + int nArg, + Fts5Tokenizer **ppOut +){ + Fts5TokenizerModule *pMod = (Fts5TokenizerModule*)pCtx; + Fts5VtoVTokenizer *pNew = 0; + int rc = SQLITE_OK; + + pNew = (Fts5VtoVTokenizer*)sqlite3Fts5MallocZero(&rc, sizeof(*pNew)); + if( rc==SQLITE_OK ){ + pNew->x1 = pMod->x1; + pNew->x2 = pMod->x2; + pNew->bV2Native = pMod->bV2Native; + if( pMod->bV2Native ){ + rc = pMod->x2.xCreate(pMod->pUserData, azArg, nArg, &pNew->pReal); + }else{ + rc = pMod->x1.xCreate(pMod->pUserData, azArg, nArg, &pNew->pReal); + } + if( rc!=SQLITE_OK ){ + sqlite3_free(pNew); + pNew = 0; + } + } + + *ppOut = (Fts5Tokenizer*)pNew; + return rc; +} + +/* +** Delete an Fts5VtoVTokenizer wrapper tokenizer. +*/ +static void fts5VtoVDelete(Fts5Tokenizer *pTok){ + Fts5VtoVTokenizer *p = (Fts5VtoVTokenizer*)pTok; + if( p ){ + if( p->bV2Native ){ + p->x2.xDelete(p->pReal); + }else{ + p->x1.xDelete(p->pReal); + } + sqlite3_free(p); + } +} + + +/* +** xTokenizer method for a wrapper tokenizer that offers the v1 interface +** (no support for locales). +*/ +static int fts5V1toV2Tokenize( + Fts5Tokenizer *pTok, + void *pCtx, int flags, + const char *pText, int nText, + int (*xToken)(void*, int, const char*, int, int, int) +){ + Fts5VtoVTokenizer *p = (Fts5VtoVTokenizer*)pTok; + assert( p->bV2Native ); + return p->x2.xTokenize(p->pReal, pCtx, flags, pText, nText, 0, 0, xToken); +} + +/* +** xTokenizer method for a wrapper tokenizer that offers the v2 interface +** (with locale support). +*/ +static int fts5V2toV1Tokenize( + Fts5Tokenizer *pTok, + void *pCtx, int flags, + const char *pText, int nText, + const char *pLocale, int nLocale, + int (*xToken)(void*, int, const char*, int, int, int) +){ + Fts5VtoVTokenizer *p = (Fts5VtoVTokenizer*)pTok; + assert( p->bV2Native==0 ); + UNUSED_PARAM2(pLocale,nLocale); + return p->x1.xTokenize(p->pReal, pCtx, flags, pText, nText, xToken); +} + +/* +** Register a new tokenizer. This is the implementation of the +** fts5_api.xCreateTokenizer_v2() method. +*/ +static int fts5CreateTokenizer_v2( + fts5_api *pApi, /* Global context (one per db handle) */ + const char *zName, /* Name of new function */ + void *pUserData, /* User data for aux. function */ + fts5_tokenizer_v2 *pTokenizer, /* Tokenizer implementation */ + void(*xDestroy)(void*) /* Destructor for pUserData */ +){ + Fts5Global *pGlobal = (Fts5Global*)pApi; + int rc = SQLITE_OK; + + if( pTokenizer->iVersion>2 ){ + rc = SQLITE_ERROR; }else{ - rc = SQLITE_NOMEM; + Fts5TokenizerModule *pNew = 0; + rc = fts5NewTokenizerModule(pGlobal, zName, pUserData, xDestroy, &pNew); + if( pNew ){ + pNew->x2 = *pTokenizer; + pNew->bV2Native = 1; + pNew->x1.xCreate = fts5VtoVCreate; + pNew->x1.xTokenize = fts5V1toV2Tokenize; + pNew->x1.xDelete = fts5VtoVDelete; + } } return rc; } +/* +** The fts5_api.xCreateTokenizer() method. +*/ +static int fts5CreateTokenizer( + fts5_api *pApi, /* Global context (one per db handle) */ + const char *zName, /* Name of new function */ + void *pUserData, /* User data for aux. function */ + fts5_tokenizer *pTokenizer, /* Tokenizer implementation */ + void(*xDestroy)(void*) /* Destructor for pUserData */ +){ + Fts5TokenizerModule *pNew = 0; + int rc = SQLITE_OK; + + rc = fts5NewTokenizerModule( + (Fts5Global*)pApi, zName, pUserData, xDestroy, &pNew + ); + if( pNew ){ + pNew->x1 = *pTokenizer; + pNew->x2.xCreate = fts5VtoVCreate; + pNew->x2.xTokenize = fts5V2toV1Tokenize; + pNew->x2.xDelete = fts5VtoVDelete; + } + return rc; +} + +/* +** Search the global context passed as the first argument for a tokenizer +** module named zName. If found, return a pointer to the Fts5TokenizerModule +** object. Otherwise, return NULL. +*/ static Fts5TokenizerModule *fts5LocateTokenizer( - Fts5Global *pGlobal, - const char *zName + Fts5Global *pGlobal, /* Global (one per db handle) object */ + const char *zName /* Name of tokenizer module to find */ ){ Fts5TokenizerModule *pMod = 0; @@ -250573,6 +260143,36 @@ static Fts5TokenizerModule *fts5LocateTokenizer( return pMod; } +/* +** Find a tokenizer. This is the implementation of the +** fts5_api.xFindTokenizer_v2() method. +*/ +static int fts5FindTokenizer_v2( + fts5_api *pApi, /* Global context (one per db handle) */ + const char *zName, /* Name of tokenizer */ + void **ppUserData, + fts5_tokenizer_v2 **ppTokenizer /* Populate this object */ +){ + int rc = SQLITE_OK; + Fts5TokenizerModule *pMod; + + pMod = fts5LocateTokenizer((Fts5Global*)pApi, zName); + if( pMod ){ + if( pMod->bV2Native ){ + *ppUserData = pMod->pUserData; + }else{ + *ppUserData = (void*)pMod; + } + *ppTokenizer = &pMod->x2; + }else{ + *ppTokenizer = 0; + *ppUserData = 0; + rc = SQLITE_ERROR; + } + + return rc; +} + /* ** Find a tokenizer. This is the implementation of the ** fts5_api.xFindTokenizer() method. @@ -250588,53 +260188,75 @@ static int fts5FindTokenizer( pMod = fts5LocateTokenizer((Fts5Global*)pApi, zName); if( pMod ){ - *pTokenizer = pMod->x; - *ppUserData = pMod->pUserData; + if( pMod->bV2Native==0 ){ + *ppUserData = pMod->pUserData; + }else{ + *ppUserData = (void*)pMod; + } + *pTokenizer = pMod->x1; }else{ - memset(pTokenizer, 0, sizeof(fts5_tokenizer)); + memset(pTokenizer, 0, sizeof(*pTokenizer)); + *ppUserData = 0; rc = SQLITE_ERROR; } return rc; } -static int sqlite3Fts5GetTokenizer( - Fts5Global *pGlobal, - const char **azArg, - int nArg, - Fts5Config *pConfig, - char **pzErr -){ - Fts5TokenizerModule *pMod; +/* +** Attempt to instantiate the tokenizer. +*/ +static int sqlite3Fts5LoadTokenizer(Fts5Config *pConfig){ + const char **azArg = pConfig->t.azArg; + const int nArg = pConfig->t.nArg; + Fts5TokenizerModule *pMod = 0; int rc = SQLITE_OK; - pMod = fts5LocateTokenizer(pGlobal, nArg==0 ? 0 : azArg[0]); + pMod = fts5LocateTokenizer(pConfig->pGlobal, nArg==0 ? 0 : azArg[0]); if( pMod==0 ){ assert( nArg>0 ); rc = SQLITE_ERROR; - *pzErr = sqlite3_mprintf("no such tokenizer: %s", azArg[0]); + sqlite3Fts5ConfigErrmsg(pConfig, "no such tokenizer: %s", azArg[0]); }else{ - rc = pMod->x.xCreate( - pMod->pUserData, (azArg?&azArg[1]:0), (nArg?nArg-1:0), &pConfig->pTok + int (*xCreate)(void*, const char**, int, Fts5Tokenizer**) = 0; + if( pMod->bV2Native ){ + xCreate = pMod->x2.xCreate; + pConfig->t.pApi2 = &pMod->x2; + }else{ + pConfig->t.pApi1 = &pMod->x1; + xCreate = pMod->x1.xCreate; + } + + rc = xCreate(pMod->pUserData, + (azArg?&azArg[1]:0), (nArg?nArg-1:0), &pConfig->t.pTok ); - pConfig->pTokApi = &pMod->x; + if( rc!=SQLITE_OK ){ - if( pzErr ) *pzErr = sqlite3_mprintf("error in tokenizer constructor"); - }else{ - pConfig->ePattern = sqlite3Fts5TokenizerPattern( - pMod->x.xCreate, pConfig->pTok + if( rc!=SQLITE_NOMEM ){ + sqlite3Fts5ConfigErrmsg(pConfig, "error in tokenizer constructor"); + } + }else if( pMod->bV2Native==0 ){ + pConfig->t.ePattern = sqlite3Fts5TokenizerPattern( + pMod->x1.xCreate, pConfig->t.pTok ); } } if( rc!=SQLITE_OK ){ - pConfig->pTokApi = 0; - pConfig->pTok = 0; + pConfig->t.pApi1 = 0; + pConfig->t.pApi2 = 0; + pConfig->t.pTok = 0; } return rc; } + +/* +** xDestroy callback passed to sqlite3_create_module(). This is invoked +** when the db handle is being closed. Free memory associated with +** tokenizers and aux functions registered with this db handle. +*/ static void fts5ModuleDestroy(void *pCtx){ Fts5TokenizerModule *pTok, *pNextTok; Fts5Auxiliary *pAux, *pNextAux; @@ -250655,6 +260277,10 @@ static void fts5ModuleDestroy(void *pCtx){ sqlite3_free(pGlobal); } +/* +** Implementation of the fts5() function used by clients to obtain the +** API pointer. +*/ static void fts5Fts5Func( sqlite3_context *pCtx, /* Function call context */ int nArg, /* Number of args */ @@ -250678,7 +260304,82 @@ static void fts5SourceIdFunc( ){ assert( nArg==0 ); UNUSED_PARAM2(nArg, apUnused); - sqlite3_result_text(pCtx, "fts5: 2024-03-12 11:06:23 d8cd6d49b46a395b13955387d05e9e1a2a47e54fb99f3c9b59835bbefad6af77", -1, SQLITE_TRANSIENT); + sqlite3_result_text(pCtx, "fts5: 2025-11-28 17:28:25 281fc0e9afc38674b9b0991943b9e9d1e64c6cbdb133d35f6f5c87ff6af38a88", -1, SQLITE_TRANSIENT); +} + +/* +** Implementation of fts5_locale(LOCALE, TEXT) function. +** +** If parameter LOCALE is NULL, or a zero-length string, then a copy of +** TEXT is returned. Otherwise, both LOCALE and TEXT are interpreted as +** text, and the value returned is a blob consisting of: +** +** * The 4 bytes 0x00, 0xE0, 0xB2, 0xEb (FTS5_LOCALE_HEADER). +** * The LOCALE, as utf-8 text, followed by +** * 0x00, followed by +** * The TEXT, as utf-8 text. +** +** There is no final nul-terminator following the TEXT value. +*/ +static void fts5LocaleFunc( + sqlite3_context *pCtx, /* Function call context */ + int nArg, /* Number of args */ + sqlite3_value **apArg /* Function arguments */ +){ + const char *zLocale = 0; + i64 nLocale = 0; + const char *zText = 0; + i64 nText = 0; + + assert( nArg==2 ); + UNUSED_PARAM(nArg); + + zLocale = (const char*)sqlite3_value_text(apArg[0]); + nLocale = sqlite3_value_bytes(apArg[0]); + + zText = (const char*)sqlite3_value_text(apArg[1]); + nText = sqlite3_value_bytes(apArg[1]); + + if( zLocale==0 || zLocale[0]=='\0' ){ + sqlite3_result_text(pCtx, zText, nText, SQLITE_TRANSIENT); + }else{ + Fts5Global *p = (Fts5Global*)sqlite3_user_data(pCtx); + u8 *pBlob = 0; + u8 *pCsr = 0; + i64 nBlob = 0; + + nBlob = FTS5_LOCALE_HDR_SIZE + nLocale + 1 + nText; + pBlob = (u8*)sqlite3_malloc64(nBlob); + if( pBlob==0 ){ + sqlite3_result_error_nomem(pCtx); + return; + } + + pCsr = pBlob; + memcpy(pCsr, (const u8*)p->aLocaleHdr, FTS5_LOCALE_HDR_SIZE); + pCsr += FTS5_LOCALE_HDR_SIZE; + memcpy(pCsr, zLocale, nLocale); + pCsr += nLocale; + (*pCsr++) = 0x00; + if( zText ) memcpy(pCsr, zText, nText); + assert( &pCsr[nText]==&pBlob[nBlob] ); + + sqlite3_result_blob(pCtx, pBlob, nBlob, sqlite3_free); + } +} + +/* +** Implementation of fts5_insttoken() function. +*/ +static void fts5InsttokenFunc( + sqlite3_context *pCtx, /* Function call context */ + int nArg, /* Number of args */ + sqlite3_value **apArg /* Function arguments */ +){ + assert( nArg==1 ); + (void)nArg; + sqlite3_result_value(pCtx, apArg[0]); + sqlite3_result_subtype(pCtx, FTS5_INSTTOKEN_SUBTYPE); } /* @@ -250713,18 +260414,26 @@ static int fts5IntegrityMethod( assert( pzErr!=0 && *pzErr==0 ); UNUSED_PARAM(isQuick); + assert( pTab->p.pConfig->pzErrmsg==0 ); + pTab->p.pConfig->pzErrmsg = pzErr; rc = sqlite3Fts5StorageIntegrity(pTab->pStorage, 0); - if( (rc&0xff)==SQLITE_CORRUPT ){ - *pzErr = sqlite3_mprintf("malformed inverted index for FTS5 table %s.%s", - zSchema, zTabname); - }else if( rc!=SQLITE_OK ){ - *pzErr = sqlite3_mprintf("unable to validate the inverted index for" - " FTS5 table %s.%s: %s", - zSchema, zTabname, sqlite3_errstr(rc)); + if( *pzErr==0 && rc!=SQLITE_OK ){ + if( (rc&0xff)==SQLITE_CORRUPT ){ + *pzErr = sqlite3_mprintf("malformed inverted index for FTS5 table %s.%s", + zSchema, zTabname); + rc = (*pzErr) ? SQLITE_OK : SQLITE_NOMEM; + }else{ + *pzErr = sqlite3_mprintf("unable to validate the inverted index for" + " FTS5 table %s.%s: %s", + zSchema, zTabname, sqlite3_errstr(rc)); + } + }else if( (rc&0xff)==SQLITE_CORRUPT ){ + rc = SQLITE_OK; } sqlite3Fts5IndexCloseReader(pTab->p.pIndex); + pTab->p.pConfig->pzErrmsg = 0; - return SQLITE_OK; + return rc; } static int fts5Init(sqlite3 *db){ @@ -250766,10 +260475,22 @@ static int fts5Init(sqlite3 *db){ void *p = (void*)pGlobal; memset(pGlobal, 0, sizeof(Fts5Global)); pGlobal->db = db; - pGlobal->api.iVersion = 2; + pGlobal->api.iVersion = 3; pGlobal->api.xCreateFunction = fts5CreateAux; pGlobal->api.xCreateTokenizer = fts5CreateTokenizer; pGlobal->api.xFindTokenizer = fts5FindTokenizer; + pGlobal->api.xCreateTokenizer_v2 = fts5CreateTokenizer_v2; + pGlobal->api.xFindTokenizer_v2 = fts5FindTokenizer_v2; + + /* Initialize pGlobal->aLocaleHdr[] to a 128-bit pseudo-random vector. + ** The constants below were generated randomly. */ + sqlite3_randomness(sizeof(pGlobal->aLocaleHdr), pGlobal->aLocaleHdr); + pGlobal->aLocaleHdr[0] ^= 0xF924976D; + pGlobal->aLocaleHdr[1] ^= 0x16596E13; + pGlobal->aLocaleHdr[2] ^= 0x7C80BEAA; + pGlobal->aLocaleHdr[3] ^= 0x9B03A67F; + assert( sizeof(pGlobal->aLocaleHdr)==16 ); + rc = sqlite3_create_module_v2(db, "fts5", &fts5Mod, p, fts5ModuleDestroy); if( rc==SQLITE_OK ) rc = sqlite3Fts5IndexInit(db); if( rc==SQLITE_OK ) rc = sqlite3Fts5ExprInit(pGlobal, db); @@ -250788,6 +260509,20 @@ static int fts5Init(sqlite3 *db){ p, fts5SourceIdFunc, 0, 0 ); } + if( rc==SQLITE_OK ){ + rc = sqlite3_create_function( + db, "fts5_locale", 2, + SQLITE_UTF8|SQLITE_INNOCUOUS|SQLITE_RESULT_SUBTYPE|SQLITE_SUBTYPE, + p, fts5LocaleFunc, 0, 0 + ); + } + if( rc==SQLITE_OK ){ + rc = sqlite3_create_function( + db, "fts5_insttoken", 1, + SQLITE_UTF8|SQLITE_INNOCUOUS|SQLITE_RESULT_SUBTYPE, + p, fts5InsttokenFunc, 0, 0 + ); + } } /* If SQLITE_FTS5_ENABLE_TEST_MI is defined, assume that the file @@ -250795,8 +260530,8 @@ static int fts5Init(sqlite3 *db){ ** its entry point to enable the matchinfo() demo. */ #ifdef SQLITE_FTS5_ENABLE_TEST_MI if( rc==SQLITE_OK ){ - extern int sqlite3Fts5TestRegisterMatchinfo(sqlite3*); - rc = sqlite3Fts5TestRegisterMatchinfo(db); + extern int sqlite3Fts5TestRegisterMatchinfoAPI(fts5_api*); + rc = sqlite3Fts5TestRegisterMatchinfoAPI(&pGlobal->api); } #endif @@ -250862,13 +260597,40 @@ SQLITE_PRIVATE int sqlite3Fts5Init(sqlite3 *db){ /* #include "fts5Int.h" */ +/* +** pSavedRow: +** SQL statement FTS5_STMT_LOOKUP2 is a copy of FTS5_STMT_LOOKUP, it +** does a by-rowid lookup to retrieve a single row from the %_content +** table or equivalent external-content table/view. +** +** However, FTS5_STMT_LOOKUP2 is only used when retrieving the original +** values for a row being UPDATEd. In that case, the SQL statement is +** not reset and pSavedRow is set to point at it. This is so that the +** insert operation that follows the delete may access the original +** row values for any new values for which sqlite3_value_nochange() returns +** true. i.e. if the user executes: +** +** CREATE VIRTUAL TABLE ft USING fts5(a, b, c, locale=1); +** ... +** UPDATE fts SET a=?, b=? WHERE rowid=?; +** +** then the value passed to the xUpdate() method of this table as the +** new.c value is an sqlite3_value_nochange() value. So in this case it +** must be read from the saved row stored in Fts5Storage.pSavedRow. +** +** This is necessary - using sqlite3_value_nochange() instead of just having +** SQLite pass the original value back via xUpdate() - so as not to discard +** any locale information associated with such values. +** +*/ struct Fts5Storage { Fts5Config *pConfig; Fts5Index *pIndex; int bTotalsValid; /* True if nTotalRow/aTotalSize[] are valid */ i64 nTotalRow; /* Total number of rows in FTS table */ i64 *aTotalSize; /* Total sizes of each column */ - sqlite3_stmt *aStmt[11]; + sqlite3_stmt *pSavedRow; + sqlite3_stmt *aStmt[12]; }; @@ -250882,14 +260644,15 @@ struct Fts5Storage { # error "FTS5_STMT_LOOKUP mismatch" #endif -#define FTS5_STMT_INSERT_CONTENT 3 -#define FTS5_STMT_REPLACE_CONTENT 4 -#define FTS5_STMT_DELETE_CONTENT 5 -#define FTS5_STMT_REPLACE_DOCSIZE 6 -#define FTS5_STMT_DELETE_DOCSIZE 7 -#define FTS5_STMT_LOOKUP_DOCSIZE 8 -#define FTS5_STMT_REPLACE_CONFIG 9 -#define FTS5_STMT_SCAN 10 +#define FTS5_STMT_LOOKUP2 3 +#define FTS5_STMT_INSERT_CONTENT 4 +#define FTS5_STMT_REPLACE_CONTENT 5 +#define FTS5_STMT_DELETE_CONTENT 6 +#define FTS5_STMT_REPLACE_DOCSIZE 7 +#define FTS5_STMT_DELETE_DOCSIZE 8 +#define FTS5_STMT_LOOKUP_DOCSIZE 9 +#define FTS5_STMT_REPLACE_CONFIG 10 +#define FTS5_STMT_SCAN 11 /* ** Prepare the two insert statements - Fts5Storage.pInsertContent and @@ -250919,6 +260682,7 @@ static int fts5StorageGetStmt( "SELECT %s FROM %s T WHERE T.%Q >= ? AND T.%Q <= ? ORDER BY T.%Q ASC", "SELECT %s FROM %s T WHERE T.%Q <= ? AND T.%Q >= ? ORDER BY T.%Q DESC", "SELECT %s FROM %s T WHERE T.%Q=?", /* LOOKUP */ + "SELECT %s FROM %s T WHERE T.%Q=?", /* LOOKUP2 */ "INSERT INTO %Q.'%q_content' VALUES(%s)", /* INSERT_CONTENT */ "REPLACE INTO %Q.'%q_content' VALUES(%s)", /* REPLACE_CONTENT */ @@ -250934,6 +260698,8 @@ static int fts5StorageGetStmt( Fts5Config *pC = p->pConfig; char *zSql = 0; + assert( ArraySize(azStmt)==ArraySize(p->aStmt) ); + switch( eStmt ){ case FTS5_STMT_SCAN: zSql = sqlite3_mprintf(azStmt[eStmt], @@ -250950,6 +260716,7 @@ static int fts5StorageGetStmt( break; case FTS5_STMT_LOOKUP: + case FTS5_STMT_LOOKUP2: zSql = sqlite3_mprintf(azStmt[eStmt], pC->zContentExprlist, pC->zContent, pC->zContentRowid ); @@ -250957,20 +260724,35 @@ static int fts5StorageGetStmt( case FTS5_STMT_INSERT_CONTENT: case FTS5_STMT_REPLACE_CONTENT: { - int nCol = pC->nCol + 1; - char *zBind; + char *zBind = 0; int i; - zBind = sqlite3_malloc64(1 + nCol*2); - if( zBind ){ - for(i=0; ieContent==FTS5_CONTENT_NORMAL + || pC->eContent==FTS5_CONTENT_UNINDEXED + ); + + /* Add bindings for the "c*" columns - those that store the actual + ** table content. If eContent==NORMAL, then there is one binding + ** for each column. Or, if eContent==UNINDEXED, then there are only + ** bindings for the UNINDEXED columns. */ + for(i=0; rc==SQLITE_OK && i<(pC->nCol+1); i++){ + if( !i || pC->eContent==FTS5_CONTENT_NORMAL || pC->abUnindexed[i-1] ){ + zBind = sqlite3Fts5Mprintf(&rc, "%z%s?%d", zBind, zBind?",":"",i+1); + } + } + + /* Add bindings for any "l*" columns. Only non-UNINDEXED columns + ** require these. */ + if( pC->bLocale && pC->eContent==FTS5_CONTENT_NORMAL ){ + for(i=0; rc==SQLITE_OK && inCol; i++){ + if( pC->abUnindexed[i]==0 ){ + zBind = sqlite3Fts5Mprintf(&rc, "%z,?%d", zBind, pC->nCol+i+2); + } } - zBind[i*2-1] = '\0'; - zSql = sqlite3_mprintf(azStmt[eStmt], pC->zDb, pC->zName, zBind); - sqlite3_free(zBind); } + + zSql = sqlite3Fts5Mprintf(&rc, azStmt[eStmt], pC->zDb, pC->zName,zBind); + sqlite3_free(zBind); break; } @@ -250996,7 +260778,7 @@ static int fts5StorageGetStmt( rc = SQLITE_NOMEM; }else{ int f = SQLITE_PREPARE_PERSISTENT; - if( eStmt>FTS5_STMT_LOOKUP ) f |= SQLITE_PREPARE_NO_VTAB; + if( eStmt>FTS5_STMT_LOOKUP2 ) f |= SQLITE_PREPARE_NO_VTAB; p->pConfig->bLock++; rc = sqlite3_prepare_v3(pC->db, zSql, -1, f, &p->aStmt[eStmt], 0); p->pConfig->bLock--; @@ -251004,6 +260786,11 @@ static int fts5StorageGetStmt( if( rc!=SQLITE_OK && pzErrMsg ){ *pzErrMsg = sqlite3_mprintf("%s", sqlite3_errmsg(pC->db)); } + if( rc==SQLITE_ERROR && eStmt>FTS5_STMT_LOOKUP2 && eStmtpIndex = pIndex; if( bCreate ){ - if( pConfig->eContent==FTS5_CONTENT_NORMAL ){ + if( pConfig->eContent==FTS5_CONTENT_NORMAL + || pConfig->eContent==FTS5_CONTENT_UNINDEXED + ){ int nDefn = 32 + pConfig->nCol*10; - char *zDefn = sqlite3_malloc64(32 + (sqlite3_int64)pConfig->nCol * 10); + char *zDefn = sqlite3_malloc64(32 + (sqlite3_int64)pConfig->nCol * 20); if( zDefn==0 ){ rc = SQLITE_NOMEM; }else{ @@ -251167,8 +260956,20 @@ static int sqlite3Fts5StorageOpen( sqlite3_snprintf(nDefn, zDefn, "id INTEGER PRIMARY KEY"); iOff = (int)strlen(zDefn); for(i=0; inCol; i++){ - sqlite3_snprintf(nDefn-iOff, &zDefn[iOff], ", c%d", i); - iOff += (int)strlen(&zDefn[iOff]); + if( pConfig->eContent==FTS5_CONTENT_NORMAL + || pConfig->abUnindexed[i] + ){ + sqlite3_snprintf(nDefn-iOff, &zDefn[iOff], ", c%d", i); + iOff += (int)strlen(&zDefn[iOff]); + } + } + if( pConfig->bLocale ){ + for(i=0; inCol; i++){ + if( pConfig->abUnindexed[i]==0 ){ + sqlite3_snprintf(nDefn-iOff, &zDefn[iOff], ", l%d", i); + iOff += (int)strlen(&zDefn[iOff]); + } + } } rc = sqlite3Fts5CreateTable(pConfig, "content", zDefn, 0, pzErr); } @@ -251245,15 +261046,49 @@ static int fts5StorageInsertCallback( return sqlite3Fts5IndexWrite(pIdx, pCtx->iCol, pCtx->szCol-1, pToken, nToken); } +/* +** This function is used as part of an UPDATE statement that modifies the +** rowid of a row. In that case, this function is called first to set +** Fts5Storage.pSavedRow to point to a statement that may be used to +** access the original values of the row being deleted - iDel. +** +** SQLITE_OK is returned if successful, or an SQLite error code otherwise. +** It is not considered an error if row iDel does not exist. In this case +** pSavedRow is not set and SQLITE_OK returned. +*/ +static int sqlite3Fts5StorageFindDeleteRow(Fts5Storage *p, i64 iDel){ + int rc = SQLITE_OK; + sqlite3_stmt *pSeek = 0; + + assert( p->pSavedRow==0 ); + rc = fts5StorageGetStmt(p, FTS5_STMT_LOOKUP+1, &pSeek, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int64(pSeek, 1, iDel); + if( sqlite3_step(pSeek)!=SQLITE_ROW ){ + rc = sqlite3_reset(pSeek); + }else{ + p->pSavedRow = pSeek; + } + } + + return rc; +} + /* ** If a row with rowid iDel is present in the %_content table, add the ** delete-markers to the FTS index necessary to delete it. Do not actually ** remove the %_content row at this time though. +** +** If parameter bSaveRow is true, then Fts5Storage.pSavedRow is left +** pointing to a statement (FTS5_STMT_LOOKUP2) that may be used to access +** the original values of the row being deleted. This is used by UPDATE +** statements. */ static int fts5StorageDeleteFromIndex( Fts5Storage *p, i64 iDel, - sqlite3_value **apVal + sqlite3_value **apVal, + int bSaveRow /* True to set pSavedRow */ ){ Fts5Config *pConfig = p->pConfig; sqlite3_stmt *pSeek = 0; /* SELECT to read row iDel from %_data */ @@ -251262,12 +261097,21 @@ static int fts5StorageDeleteFromIndex( int iCol; Fts5InsertCtx ctx; + assert( bSaveRow==0 || apVal==0 ); + assert( bSaveRow==0 || bSaveRow==1 ); + assert( FTS5_STMT_LOOKUP2==FTS5_STMT_LOOKUP+1 ); + if( apVal==0 ){ - rc = fts5StorageGetStmt(p, FTS5_STMT_LOOKUP, &pSeek, 0); - if( rc!=SQLITE_OK ) return rc; - sqlite3_bind_int64(pSeek, 1, iDel); - if( sqlite3_step(pSeek)!=SQLITE_ROW ){ - return sqlite3_reset(pSeek); + if( p->pSavedRow && bSaveRow ){ + pSeek = p->pSavedRow; + p->pSavedRow = 0; + }else{ + rc = fts5StorageGetStmt(p, FTS5_STMT_LOOKUP+bSaveRow, &pSeek, 0); + if( rc!=SQLITE_OK ) return rc; + sqlite3_bind_int64(pSeek, 1, iDel); + if( sqlite3_step(pSeek)!=SQLITE_ROW ){ + return sqlite3_reset(pSeek); + } } } @@ -251275,27 +261119,56 @@ static int fts5StorageDeleteFromIndex( ctx.iCol = -1; for(iCol=1; rc==SQLITE_OK && iCol<=pConfig->nCol; iCol++){ if( pConfig->abUnindexed[iCol-1]==0 ){ - const char *zText; - int nText; + sqlite3_value *pVal = 0; + sqlite3_value *pFree = 0; + const char *pText = 0; + int nText = 0; + const char *pLoc = 0; + int nLoc = 0; + assert( pSeek==0 || apVal==0 ); assert( pSeek!=0 || apVal!=0 ); if( pSeek ){ - zText = (const char*)sqlite3_column_text(pSeek, iCol); - nText = sqlite3_column_bytes(pSeek, iCol); - }else if( ALWAYS(apVal) ){ - zText = (const char*)sqlite3_value_text(apVal[iCol-1]); - nText = sqlite3_value_bytes(apVal[iCol-1]); + pVal = sqlite3_column_value(pSeek, iCol); }else{ - continue; + pVal = apVal[iCol-1]; } - ctx.szCol = 0; - rc = sqlite3Fts5Tokenize(pConfig, FTS5_TOKENIZE_DOCUMENT, - zText, nText, (void*)&ctx, fts5StorageInsertCallback - ); - p->aTotalSize[iCol-1] -= (i64)ctx.szCol; - if( p->aTotalSize[iCol-1]<0 ){ - rc = FTS5_CORRUPT; + + if( pConfig->bLocale && sqlite3Fts5IsLocaleValue(pConfig, pVal) ){ + rc = sqlite3Fts5DecodeLocaleValue(pVal, &pText, &nText, &pLoc, &nLoc); + }else{ + if( sqlite3_value_type(pVal)!=SQLITE_TEXT ){ + /* Make a copy of the value to work with. This is because the call + ** to sqlite3_value_text() below forces the type of the value to + ** SQLITE_TEXT, and we may need to use it again later. */ + pFree = pVal = sqlite3_value_dup(pVal); + if( pVal==0 ){ + rc = SQLITE_NOMEM; + } + } + if( rc==SQLITE_OK ){ + pText = (const char*)sqlite3_value_text(pVal); + nText = sqlite3_value_bytes(pVal); + if( pConfig->bLocale && pSeek ){ + pLoc = (const char*)sqlite3_column_text(pSeek, iCol+pConfig->nCol); + nLoc = sqlite3_column_bytes(pSeek, iCol + pConfig->nCol); + } + } + } + + if( rc==SQLITE_OK ){ + sqlite3Fts5SetLocale(pConfig, pLoc, nLoc); + ctx.szCol = 0; + rc = sqlite3Fts5Tokenize(pConfig, FTS5_TOKENIZE_DOCUMENT, + pText, nText, (void*)&ctx, fts5StorageInsertCallback + ); + p->aTotalSize[iCol-1] -= (i64)ctx.szCol; + if( rc==SQLITE_OK && p->aTotalSize[iCol-1]<0 ){ + rc = FTS5_CORRUPT; + } + sqlite3Fts5ClearLocale(pConfig); } + sqlite3_value_free(pFree); } } if( rc==SQLITE_OK && p->nTotalRow<1 ){ @@ -251304,11 +261177,29 @@ static int fts5StorageDeleteFromIndex( p->nTotalRow--; } - rc2 = sqlite3_reset(pSeek); - if( rc==SQLITE_OK ) rc = rc2; + if( rc==SQLITE_OK && bSaveRow ){ + assert( p->pSavedRow==0 ); + p->pSavedRow = pSeek; + }else{ + rc2 = sqlite3_reset(pSeek); + if( rc==SQLITE_OK ) rc = rc2; + } return rc; } +/* +** Reset any saved statement pSavedRow. Zero pSavedRow as well. This +** should be called by the xUpdate() method of the fts5 table before +** returning from any operation that may have set Fts5Storage.pSavedRow. +*/ +static void sqlite3Fts5StorageReleaseDeleteRow(Fts5Storage *pStorage){ + assert( pStorage->pSavedRow==0 + || pStorage->pSavedRow==pStorage->aStmt[FTS5_STMT_LOOKUP2] + ); + sqlite3_reset(pStorage->pSavedRow); + pStorage->pSavedRow = 0; +} + /* ** This function is called to process a DELETE on a contentless_delete=1 ** table. It adds the tombstone required to delete the entry with rowid @@ -251321,7 +261212,9 @@ static int fts5StorageContentlessDelete(Fts5Storage *p, i64 iDel){ int rc = SQLITE_OK; assert( p->pConfig->bContentlessDelete ); - assert( p->pConfig->eContent==FTS5_CONTENT_NONE ); + assert( p->pConfig->eContent==FTS5_CONTENT_NONE + || p->pConfig->eContent==FTS5_CONTENT_UNINDEXED + ); /* Look up the origin of the document in the %_docsize table. Store ** this in stack variable iOrigin. */ @@ -251365,12 +261258,12 @@ static int fts5StorageInsertDocsize( rc = sqlite3Fts5IndexGetOrigin(p->pIndex, &iOrigin); sqlite3_bind_int64(pReplace, 3, iOrigin); } - if( rc==SQLITE_OK ){ - sqlite3_bind_blob(pReplace, 2, pBuf->p, pBuf->n, SQLITE_STATIC); - sqlite3_step(pReplace); - rc = sqlite3_reset(pReplace); - sqlite3_bind_null(pReplace, 2); - } + } + if( rc==SQLITE_OK ){ + sqlite3_bind_blob(pReplace, 2, pBuf->p, pBuf->n, SQLITE_STATIC); + sqlite3_step(pReplace); + rc = sqlite3_reset(pReplace); + sqlite3_bind_null(pReplace, 2); } } return rc; @@ -251424,7 +261317,12 @@ static int fts5StorageSaveTotals(Fts5Storage *p){ /* ** Remove a row from the FTS table. */ -static int sqlite3Fts5StorageDelete(Fts5Storage *p, i64 iDel, sqlite3_value **apVal){ +static int sqlite3Fts5StorageDelete( + Fts5Storage *p, /* Storage object */ + i64 iDel, /* Rowid to delete from table */ + sqlite3_value **apVal, /* Optional - values to remove from index */ + int bSaveRow /* If true, set pSavedRow for deleted row */ +){ Fts5Config *pConfig = p->pConfig; int rc; sqlite3_stmt *pDel = 0; @@ -251440,8 +261338,14 @@ static int sqlite3Fts5StorageDelete(Fts5Storage *p, i64 iDel, sqlite3_value **ap if( rc==SQLITE_OK ){ if( p->pConfig->bContentlessDelete ){ rc = fts5StorageContentlessDelete(p, iDel); + if( rc==SQLITE_OK + && bSaveRow + && p->pConfig->eContent==FTS5_CONTENT_UNINDEXED + ){ + rc = sqlite3Fts5StorageFindDeleteRow(p, iDel); + } }else{ - rc = fts5StorageDeleteFromIndex(p, iDel, apVal); + rc = fts5StorageDeleteFromIndex(p, iDel, apVal, bSaveRow); } } @@ -251456,7 +261360,9 @@ static int sqlite3Fts5StorageDelete(Fts5Storage *p, i64 iDel, sqlite3_value **ap } /* Delete the %_content record */ - if( pConfig->eContent==FTS5_CONTENT_NORMAL ){ + if( pConfig->eContent==FTS5_CONTENT_NORMAL + || pConfig->eContent==FTS5_CONTENT_UNINDEXED + ){ if( rc==SQLITE_OK ){ rc = fts5StorageGetStmt(p, FTS5_STMT_DELETE_CONTENT, &pDel, 0); } @@ -251488,8 +261394,13 @@ static int sqlite3Fts5StorageDeleteAll(Fts5Storage *p){ ); if( rc==SQLITE_OK && pConfig->bColumnsize ){ rc = fts5ExecPrintf(pConfig->db, 0, - "DELETE FROM %Q.'%q_docsize';", - pConfig->zDb, pConfig->zName + "DELETE FROM %Q.'%q_docsize';", pConfig->zDb, pConfig->zName + ); + } + + if( rc==SQLITE_OK && pConfig->eContent==FTS5_CONTENT_UNINDEXED ){ + rc = fts5ExecPrintf(pConfig->db, 0, + "DELETE FROM %Q.'%q_content';", pConfig->zDb, pConfig->zName ); } @@ -251530,14 +261441,36 @@ static int sqlite3Fts5StorageRebuild(Fts5Storage *p){ for(ctx.iCol=0; rc==SQLITE_OK && ctx.iColnCol; ctx.iCol++){ ctx.szCol = 0; if( pConfig->abUnindexed[ctx.iCol]==0 ){ - const char *zText = (const char*)sqlite3_column_text(pScan, ctx.iCol+1); - int nText = sqlite3_column_bytes(pScan, ctx.iCol+1); - rc = sqlite3Fts5Tokenize(pConfig, - FTS5_TOKENIZE_DOCUMENT, - zText, nText, - (void*)&ctx, - fts5StorageInsertCallback - ); + int nText = 0; /* Size of pText in bytes */ + const char *pText = 0; /* Pointer to buffer containing text value */ + int nLoc = 0; /* Size of pLoc in bytes */ + const char *pLoc = 0; /* Pointer to buffer containing text value */ + + sqlite3_value *pVal = sqlite3_column_value(pScan, ctx.iCol+1); + if( pConfig->eContent==FTS5_CONTENT_EXTERNAL + && sqlite3Fts5IsLocaleValue(pConfig, pVal) + ){ + rc = sqlite3Fts5DecodeLocaleValue(pVal, &pText, &nText, &pLoc, &nLoc); + }else{ + pText = (const char*)sqlite3_value_text(pVal); + nText = sqlite3_value_bytes(pVal); + if( pConfig->bLocale ){ + int iCol = ctx.iCol + 1 + pConfig->nCol; + pLoc = (const char*)sqlite3_column_text(pScan, iCol); + nLoc = sqlite3_column_bytes(pScan, iCol); + } + } + + if( rc==SQLITE_OK ){ + sqlite3Fts5SetLocale(pConfig, pLoc, nLoc); + rc = sqlite3Fts5Tokenize(pConfig, + FTS5_TOKENIZE_DOCUMENT, + pText, nText, + (void*)&ctx, + fts5StorageInsertCallback + ); + sqlite3Fts5ClearLocale(pConfig); + } } sqlite3Fts5BufferAppendVarint(&rc, &buf, ctx.szCol); p->aTotalSize[ctx.iCol] += (i64)ctx.szCol; @@ -251603,6 +261536,7 @@ static int fts5StorageNewRowid(Fts5Storage *p, i64 *piRowid){ */ static int sqlite3Fts5StorageContentInsert( Fts5Storage *p, + int bReplace, /* True to use REPLACE instead of INSERT */ sqlite3_value **apVal, i64 *piRowid ){ @@ -251610,7 +261544,9 @@ static int sqlite3Fts5StorageContentInsert( int rc = SQLITE_OK; /* Insert the new row into the %_content table. */ - if( pConfig->eContent!=FTS5_CONTENT_NORMAL ){ + if( pConfig->eContent!=FTS5_CONTENT_NORMAL + && pConfig->eContent!=FTS5_CONTENT_UNINDEXED + ){ if( sqlite3_value_type(apVal[1])==SQLITE_INTEGER ){ *piRowid = sqlite3_value_int64(apVal[1]); }else{ @@ -251619,9 +261555,52 @@ static int sqlite3Fts5StorageContentInsert( }else{ sqlite3_stmt *pInsert = 0; /* Statement to write %_content table */ int i; /* Counter variable */ - rc = fts5StorageGetStmt(p, FTS5_STMT_INSERT_CONTENT, &pInsert, 0); - for(i=1; rc==SQLITE_OK && i<=pConfig->nCol+1; i++){ - rc = sqlite3_bind_value(pInsert, i, apVal[i]); + + assert( FTS5_STMT_INSERT_CONTENT+1==FTS5_STMT_REPLACE_CONTENT ); + assert( bReplace==0 || bReplace==1 ); + rc = fts5StorageGetStmt(p, FTS5_STMT_INSERT_CONTENT+bReplace, &pInsert, 0); + if( pInsert ) sqlite3_clear_bindings(pInsert); + + /* Bind the rowid value */ + sqlite3_bind_value(pInsert, 1, apVal[1]); + + /* Loop through values for user-defined columns. i=2 is the leftmost + ** user-defined column. As is column 1 of pSavedRow. */ + for(i=2; rc==SQLITE_OK && i<=pConfig->nCol+1; i++){ + int bUnindexed = pConfig->abUnindexed[i-2]; + if( pConfig->eContent==FTS5_CONTENT_NORMAL || bUnindexed ){ + sqlite3_value *pVal = apVal[i]; + + if( sqlite3_value_nochange(pVal) && p->pSavedRow ){ + /* This is an UPDATE statement, and user-defined column (i-2) was not + ** modified. Retrieve the value from Fts5Storage.pSavedRow. */ + pVal = sqlite3_column_value(p->pSavedRow, i-1); + if( pConfig->bLocale && bUnindexed==0 ){ + sqlite3_bind_value(pInsert, pConfig->nCol + i, + sqlite3_column_value(p->pSavedRow, pConfig->nCol + i - 1) + ); + } + }else if( sqlite3Fts5IsLocaleValue(pConfig, pVal) ){ + const char *pText = 0; + const char *pLoc = 0; + int nText = 0; + int nLoc = 0; + assert( pConfig->bLocale ); + + rc = sqlite3Fts5DecodeLocaleValue(pVal, &pText, &nText, &pLoc, &nLoc); + if( rc==SQLITE_OK ){ + sqlite3_bind_text(pInsert, i, pText, nText, SQLITE_TRANSIENT); + if( bUnindexed==0 ){ + int iLoc = pConfig->nCol + i; + sqlite3_bind_text(pInsert, iLoc, pLoc, nLoc, SQLITE_TRANSIENT); + } + } + + continue; + } + + rc = sqlite3_bind_value(pInsert, i, pVal); + } } if( rc==SQLITE_OK ){ sqlite3_step(pInsert); @@ -251656,14 +261635,38 @@ static int sqlite3Fts5StorageIndexInsert( for(ctx.iCol=0; rc==SQLITE_OK && ctx.iColnCol; ctx.iCol++){ ctx.szCol = 0; if( pConfig->abUnindexed[ctx.iCol]==0 ){ - const char *zText = (const char*)sqlite3_value_text(apVal[ctx.iCol+2]); - int nText = sqlite3_value_bytes(apVal[ctx.iCol+2]); - rc = sqlite3Fts5Tokenize(pConfig, - FTS5_TOKENIZE_DOCUMENT, - zText, nText, - (void*)&ctx, - fts5StorageInsertCallback - ); + int nText = 0; /* Size of pText in bytes */ + const char *pText = 0; /* Pointer to buffer containing text value */ + int nLoc = 0; /* Size of pText in bytes */ + const char *pLoc = 0; /* Pointer to buffer containing text value */ + + sqlite3_value *pVal = apVal[ctx.iCol+2]; + if( p->pSavedRow && sqlite3_value_nochange(pVal) ){ + pVal = sqlite3_column_value(p->pSavedRow, ctx.iCol+1); + if( pConfig->eContent==FTS5_CONTENT_NORMAL && pConfig->bLocale ){ + int iCol = ctx.iCol + 1 + pConfig->nCol; + pLoc = (const char*)sqlite3_column_text(p->pSavedRow, iCol); + nLoc = sqlite3_column_bytes(p->pSavedRow, iCol); + } + }else{ + pVal = apVal[ctx.iCol+2]; + } + + if( pConfig->bLocale && sqlite3Fts5IsLocaleValue(pConfig, pVal) ){ + rc = sqlite3Fts5DecodeLocaleValue(pVal, &pText, &nText, &pLoc, &nLoc); + }else{ + pText = (const char*)sqlite3_value_text(pVal); + nText = sqlite3_value_bytes(pVal); + } + + if( rc==SQLITE_OK ){ + sqlite3Fts5SetLocale(pConfig, pLoc, nLoc); + rc = sqlite3Fts5Tokenize(pConfig, + FTS5_TOKENIZE_DOCUMENT, pText, nText, (void*)&ctx, + fts5StorageInsertCallback + ); + sqlite3Fts5ClearLocale(pConfig); + } } sqlite3Fts5BufferAppendVarint(&rc, &buf, ctx.szCol); p->aTotalSize[ctx.iCol] += (i64)ctx.szCol; @@ -251827,29 +261830,61 @@ static int sqlite3Fts5StorageIntegrity(Fts5Storage *p, int iArg){ rc = sqlite3Fts5TermsetNew(&ctx.pTermset); } for(i=0; rc==SQLITE_OK && inCol; i++){ - if( pConfig->abUnindexed[i] ) continue; - ctx.iCol = i; - ctx.szCol = 0; - if( pConfig->eDetail==FTS5_DETAIL_COLUMNS ){ - rc = sqlite3Fts5TermsetNew(&ctx.pTermset); - } - if( rc==SQLITE_OK ){ - const char *zText = (const char*)sqlite3_column_text(pScan, i+1); - int nText = sqlite3_column_bytes(pScan, i+1); - rc = sqlite3Fts5Tokenize(pConfig, - FTS5_TOKENIZE_DOCUMENT, - zText, nText, - (void*)&ctx, - fts5StorageIntegrityCallback - ); - } - if( rc==SQLITE_OK && pConfig->bColumnsize && ctx.szCol!=aColSize[i] ){ - rc = FTS5_CORRUPT; - } - aTotalSize[i] += ctx.szCol; - if( pConfig->eDetail==FTS5_DETAIL_COLUMNS ){ - sqlite3Fts5TermsetFree(ctx.pTermset); - ctx.pTermset = 0; + if( pConfig->abUnindexed[i]==0 ){ + const char *pText = 0; + int nText = 0; + const char *pLoc = 0; + int nLoc = 0; + sqlite3_value *pVal = sqlite3_column_value(pScan, i+1); + + if( pConfig->eContent==FTS5_CONTENT_EXTERNAL + && sqlite3Fts5IsLocaleValue(pConfig, pVal) + ){ + rc = sqlite3Fts5DecodeLocaleValue( + pVal, &pText, &nText, &pLoc, &nLoc + ); + }else{ + if( pConfig->eContent==FTS5_CONTENT_NORMAL && pConfig->bLocale ){ + int iCol = i + 1 + pConfig->nCol; + pLoc = (const char*)sqlite3_column_text(pScan, iCol); + nLoc = sqlite3_column_bytes(pScan, iCol); + } + pText = (const char*)sqlite3_value_text(pVal); + nText = sqlite3_value_bytes(pVal); + } + + ctx.iCol = i; + ctx.szCol = 0; + + if( rc==SQLITE_OK && pConfig->eDetail==FTS5_DETAIL_COLUMNS ){ + rc = sqlite3Fts5TermsetNew(&ctx.pTermset); + } + + if( rc==SQLITE_OK ){ + sqlite3Fts5SetLocale(pConfig, pLoc, nLoc); + rc = sqlite3Fts5Tokenize(pConfig, + FTS5_TOKENIZE_DOCUMENT, + pText, nText, + (void*)&ctx, + fts5StorageIntegrityCallback + ); + sqlite3Fts5ClearLocale(pConfig); + } + + /* If this is not a columnsize=0 database, check that the number + ** of tokens in the value matches the aColSize[] value read from + ** the %_docsize table. */ + if( rc==SQLITE_OK + && pConfig->bColumnsize + && ctx.szCol!=aColSize[i] + ){ + rc = FTS5_CORRUPT; + } + aTotalSize[i] += ctx.szCol; + if( pConfig->eDetail==FTS5_DETAIL_COLUMNS ){ + sqlite3Fts5TermsetFree(ctx.pTermset); + ctx.pTermset = 0; + } } } sqlite3Fts5TermsetFree(ctx.pTermset); @@ -252275,7 +262310,7 @@ static const unsigned char sqlite3Utf8Trans1[] = { c = *(zIn++); \ if( c>=0xc0 ){ \ c = sqlite3Utf8Trans1[c-0xc0]; \ - while( zIn!=zTerm && (*zIn & 0xc0)==0x80 ){ \ + while( zInpTokenizer ){ - p->tokenizer.xDelete(p->pTokenizer); + p->tokenizer_v2.xDelete(p->pTokenizer); } sqlite3_free(p); } @@ -252663,6 +262696,7 @@ static int fts5PorterCreate( PorterTokenizer *pRet; void *pUserdata = 0; const char *zBase = "unicode61"; + fts5_tokenizer_v2 *pV2 = 0; if( nArg>0 ){ zBase = azArg[0]; @@ -252671,14 +262705,15 @@ static int fts5PorterCreate( pRet = (PorterTokenizer*)sqlite3_malloc(sizeof(PorterTokenizer)); if( pRet ){ memset(pRet, 0, sizeof(PorterTokenizer)); - rc = pApi->xFindTokenizer(pApi, zBase, &pUserdata, &pRet->tokenizer); + rc = pApi->xFindTokenizer_v2(pApi, zBase, &pUserdata, &pV2); }else{ rc = SQLITE_NOMEM; } if( rc==SQLITE_OK ){ int nArg2 = (nArg>0 ? nArg-1 : 0); - const char **azArg2 = (nArg2 ? &azArg[1] : 0); - rc = pRet->tokenizer.xCreate(pUserdata, azArg2, nArg2, &pRet->pTokenizer); + const char **az2 = (nArg2 ? &azArg[1] : 0); + memcpy(&pRet->tokenizer_v2, pV2, sizeof(fts5_tokenizer_v2)); + rc = pRet->tokenizer_v2.xCreate(pUserdata, az2, nArg2, &pRet->pTokenizer); } if( rc!=SQLITE_OK ){ @@ -253329,6 +263364,7 @@ static int fts5PorterTokenize( void *pCtx, int flags, const char *pText, int nText, + const char *pLoc, int nLoc, int (*xToken)(void*, int, const char*, int nToken, int iStart, int iEnd) ){ PorterTokenizer *p = (PorterTokenizer*)pTokenizer; @@ -253336,8 +263372,8 @@ static int fts5PorterTokenize( sCtx.xToken = xToken; sCtx.pCtx = pCtx; sCtx.aBuf = p->aBuf; - return p->tokenizer.xTokenize( - p->pTokenizer, (void*)&sCtx, flags, pText, nText, fts5PorterCb + return p->tokenizer_v2.xTokenize( + p->pTokenizer, (void*)&sCtx, flags, pText, nText, pLoc, nLoc, fts5PorterCb ); } @@ -253367,40 +263403,46 @@ static int fts5TriCreate( Fts5Tokenizer **ppOut ){ int rc = SQLITE_OK; - TrigramTokenizer *pNew = (TrigramTokenizer*)sqlite3_malloc(sizeof(*pNew)); + TrigramTokenizer *pNew = 0; UNUSED_PARAM(pUnused); - if( pNew==0 ){ - rc = SQLITE_NOMEM; + if( nArg%2 ){ + rc = SQLITE_ERROR; }else{ int i; - pNew->bFold = 1; - pNew->iFoldParam = 0; - for(i=0; rc==SQLITE_OK && ibFold = 1; + pNew->iFoldParam = 0; + + for(i=0; rc==SQLITE_OK && ibFold = (zArg[0]=='0'); + } + }else if( 0==sqlite3_stricmp(azArg[i], "remove_diacritics") ){ + if( (zArg[0]!='0' && zArg[0]!='1' && zArg[0]!='2') || zArg[1] ){ + rc = SQLITE_ERROR; + }else{ + pNew->iFoldParam = (zArg[0]!='0') ? 2 : 0; + } }else{ - pNew->bFold = (zArg[0]=='0'); - } - }else if( 0==sqlite3_stricmp(azArg[i], "remove_diacritics") ){ - if( (zArg[0]!='0' && zArg[0]!='1' && zArg[0]!='2') || zArg[1] ){ rc = SQLITE_ERROR; - }else{ - pNew->iFoldParam = (zArg[0]!='0') ? 2 : 0; } - }else{ - rc = SQLITE_ERROR; } - } - if( pNew->iFoldParam!=0 && pNew->bFold==0 ){ - rc = SQLITE_ERROR; - } + if( pNew->iFoldParam!=0 && pNew->bFold==0 ){ + rc = SQLITE_ERROR; + } - if( rc!=SQLITE_OK ){ - fts5TriDelete((Fts5Tokenizer*)pNew); - pNew = 0; + if( rc!=SQLITE_OK ){ + fts5TriDelete((Fts5Tokenizer*)pNew); + pNew = 0; + } } } *ppOut = (Fts5Tokenizer*)pNew; @@ -253423,8 +263465,8 @@ static int fts5TriTokenize( char *zOut = aBuf; int ii; const unsigned char *zIn = (const unsigned char*)pText; - const unsigned char *zEof = &zIn[nText]; - u32 iCode; + const unsigned char *zEof = (zIn ? &zIn[nText] : 0); + u32 iCode = 0; int aStart[3]; /* Input offset of each character in aBuf[] */ UNUSED_PARAM(unusedFlags); @@ -253433,8 +263475,8 @@ static int fts5TriTokenize( for(ii=0; ii<3; ii++){ do { aStart[ii] = zIn - (const unsigned char*)pText; + if( zIn>=zEof ) return SQLITE_OK; READ_UTF8(zIn, zEof, iCode); - if( iCode==0 ) return SQLITE_OK; if( p->bFold ) iCode = sqlite3Fts5UnicodeFold(iCode, p->iFoldParam); }while( iCode==0 ); WRITE_UTF8(zOut, iCode); @@ -253455,8 +263497,11 @@ static int fts5TriTokenize( /* Read characters from the input up until the first non-diacritic */ do { iNext = zIn - (const unsigned char*)pText; + if( zIn>=zEof ){ + iCode = 0; + break; + } READ_UTF8(zIn, zEof, iCode); - if( iCode==0 ) break; if( p->bFold ) iCode = sqlite3Fts5UnicodeFold(iCode, p->iFoldParam); }while( iCode==0 ); @@ -253505,6 +263550,16 @@ static int sqlite3Fts5TokenizerPattern( return FTS5_PATTERN_NONE; } +/* +** Return true if the tokenizer described by p->azArg[] is the trigram +** tokenizer. This tokenizer needs to be loaded before xBestIndex is +** called for the first time in order to correctly handle LIKE/GLOB. +*/ +static int sqlite3Fts5TokenizerPreload(Fts5TokenizerConfig *p){ + return (p->nArg>=1 && 0==sqlite3_stricmp(p->azArg[0], "trigram")); +} + + /* ** Register all built-in tokenizers with FTS5. */ @@ -253515,7 +263570,6 @@ static int sqlite3Fts5TokenizerInit(fts5_api *pApi){ } aBuiltin[] = { { "unicode61", {fts5UnicodeCreate, fts5UnicodeDelete, fts5UnicodeTokenize}}, { "ascii", {fts5AsciiCreate, fts5AsciiDelete, fts5AsciiTokenize }}, - { "porter", {fts5PorterCreate, fts5PorterDelete, fts5PorterTokenize }}, { "trigram", {fts5TriCreate, fts5TriDelete, fts5TriTokenize}}, }; @@ -253530,7 +263584,20 @@ static int sqlite3Fts5TokenizerInit(fts5_api *pApi){ 0 ); } - + if( rc==SQLITE_OK ){ + fts5_tokenizer_v2 sPorter = { + 2, + fts5PorterCreate, + fts5PorterDelete, + fts5PorterTokenize + }; + rc = pApi->xCreateTokenizer_v2(pApi, + "porter", + (void*)pApi, + &sPorter, + 0 + ); + } return rc; } @@ -253900,6 +263967,9 @@ static int sqlite3Fts5UnicodeCatParse(const char *zCat, u8 *aArray){ default: return 1; } break; + + default: + return 1; } return 0; } @@ -254312,7 +264382,6 @@ static void sqlite3Fts5UnicodeAscii(u8 *aArray, u8 *aAscii){ aAscii[0] = 0; /* 0x00 is never a token character */ } - /* ** 2015 May 30 ** @@ -254724,6 +264793,7 @@ struct Fts5VocabCursor { int nLeTerm; /* Size of zLeTerm in bytes */ char *zLeTerm; /* (term <= $zLeTerm) paramater, or NULL */ + int colUsed; /* Copy of sqlite3_index_info.colUsed */ /* These are used by 'col' tables only */ int iCol; @@ -254750,9 +264820,11 @@ struct Fts5VocabCursor { /* ** Bits for the mask used as the idxNum value by xBestIndex/xFilter. */ -#define FTS5_VOCAB_TERM_EQ 0x01 -#define FTS5_VOCAB_TERM_GE 0x02 -#define FTS5_VOCAB_TERM_LE 0x04 +#define FTS5_VOCAB_TERM_EQ 0x0100 +#define FTS5_VOCAB_TERM_GE 0x0200 +#define FTS5_VOCAB_TERM_LE 0x0400 + +#define FTS5_VOCAB_COLUSED_MASK 0xFF /* @@ -254850,12 +264922,12 @@ static int fts5VocabInitVtab( *pzErr = sqlite3_mprintf("wrong number of vtable arguments"); rc = SQLITE_ERROR; }else{ - int nByte; /* Bytes of space to allocate */ + i64 nByte; /* Bytes of space to allocate */ const char *zDb = bDb ? argv[3] : argv[1]; const char *zTab = bDb ? argv[4] : argv[3]; const char *zType = bDb ? argv[5] : argv[4]; - int nDb = (int)strlen(zDb)+1; - int nTab = (int)strlen(zTab)+1; + i64 nDb = strlen(zDb)+1; + i64 nTab = strlen(zTab)+1; int eType = 0; rc = fts5VocabTableType(zType, pzErr, &eType); @@ -254929,11 +265001,13 @@ static int fts5VocabBestIndexMethod( int iTermEq = -1; int iTermGe = -1; int iTermLe = -1; - int idxNum = 0; + int idxNum = (int)pInfo->colUsed; int nArg = 0; UNUSED_PARAM(pUnused); + assert( (pInfo->colUsed & FTS5_VOCAB_COLUSED_MASK)==pInfo->colUsed ); + for(i=0; inConstraint; i++){ struct sqlite3_index_constraint *p = &pInfo->aConstraint[i]; if( p->usable==0 ) continue; @@ -255025,7 +265099,7 @@ static int fts5VocabOpenMethod( if( rc==SQLITE_OK ){ pVTab->zErrMsg = sqlite3_mprintf( "no such fts5 table: %s.%s", pTab->zFts5Db, pTab->zFts5Tbl - ); + ); rc = SQLITE_ERROR; } }else{ @@ -255051,7 +265125,12 @@ static int fts5VocabOpenMethod( return rc; } +/* +** Restore cursor pCsr to the state it was in immediately after being +** created by the xOpen() method. +*/ static void fts5VocabResetCursor(Fts5VocabCursor *pCsr){ + int nCol = pCsr->pFts5->pConfig->nCol; pCsr->rowid = 0; sqlite3Fts5IterClose(pCsr->pIter); sqlite3Fts5StructureRelease(pCsr->pStruct); @@ -255061,6 +265140,12 @@ static void fts5VocabResetCursor(Fts5VocabCursor *pCsr){ pCsr->nLeTerm = -1; pCsr->zLeTerm = 0; pCsr->bEof = 0; + pCsr->iCol = 0; + pCsr->iInstPos = 0; + pCsr->iInstOff = 0; + pCsr->colUsed = 0; + memset(pCsr->aCnt, 0, sizeof(i64)*nCol); + memset(pCsr->aDoc, 0, sizeof(i64)*nCol); } /* @@ -255185,9 +265270,19 @@ static int fts5VocabNextMethod(sqlite3_vtab_cursor *pCursor){ switch( pTab->eType ){ case FTS5_VOCAB_ROW: - if( eDetail==FTS5_DETAIL_FULL ){ - while( 0==sqlite3Fts5PoslistNext64(pPos, nPos, &iOff, &iPos) ){ - pCsr->aCnt[0]++; + /* Do not bother counting the number of instances if the "cnt" + ** column is not being read (according to colUsed). */ + if( eDetail==FTS5_DETAIL_FULL && (pCsr->colUsed & 0x04) ){ + while( iPosaCnt[] */ + pCsr->aCnt[0]++; + } } } pCsr->aDoc[0]++; @@ -255285,6 +265380,7 @@ static int fts5VocabFilterMethod( if( idxNum & FTS5_VOCAB_TERM_EQ ) pEq = apVal[iVal++]; if( idxNum & FTS5_VOCAB_TERM_GE ) pGe = apVal[iVal++]; if( idxNum & FTS5_VOCAB_TERM_LE ) pLe = apVal[iVal++]; + pCsr->colUsed = (idxNum & FTS5_VOCAB_COLUSED_MASK); if( pEq ){ zTerm = (const char *)sqlite3_value_text(pEq); @@ -255452,7 +265548,7 @@ static int sqlite3Fts5VocabInit(Fts5Global *pGlobal, sqlite3 *db){ } - +/* Here ends the fts5.c composite file. */ #endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS5) */ /************** End of fts5.c ************************************************/ @@ -255808,4 +265904,5 @@ SQLITE_API int sqlite3_stmt_init( /************** End of stmt.c ************************************************/ /* Return the source-id for this library */ SQLITE_API const char *sqlite3_sourceid(void){ return SQLITE_SOURCE_ID; } +#endif /* SQLITE_AMALGAMATION */ /************************** End of sqlite3.c ******************************/ diff --git a/source/lib/sqlite-amalgamation-3450200/sqlite3.h b/source/lib/sqlite-amalgamation-3510100/sqlite3.h similarity index 92% rename from source/lib/sqlite-amalgamation-3450200/sqlite3.h rename to source/lib/sqlite-amalgamation-3510100/sqlite3.h index c9fc77fb..76c567d0 100644 --- a/source/lib/sqlite-amalgamation-3450200/sqlite3.h +++ b/source/lib/sqlite-amalgamation-3510100/sqlite3.h @@ -133,7 +133,7 @@ extern "C" { ** ** Since [version 3.6.18] ([dateof:3.6.18]), ** SQLite source code has been stored in the -** Fossil configuration management +** Fossil configuration management ** system. ^The SQLITE_SOURCE_ID macro evaluates to ** a string which identifies a particular check-in of SQLite ** within its configuration management system. ^The SQLITE_SOURCE_ID @@ -146,9 +146,12 @@ extern "C" { ** [sqlite3_libversion_number()], [sqlite3_sourceid()], ** [sqlite_version()] and [sqlite_source_id()]. */ -#define SQLITE_VERSION "3.45.2" -#define SQLITE_VERSION_NUMBER 3045002 -#define SQLITE_SOURCE_ID "2024-03-12 11:06:23 d8cd6d49b46a395b13955387d05e9e1a2a47e54fb99f3c9b59835bbefad6af77" +#define SQLITE_VERSION "3.51.1" +#define SQLITE_VERSION_NUMBER 3051001 +#define SQLITE_SOURCE_ID "2025-11-28 17:28:25 281fc0e9afc38674b9b0991943b9e9d1e64c6cbdb133d35f6f5c87ff6af38a88" +#define SQLITE_SCM_BRANCH "branch-3.51" +#define SQLITE_SCM_TAGS "release version-3.51.1" +#define SQLITE_SCM_DATETIME "2025-11-28T17:28:25.933Z" /* ** CAPI3REF: Run-Time Library Version Numbers @@ -168,9 +171,9 @@ extern "C" { ** assert( strcmp(sqlite3_libversion(),SQLITE_VERSION)==0 ); ** )^ ** -** ^The sqlite3_version[] string constant contains the text of [SQLITE_VERSION] -** macro. ^The sqlite3_libversion() function returns a pointer to the -** to the sqlite3_version[] string constant. The sqlite3_libversion() +** ^The sqlite3_version[] string constant contains the text of the +** [SQLITE_VERSION] macro. ^The sqlite3_libversion() function returns a +** pointer to the sqlite3_version[] string constant. The sqlite3_libversion() ** function is provided for use in DLLs since DLL users usually do not have ** direct access to string constants within the DLL. ^The ** sqlite3_libversion_number() function returns an integer equal to @@ -370,7 +373,7 @@ typedef int (*sqlite3_callback)(void*,int,char**, char**); ** without having to use a lot of C code. ** ** ^The sqlite3_exec() interface runs zero or more UTF-8 encoded, -** semicolon-separate SQL statements passed into its 2nd argument, +** semicolon-separated SQL statements passed into its 2nd argument, ** in the context of the [database connection] passed in as its 1st ** argument. ^If the callback function of the 3rd argument to ** sqlite3_exec() is not NULL, then it is invoked for each result row @@ -403,7 +406,7 @@ typedef int (*sqlite3_callback)(void*,int,char**, char**); ** result row is NULL then the corresponding string pointer for the ** sqlite3_exec() callback is a NULL pointer. ^The 4th argument to the ** sqlite3_exec() callback is an array of pointers to strings where each -** entry represents the name of corresponding result column as obtained +** entry represents the name of a corresponding result column as obtained ** from [sqlite3_column_name()]. ** ** ^If the 2nd parameter to sqlite3_exec() is a NULL pointer, a pointer @@ -497,6 +500,9 @@ SQLITE_API int sqlite3_exec( #define SQLITE_ERROR_MISSING_COLLSEQ (SQLITE_ERROR | (1<<8)) #define SQLITE_ERROR_RETRY (SQLITE_ERROR | (2<<8)) #define SQLITE_ERROR_SNAPSHOT (SQLITE_ERROR | (3<<8)) +#define SQLITE_ERROR_RESERVESIZE (SQLITE_ERROR | (4<<8)) +#define SQLITE_ERROR_KEY (SQLITE_ERROR | (5<<8)) +#define SQLITE_ERROR_UNABLE (SQLITE_ERROR | (6<<8)) #define SQLITE_IOERR_READ (SQLITE_IOERR | (1<<8)) #define SQLITE_IOERR_SHORT_READ (SQLITE_IOERR | (2<<8)) #define SQLITE_IOERR_WRITE (SQLITE_IOERR | (3<<8)) @@ -531,6 +537,8 @@ SQLITE_API int sqlite3_exec( #define SQLITE_IOERR_DATA (SQLITE_IOERR | (32<<8)) #define SQLITE_IOERR_CORRUPTFS (SQLITE_IOERR | (33<<8)) #define SQLITE_IOERR_IN_PAGE (SQLITE_IOERR | (34<<8)) +#define SQLITE_IOERR_BADKEY (SQLITE_IOERR | (35<<8)) +#define SQLITE_IOERR_CODEC (SQLITE_IOERR | (36<<8)) #define SQLITE_LOCKED_SHAREDCACHE (SQLITE_LOCKED | (1<<8)) #define SQLITE_LOCKED_VTAB (SQLITE_LOCKED | (2<<8)) #define SQLITE_BUSY_RECOVERY (SQLITE_BUSY | (1<<8)) @@ -589,7 +597,7 @@ SQLITE_API int sqlite3_exec( ** Note in particular that passing the SQLITE_OPEN_EXCLUSIVE flag into ** [sqlite3_open_v2()] does *not* cause the underlying database file ** to be opened using O_EXCL. Passing SQLITE_OPEN_EXCLUSIVE into -** [sqlite3_open_v2()] has historically be a no-op and might become an +** [sqlite3_open_v2()] has historically been a no-op and might become an ** error in future versions of SQLite. */ #define SQLITE_OPEN_READONLY 0x00000001 /* Ok for sqlite3_open_v2() */ @@ -652,6 +660,13 @@ SQLITE_API int sqlite3_exec( ** filesystem supports doing multiple write operations atomically when those ** write operations are bracketed by [SQLITE_FCNTL_BEGIN_ATOMIC_WRITE] and ** [SQLITE_FCNTL_COMMIT_ATOMIC_WRITE]. +** +** The SQLITE_IOCAP_SUBPAGE_READ property means that it is ok to read +** from the database file in amounts that are not a multiple of the +** page size and that do not begin at a page boundary. Without this +** property, SQLite is careful to only do full-page reads and write +** on aligned pages, with the one exception that it will do a sub-page +** read of the first page to access the database header. */ #define SQLITE_IOCAP_ATOMIC 0x00000001 #define SQLITE_IOCAP_ATOMIC512 0x00000002 @@ -668,6 +683,7 @@ SQLITE_API int sqlite3_exec( #define SQLITE_IOCAP_POWERSAFE_OVERWRITE 0x00001000 #define SQLITE_IOCAP_IMMUTABLE 0x00002000 #define SQLITE_IOCAP_BATCH_ATOMIC 0x00004000 +#define SQLITE_IOCAP_SUBPAGE_READ 0x00008000 /* ** CAPI3REF: File Locking Levels @@ -675,7 +691,7 @@ SQLITE_API int sqlite3_exec( ** SQLite uses one of these integer values as the second ** argument to calls it makes to the xLock() and xUnlock() methods ** of an [sqlite3_io_methods] object. These values are ordered from -** lest restrictive to most restrictive. +** least restrictive to most restrictive. ** ** The argument to xLock() is always SHARED or higher. The argument to ** xUnlock is either SHARED or NONE. @@ -764,16 +780,16 @@ struct sqlite3_file { ** ** xLock() upgrades the database file lock. In other words, xLock() moves the ** database file lock in the direction NONE toward EXCLUSIVE. The argument to -** xLock() is always on of SHARED, RESERVED, PENDING, or EXCLUSIVE, never +** xLock() is always one of SHARED, RESERVED, PENDING, or EXCLUSIVE, never ** SQLITE_LOCK_NONE. If the database file lock is already at or above the ** requested lock, then the call to xLock() is a no-op. ** xUnlock() downgrades the database file lock to either SHARED or NONE. -* If the lock is already at or below the requested lock state, then the call +** If the lock is already at or below the requested lock state, then the call ** to xUnlock() is a no-op. ** The xCheckReservedLock() method checks whether any database connection, ** either in this process or in some other process, is holding a RESERVED, -** PENDING, or EXCLUSIVE lock on the file. It returns true -** if such a lock exists and false otherwise. +** PENDING, or EXCLUSIVE lock on the file. It returns, via its output +** pointer parameter, true if such a lock exists and false otherwise. ** ** The xFileControl() method is a generic interface that allows custom ** VFS implementations to directly control an open file using the @@ -814,6 +830,7 @@ struct sqlite3_file { **
        • [SQLITE_IOCAP_POWERSAFE_OVERWRITE] **
        • [SQLITE_IOCAP_IMMUTABLE] **
        • [SQLITE_IOCAP_BATCH_ATOMIC] +**
        • [SQLITE_IOCAP_SUBPAGE_READ] ** ** ** The SQLITE_IOCAP_ATOMIC property means that all writes of @@ -915,7 +932,7 @@ struct sqlite3_io_methods { ** connection. See also [SQLITE_FCNTL_FILE_POINTER]. ** **
        • [[SQLITE_FCNTL_SYNC_OMITTED]] -** No longer in use. +** The SQLITE_FCNTL_SYNC_OMITTED file-control is no longer used. ** **
        • [[SQLITE_FCNTL_SYNC]] ** The [SQLITE_FCNTL_SYNC] opcode is generated internally by SQLite and @@ -990,7 +1007,7 @@ struct sqlite3_io_methods { ** **
        • [[SQLITE_FCNTL_VFSNAME]] ** ^The [SQLITE_FCNTL_VFSNAME] opcode can be used to obtain the names of -** all [VFSes] in the VFS stack. The names are of all VFS shims and the +** all [VFSes] in the VFS stack. The names of all VFS shims and the ** final bottom-level VFS are written into memory obtained from ** [sqlite3_malloc()] and the result is stored in the char* variable ** that the fourth parameter of [sqlite3_file_control()] points to. @@ -1004,7 +1021,7 @@ struct sqlite3_io_methods { ** ^The [SQLITE_FCNTL_VFS_POINTER] opcode finds a pointer to the top-level ** [VFSes] currently in use. ^(The argument X in ** sqlite3_file_control(db,SQLITE_FCNTL_VFS_POINTER,X) must be -** of type "[sqlite3_vfs] **". This opcodes will set *X +** of type "[sqlite3_vfs] **". This opcode will set *X ** to a pointer to the top-level VFS.)^ ** ^When there are multiple VFS shims in the stack, this opcode finds the ** upper-most shim only. @@ -1091,6 +1108,11 @@ struct sqlite3_io_methods { ** pointed to by the pArg argument. This capability is used during testing ** and only needs to be supported when SQLITE_TEST is defined. ** +**
        • [[SQLITE_FCNTL_NULL_IO]] +** The [SQLITE_FCNTL_NULL_IO] opcode sets the low-level file descriptor +** or file handle for the [sqlite3_file] object such that it will no longer +** read or write to the database file. +** **
        • [[SQLITE_FCNTL_WAL_BLOCK]] ** The [SQLITE_FCNTL_WAL_BLOCK] is a signal to the VFS layer that it might ** be advantageous to block on the next WAL lock if the lock is not immediately @@ -1149,6 +1171,12 @@ struct sqlite3_io_methods { ** the value that M is to be set to. Before returning, the 32-bit signed ** integer is overwritten with the previous value of M. ** +**
        • [[SQLITE_FCNTL_BLOCK_ON_CONNECT]] +** The [SQLITE_FCNTL_BLOCK_ON_CONNECT] opcode is used to configure the +** VFS to block when taking a SHARED lock to connect to a wal mode database. +** This is used to implement the functionality associated with +** SQLITE_SETLK_BLOCK_ON_CONNECT. +** **
        • [[SQLITE_FCNTL_DATA_VERSION]] ** The [SQLITE_FCNTL_DATA_VERSION] opcode is used to detect changes to ** a database file. The argument is a pointer to a 32-bit unsigned integer. @@ -1183,7 +1211,7 @@ struct sqlite3_io_methods { **
        • [[SQLITE_FCNTL_EXTERNAL_READER]] ** The EXPERIMENTAL [SQLITE_FCNTL_EXTERNAL_READER] opcode is used to detect ** whether or not there is a database client in another process with a wal-mode -** transaction open on the database or not. It is only available on unix.The +** transaction open on the database or not. It is only available on unix. The ** (void*) argument passed with this file-control should be a pointer to a ** value of type (int). The integer value is set to 1 if the database is a wal ** mode database and there exists at least one client in another process that @@ -1201,6 +1229,15 @@ struct sqlite3_io_methods { ** database is not a temp db, then the [SQLITE_FCNTL_RESET_CACHE] file-control ** purges the contents of the in-memory page cache. If there is an open ** transaction, or if the db is a temp-db, this opcode is a no-op, not an error. +** +**
        • [[SQLITE_FCNTL_FILESTAT]] +** The [SQLITE_FCNTL_FILESTAT] opcode returns low-level diagnostic information +** about the [sqlite3_file] objects used access the database and journal files +** for the given schema. The fourth parameter to [sqlite3_file_control()] +** should be an initialized [sqlite3_str] pointer. JSON text describing +** various aspects of the sqlite3_file object is appended to the sqlite3_str. +** The SQLITE_FCNTL_FILESTAT opcode is usually a no-op, unless compile-time +** options are used to enable it. ** */ #define SQLITE_FCNTL_LOCKSTATE 1 @@ -1244,6 +1281,9 @@ struct sqlite3_io_methods { #define SQLITE_FCNTL_EXTERNAL_READER 40 #define SQLITE_FCNTL_CKSM_FILE 41 #define SQLITE_FCNTL_RESET_CACHE 42 +#define SQLITE_FCNTL_NULL_IO 43 +#define SQLITE_FCNTL_BLOCK_ON_CONNECT 44 +#define SQLITE_FCNTL_FILESTAT 45 /* deprecated names */ #define SQLITE_GET_LOCKPROXYFILE SQLITE_FCNTL_GET_LOCKPROXYFILE @@ -1606,7 +1646,7 @@ struct sqlite3_vfs { ** SQLite interfaces so that an application usually does not need to ** invoke sqlite3_initialize() directly. For example, [sqlite3_open()] ** calls sqlite3_initialize() so the SQLite library will be automatically -** initialized when [sqlite3_open()] is called if it has not be initialized +** initialized when [sqlite3_open()] is called if it has not been initialized ** already. ^However, if SQLite is compiled with the [SQLITE_OMIT_AUTOINIT] ** compile-time option, then the automatic calls to sqlite3_initialize() ** are omitted and the application must call sqlite3_initialize() directly @@ -1863,21 +1903,21 @@ struct sqlite3_mem_methods { ** The [sqlite3_mem_methods] ** structure is filled with the currently defined memory allocation routines.)^ ** This option can be used to overload the default memory allocation -** routines with a wrapper that simulations memory allocation failure or +** routines with a wrapper that simulates memory allocation failure or ** tracks memory usage, for example. ** ** [[SQLITE_CONFIG_SMALL_MALLOC]]
          SQLITE_CONFIG_SMALL_MALLOC
          -**
          ^The SQLITE_CONFIG_SMALL_MALLOC option takes single argument of +**
          ^The SQLITE_CONFIG_SMALL_MALLOC option takes a single argument of ** type int, interpreted as a boolean, which if true provides a hint to ** SQLite that it should avoid large memory allocations if possible. ** SQLite will run faster if it is free to make large memory allocations, -** but some application might prefer to run slower in exchange for +** but some applications might prefer to run slower in exchange for ** guarantees about memory fragmentation that are possible if large ** allocations are avoided. This hint is normally off. **
          ** ** [[SQLITE_CONFIG_MEMSTATUS]]
          SQLITE_CONFIG_MEMSTATUS
          -**
          ^The SQLITE_CONFIG_MEMSTATUS option takes single argument of type int, +**
          ^The SQLITE_CONFIG_MEMSTATUS option takes a single argument of type int, ** interpreted as a boolean, which enables or disables the collection of ** memory allocation statistics. ^(When memory allocation statistics are ** disabled, the following SQLite interfaces become non-operational: @@ -1922,7 +1962,7 @@ struct sqlite3_mem_methods { ** ^If pMem is NULL and N is non-zero, then each database connection ** does an initial bulk allocation for page cache memory ** from [sqlite3_malloc()] sufficient for N cache lines if N is positive or -** of -1024*N bytes if N is negative, . ^If additional +** of -1024*N bytes if N is negative. ^If additional ** page cache memory is needed beyond what is provided by the initial ** allocation, then SQLite goes to [sqlite3_malloc()] separately for each ** additional cache line.
          @@ -1951,7 +1991,7 @@ struct sqlite3_mem_methods { **
          ^(The SQLITE_CONFIG_MUTEX option takes a single argument which is a ** pointer to an instance of the [sqlite3_mutex_methods] structure. ** The argument specifies alternative low-level mutex routines to be used -** in place the mutex routines built into SQLite.)^ ^SQLite makes a copy of +** in place of the mutex routines built into SQLite.)^ ^SQLite makes a copy of ** the content of the [sqlite3_mutex_methods] structure before the call to ** [sqlite3_config()] returns. ^If SQLite is compiled with ** the [SQLITE_THREADSAFE | SQLITE_THREADSAFE=0] compile-time option then @@ -1974,13 +2014,16 @@ struct sqlite3_mem_methods { ** ** [[SQLITE_CONFIG_LOOKASIDE]]
          SQLITE_CONFIG_LOOKASIDE
          **
          ^(The SQLITE_CONFIG_LOOKASIDE option takes two arguments that determine -** the default size of lookaside memory on each [database connection]. +** the default size of [lookaside memory] on each [database connection]. ** The first argument is the -** size of each lookaside buffer slot and the second is the number of -** slots allocated to each database connection.)^ ^(SQLITE_CONFIG_LOOKASIDE -** sets the default lookaside size. The [SQLITE_DBCONFIG_LOOKASIDE] -** option to [sqlite3_db_config()] can be used to change the lookaside -** configuration on individual connections.)^
          +** size of each lookaside buffer slot ("sz") and the second is the number of +** slots allocated to each database connection ("cnt").)^ +** ^(SQLITE_CONFIG_LOOKASIDE sets the default lookaside size. +** The [SQLITE_DBCONFIG_LOOKASIDE] option to [sqlite3_db_config()] can +** be used to change the lookaside configuration on individual connections.)^ +** The [-DSQLITE_DEFAULT_LOOKASIDE] option can be used to change the +** default lookaside configuration at compile-time. +** ** ** [[SQLITE_CONFIG_PCACHE2]]
          SQLITE_CONFIG_PCACHE2
          **
          ^(The SQLITE_CONFIG_PCACHE2 option takes a single argument which is @@ -1990,7 +2033,7 @@ struct sqlite3_mem_methods { ** ** [[SQLITE_CONFIG_GETPCACHE2]]
          SQLITE_CONFIG_GETPCACHE2
          **
          ^(The SQLITE_CONFIG_GETPCACHE2 option takes a single argument which -** is a pointer to an [sqlite3_pcache_methods2] object. SQLite copies of +** is a pointer to an [sqlite3_pcache_methods2] object. SQLite copies off ** the current page cache implementation into that object.)^
          ** ** [[SQLITE_CONFIG_LOG]]
          SQLITE_CONFIG_LOG
          @@ -2007,7 +2050,7 @@ struct sqlite3_mem_methods { ** the logger function is a copy of the first parameter to the corresponding ** [sqlite3_log()] call and is intended to be a [result code] or an ** [extended result code]. ^The third parameter passed to the logger is -** log message after formatting via [sqlite3_snprintf()]. +** a log message after formatting via [sqlite3_snprintf()]. ** The SQLite logging interface is not reentrant; the logger function ** supplied by the application must not invoke any SQLite interface. ** In a multi-threaded application, the application-defined logger @@ -2143,6 +2186,22 @@ struct sqlite3_mem_methods { ** configuration setting is never used, then the default maximum is determined ** by the [SQLITE_MEMDB_DEFAULT_MAXSIZE] compile-time option. If that ** compile-time option is not set, then the default maximum is 1073741824. +** +** [[SQLITE_CONFIG_ROWID_IN_VIEW]] +**
          SQLITE_CONFIG_ROWID_IN_VIEW +**
          The SQLITE_CONFIG_ROWID_IN_VIEW option enables or disables the ability +** for VIEWs to have a ROWID. The capability can only be enabled if SQLite is +** compiled with -DSQLITE_ALLOW_ROWID_IN_VIEW, in which case the capability +** defaults to on. This configuration option queries the current setting or +** changes the setting to off or on. The argument is a pointer to an integer. +** If that integer initially holds a value of 1, then the ability for VIEWs to +** have ROWIDs is activated. If the integer initially holds zero, then the +** ability is deactivated. Any other initial value for the integer leaves the +** setting unchanged. After changes, if any, the integer is written with +** a 1 or 0, if the ability for VIEWs to have ROWIDs is on or off. If SQLite +** is compiled without -DSQLITE_ALLOW_ROWID_IN_VIEW (which is the usual and +** recommended case) then the integer is always filled with zero, regardless +** if its initial value. ** */ #define SQLITE_CONFIG_SINGLETHREAD 1 /* nil */ @@ -2174,12 +2233,21 @@ struct sqlite3_mem_methods { #define SQLITE_CONFIG_SMALL_MALLOC 27 /* boolean */ #define SQLITE_CONFIG_SORTERREF_SIZE 28 /* int nByte */ #define SQLITE_CONFIG_MEMDB_MAXSIZE 29 /* sqlite3_int64 */ +#define SQLITE_CONFIG_ROWID_IN_VIEW 30 /* int* */ /* ** CAPI3REF: Database Connection Configuration Options ** ** These constants are the available integer configuration options that -** can be passed as the second argument to the [sqlite3_db_config()] interface. +** can be passed as the second parameter to the [sqlite3_db_config()] interface. +** +** The [sqlite3_db_config()] interface is a var-args function. It takes a +** variable number of parameters, though always at least two. The number of +** parameters passed into sqlite3_db_config() depends on which of these +** constants is given as the second parameter. This documentation page +** refers to parameters beyond the second as "arguments". Thus, when this +** page says "the N-th argument" it means "the N-th parameter past the +** configuration option" or "the (N+2)-th parameter to sqlite3_db_config()". ** ** New configuration options may be added in future releases of SQLite. ** Existing configuration options might be discontinued. Applications @@ -2191,31 +2259,57 @@ struct sqlite3_mem_methods { **
          ** [[SQLITE_DBCONFIG_LOOKASIDE]] **
          SQLITE_DBCONFIG_LOOKASIDE
          -**
          ^This option takes three additional arguments that determine the -** [lookaside memory allocator] configuration for the [database connection]. -** ^The first argument (the third parameter to [sqlite3_db_config()] is a +**
          The SQLITE_DBCONFIG_LOOKASIDE option is used to adjust the +** configuration of the [lookaside memory allocator] within a database +** connection. +** The arguments to the SQLITE_DBCONFIG_LOOKASIDE option are not +** in the [DBCONFIG arguments|usual format]. +** The SQLITE_DBCONFIG_LOOKASIDE option takes three arguments, not two, +** so that a call to [sqlite3_db_config()] that uses SQLITE_DBCONFIG_LOOKASIDE +** should have a total of five parameters. +**
            +**
          1. The first argument ("buf") is a ** pointer to a memory buffer to use for lookaside memory. -** ^The first argument after the SQLITE_DBCONFIG_LOOKASIDE verb -** may be NULL in which case SQLite will allocate the -** lookaside buffer itself using [sqlite3_malloc()]. ^The second argument is the -** size of each lookaside buffer slot. ^The third argument is the number of -** slots. The size of the buffer in the first argument must be greater than -** or equal to the product of the second and third arguments. The buffer -** must be aligned to an 8-byte boundary. ^If the second argument to -** SQLITE_DBCONFIG_LOOKASIDE is not a multiple of 8, it is internally -** rounded down to the next smaller multiple of 8. ^(The lookaside memory +** The first argument may be NULL in which case SQLite will allocate the +** lookaside buffer itself using [sqlite3_malloc()]. +**

          2. The second argument ("sz") is the +** size of each lookaside buffer slot. Lookaside is disabled if "sz" +** is less than 8. The "sz" argument should be a multiple of 8 less than +** 65536. If "sz" does not meet this constraint, it is reduced in size until +** it does. +**

          3. The third argument ("cnt") is the number of slots. Lookaside is disabled +** if "cnt"is less than 1. The "cnt" value will be reduced, if necessary, so +** that the product of "sz" and "cnt" does not exceed 2,147,418,112. The "cnt" +** parameter is usually chosen so that the product of "sz" and "cnt" is less +** than 1,000,000. +**

          +**

          If the "buf" argument is not NULL, then it must +** point to a memory buffer with a size that is greater than +** or equal to the product of "sz" and "cnt". +** The buffer must be aligned to an 8-byte boundary. +** The lookaside memory ** configuration for a database connection can only be changed when that ** connection is not currently using lookaside memory, or in other words -** when the "current value" returned by -** [sqlite3_db_status](D,[SQLITE_DBSTATUS_LOOKASIDE_USED],...) is zero. +** when the value returned by [SQLITE_DBSTATUS_LOOKASIDE_USED] is zero. ** Any attempt to change the lookaside memory configuration when lookaside ** memory is in use leaves the configuration unchanged and returns -** [SQLITE_BUSY].)^

          +** [SQLITE_BUSY]. +** If the "buf" argument is NULL and an attempt +** to allocate memory based on "sz" and "cnt" fails, then +** lookaside is silently disabled. +**

          +** The [SQLITE_CONFIG_LOOKASIDE] configuration option can be used to set the +** default lookaside configuration at initialization. The +** [-DSQLITE_DEFAULT_LOOKASIDE] option can be used to set the default lookaside +** configuration at compile-time. Typical values for lookaside are 1200 for +** "sz" and 40 to 100 for "cnt". +**

          ** ** [[SQLITE_DBCONFIG_ENABLE_FKEY]] **
          SQLITE_DBCONFIG_ENABLE_FKEY
          **
          ^This option is used to enable or disable the enforcement of -** [foreign key constraints]. There should be two additional arguments. +** [foreign key constraints]. This is the same setting that is +** enabled or disabled by the [PRAGMA foreign_keys] statement. ** The first argument is an integer which is 0 to disable FK enforcement, ** positive to enable FK enforcement or negative to leave FK enforcement ** unchanged. The second parameter is a pointer to an integer into which @@ -2237,13 +2331,13 @@ struct sqlite3_mem_methods { **

          Originally this option disabled all triggers. ^(However, since ** SQLite version 3.35.0, TEMP triggers are still allowed even if ** this option is off. So, in other words, this option now only disables -** triggers in the main database schema or in the schemas of ATTACH-ed +** triggers in the main database schema or in the schemas of [ATTACH]-ed ** databases.)^

          ** ** [[SQLITE_DBCONFIG_ENABLE_VIEW]] **
          SQLITE_DBCONFIG_ENABLE_VIEW
          **
          ^This option is used to enable or disable [CREATE VIEW | views]. -** There should be two additional arguments. +** There must be two additional arguments. ** The first argument is an integer which is 0 to disable views, ** positive to enable views or negative to leave the setting unchanged. ** The second parameter is a pointer to an integer into which @@ -2259,17 +2353,20 @@ struct sqlite3_mem_methods { ** ** [[SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER]] **
          SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER
          -**
          ^This option is used to enable or disable the -** [fts3_tokenizer()] function which is part of the -** [FTS3] full-text search engine extension. -** There should be two additional arguments. -** The first argument is an integer which is 0 to disable fts3_tokenizer() or -** positive to enable fts3_tokenizer() or negative to leave the setting -** unchanged. -** The second parameter is a pointer to an integer into which -** is written 0 or 1 to indicate whether fts3_tokenizer is disabled or enabled -** following this call. The second parameter may be a NULL pointer, in -** which case the new setting is not reported back.
          +**
          ^This option is used to enable or disable using the +** [fts3_tokenizer()] function - part of the [FTS3] full-text search engine +** extension - without using bound parameters as the parameters. Doing so +** is disabled by default. There must be two additional arguments. The first +** argument is an integer. If it is passed 0, then using fts3_tokenizer() +** without bound parameters is disabled. If it is passed a positive value, +** then calling fts3_tokenizer without bound parameters is enabled. If it +** is passed a negative value, this setting is not modified - this can be +** used to query for the current setting. The second parameter is a pointer +** to an integer into which is written 0 or 1 to indicate the current value +** of this setting (after it is modified, if applicable). The second +** parameter may be a NULL pointer, in which case the value of the setting +** is not reported back. Refer to [FTS3] documentation for further details. +**
          ** ** [[SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION]] **
          SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION
          @@ -2277,12 +2374,12 @@ struct sqlite3_mem_methods { ** interface independently of the [load_extension()] SQL function. ** The [sqlite3_enable_load_extension()] API enables or disables both the ** C-API [sqlite3_load_extension()] and the SQL function [load_extension()]. -** There should be two additional arguments. +** There must be two additional arguments. ** When the first argument to this interface is 1, then only the C-API is ** enabled and the SQL function remains disabled. If the first argument to ** this interface is 0, then both the C-API and the SQL function are disabled. -** If the first argument is -1, then no changes are made to state of either the -** C-API or the SQL function. +** If the first argument is -1, then no changes are made to the state of either +** the C-API or the SQL function. ** The second parameter is a pointer to an integer into which ** is written 0 or 1 to indicate whether [sqlite3_load_extension()] interface ** is disabled or enabled following this call. The second parameter may @@ -2291,23 +2388,30 @@ struct sqlite3_mem_methods { ** ** [[SQLITE_DBCONFIG_MAINDBNAME]]
          SQLITE_DBCONFIG_MAINDBNAME
          **
          ^This option is used to change the name of the "main" database -** schema. ^The sole argument is a pointer to a constant UTF8 string -** which will become the new schema name in place of "main". ^SQLite -** does not make a copy of the new main schema name string, so the application -** must ensure that the argument passed into this DBCONFIG option is unchanged -** until after the database connection closes. +** schema. This option does not follow the +** [DBCONFIG arguments|usual SQLITE_DBCONFIG argument format]. +** This option takes exactly one additional argument so that the +** [sqlite3_db_config()] call has a total of three parameters. The +** extra argument must be a pointer to a constant UTF8 string which +** will become the new schema name in place of "main". ^SQLite does +** not make a copy of the new main schema name string, so the application +** must ensure that the argument passed into SQLITE_DBCONFIG MAINDBNAME +** is unchanged until after the database connection closes. **
          ** ** [[SQLITE_DBCONFIG_NO_CKPT_ON_CLOSE]] **
          SQLITE_DBCONFIG_NO_CKPT_ON_CLOSE
          -**
          Usually, when a database in wal mode is closed or detached from a -** database handle, SQLite checks if this will mean that there are now no -** connections at all to the database. If so, it performs a checkpoint -** operation before closing the connection. This option may be used to -** override this behavior. The first parameter passed to this operation -** is an integer - positive to disable checkpoints-on-close, or zero (the -** default) to enable them, and negative to leave the setting unchanged. -** The second parameter is a pointer to an integer +**
          Usually, when a database in [WAL mode] is closed or detached from a +** database handle, SQLite checks if if there are other connections to the +** same database, and if there are no other database connection (if the +** connection being closed is the last open connection to the database), +** then SQLite performs a [checkpoint] before closing the connection and +** deletes the WAL file. The SQLITE_DBCONFIG_NO_CKPT_ON_CLOSE option can +** be used to override that behavior. The first argument passed to this +** operation (the third parameter to [sqlite3_db_config()]) is an integer +** which is positive to disable checkpoints-on-close, or zero (the default) +** to enable them, and negative to leave the setting unchanged. +** The second argument (the fourth parameter) is a pointer to an integer ** into which is written 0 or 1 to indicate whether checkpoints-on-close ** have been disabled - 0 if they are not disabled, 1 if they are. **
          @@ -2393,7 +2497,7 @@ struct sqlite3_mem_methods { ** [[SQLITE_DBCONFIG_LEGACY_ALTER_TABLE]] **
          SQLITE_DBCONFIG_LEGACY_ALTER_TABLE
          **
          The SQLITE_DBCONFIG_LEGACY_ALTER_TABLE option activates or deactivates -** the legacy behavior of the [ALTER TABLE RENAME] command such it +** the legacy behavior of the [ALTER TABLE RENAME] command such that it ** behaves as it did prior to [version 3.24.0] (2018-06-04). See the ** "Compatibility Notice" on the [ALTER TABLE RENAME documentation] for ** additional information. This feature can also be turned on and off @@ -2442,7 +2546,7 @@ struct sqlite3_mem_methods { **
          SQLITE_DBCONFIG_LEGACY_FILE_FORMAT
          **
          The SQLITE_DBCONFIG_LEGACY_FILE_FORMAT option activates or deactivates ** the legacy file format flag. When activated, this flag causes all newly -** created database file to have a schema format version number (the 4-byte +** created database files to have a schema format version number (the 4-byte ** integer found at offset 44 into the database header) of 1. This in turn ** means that the resulting database file will be readable and writable by ** any SQLite version back to 3.0.0 ([dateof:3.0.0]). Without this setting, @@ -2468,8 +2572,8 @@ struct sqlite3_mem_methods { ** statistics. For statistics to be collected, the flag must be set on ** the database handle both when the SQL statement is prepared and when it ** is stepped. The flag is set (collection of statistics is enabled) -** by default. This option takes two arguments: an integer and a pointer to -** an integer.. The first argument is 1, 0, or -1 to enable, disable, or +** by default.

          This option takes two arguments: an integer and a pointer to +** an integer. The first argument is 1, 0, or -1 to enable, disable, or ** leave unchanged the statement scanstatus option. If the second argument ** is not NULL, then the value of the statement scanstatus setting after ** processing the first argument is written into the integer that the second @@ -2482,7 +2586,7 @@ struct sqlite3_mem_methods { ** in which tables and indexes are scanned so that the scans start at the end ** and work toward the beginning rather than starting at the beginning and ** working toward the end. Setting SQLITE_DBCONFIG_REVERSE_SCANORDER is the -** same as setting [PRAGMA reverse_unordered_selects]. This option takes +** same as setting [PRAGMA reverse_unordered_selects].

          This option takes ** two arguments which are an integer and a pointer to an integer. The first ** argument is 1, 0, or -1 to enable, disable, or leave unchanged the ** reverse scan order flag, respectively. If the second argument is not NULL, @@ -2491,7 +2595,76 @@ struct sqlite3_mem_methods { ** first argument. **

          ** +** [[SQLITE_DBCONFIG_ENABLE_ATTACH_CREATE]] +**
          SQLITE_DBCONFIG_ENABLE_ATTACH_CREATE
          +**
          The SQLITE_DBCONFIG_ENABLE_ATTACH_CREATE option enables or disables +** the ability of the [ATTACH DATABASE] SQL command to create a new database +** file if the database filed named in the ATTACH command does not already +** exist. This ability of ATTACH to create a new database is enabled by +** default. Applications can disable or reenable the ability for ATTACH to +** create new database files using this DBCONFIG option.

          +** This option takes two arguments which are an integer and a pointer +** to an integer. The first argument is 1, 0, or -1 to enable, disable, or +** leave unchanged the attach-create flag, respectively. If the second +** argument is not NULL, then 0 or 1 is written into the integer that the +** second argument points to depending on if the attach-create flag is set +** after processing the first argument. +**

          +** +** [[SQLITE_DBCONFIG_ENABLE_ATTACH_WRITE]] +**
          SQLITE_DBCONFIG_ENABLE_ATTACH_WRITE
          +**
          The SQLITE_DBCONFIG_ENABLE_ATTACH_WRITE option enables or disables the +** ability of the [ATTACH DATABASE] SQL command to open a database for writing. +** This capability is enabled by default. Applications can disable or +** reenable this capability using the current DBCONFIG option. If +** this capability is disabled, the [ATTACH] command will still work, +** but the database will be opened read-only. If this option is disabled, +** then the ability to create a new database using [ATTACH] is also disabled, +** regardless of the value of the [SQLITE_DBCONFIG_ENABLE_ATTACH_CREATE] +** option.

          +** This option takes two arguments which are an integer and a pointer +** to an integer. The first argument is 1, 0, or -1 to enable, disable, or +** leave unchanged the ability to ATTACH another database for writing, +** respectively. If the second argument is not NULL, then 0 or 1 is written +** into the integer to which the second argument points, depending on whether +** the ability to ATTACH a read/write database is enabled or disabled +** after processing the first argument. +**

          +** +** [[SQLITE_DBCONFIG_ENABLE_COMMENTS]] +**
          SQLITE_DBCONFIG_ENABLE_COMMENTS
          +**
          The SQLITE_DBCONFIG_ENABLE_COMMENTS option enables or disables the +** ability to include comments in SQL text. Comments are enabled by default. +** An application can disable or reenable comments in SQL text using this +** DBCONFIG option.

          +** This option takes two arguments which are an integer and a pointer +** to an integer. The first argument is 1, 0, or -1 to enable, disable, or +** leave unchanged the ability to use comments in SQL text, +** respectively. If the second argument is not NULL, then 0 or 1 is written +** into the integer that the second argument points to depending on if +** comments are allowed in SQL text after processing the first argument. +**

          +** ** +** +** [[DBCONFIG arguments]]

          Arguments To SQLITE_DBCONFIG Options

          +** +**

          Most of the SQLITE_DBCONFIG options take two arguments, so that the +** overall call to [sqlite3_db_config()] has a total of four parameters. +** The first argument (the third parameter to sqlite3_db_config()) is an integer. +** The second argument is a pointer to an integer. If the first argument is 1, +** then the option becomes enabled. If the first integer argument is 0, then the +** option is disabled. If the first argument is -1, then the option setting +** is unchanged. The second argument, the pointer to an integer, may be NULL. +** If the second argument is not NULL, then a value of 0 or 1 is written into +** the integer to which the second argument points, depending on whether the +** setting is disabled or enabled after applying any changes specified by +** the first argument. +** +**

          While most SQLITE_DBCONFIG options use the argument format +** described in the previous paragraph, the [SQLITE_DBCONFIG_MAINDBNAME] +** and [SQLITE_DBCONFIG_LOOKASIDE] options are different. See the +** documentation of those exceptional options for details. */ #define SQLITE_DBCONFIG_MAINDBNAME 1000 /* const char* */ #define SQLITE_DBCONFIG_LOOKASIDE 1001 /* void* int int */ @@ -2513,7 +2686,10 @@ struct sqlite3_mem_methods { #define SQLITE_DBCONFIG_TRUSTED_SCHEMA 1017 /* int int* */ #define SQLITE_DBCONFIG_STMT_SCANSTATUS 1018 /* int int* */ #define SQLITE_DBCONFIG_REVERSE_SCANORDER 1019 /* int int* */ -#define SQLITE_DBCONFIG_MAX 1019 /* Largest DBCONFIG */ +#define SQLITE_DBCONFIG_ENABLE_ATTACH_CREATE 1020 /* int int* */ +#define SQLITE_DBCONFIG_ENABLE_ATTACH_WRITE 1021 /* int int* */ +#define SQLITE_DBCONFIG_ENABLE_COMMENTS 1022 /* int int* */ +#define SQLITE_DBCONFIG_MAX 1022 /* Largest DBCONFIG */ /* ** CAPI3REF: Enable Or Disable Extended Result Codes @@ -2605,10 +2781,14 @@ SQLITE_API void sqlite3_set_last_insert_rowid(sqlite3*,sqlite3_int64); ** deleted by the most recently completed INSERT, UPDATE or DELETE ** statement on the database connection specified by the only parameter. ** The two functions are identical except for the type of the return value -** and that if the number of rows modified by the most recent INSERT, UPDATE +** and that if the number of rows modified by the most recent INSERT, UPDATE, ** or DELETE is greater than the maximum value supported by type "int", then ** the return value of sqlite3_changes() is undefined. ^Executing any other ** type of SQL statement does not modify the value returned by these functions. +** For the purposes of this interface, a CREATE TABLE AS SELECT statement +** does not count as an INSERT, UPDATE or DELETE statement and hence the rows +** added to the new table by the CREATE TABLE AS SELECT statement are not +** counted. ** ** ^Only changes made directly by the INSERT, UPDATE or DELETE statement are ** considered - auxiliary changes caused by [CREATE TRIGGER | triggers], @@ -2761,7 +2941,7 @@ SQLITE_API int sqlite3_is_interrupted(sqlite3*); ** ^These routines return 0 if the statement is incomplete. ^If a ** memory allocation fails, then SQLITE_NOMEM is returned. ** -** ^These routines do not parse the SQL statements thus +** ^These routines do not parse the SQL statements and thus ** will not detect syntactically incorrect SQL. ** ** ^(If SQLite has not been initialized using [sqlite3_initialize()] prior @@ -2863,6 +3043,44 @@ SQLITE_API int sqlite3_busy_handler(sqlite3*,int(*)(void*,int),void*); */ SQLITE_API int sqlite3_busy_timeout(sqlite3*, int ms); +/* +** CAPI3REF: Set the Setlk Timeout +** METHOD: sqlite3 +** +** This routine is only useful in SQLITE_ENABLE_SETLK_TIMEOUT builds. If +** the VFS supports blocking locks, it sets the timeout in ms used by +** eligible locks taken on wal mode databases by the specified database +** handle. In non-SQLITE_ENABLE_SETLK_TIMEOUT builds, or if the VFS does +** not support blocking locks, this function is a no-op. +** +** Passing 0 to this function disables blocking locks altogether. Passing +** -1 to this function requests that the VFS blocks for a long time - +** indefinitely if possible. The results of passing any other negative value +** are undefined. +** +** Internally, each SQLite database handle stores two timeout values - the +** busy-timeout (used for rollback mode databases, or if the VFS does not +** support blocking locks) and the setlk-timeout (used for blocking locks +** on wal-mode databases). The sqlite3_busy_timeout() method sets both +** values, this function sets only the setlk-timeout value. Therefore, +** to configure separate busy-timeout and setlk-timeout values for a single +** database handle, call sqlite3_busy_timeout() followed by this function. +** +** Whenever the number of connections to a wal mode database falls from +** 1 to 0, the last connection takes an exclusive lock on the database, +** then checkpoints and deletes the wal file. While it is doing this, any +** new connection that tries to read from the database fails with an +** SQLITE_BUSY error. Or, if the SQLITE_SETLK_BLOCK_ON_CONNECT flag is +** passed to this API, the new connection blocks until the exclusive lock +** has been released. +*/ +SQLITE_API int sqlite3_setlk_timeout(sqlite3*, int ms, int flags); + +/* +** CAPI3REF: Flags for sqlite3_setlk_timeout() +*/ +#define SQLITE_SETLK_BLOCK_ON_CONNECT 0x01 + /* ** CAPI3REF: Convenience Routines For Running Queries ** METHOD: sqlite3 @@ -2870,7 +3088,7 @@ SQLITE_API int sqlite3_busy_timeout(sqlite3*, int ms); ** This is a legacy interface that is preserved for backwards compatibility. ** Use of this interface is not recommended. ** -** Definition: A result table is memory data structure created by the +** Definition: A result table is a memory data structure created by the ** [sqlite3_get_table()] interface. A result table records the ** complete query results from one or more queries. ** @@ -3013,7 +3231,7 @@ SQLITE_API char *sqlite3_vsnprintf(int,char*,const char*, va_list); ** ^Calling sqlite3_free() with a pointer previously returned ** by sqlite3_malloc() or sqlite3_realloc() releases that memory so ** that it might be reused. ^The sqlite3_free() routine is -** a no-op if is called with a NULL pointer. Passing a NULL pointer +** a no-op if it is called with a NULL pointer. Passing a NULL pointer ** to sqlite3_free() is harmless. After being freed, memory ** should neither be read nor written. Even reading previously freed ** memory might result in a segmentation fault or other severe error. @@ -3031,13 +3249,13 @@ SQLITE_API char *sqlite3_vsnprintf(int,char*,const char*, va_list); ** sqlite3_free(X). ** ^sqlite3_realloc(X,N) returns a pointer to a memory allocation ** of at least N bytes in size or NULL if insufficient memory is available. -** ^If M is the size of the prior allocation, then min(N,M) bytes -** of the prior allocation are copied into the beginning of buffer returned +** ^If M is the size of the prior allocation, then min(N,M) bytes of the +** prior allocation are copied into the beginning of the buffer returned ** by sqlite3_realloc(X,N) and the prior allocation is freed. ** ^If sqlite3_realloc(X,N) returns NULL and N is positive, then the ** prior allocation is not freed. ** -** ^The sqlite3_realloc64(X,N) interfaces works the same as +** ^The sqlite3_realloc64(X,N) interface works the same as ** sqlite3_realloc(X,N) except that N is a 64-bit unsigned integer instead ** of a 32-bit signed integer. ** @@ -3087,7 +3305,7 @@ SQLITE_API sqlite3_uint64 sqlite3_msize(void*); ** was last reset. ^The values returned by [sqlite3_memory_used()] and ** [sqlite3_memory_highwater()] include any overhead ** added by SQLite in its implementation of [sqlite3_malloc()], -** but not overhead added by the any underlying system library +** but not overhead added by any underlying system library ** routines that [sqlite3_malloc()] may call. ** ** ^The memory high-water mark is reset to the current value of @@ -3288,8 +3506,8 @@ SQLITE_API int sqlite3_set_authorizer( #define SQLITE_RECURSIVE 33 /* NULL NULL */ /* -** CAPI3REF: Tracing And Profiling Functions -** METHOD: sqlite3 +** CAPI3REF: Deprecated Tracing And Profiling Functions +** DEPRECATED ** ** These routines are deprecated. Use the [sqlite3_trace_v2()] interface ** instead of the routines described here. @@ -3539,7 +3757,7 @@ SQLITE_API void sqlite3_progress_handler(sqlite3*, int, int(*)(void*), void*); ** there is no harm in trying.) ** ** ^(

          [SQLITE_OPEN_SHAREDCACHE]
          -**
          The database is opened [shared cache] enabled, overriding +**
          The database is opened with [shared cache] enabled, overriding ** the default shared cache setting provided by ** [sqlite3_enable_shared_cache()].)^ ** The [use of shared cache mode is discouraged] and hence shared cache @@ -3547,14 +3765,14 @@ SQLITE_API void sqlite3_progress_handler(sqlite3*, int, int(*)(void*), void*); ** this option is a no-op. ** ** ^(
          [SQLITE_OPEN_PRIVATECACHE]
          -**
          The database is opened [shared cache] disabled, overriding +**
          The database is opened with [shared cache] disabled, overriding ** the default shared cache setting provided by ** [sqlite3_enable_shared_cache()].)^ ** ** [[OPEN_EXRESCODE]] ^(
          [SQLITE_OPEN_EXRESCODE]
          **
          The database connection comes up in "extended result code mode". -** In other words, the database behaves has if -** [sqlite3_extended_result_codes(db,1)] where called on the database +** In other words, the database behaves as if +** [sqlite3_extended_result_codes(db,1)] were called on the database ** connection as soon as the connection is created. In addition to setting ** the extended result code mode, this flag also causes [sqlite3_open_v2()] ** to return an extended result code.
          @@ -3882,7 +4100,7 @@ SQLITE_API sqlite3_file *sqlite3_database_file_object(const char*); ** ** The sqlite3_create_filename(D,J,W,N,P) allocates memory to hold a version of ** database filename D with corresponding journal file J and WAL file W and -** with N URI parameters key/values pairs in the array P. The result from +** an array P of N URI Key/Value pairs. The result from ** sqlite3_create_filename(D,J,W,N,P) is a pointer to a database filename that ** is safe to pass to routines like: **
            @@ -3965,7 +4183,7 @@ SQLITE_API void sqlite3_free_filename(sqlite3_filename); ** subsequent calls to other SQLite interface functions.)^ ** ** ^The sqlite3_errstr(E) interface returns the English-language text -** that describes the [result code] E, as UTF-8, or NULL if E is not an +** that describes the [result code] E, as UTF-8, or NULL if E is not a ** result code for which a text error message is available. ** ^(Memory to hold the error message string is managed internally ** and must not be freed by the application)^. @@ -3973,7 +4191,7 @@ SQLITE_API void sqlite3_free_filename(sqlite3_filename); ** ^If the most recent error references a specific token in the input ** SQL, the sqlite3_error_offset() interface returns the byte offset ** of the start of that token. ^The byte offset returned by -** sqlite3_error_offset() assumes that the input SQL is UTF8. +** sqlite3_error_offset() assumes that the input SQL is UTF-8. ** ^If the most recent error does not reference a specific token in the input ** SQL, then the sqlite3_error_offset() function returns -1. ** @@ -3998,6 +4216,34 @@ SQLITE_API const void *sqlite3_errmsg16(sqlite3*); SQLITE_API const char *sqlite3_errstr(int); SQLITE_API int sqlite3_error_offset(sqlite3 *db); +/* +** CAPI3REF: Set Error Codes And Message +** METHOD: sqlite3 +** +** Set the error code of the database handle passed as the first argument +** to errcode, and the error message to a copy of nul-terminated string +** zErrMsg. If zErrMsg is passed NULL, then the error message is set to +** the default message associated with the supplied error code. Subsequent +** calls to [sqlite3_errcode()] and [sqlite3_errmsg()] and similar will +** return the values set by this routine in place of what was previously +** set by SQLite itself. +** +** This function returns SQLITE_OK if the error code and error message are +** successfully set, SQLITE_NOMEM if an OOM occurs, and SQLITE_MISUSE if +** the database handle is NULL or invalid. +** +** The error code and message set by this routine remains in effect until +** they are changed, either by another call to this routine or until they are +** changed to by SQLite itself to reflect the result of some subsquent +** API call. +** +** This function is intended for use by SQLite extensions or wrappers. The +** idea is that an extension or wrapper can use this routine to set error +** messages and error codes and thus behave more like a core SQLite +** feature from the point of view of an application. +*/ +SQLITE_API int sqlite3_set_errmsg(sqlite3 *db, int errcode, const char *zErrMsg); + /* ** CAPI3REF: Prepared Statement Object ** KEYWORDS: {prepared statement} {prepared statements} @@ -4072,8 +4318,8 @@ SQLITE_API int sqlite3_limit(sqlite3*, int id, int newVal); ** ** These constants define various performance limits ** that can be lowered at run-time using [sqlite3_limit()]. -** The synopsis of the meanings of the various limits is shown below. -** Additional information is available at [limits | Limits in SQLite]. +** A concise description of these limits follows, and additional information +** is available at [limits | Limits in SQLite]. ** **
            ** [[SQLITE_LIMIT_LENGTH]] ^(
            SQLITE_LIMIT_LENGTH
            @@ -4138,7 +4384,7 @@ SQLITE_API int sqlite3_limit(sqlite3*, int id, int newVal); /* ** CAPI3REF: Prepare Flags ** -** These constants define various flags that can be passed into +** These constants define various flags that can be passed into the ** "prepFlags" parameter of the [sqlite3_prepare_v3()] and ** [sqlite3_prepare16_v3()] interfaces. ** @@ -4168,11 +4414,22 @@ SQLITE_API int sqlite3_limit(sqlite3*, int id, int newVal); **
            The SQLITE_PREPARE_NO_VTAB flag causes the SQL compiler ** to return an error (error code SQLITE_ERROR) if the statement uses ** any virtual tables. +** +** [[SQLITE_PREPARE_DONT_LOG]]
            SQLITE_PREPARE_DONT_LOG
            +**
            The SQLITE_PREPARE_DONT_LOG flag prevents SQL compiler +** errors from being sent to the error log defined by +** [SQLITE_CONFIG_LOG]. This can be used, for example, to do test +** compiles to see if some SQL syntax is well-formed, without generating +** messages on the global error log when it is not. If the test compile +** fails, the sqlite3_prepare_v3() call returns the same error indications +** with or without this flag; it just omits the call to [sqlite3_log()] that +** logs the error. **
            */ #define SQLITE_PREPARE_PERSISTENT 0x01 #define SQLITE_PREPARE_NORMALIZE 0x02 #define SQLITE_PREPARE_NO_VTAB 0x04 +#define SQLITE_PREPARE_DONT_LOG 0x10 /* ** CAPI3REF: Compiling An SQL Statement @@ -4205,13 +4462,17 @@ SQLITE_API int sqlite3_limit(sqlite3*, int id, int newVal); ** and sqlite3_prepare16_v3() use UTF-16. ** ** ^If the nByte argument is negative, then zSql is read up to the -** first zero terminator. ^If nByte is positive, then it is the -** number of bytes read from zSql. ^If nByte is zero, then no prepared +** first zero terminator. ^If nByte is positive, then it is the maximum +** number of bytes read from zSql. When nByte is positive, zSql is read +** up to the first zero terminator or until the nByte bytes have been read, +** whichever comes first. ^If nByte is zero, then no prepared ** statement is generated. ** If the caller knows that the supplied string is nul-terminated, then ** there is a small performance advantage to passing an nByte parameter that ** is the number of bytes in the input string including ** the nul-terminator. +** Note that nByte measures the length of the input in bytes, not +** characters, even for the UTF-16 interfaces. ** ** ^If pzTail is not NULL then *pzTail is made to point to the first byte ** past the end of the first SQL statement in zSql. These routines only @@ -4344,7 +4605,7 @@ SQLITE_API int sqlite3_prepare16_v3( ** ** ^The sqlite3_expanded_sql() interface returns NULL if insufficient memory ** is available to hold the result, or if the result would exceed the -** the maximum string length determined by the [SQLITE_LIMIT_LENGTH]. +** maximum string length determined by the [SQLITE_LIMIT_LENGTH]. ** ** ^The [SQLITE_TRACE_SIZE_LIMIT] compile-time option limits the size of ** bound parameter expansions. ^The [SQLITE_OMIT_TRACE] compile-time @@ -4532,7 +4793,7 @@ typedef struct sqlite3_value sqlite3_value; ** ** The context in which an SQL function executes is stored in an ** sqlite3_context object. ^A pointer to an sqlite3_context object -** is always first parameter to [application-defined SQL functions]. +** is always the first parameter to [application-defined SQL functions]. ** The application-defined SQL function implementation will pass this ** pointer through into calls to [sqlite3_result_int | sqlite3_result()], ** [sqlite3_aggregate_context()], [sqlite3_user_data()], @@ -4548,7 +4809,7 @@ typedef struct sqlite3_context sqlite3_context; ** METHOD: sqlite3_stmt ** ** ^(In the SQL statement text input to [sqlite3_prepare_v2()] and its variants, -** literals may be replaced by a [parameter] that matches one of following +** literals may be replaced by a [parameter] that matches one of the following ** templates: ** **
              @@ -4593,7 +4854,7 @@ typedef struct sqlite3_context sqlite3_context; ** ** [[byte-order determination rules]] ^The byte-order of ** UTF16 input text is determined by the byte-order mark (BOM, U+FEFF) -** found in first character, which is removed, or in the absence of a BOM +** found in the first character, which is removed, or in the absence of a BOM ** the byte order is the native byte order of the host ** machine for sqlite3_bind_text16() or the byte order specified in ** the 6th parameter for sqlite3_bind_text64().)^ @@ -4613,7 +4874,7 @@ typedef struct sqlite3_context sqlite3_context; ** or sqlite3_bind_text16() or sqlite3_bind_text64() then ** that parameter must be the byte offset ** where the NUL terminator would occur assuming the string were NUL -** terminated. If any NUL characters occurs at byte offsets less than +** terminated. If any NUL characters occur at byte offsets less than ** the value of the fourth parameter then the resulting string value will ** contain embedded NULs. The result of expressions involving strings ** with embedded NULs is undefined. @@ -4656,9 +4917,11 @@ typedef struct sqlite3_context sqlite3_context; ** associated with the pointer P of type T. ^D is either a NULL pointer or ** a pointer to a destructor function for P. ^SQLite will invoke the ** destructor D with a single argument of P when it is finished using -** P. The T parameter should be a static string, preferably a string -** literal. The sqlite3_bind_pointer() routine is part of the -** [pointer passing interface] added for SQLite 3.20.0. +** P, even if the call to sqlite3_bind_pointer() fails. Due to a +** historical design quirk, results are undefined if D is +** SQLITE_TRANSIENT. The T parameter should be a static string, +** preferably a string literal. The sqlite3_bind_pointer() routine is +** part of the [pointer passing interface] added for SQLite 3.20.0. ** ** ^If any of the sqlite3_bind_*() routines are called with a NULL pointer ** for the [prepared statement] or with a prepared statement for which @@ -4825,7 +5088,7 @@ SQLITE_API const void *sqlite3_column_name16(sqlite3_stmt*, int N); ** METHOD: sqlite3_stmt ** ** ^These routines provide a means to determine the database, table, and -** table column that is the origin of a particular result column in +** table column that is the origin of a particular result column in a ** [SELECT] statement. ** ^The name of the database or table or column can be returned as ** either a UTF-8 or UTF-16 string. ^The _database_ routines return @@ -4963,7 +5226,7 @@ SQLITE_API const void *sqlite3_column_decltype16(sqlite3_stmt*,int); ** other than [SQLITE_ROW] before any subsequent invocation of ** sqlite3_step(). Failure to reset the prepared statement using ** [sqlite3_reset()] would result in an [SQLITE_MISUSE] return from -** sqlite3_step(). But after [version 3.6.23.1] ([dateof:3.6.23.1], +** sqlite3_step(). But after [version 3.6.23.1] ([dateof:3.6.23.1]), ** sqlite3_step() began ** calling [sqlite3_reset()] automatically in this circumstance rather ** than returning [SQLITE_MISUSE]. This is not considered a compatibility @@ -5269,7 +5532,7 @@ SQLITE_API int sqlite3_column_type(sqlite3_stmt*, int iCol); ** ** ^The sqlite3_finalize() function is called to delete a [prepared statement]. ** ^If the most recent evaluation of the statement encountered no errors -** or if the statement is never been evaluated, then sqlite3_finalize() returns +** or if the statement has never been evaluated, then sqlite3_finalize() returns ** SQLITE_OK. ^If the most recent evaluation of statement S failed, then ** sqlite3_finalize(S) returns the appropriate [error code] or ** [extended error code]. @@ -5394,8 +5657,8 @@ SQLITE_API int sqlite3_reset(sqlite3_stmt *pStmt); ** ** For best security, the [SQLITE_DIRECTONLY] flag is recommended for ** all application-defined SQL functions that do not need to be -** used inside of triggers, view, CHECK constraints, or other elements of -** the database schema. This flags is especially recommended for SQL +** used inside of triggers, views, CHECK constraints, or other elements of +** the database schema. This flag is especially recommended for SQL ** functions that have side effects or reveal internal application state. ** Without this flag, an attacker might be able to modify the schema of ** a database file to include invocations of the function with parameters @@ -5426,7 +5689,7 @@ SQLITE_API int sqlite3_reset(sqlite3_stmt *pStmt); ** [user-defined window functions|available here]. ** ** ^(If the final parameter to sqlite3_create_function_v2() or -** sqlite3_create_window_function() is not NULL, then it is destructor for +** sqlite3_create_window_function() is not NULL, then it is the destructor for ** the application data pointer. The destructor is invoked when the function ** is deleted, either by being overloaded or when the database connection ** closes.)^ ^The destructor is also invoked if the call to @@ -5501,7 +5764,7 @@ SQLITE_API int sqlite3_create_window_function( /* ** CAPI3REF: Text Encodings ** -** These constant define integer codes that represent the various +** These constants define integer codes that represent the various ** text encodings supported by SQLite. */ #define SQLITE_UTF8 1 /* IMP: R-37514-35566 */ @@ -5582,7 +5845,7 @@ SQLITE_API int sqlite3_create_window_function( ** This flag instructs SQLite to omit some corner-case optimizations that ** might disrupt the operation of the [sqlite3_value_subtype()] function, ** causing it to return zero rather than the correct subtype(). -** SQL functions that invokes [sqlite3_value_subtype()] should have this +** All SQL functions that invoke [sqlite3_value_subtype()] should have this ** property. If the SQLITE_SUBTYPE property is omitted, then the return ** value from [sqlite3_value_subtype()] might sometimes be zero even though ** a non-zero subtype was specified by the function argument expression. @@ -5593,11 +5856,20 @@ SQLITE_API int sqlite3_create_window_function( ** result. ** Every function that invokes [sqlite3_result_subtype()] should have this ** property. If it does not, then the call to [sqlite3_result_subtype()] -** might become a no-op if the function is used as term in an +** might become a no-op if the function is used as a term in an ** [expression index]. On the other hand, SQL functions that never invoke ** [sqlite3_result_subtype()] should avoid setting this property, as the ** purpose of this property is to disable certain optimizations that are ** incompatible with subtypes. +** +** [[SQLITE_SELFORDER1]]
              SQLITE_SELFORDER1
              +** The SQLITE_SELFORDER1 flag indicates that the function is an aggregate +** that internally orders the values provided to the first argument. The +** ordered-set aggregate SQL notation with a single ORDER BY term can be +** used to invoke this function. If the ordered-set aggregate notation is +** used on a function that lacks this flag, then an error is raised. Note +** that the ordered-set aggregate syntax is only available if SQLite is +** built using the -DSQLITE_ENABLE_ORDERED_SET_AGGREGATES compile-time option. **
              ** */ @@ -5606,6 +5878,7 @@ SQLITE_API int sqlite3_create_window_function( #define SQLITE_SUBTYPE 0x000100000 #define SQLITE_INNOCUOUS 0x000200000 #define SQLITE_RESULT_SUBTYPE 0x001000000 +#define SQLITE_SELFORDER1 0x002000000 /* ** CAPI3REF: Deprecated Functions @@ -5710,7 +5983,7 @@ SQLITE_API SQLITE_DEPRECATED int sqlite3_memory_alarm(void(*)(void*,sqlite3_int6 ** sqlite3_value_nochange(X) interface returns true if and only if ** the column corresponding to X is unchanged by the UPDATE operation ** that the xUpdate method call was invoked to implement and if -** and the prior [xColumn] method call that was invoked to extracted +** the prior [xColumn] method call that was invoked to extract ** the value for that column returned without setting a result (probably ** because it queried [sqlite3_vtab_nochange()] and found that the column ** was unchanging). ^Within an [xUpdate] method, any value for which @@ -5803,7 +6076,7 @@ SQLITE_API int sqlite3_value_encoding(sqlite3_value*); ** one SQL function to another. Use the [sqlite3_result_subtype()] ** routine to set the subtype for the return value of an SQL function. ** -** Every [application-defined SQL function] that invoke this interface +** Every [application-defined SQL function] that invokes this interface ** should include the [SQLITE_SUBTYPE] property in the text ** encoding argument when the function is [sqlite3_create_function|registered]. ** If the [SQLITE_SUBTYPE] property is omitted, then sqlite3_value_subtype() @@ -5816,7 +6089,7 @@ SQLITE_API unsigned int sqlite3_value_subtype(sqlite3_value*); ** METHOD: sqlite3_value ** ** ^The sqlite3_value_dup(V) interface makes a copy of the [sqlite3_value] -** object D and returns a pointer to that copy. ^The [sqlite3_value] returned +** object V and returns a pointer to that copy. ^The [sqlite3_value] returned ** is a [protected sqlite3_value] object even if the input is not. ** ^The sqlite3_value_dup(V) interface returns NULL if V is NULL or if a ** memory allocation fails. ^If V is a [pointer value], then the result @@ -5854,7 +6127,7 @@ SQLITE_API void sqlite3_value_free(sqlite3_value*); ** allocation error occurs. ** ** ^(The amount of space allocated by sqlite3_aggregate_context(C,N) is -** determined by the N parameter on first successful call. Changing the +** determined by the N parameter on the first successful call. Changing the ** value of N in any subsequent call to sqlite3_aggregate_context() within ** the same aggregate function instance will not resize the memory ** allocation.)^ Within the xFinal callback, it is customary to set @@ -5983,6 +6256,7 @@ SQLITE_API void sqlite3_set_auxdata(sqlite3_context*, int N, void*, void (*)(voi ** or a NULL pointer if there were no prior calls to ** sqlite3_set_clientdata() with the same values of D and N. ** Names are compared using strcmp() and are thus case sensitive. +** It returns 0 on success and SQLITE_NOMEM on allocation failure. ** ** If P and X are both non-NULL, then the destructor X is invoked with ** argument P on the first of the following occurrences: @@ -6016,7 +6290,7 @@ SQLITE_API void sqlite3_set_auxdata(sqlite3_context*, int N, void*, void (*)(voi ** ** Security Warning: These interfaces should not be exposed in scripting ** languages or in other circumstances where it might be possible for an -** an attacker to invoke them. Any agent that can invoke these interfaces +** attacker to invoke them. Any agent that can invoke these interfaces ** can probably also take control of the process. ** ** Database connection client data is only available for SQLite @@ -6130,7 +6404,7 @@ typedef void (*sqlite3_destructor_type)(void*); ** pointed to by the 2nd parameter are taken as the application-defined ** function result. If the 3rd parameter is non-negative, then it ** must be the byte offset into the string where the NUL terminator would -** appear if the string where NUL terminated. If any NUL characters occur +** appear if the string were NUL terminated. If any NUL characters occur ** in the string at a byte offset that is less than the value of the 3rd ** parameter, then the resulting string will contain embedded NULs and the ** result of expressions operating on strings with embedded NULs is undefined. @@ -6188,7 +6462,7 @@ typedef void (*sqlite3_destructor_type)(void*); ** string and preferably a string literal. The sqlite3_result_pointer() ** routine is part of the [pointer passing interface] added for SQLite 3.20.0. ** -** If these routines are called from within the different thread +** If these routines are called from within a different thread ** than the one containing the application-defined function that received ** the [sqlite3_context] pointer, the results are undefined. */ @@ -6594,7 +6868,7 @@ SQLITE_API sqlite3 *sqlite3_db_handle(sqlite3_stmt*); ** METHOD: sqlite3 ** ** ^The sqlite3_db_name(D,N) interface returns a pointer to the schema name -** for the N-th database on database connection D, or a NULL pointer of N is +** for the N-th database on database connection D, or a NULL pointer if N is ** out of range. An N value of 0 means the main database file. An N of 1 is ** the "temp" schema. Larger values of N correspond to various ATTACH-ed ** databases. @@ -6689,7 +6963,7 @@ SQLITE_API int sqlite3_txn_state(sqlite3*,const char *zSchema); **
              The SQLITE_TXN_READ state means that the database is currently ** in a read transaction. Content has been read from the database file ** but nothing in the database file has changed. The transaction state -** will advanced to SQLITE_TXN_WRITE if any changes occur and there are +** will be advanced to SQLITE_TXN_WRITE if any changes occur and there are ** no other conflicting concurrent write transactions. The transaction ** state will revert to SQLITE_TXN_NONE following a [ROLLBACK] or ** [COMMIT].
              @@ -6698,7 +6972,7 @@ SQLITE_API int sqlite3_txn_state(sqlite3*,const char *zSchema); **
              The SQLITE_TXN_WRITE state means that the database is currently ** in a write transaction. Content has been written to the database file ** but has not yet committed. The transaction state will change to -** to SQLITE_TXN_NONE at the next [ROLLBACK] or [COMMIT].
              +** SQLITE_TXN_NONE at the next [ROLLBACK] or [COMMIT]. */ #define SQLITE_TXN_NONE 0 #define SQLITE_TXN_READ 1 @@ -6849,6 +7123,8 @@ SQLITE_API int sqlite3_autovacuum_pages( ** ** ^The second argument is a pointer to the function to invoke when a ** row is updated, inserted or deleted in a rowid table. +** ^The update hook is disabled by invoking sqlite3_update_hook() +** with a NULL pointer as the second parameter. ** ^The first argument to the callback is a copy of the third argument ** to sqlite3_update_hook(). ** ^The second callback argument is one of [SQLITE_INSERT], [SQLITE_DELETE], @@ -6870,6 +7146,12 @@ SQLITE_API int sqlite3_autovacuum_pages( ** The exceptions defined in this paragraph might change in a future ** release of SQLite. ** +** Whether the update hook is invoked before or after the +** corresponding change is currently unspecified and may differ +** depending on the type of change. Do not rely on the order of the +** hook call with regards to the final result of the operation which +** triggers the hook. +** ** The update hook implementation must not do anything that will modify ** the database connection that invoked the update hook. Any actions ** to modify the database connection must be deferred until after the @@ -6971,7 +7253,7 @@ SQLITE_API int sqlite3_db_release_memory(sqlite3*); ** CAPI3REF: Impose A Limit On Heap Size ** ** These interfaces impose limits on the amount of heap memory that will be -** by all database connections within a single process. +** used by all database connections within a single process. ** ** ^The sqlite3_soft_heap_limit64() interface sets and/or queries the ** soft limit on the amount of heap memory that may be allocated by SQLite. @@ -7029,7 +7311,7 @@ SQLITE_API int sqlite3_db_release_memory(sqlite3*); **
            )^ ** ** The circumstances under which SQLite will enforce the heap limits may -** changes in future releases of SQLite. +** change in future releases of SQLite. */ SQLITE_API sqlite3_int64 sqlite3_soft_heap_limit64(sqlite3_int64 N); SQLITE_API sqlite3_int64 sqlite3_hard_heap_limit64(sqlite3_int64 N); @@ -7144,8 +7426,8 @@ SQLITE_API int sqlite3_table_column_metadata( ** ^The entry point is zProc. ** ^(zProc may be 0, in which case SQLite will try to come up with an ** entry point name on its own. It first tries "sqlite3_extension_init". -** If that does not work, it constructs a name "sqlite3_X_init" where the -** X is consists of the lower-case equivalent of all ASCII alphabetic +** If that does not work, it constructs a name "sqlite3_X_init" where +** X consists of the lower-case equivalent of all ASCII alphabetic ** characters in the filename from the last "/" to the first following ** "." and omitting any initial "lib".)^ ** ^The sqlite3_load_extension() interface returns @@ -7216,7 +7498,7 @@ SQLITE_API int sqlite3_enable_load_extension(sqlite3 *db, int onoff); ** ^(Even though the function prototype shows that xEntryPoint() takes ** no arguments and returns void, SQLite invokes xEntryPoint() with three ** arguments and expects an integer result as if the signature of the -** entry point where as follows: +** entry point were as follows: ** **
             **    int xEntryPoint(
            @@ -7380,7 +7662,7 @@ struct sqlite3_module {
             ** virtual table and might not be checked again by the byte code.)^ ^(The
             ** aConstraintUsage[].omit flag is an optimization hint. When the omit flag
             ** is left in its default setting of false, the constraint will always be
            -** checked separately in byte code.  If the omit flag is change to true, then
            +** checked separately in byte code.  If the omit flag is changed to true, then
             ** the constraint may or may not be checked in byte code.  In other words,
             ** when the omit flag is true there is no guarantee that the constraint will
             ** not be checked again using byte code.)^
            @@ -7404,9 +7686,11 @@ struct sqlite3_module {
             ** will be returned by the strategy.
             **
             ** The xBestIndex method may optionally populate the idxFlags field with a
            -** mask of SQLITE_INDEX_SCAN_* flags. Currently there is only one such flag -
            -** SQLITE_INDEX_SCAN_UNIQUE. If the xBestIndex method sets this flag, SQLite
            -** assumes that the strategy may visit at most one row.
            +** mask of SQLITE_INDEX_SCAN_* flags. One such flag is
            +** [SQLITE_INDEX_SCAN_HEX], which if set causes the [EXPLAIN QUERY PLAN]
            +** output to show the idxNum as hex instead of as decimal.  Another flag is
            +** SQLITE_INDEX_SCAN_UNIQUE, which if set indicates that the query plan will
            +** return at most one row.
             **
             ** Additionally, if xBestIndex sets the SQLITE_INDEX_SCAN_UNIQUE flag, then
             ** SQLite also assumes that if a call to the xUpdate() method is made as
            @@ -7470,7 +7754,9 @@ struct sqlite3_index_info {
             ** [sqlite3_index_info].idxFlags field to some combination of
             ** these bits.
             */
            -#define SQLITE_INDEX_SCAN_UNIQUE      1     /* Scan visits at most 1 row */
            +#define SQLITE_INDEX_SCAN_UNIQUE 0x00000001 /* Scan visits at most 1 row */
            +#define SQLITE_INDEX_SCAN_HEX    0x00000002 /* Display idxNum as hex */
            +                                            /* in EXPLAIN QUERY PLAN */
             
             /*
             ** CAPI3REF: Virtual Table Constraint Operator Codes
            @@ -7543,7 +7829,7 @@ struct sqlite3_index_info {
             ** the implementation of the [virtual table module].   ^The fourth
             ** parameter is an arbitrary client data pointer that is passed through
             ** into the [xCreate] and [xConnect] methods of the virtual table module
            -** when a new virtual table is be being created or reinitialized.
            +** when a new virtual table is being created or reinitialized.
             **
             ** ^The sqlite3_create_module_v2() interface has a fifth parameter which
             ** is a pointer to a destructor for the pClientData.  ^SQLite will
            @@ -7708,7 +7994,7 @@ typedef struct sqlite3_blob sqlite3_blob;
             ** in *ppBlob. Otherwise an [error code] is returned and, unless the error
             ** code is SQLITE_MISUSE, *ppBlob is set to NULL.)^ ^This means that, provided
             ** the API is not misused, it is always safe to call [sqlite3_blob_close()]
            -** on *ppBlob after this function it returns.
            +** on *ppBlob after this function returns.
             **
             ** This function fails with SQLITE_ERROR if any of the following are true:
             ** 
              @@ -7828,7 +8114,7 @@ SQLITE_API int sqlite3_blob_close(sqlite3_blob *); ** ** ^Returns the size in bytes of the BLOB accessible via the ** successfully opened [BLOB handle] in its only argument. ^The -** incremental blob I/O routines can only read or overwriting existing +** incremental blob I/O routines can only read or overwrite existing ** blob content; they cannot change the size of a blob. ** ** This routine only works on a [BLOB handle] which has been created @@ -7978,7 +8264,7 @@ SQLITE_API int sqlite3_vfs_unregister(sqlite3_vfs*); ** ^The sqlite3_mutex_alloc() routine allocates a new ** mutex and returns a pointer to it. ^The sqlite3_mutex_alloc() ** routine returns NULL if it is unable to allocate the requested -** mutex. The argument to sqlite3_mutex_alloc() must one of these +** mutex. The argument to sqlite3_mutex_alloc() must be one of these ** integer constants: ** **
                @@ -8211,7 +8497,7 @@ SQLITE_API int sqlite3_mutex_notheld(sqlite3_mutex*); ** CAPI3REF: Retrieve the mutex for a database connection ** METHOD: sqlite3 ** -** ^This interface returns a pointer the [sqlite3_mutex] object that +** ^This interface returns a pointer to the [sqlite3_mutex] object that ** serializes access to the [database connection] given in the argument ** when the [threading mode] is Serialized. ** ^If the [threading mode] is Single-thread or Multi-thread then this @@ -8307,6 +8593,7 @@ SQLITE_API int sqlite3_test_control(int op, ...); #define SQLITE_TESTCTRL_JSON_SELFCHECK 14 #define SQLITE_TESTCTRL_OPTIMIZATIONS 15 #define SQLITE_TESTCTRL_ISKEYWORD 16 /* NOT USED */ +#define SQLITE_TESTCTRL_GETOPT 16 #define SQLITE_TESTCTRL_SCRATCHMALLOC 17 /* NOT USED */ #define SQLITE_TESTCTRL_INTERNAL_FUNCTIONS 17 #define SQLITE_TESTCTRL_LOCALTIME_FAULT 18 @@ -8326,21 +8613,21 @@ SQLITE_API int sqlite3_test_control(int op, ...); #define SQLITE_TESTCTRL_TRACEFLAGS 31 #define SQLITE_TESTCTRL_TUNE 32 #define SQLITE_TESTCTRL_LOGEST 33 -#define SQLITE_TESTCTRL_USELONGDOUBLE 34 +#define SQLITE_TESTCTRL_USELONGDOUBLE 34 /* NOT USED */ #define SQLITE_TESTCTRL_LAST 34 /* Largest TESTCTRL */ /* ** CAPI3REF: SQL Keyword Checking ** ** These routines provide access to the set of SQL language keywords -** recognized by SQLite. Applications can uses these routines to determine +** recognized by SQLite. Applications can use these routines to determine ** whether or not a specific identifier needs to be escaped (for example, ** by enclosing in double-quotes) so as not to confuse the parser. ** ** The sqlite3_keyword_count() interface returns the number of distinct ** keywords understood by SQLite. ** -** The sqlite3_keyword_name(N,Z,L) interface finds the N-th keyword and +** The sqlite3_keyword_name(N,Z,L) interface finds the 0-based N-th keyword and ** makes *Z point to that keyword expressed as UTF8 and writes the number ** of bytes in the keyword into *L. The string that *Z points to is not ** zero-terminated. The sqlite3_keyword_name(N,Z,L) routine returns @@ -8501,7 +8788,7 @@ SQLITE_API void sqlite3_str_reset(sqlite3_str*); ** content of the dynamic string under construction in X. The value ** returned by [sqlite3_str_value(X)] is managed by the sqlite3_str object X ** and might be freed or altered by any subsequent method on the same -** [sqlite3_str] object. Applications must not used the pointer returned +** [sqlite3_str] object. Applications must not use the pointer returned by ** [sqlite3_str_value(X)] after any subsequent method call on the same ** object. ^Applications may change the content of the string returned ** by [sqlite3_str_value(X)] as long as they do not write into any bytes @@ -8587,7 +8874,7 @@ SQLITE_API int sqlite3_status64( ** allocation which could not be satisfied by the [SQLITE_CONFIG_PAGECACHE] ** buffer and where forced to overflow to [sqlite3_malloc()]. The ** returned value includes allocations that overflowed because they -** where too large (they were larger than the "sz" parameter to +** were too large (they were larger than the "sz" parameter to ** [SQLITE_CONFIG_PAGECACHE]) and allocations that overflowed because ** no space was left in the page cache.)^ ** @@ -8646,9 +8933,18 @@ SQLITE_API int sqlite3_status64( ** ^The sqlite3_db_status() routine returns SQLITE_OK on success and a ** non-zero [error code] on failure. ** +** ^The sqlite3_db_status64(D,O,C,H,R) routine works exactly the same +** way as sqlite3_db_status(D,O,C,H,R) routine except that the C and H +** parameters are pointer to 64-bit integers (type: sqlite3_int64) instead +** of pointers to 32-bit integers, which allows larger status values +** to be returned. If a status value exceeds 2,147,483,647 then +** sqlite3_db_status() will truncate the value whereas sqlite3_db_status64() +** will return the full value. +** ** See also: [sqlite3_status()] and [sqlite3_stmt_status()]. */ SQLITE_API int sqlite3_db_status(sqlite3*, int op, int *pCur, int *pHiwtr, int resetFlg); +SQLITE_API int sqlite3_db_status64(sqlite3*,int,sqlite3_int64*,sqlite3_int64*,int); /* ** CAPI3REF: Status Parameters for database connections @@ -8671,28 +8967,29 @@ SQLITE_API int sqlite3_db_status(sqlite3*, int op, int *pCur, int *pHiwtr, int r ** [[SQLITE_DBSTATUS_LOOKASIDE_HIT]] ^(
                SQLITE_DBSTATUS_LOOKASIDE_HIT
                **
                This parameter returns the number of malloc attempts that were ** satisfied using lookaside memory. Only the high-water value is meaningful; -** the current value is always zero.)^ +** the current value is always zero.
                )^ ** ** [[SQLITE_DBSTATUS_LOOKASIDE_MISS_SIZE]] ** ^(
                SQLITE_DBSTATUS_LOOKASIDE_MISS_SIZE
                -**
                This parameter returns the number malloc attempts that might have +**
                This parameter returns the number of malloc attempts that might have ** been satisfied using lookaside memory but failed due to the amount of ** memory requested being larger than the lookaside slot size. ** Only the high-water value is meaningful; -** the current value is always zero.)^ +** the current value is always zero.
                )^ ** ** [[SQLITE_DBSTATUS_LOOKASIDE_MISS_FULL]] ** ^(
                SQLITE_DBSTATUS_LOOKASIDE_MISS_FULL
                -**
                This parameter returns the number malloc attempts that might have +**
                This parameter returns the number of malloc attempts that might have ** been satisfied using lookaside memory but failed due to all lookaside ** memory already being in use. ** Only the high-water value is meaningful; -** the current value is always zero.)^ +** the current value is always zero.
                )^ ** ** [[SQLITE_DBSTATUS_CACHE_USED]] ^(
                SQLITE_DBSTATUS_CACHE_USED
                **
                This parameter returns the approximate number of bytes of heap ** memory used by all pager caches associated with the database connection.)^ ** ^The highwater mark associated with SQLITE_DBSTATUS_CACHE_USED is always 0. +**
                ** ** [[SQLITE_DBSTATUS_CACHE_USED_SHARED]] ** ^(
                SQLITE_DBSTATUS_CACHE_USED_SHARED
                @@ -8701,10 +8998,10 @@ SQLITE_API int sqlite3_db_status(sqlite3*, int op, int *pCur, int *pHiwtr, int r ** memory used by that pager cache is divided evenly between the attached ** connections.)^ In other words, if none of the pager caches associated ** with the database connection are shared, this request returns the same -** value as DBSTATUS_CACHE_USED. Or, if one or more or the pager caches are +** value as DBSTATUS_CACHE_USED. Or, if one or more of the pager caches are ** shared, the value returned by this call will be smaller than that returned ** by DBSTATUS_CACHE_USED. ^The highwater mark associated with -** SQLITE_DBSTATUS_CACHE_USED_SHARED is always 0. +** SQLITE_DBSTATUS_CACHE_USED_SHARED is always 0. ** ** [[SQLITE_DBSTATUS_SCHEMA_USED]] ^(
                SQLITE_DBSTATUS_SCHEMA_USED
                **
                This parameter returns the approximate number of bytes of heap @@ -8714,6 +9011,7 @@ SQLITE_API int sqlite3_db_status(sqlite3*, int op, int *pCur, int *pHiwtr, int r ** schema memory is shared with other database connections due to ** [shared cache mode] being enabled. ** ^The highwater mark associated with SQLITE_DBSTATUS_SCHEMA_USED is always 0. +**
                ** ** [[SQLITE_DBSTATUS_STMT_USED]] ^(
                SQLITE_DBSTATUS_STMT_USED
                **
                This parameter returns the approximate number of bytes of heap @@ -8743,6 +9041,10 @@ SQLITE_API int sqlite3_db_status(sqlite3*, int op, int *pCur, int *pHiwtr, int r ** If an IO or other error occurs while writing a page to disk, the effect ** on subsequent SQLITE_DBSTATUS_CACHE_WRITE requests is undefined.)^ ^The ** highwater mark associated with SQLITE_DBSTATUS_CACHE_WRITE is always 0. +**

                +** ^(There is overlap between the quantities measured by this parameter +** (SQLITE_DBSTATUS_CACHE_WRITE) and SQLITE_DBSTATUS_TEMPBUF_SPILL. +** Resetting one will reduce the other.)^ **

                ** ** [[SQLITE_DBSTATUS_CACHE_SPILL]] ^(
                SQLITE_DBSTATUS_CACHE_SPILL
                @@ -8750,7 +9052,7 @@ SQLITE_API int sqlite3_db_status(sqlite3*, int op, int *pCur, int *pHiwtr, int r ** been written to disk in the middle of a transaction due to the page ** cache overflowing. Transactions are more efficient if they are written ** to disk all at once. When pages spill mid-transaction, that introduces -** additional overhead. This parameter can be used help identify +** additional overhead. This parameter can be used to help identify ** inefficiencies that can be resolved by increasing the cache size. ** ** @@ -8758,6 +9060,18 @@ SQLITE_API int sqlite3_db_status(sqlite3*, int op, int *pCur, int *pHiwtr, int r **
                This parameter returns zero for the current value if and only if ** all foreign key constraints (deferred or immediate) have been ** resolved.)^ ^The highwater mark is always 0. +** +** [[SQLITE_DBSTATUS_TEMPBUF_SPILL] ^(
                SQLITE_DBSTATUS_TEMPBUF_SPILL
                +**
                ^(This parameter returns the number of bytes written to temporary +** files on disk that could have been kept in memory had sufficient memory +** been available. This value includes writes to intermediate tables that +** are part of complex queries, external sorts that spill to disk, and +** writes to TEMP tables.)^ +** ^The highwater mark is always 0. +**

                +** ^(There is overlap between the quantities measured by this parameter +** (SQLITE_DBSTATUS_TEMPBUF_SPILL) and SQLITE_DBSTATUS_CACHE_WRITE. +** Resetting one will reduce the other.)^ **

                ** */ @@ -8774,7 +9088,8 @@ SQLITE_API int sqlite3_db_status(sqlite3*, int op, int *pCur, int *pHiwtr, int r #define SQLITE_DBSTATUS_DEFERRED_FKS 10 #define SQLITE_DBSTATUS_CACHE_USED_SHARED 11 #define SQLITE_DBSTATUS_CACHE_SPILL 12 -#define SQLITE_DBSTATUS_MAX 12 /* Largest defined DBSTATUS */ +#define SQLITE_DBSTATUS_TEMPBUF_SPILL 13 +#define SQLITE_DBSTATUS_MAX 13 /* Largest defined DBSTATUS */ /* @@ -8821,13 +9136,13 @@ SQLITE_API int sqlite3_stmt_status(sqlite3_stmt*, int op,int resetFlg); ** [[SQLITE_STMTSTATUS_SORT]]
                SQLITE_STMTSTATUS_SORT
                **
                ^This is the number of sort operations that have occurred. ** A non-zero value in this counter may indicate an opportunity to -** improvement performance through careful use of indices.
                +** improve performance through careful use of indices. ** ** [[SQLITE_STMTSTATUS_AUTOINDEX]]
                SQLITE_STMTSTATUS_AUTOINDEX
                **
                ^This is the number of rows inserted into transient indices that ** were created automatically in order to help joins run faster. ** A non-zero value in this counter may indicate an opportunity to -** improvement performance by adding permanent indices that do not +** improve performance by adding permanent indices that do not ** need to be reinitialized each time the statement is run.
                ** ** [[SQLITE_STMTSTATUS_VM_STEP]]
                SQLITE_STMTSTATUS_VM_STEP
                @@ -8836,19 +9151,19 @@ SQLITE_API int sqlite3_stmt_status(sqlite3_stmt*, int op,int resetFlg); ** to 2147483647. The number of virtual machine operations can be ** used as a proxy for the total work done by the prepared statement. ** If the number of virtual machine operations exceeds 2147483647 -** then the value returned by this statement status code is undefined. +** then the value returned by this statement status code is undefined. ** ** [[SQLITE_STMTSTATUS_REPREPARE]]
                SQLITE_STMTSTATUS_REPREPARE
                **
                ^This is the number of times that the prepare statement has been ** automatically regenerated due to schema changes or changes to -** [bound parameters] that might affect the query plan. +** [bound parameters] that might affect the query plan.
                ** ** [[SQLITE_STMTSTATUS_RUN]]
                SQLITE_STMTSTATUS_RUN
                **
                ^This is the number of times that the prepared statement has ** been run. A single "run" for the purposes of this counter is one ** or more calls to [sqlite3_step()] followed by a call to [sqlite3_reset()]. ** The counter is incremented on the first [sqlite3_step()] call of each -** cycle. +** cycle.
                ** ** [[SQLITE_STMTSTATUS_FILTER_MISS]] ** [[SQLITE_STMTSTATUS_FILTER HIT]] @@ -8858,7 +9173,7 @@ SQLITE_API int sqlite3_stmt_status(sqlite3_stmt*, int op,int resetFlg); ** step was bypassed because a Bloom filter returned not-found. The ** corresponding SQLITE_STMTSTATUS_FILTER_MISS value is the number of ** times that the Bloom filter returned a find, and thus the join step -** had to be processed as normal. +** had to be processed as normal. ** ** [[SQLITE_STMTSTATUS_MEMUSED]]
                SQLITE_STMTSTATUS_MEMUSED
                **
                ^This is the approximate number of bytes of heap memory @@ -8963,9 +9278,9 @@ struct sqlite3_pcache_page { ** SQLite will typically create one cache instance for each open database file, ** though this is not guaranteed. ^The ** first parameter, szPage, is the size in bytes of the pages that must -** be allocated by the cache. ^szPage will always a power of two. ^The +** be allocated by the cache. ^szPage will always be a power of two. ^The ** second parameter szExtra is a number of bytes of extra storage -** associated with each page cache entry. ^The szExtra parameter will +** associated with each page cache entry. ^The szExtra parameter will be ** a number less than 250. SQLite will use the ** extra szExtra bytes on each page to store metadata about the underlying ** database page on disk. The value passed into szExtra depends @@ -8973,17 +9288,17 @@ struct sqlite3_pcache_page { ** ^The third argument to xCreate(), bPurgeable, is true if the cache being ** created will be used to cache database pages of a file stored on disk, or ** false if it is used for an in-memory database. The cache implementation -** does not have to do anything special based with the value of bPurgeable; +** does not have to do anything special based upon the value of bPurgeable; ** it is purely advisory. ^On a cache where bPurgeable is false, SQLite will ** never invoke xUnpin() except to deliberately delete a page. ** ^In other words, calls to xUnpin() on a cache with bPurgeable set to ** false will always have the "discard" flag set to true. -** ^Hence, a cache created with bPurgeable false will +** ^Hence, a cache created with bPurgeable set to false will ** never contain any unpinned pages. ** ** [[the xCachesize() page cache method]] ** ^(The xCachesize() method may be called at any time by SQLite to set the -** suggested maximum cache-size (number of pages stored by) the cache +** suggested maximum cache-size (number of pages stored) for the cache ** instance passed as the first argument. This is the value configured using ** the SQLite "[PRAGMA cache_size]" command.)^ As with the bPurgeable ** parameter, the implementation is not required to do anything with this @@ -9010,12 +9325,12 @@ struct sqlite3_pcache_page { ** implementation must return a pointer to the page buffer with its content ** intact. If the requested page is not already in the cache, then the ** cache implementation should use the value of the createFlag -** parameter to help it determined what action to take: +** parameter to help it determine what action to take: ** **
        • ** - + - + + + - + - + - + - - + + - + - + - + - - + + - + - - - + + + - + - +
          createFlag Behavior when page is not already in cache **
          0 Do not allocate a new page. Return NULL. -**
          1 Allocate a new page if it easy and convenient to do so. +**
          1 Allocate a new page if it is easy and convenient to do so. ** Otherwise return NULL. **
          2 Make every effort to allocate a new page. Only return ** NULL if allocating a new page is effectively impossible. @@ -9032,7 +9347,7 @@ struct sqlite3_pcache_page { ** as its second argument. If the third parameter, discard, is non-zero, ** then the page must be evicted from the cache. ** ^If the discard parameter is -** zero, then the page may be discarded or retained at the discretion of +** zero, then the page may be discarded or retained at the discretion of the ** page cache implementation. ^The page cache implementation ** may choose to evict unpinned pages at any time. ** @@ -9050,7 +9365,7 @@ struct sqlite3_pcache_page { ** When SQLite calls the xTruncate() method, the cache must discard all ** existing cache entries with page numbers (keys) greater than or equal ** to the value of the iLimit parameter passed to xTruncate(). If any -** of these pages are pinned, they are implicitly unpinned, meaning that +** of these pages are pinned, they become implicitly unpinned, meaning that ** they can be safely discarded. ** ** [[the xDestroy() page cache method]] @@ -9230,7 +9545,7 @@ typedef struct sqlite3_backup sqlite3_backup; ** external process or via a database connection other than the one being ** used by the backup operation, then the backup will be automatically ** restarted by the next call to sqlite3_backup_step(). ^If the source -** database is modified by the using the same database connection as is used +** database is modified by using the same database connection as is used ** by the backup operation, then the backup database is automatically ** updated at the same time. ** @@ -9247,7 +9562,7 @@ typedef struct sqlite3_backup sqlite3_backup; ** and may not be used following a call to sqlite3_backup_finish(). ** ** ^The value returned by sqlite3_backup_finish is [SQLITE_OK] if no -** sqlite3_backup_step() errors occurred, regardless or whether or not +** sqlite3_backup_step() errors occurred, regardless of whether or not ** sqlite3_backup_step() completed. ** ^If an out-of-memory condition or IO error occurred during any prior ** sqlite3_backup_step() call on the same [sqlite3_backup] object, then @@ -9302,6 +9617,16 @@ typedef struct sqlite3_backup sqlite3_backup; ** APIs are not strictly speaking threadsafe. If they are invoked at the ** same time as another thread is invoking sqlite3_backup_step() it is ** possible that they return invalid values. +** +** Alternatives To Using The Backup API +** +** Other techniques for safely creating a consistent backup of an SQLite +** database include: +** +**
            +**
          • The [VACUUM INTO] command. +**
          • The [sqlite3_rsync] utility program. +**
          */ SQLITE_API sqlite3_backup *sqlite3_backup_init( sqlite3 *pDest, /* Destination database handle */ @@ -9339,7 +9664,7 @@ SQLITE_API int sqlite3_backup_pagecount(sqlite3_backup *p); ** application receives an SQLITE_LOCKED error, it may call the ** sqlite3_unlock_notify() method with the blocked connection handle as ** the first argument to register for a callback that will be invoked -** when the blocking connections current transaction is concluded. ^The +** when the blocking connection's current transaction is concluded. ^The ** callback is invoked from within the [sqlite3_step] or [sqlite3_close] ** call that concludes the blocking connection's transaction. ** @@ -9359,7 +9684,7 @@ SQLITE_API int sqlite3_backup_pagecount(sqlite3_backup *p); ** blocked connection already has a registered unlock-notify callback, ** then the new callback replaces the old.)^ ^If sqlite3_unlock_notify() is ** called with a NULL pointer as its second argument, then any existing -** unlock-notify callback is canceled. ^The blocked connections +** unlock-notify callback is canceled. ^The blocked connection's ** unlock-notify callback may also be canceled by closing the blocked ** connection using [sqlite3_close()]. ** @@ -9529,7 +9854,7 @@ SQLITE_API void sqlite3_log(int iErrCode, const char *zFormat, ...); ** is the number of pages currently in the write-ahead log file, ** including those that were just committed. ** -** The callback function should normally return [SQLITE_OK]. ^If an error +** ^The callback function should normally return [SQLITE_OK]. ^If an error ** code is returned, that error will propagate back up through the ** SQLite code base to cause the statement that provoked the callback ** to report an error, though the commit will have still occurred. If the @@ -9537,13 +9862,26 @@ SQLITE_API void sqlite3_log(int iErrCode, const char *zFormat, ...); ** that does not correspond to any valid SQLite error code, the results ** are undefined. ** -** A single database handle may have at most a single write-ahead log callback -** registered at one time. ^Calling [sqlite3_wal_hook()] replaces any -** previously registered write-ahead log callback. ^The return value is -** a copy of the third parameter from the previous call, if any, or 0. -** ^Note that the [sqlite3_wal_autocheckpoint()] interface and the -** [wal_autocheckpoint pragma] both invoke [sqlite3_wal_hook()] and will -** overwrite any prior [sqlite3_wal_hook()] settings. +** ^A single database handle may have at most a single write-ahead log +** callback registered at one time. ^Calling [sqlite3_wal_hook()] +** replaces the default behavior or previously registered write-ahead +** log callback. +** +** ^The return value is a copy of the third parameter from the +** previous call, if any, or 0. +** +** ^The [sqlite3_wal_autocheckpoint()] interface and the +** [wal_autocheckpoint pragma] both invoke [sqlite3_wal_hook()] and +** will overwrite any prior [sqlite3_wal_hook()] settings. +** +** ^If a write-ahead log callback is set using this function then +** [sqlite3_wal_checkpoint_v2()] or [PRAGMA wal_checkpoint] +** should be invoked periodically to keep the write-ahead log file +** from growing without bound. +** +** ^Passing a NULL pointer for the callback disables automatic +** checkpointing entirely. To re-enable the default behavior, call +** sqlite3_wal_autocheckpoint(db,1000) or use [PRAGMA wal_checkpoint]. */ SQLITE_API void *sqlite3_wal_hook( sqlite3*, @@ -9560,7 +9898,7 @@ SQLITE_API void *sqlite3_wal_hook( ** to automatically [checkpoint] ** after committing a transaction if there are N or ** more frames in the [write-ahead log] file. ^Passing zero or -** a negative value as the nFrame parameter disables automatic +** a negative value as the N parameter disables automatic ** checkpoints entirely. ** ** ^The callback registered by this function replaces any existing callback @@ -9576,9 +9914,10 @@ SQLITE_API void *sqlite3_wal_hook( ** ** ^Every new [database connection] defaults to having the auto-checkpoint ** enabled with a threshold of 1000 or [SQLITE_DEFAULT_WAL_AUTOCHECKPOINT] -** pages. The use of this interface -** is only necessary if the default setting is found to be suboptimal -** for a particular application. +** pages. +** +** ^The use of this interface is only necessary if the default setting +** is found to be suboptimal for a particular application. */ SQLITE_API int sqlite3_wal_autocheckpoint(sqlite3 *db, int N); @@ -9643,6 +9982,11 @@ SQLITE_API int sqlite3_wal_checkpoint(sqlite3 *db, const char *zDb); ** ^This mode works the same way as SQLITE_CHECKPOINT_RESTART with the ** addition that it also truncates the log file to zero bytes just prior ** to a successful return. +** +**
          SQLITE_CHECKPOINT_NOOP
          +** ^This mode always checkpoints zero frames. The only reason to invoke +** a NOOP checkpoint is to access the values returned by +** sqlite3_wal_checkpoint_v2() via output parameters *pnLog and *pnCkpt. ** ** ** ^If pnLog is not NULL, then *pnLog is set to the total number of frames in @@ -9713,6 +10057,7 @@ SQLITE_API int sqlite3_wal_checkpoint_v2( ** See the [sqlite3_wal_checkpoint_v2()] documentation for details on the ** meaning of each of these checkpoint modes. */ +#define SQLITE_CHECKPOINT_NOOP -1 /* Do no work at all */ #define SQLITE_CHECKPOINT_PASSIVE 0 /* Do as much as possible w/o blocking */ #define SQLITE_CHECKPOINT_FULL 1 /* Wait for writers, then checkpoint */ #define SQLITE_CHECKPOINT_RESTART 2 /* Like FULL but wait for readers */ @@ -9757,7 +10102,7 @@ SQLITE_API int sqlite3_vtab_config(sqlite3*, int op, ...); ** support constraints. In this configuration (which is the default) if ** a call to the [xUpdate] method returns [SQLITE_CONSTRAINT], then the entire ** statement is rolled back as if [ON CONFLICT | OR ABORT] had been -** specified as part of the users SQL statement, regardless of the actual +** specified as part of the user's SQL statement, regardless of the actual ** ON CONFLICT mode specified. ** ** If X is non-zero, then the virtual table implementation guarantees @@ -9791,7 +10136,7 @@ SQLITE_API int sqlite3_vtab_config(sqlite3*, int op, ...); ** [[SQLITE_VTAB_INNOCUOUS]]
          SQLITE_VTAB_INNOCUOUS
          **
          Calls of the form ** [sqlite3_vtab_config](db,SQLITE_VTAB_INNOCUOUS) from within the -** the [xConnect] or [xCreate] methods of a [virtual table] implementation +** [xConnect] or [xCreate] methods of a [virtual table] implementation ** identify that virtual table as being safe to use from within triggers ** and views. Conceptually, the SQLITE_VTAB_INNOCUOUS tag means that the ** virtual table can do no serious harm even if it is controlled by a @@ -9919,26 +10264,47 @@ SQLITE_API const char *sqlite3_vtab_collation(sqlite3_index_info*,int); **
        • ** ^(If the sqlite3_vtab_distinct() interface returns 2, that means ** that the query planner does not need the rows returned in any particular -** order, as long as rows with the same values in all "aOrderBy" columns -** are adjacent.)^ ^(Furthermore, only a single row for each particular -** combination of values in the columns identified by the "aOrderBy" field -** needs to be returned.)^ ^It is always ok for two or more rows with the same -** values in all "aOrderBy" columns to be returned, as long as all such rows -** are adjacent. ^The virtual table may, if it chooses, omit extra rows -** that have the same value for all columns identified by "aOrderBy". -** ^However omitting the extra rows is optional. +** order, as long as rows with the same values in all columns identified +** by "aOrderBy" are adjacent.)^ ^(Furthermore, when two or more rows +** contain the same values for all columns identified by "colUsed", all but +** one such row may optionally be omitted from the result.)^ +** The virtual table is not required to omit rows that are duplicates +** over the "colUsed" columns, but if the virtual table can do that without +** too much extra effort, it could potentially help the query to run faster. ** This mode is used for a DISTINCT query. **

        • -** ^(If the sqlite3_vtab_distinct() interface returns 3, that means -** that the query planner needs only distinct rows but it does need the -** rows to be sorted.)^ ^The virtual table implementation is free to omit -** rows that are identical in all aOrderBy columns, if it wants to, but -** it is not required to omit any rows. This mode is used for queries +** ^(If the sqlite3_vtab_distinct() interface returns 3, that means the +** virtual table must return rows in the order defined by "aOrderBy" as +** if the sqlite3_vtab_distinct() interface had returned 0. However if +** two or more rows in the result have the same values for all columns +** identified by "colUsed", then all but one such row may optionally be +** omitted.)^ Like when the return value is 2, the virtual table +** is not required to omit rows that are duplicates over the "colUsed" +** columns, but if the virtual table can do that without +** too much extra effort, it could potentially help the query to run faster. +** This mode is used for queries ** that have both DISTINCT and ORDER BY clauses. ** ** +**

          The following table summarizes the conditions under which the +** virtual table is allowed to set the "orderByConsumed" flag based on +** the value returned by sqlite3_vtab_distinct(). This table is a +** restatement of the previous four paragraphs: +** +** +** +**
          sqlite3_vtab_distinct() return value +** Rows are returned in aOrderBy order +** Rows with the same value in all aOrderBy columns are adjacent +** Duplicates over all colUsed columns may be omitted +**
          0yesyesno +**
          1noyesno +**
          2noyesyes +**
          3yesyesyes +**
          +** ** ^For the purposes of comparing virtual table output values to see if the -** values are same value for sorting purposes, two NULL values are considered +** values are the same value for sorting purposes, two NULL values are considered ** to be the same. In other words, the comparison operator is "IS" ** (or "IS NOT DISTINCT FROM") and not "==". ** @@ -9948,7 +10314,7 @@ SQLITE_API const char *sqlite3_vtab_collation(sqlite3_index_info*,int); ** ** ^A virtual table implementation is always free to return rows in any order ** it wants, as long as the "orderByConsumed" flag is not set. ^When the -** the "orderByConsumed" flag is unset, the query planner will add extra +** "orderByConsumed" flag is unset, the query planner will add extra ** [bytecode] to ensure that the final results returned by the SQL query are ** ordered correctly. The use of the "orderByConsumed" flag and the ** sqlite3_vtab_distinct() interface is merely an optimization. ^Careful @@ -10045,7 +10411,7 @@ SQLITE_API int sqlite3_vtab_in(sqlite3_index_info*, int iCons, int bHandle); ** sqlite3_vtab_in_next(X,P) should be one of the parameters to the ** xFilter method which invokes these routines, and specifically ** a parameter that was previously selected for all-at-once IN constraint -** processing use the [sqlite3_vtab_in()] interface in the +** processing using the [sqlite3_vtab_in()] interface in the ** [xBestIndex|xBestIndex method]. ^(If the X parameter is not ** an xFilter argument that was selected for all-at-once IN constraint ** processing, then these routines return [SQLITE_ERROR].)^ @@ -10060,7 +10426,7 @@ SQLITE_API int sqlite3_vtab_in(sqlite3_index_info*, int iCons, int bHandle); **   ){ **   // do something with pVal **   } -**   if( rc!=SQLITE_OK ){ +**   if( rc!=SQLITE_DONE ){ **   // an error has occurred **   } ** )^ @@ -10100,7 +10466,7 @@ SQLITE_API int sqlite3_vtab_in_next(sqlite3_value *pVal, sqlite3_value **ppOut); ** and only if *V is set to a value. ^The sqlite3_vtab_rhs_value(P,J,V) ** inteface returns SQLITE_NOTFOUND if the right-hand side of the J-th ** constraint is not available. ^The sqlite3_vtab_rhs_value() interface -** can return an result code other than SQLITE_OK or SQLITE_NOTFOUND if +** can return a result code other than SQLITE_OK or SQLITE_NOTFOUND if ** something goes wrong. ** ** The sqlite3_vtab_rhs_value() interface is usually only successful if @@ -10128,8 +10494,8 @@ SQLITE_API int sqlite3_vtab_rhs_value(sqlite3_index_info*, int, sqlite3_value ** ** KEYWORDS: {conflict resolution mode} ** ** These constants are returned by [sqlite3_vtab_on_conflict()] to -** inform a [virtual table] implementation what the [ON CONFLICT] mode -** is for the SQL statement being evaluated. +** inform a [virtual table] implementation of the [ON CONFLICT] mode +** for the SQL statement being evaluated. ** ** Note that the [SQLITE_IGNORE] constant is also used as a potential ** return value from the [sqlite3_set_authorizer()] callback and that @@ -10169,39 +10535,39 @@ SQLITE_API int sqlite3_vtab_rhs_value(sqlite3_index_info*, int, sqlite3_value ** ** [[SQLITE_SCANSTAT_EST]]

          SQLITE_SCANSTAT_EST
          **
          ^The "double" variable pointed to by the V parameter will be set to the ** query planner's estimate for the average number of rows output from each -** iteration of the X-th loop. If the query planner's estimates was accurate, +** iteration of the X-th loop. If the query planner's estimate was accurate, ** then this value will approximate the quotient NVISIT/NLOOP and the ** product of this value for all prior loops with the same SELECTID will -** be the NLOOP value for the current loop. +** be the NLOOP value for the current loop.
          ** ** [[SQLITE_SCANSTAT_NAME]]
          SQLITE_SCANSTAT_NAME
          **
          ^The "const char *" variable pointed to by the V parameter will be set ** to a zero-terminated UTF-8 string containing the name of the index or table -** used for the X-th loop. +** used for the X-th loop.
          ** ** [[SQLITE_SCANSTAT_EXPLAIN]]
          SQLITE_SCANSTAT_EXPLAIN
          **
          ^The "const char *" variable pointed to by the V parameter will be set ** to a zero-terminated UTF-8 string containing the [EXPLAIN QUERY PLAN] -** description for the X-th loop. +** description for the X-th loop.
          ** ** [[SQLITE_SCANSTAT_SELECTID]]
          SQLITE_SCANSTAT_SELECTID
          **
          ^The "int" variable pointed to by the V parameter will be set to the ** id for the X-th query plan element. The id value is unique within the ** statement. The select-id is the same value as is output in the first -** column of an [EXPLAIN QUERY PLAN] query. +** column of an [EXPLAIN QUERY PLAN] query.
          ** ** [[SQLITE_SCANSTAT_PARENTID]]
          SQLITE_SCANSTAT_PARENTID
          **
          The "int" variable pointed to by the V parameter will be set to the -** the id of the parent of the current query element, if applicable, or +** id of the parent of the current query element, if applicable, or ** to zero if the query element has no parent. This is the same value as -** returned in the second column of an [EXPLAIN QUERY PLAN] query. +** returned in the second column of an [EXPLAIN QUERY PLAN] query.
          ** ** [[SQLITE_SCANSTAT_NCYCLE]]
          SQLITE_SCANSTAT_NCYCLE
          **
          The sqlite3_int64 output value is set to the number of cycles, ** according to the processor time-stamp counter, that elapsed while the ** query element was being processed. This value is not available for ** all query elements - if it is unavailable the output variable is -** set to -1. +** set to -1.
          ** */ #define SQLITE_SCANSTAT_NLOOP 0 @@ -10242,8 +10608,8 @@ SQLITE_API int sqlite3_vtab_rhs_value(sqlite3_index_info*, int, sqlite3_value ** ** sqlite3_stmt_scanstatus_v2() with a zeroed flags parameter. ** ** Parameter "idx" identifies the specific query element to retrieve statistics -** for. Query elements are numbered starting from zero. A value of -1 may be -** to query for statistics regarding the entire query. ^If idx is out of range +** for. Query elements are numbered starting from zero. A value of -1 may +** retrieve statistics for the entire query. ^If idx is out of range ** - less than -1 or greater than or equal to the total number of query ** elements used to implement the statement - a non-zero value is returned and ** the variable that pOut points to is unchanged. @@ -10286,7 +10652,7 @@ SQLITE_API void sqlite3_stmt_scanstatus_reset(sqlite3_stmt*); ** METHOD: sqlite3 ** ** ^If a write-transaction is open on [database connection] D when the -** [sqlite3_db_cacheflush(D)] interface invoked, any dirty +** [sqlite3_db_cacheflush(D)] interface is invoked, any dirty ** pages in the pager-cache that are not currently in use are written out ** to disk. A dirty page may be in use if a database cursor created by an ** active SQL statement is reading from it, or if it is page 1 of a database @@ -10400,8 +10766,8 @@ SQLITE_API int sqlite3_db_cacheflush(sqlite3*); ** triggers; and so forth. ** ** When the [sqlite3_blob_write()] API is used to update a blob column, -** the pre-update hook is invoked with SQLITE_DELETE. This is because the -** in this case the new values are not available. In this case, when a +** the pre-update hook is invoked with SQLITE_DELETE, because +** the new values are not yet available. In this case, when a ** callback made with op==SQLITE_DELETE is actually a write using the ** sqlite3_blob_write() API, the [sqlite3_preupdate_blobwrite()] returns ** the index of the column being written. In other cases, where the @@ -10480,6 +10846,14 @@ typedef struct sqlite3_snapshot { ** If there is not already a read-transaction open on schema S when ** this function is called, one is opened automatically. ** +** If a read-transaction is opened by this function, then it is guaranteed +** that the returned snapshot object may not be invalidated by a database +** writer or checkpointer until after the read-transaction is closed. This +** is not guaranteed if a read-transaction is already open when this +** function is called. In that case, any subsequent write or checkpoint +** operation on the database may invalidate the returned snapshot handle, +** even while the read-transaction remains open. +** ** The following must be true for this function to succeed. If any of ** the following statements are false when sqlite3_snapshot_get() is ** called, SQLITE_ERROR is returned. The final value of *P is undefined @@ -10511,7 +10885,7 @@ typedef struct sqlite3_snapshot { ** The [sqlite3_snapshot_get()] interface is only available when the ** [SQLITE_ENABLE_SNAPSHOT] compile-time option is used. */ -SQLITE_API SQLITE_EXPERIMENTAL int sqlite3_snapshot_get( +SQLITE_API int sqlite3_snapshot_get( sqlite3 *db, const char *zSchema, sqlite3_snapshot **ppSnapshot @@ -10560,7 +10934,7 @@ SQLITE_API SQLITE_EXPERIMENTAL int sqlite3_snapshot_get( ** The [sqlite3_snapshot_open()] interface is only available when the ** [SQLITE_ENABLE_SNAPSHOT] compile-time option is used. */ -SQLITE_API SQLITE_EXPERIMENTAL int sqlite3_snapshot_open( +SQLITE_API int sqlite3_snapshot_open( sqlite3 *db, const char *zSchema, sqlite3_snapshot *pSnapshot @@ -10577,7 +10951,7 @@ SQLITE_API SQLITE_EXPERIMENTAL int sqlite3_snapshot_open( ** The [sqlite3_snapshot_free()] interface is only available when the ** [SQLITE_ENABLE_SNAPSHOT] compile-time option is used. */ -SQLITE_API SQLITE_EXPERIMENTAL void sqlite3_snapshot_free(sqlite3_snapshot*); +SQLITE_API void sqlite3_snapshot_free(sqlite3_snapshot*); /* ** CAPI3REF: Compare the ages of two snapshot handles. @@ -10604,7 +10978,7 @@ SQLITE_API SQLITE_EXPERIMENTAL void sqlite3_snapshot_free(sqlite3_snapshot*); ** This interface is only available if SQLite is compiled with the ** [SQLITE_ENABLE_SNAPSHOT] option. */ -SQLITE_API SQLITE_EXPERIMENTAL int sqlite3_snapshot_cmp( +SQLITE_API int sqlite3_snapshot_cmp( sqlite3_snapshot *p1, sqlite3_snapshot *p2 ); @@ -10632,20 +11006,21 @@ SQLITE_API SQLITE_EXPERIMENTAL int sqlite3_snapshot_cmp( ** This interface is only available if SQLite is compiled with the ** [SQLITE_ENABLE_SNAPSHOT] option. */ -SQLITE_API SQLITE_EXPERIMENTAL int sqlite3_snapshot_recover(sqlite3 *db, const char *zDb); +SQLITE_API int sqlite3_snapshot_recover(sqlite3 *db, const char *zDb); /* ** CAPI3REF: Serialize a database ** -** The sqlite3_serialize(D,S,P,F) interface returns a pointer to memory -** that is a serialization of the S database on [database connection] D. +** The sqlite3_serialize(D,S,P,F) interface returns a pointer to +** memory that is a serialization of the S database on +** [database connection] D. If S is a NULL pointer, the main database is used. ** If P is not a NULL pointer, then the size of the database in bytes ** is written into *P. ** ** For an ordinary on-disk database file, the serialization is just a ** copy of the disk file. For an in-memory database or a "TEMP" database, ** the serialization is the same sequence of bytes which would be written -** to disk if that database where backed up to disk. +** to disk if that database were backed up to disk. ** ** The usual case is that sqlite3_serialize() copies the serialization of ** the database into memory obtained from [sqlite3_malloc64()] and returns @@ -10654,7 +11029,7 @@ SQLITE_API SQLITE_EXPERIMENTAL int sqlite3_snapshot_recover(sqlite3 *db, const c ** contains the SQLITE_SERIALIZE_NOCOPY bit, then no memory allocations ** are made, and the sqlite3_serialize() function will return a pointer ** to the contiguous memory representation of the database that SQLite -** is currently using for that database, or NULL if the no such contiguous +** is currently using for that database, or NULL if no such contiguous ** memory representation of the database exists. A contiguous memory ** representation of the database will usually only exist if there has ** been a prior call to [sqlite3_deserialize(D,S,...)] with the same @@ -10705,12 +11080,13 @@ SQLITE_API unsigned char *sqlite3_serialize( ** ** The sqlite3_deserialize(D,S,P,N,M,F) interface causes the ** [database connection] D to disconnect from database S and then -** reopen S as an in-memory database based on the serialization contained -** in P. The serialized database P is N bytes in size. M is the size of -** the buffer P, which might be larger than N. If M is larger than N, and -** the SQLITE_DESERIALIZE_READONLY bit is not set in F, then SQLite is -** permitted to add content to the in-memory database as long as the total -** size does not exceed M bytes. +** reopen S as an in-memory database based on the serialization +** contained in P. If S is a NULL pointer, the main database is +** used. The serialized database P is N bytes in size. M is the size +** of the buffer P, which might be larger than N. If M is larger than +** N, and the SQLITE_DESERIALIZE_READONLY bit is not set in F, then +** SQLite is permitted to add content to the in-memory database as +** long as the total size does not exceed M bytes. ** ** If the SQLITE_DESERIALIZE_FREEONCLOSE bit is set in F, then SQLite will ** invoke sqlite3_free() on the serialization buffer when the database @@ -10725,7 +11101,7 @@ SQLITE_API unsigned char *sqlite3_serialize( ** database is currently in a read transaction or is involved in a backup ** operation. ** -** It is not possible to deserialized into the TEMP database. If the +** It is not possible to deserialize into the TEMP database. If the ** S argument to sqlite3_deserialize(D,S,P,N,M,F) is "temp" then the ** function returns SQLITE_ERROR. ** @@ -10747,7 +11123,7 @@ SQLITE_API int sqlite3_deserialize( sqlite3 *db, /* The database connection */ const char *zSchema, /* Which DB to reopen with the deserialization */ unsigned char *pData, /* The serialized database content */ - sqlite3_int64 szDb, /* Number bytes in the deserialization */ + sqlite3_int64 szDb, /* Number of bytes in the deserialization */ sqlite3_int64 szBuf, /* Total size of buffer pData[] */ unsigned mFlags /* Zero or more SQLITE_DESERIALIZE_* flags */ ); @@ -10755,7 +11131,7 @@ SQLITE_API int sqlite3_deserialize( /* ** CAPI3REF: Flags for sqlite3_deserialize() ** -** The following are allowed values for 6th argument (the F argument) to +** The following are allowed values for the 6th argument (the F argument) to ** the [sqlite3_deserialize(D,S,P,N,M,F)] interface. ** ** The SQLITE_DESERIALIZE_FREEONCLOSE means that the database serialization @@ -10777,6 +11153,54 @@ SQLITE_API int sqlite3_deserialize( #define SQLITE_DESERIALIZE_RESIZEABLE 2 /* Resize using sqlite3_realloc64() */ #define SQLITE_DESERIALIZE_READONLY 4 /* Database is read-only */ +/* +** CAPI3REF: Bind array values to the CARRAY table-valued function +** +** The sqlite3_carray_bind(S,I,P,N,F,X) interface binds an array value to +** one of the first argument of the [carray() table-valued function]. The +** S parameter is a pointer to the [prepared statement] that uses the carray() +** functions. I is the parameter index to be bound. P is a pointer to the +** array to be bound, and N is the number of eements in the array. The +** F argument is one of constants [SQLITE_CARRAY_INT32], [SQLITE_CARRAY_INT64], +** [SQLITE_CARRAY_DOUBLE], [SQLITE_CARRAY_TEXT], or [SQLITE_CARRAY_BLOB] to +** indicate the datatype of the array being bound. The X argument is not a +** NULL pointer, then SQLite will invoke the function X on the P parameter +** after it has finished using P, even if the call to +** sqlite3_carray_bind() fails. The special-case finalizer +** SQLITE_TRANSIENT has no effect here. +*/ +SQLITE_API int sqlite3_carray_bind( + sqlite3_stmt *pStmt, /* Statement to be bound */ + int i, /* Parameter index */ + void *aData, /* Pointer to array data */ + int nData, /* Number of data elements */ + int mFlags, /* CARRAY flags */ + void (*xDel)(void*) /* Destructor for aData */ +); + +/* +** CAPI3REF: Datatypes for the CARRAY table-valued function +** +** The fifth argument to the [sqlite3_carray_bind()] interface musts be +** one of the following constants, to specify the datatype of the array +** that is being bound into the [carray table-valued function]. +*/ +#define SQLITE_CARRAY_INT32 0 /* Data is 32-bit signed integers */ +#define SQLITE_CARRAY_INT64 1 /* Data is 64-bit signed integers */ +#define SQLITE_CARRAY_DOUBLE 2 /* Data is doubles */ +#define SQLITE_CARRAY_TEXT 3 /* Data is char* */ +#define SQLITE_CARRAY_BLOB 4 /* Data is struct iovec */ + +/* +** Versions of the above #defines that omit the initial SQLITE_, for +** legacy compatibility. +*/ +#define CARRAY_INT32 0 /* Data is 32-bit signed integers */ +#define CARRAY_INT64 1 /* Data is 64-bit signed integers */ +#define CARRAY_DOUBLE 2 /* Data is doubles */ +#define CARRAY_TEXT 3 /* Data is char* */ +#define CARRAY_BLOB 4 /* Data is struct iovec */ + /* ** Undo the hack that converts floating point types to integer for ** builds on processors without floating point support. @@ -10788,8 +11212,6 @@ SQLITE_API int sqlite3_deserialize( #if defined(__wasi__) # undef SQLITE_WASI # define SQLITE_WASI 1 -# undef SQLITE_OMIT_WAL -# define SQLITE_OMIT_WAL 1/* because it requires shared memory APIs */ # ifndef SQLITE_OMIT_LOAD_EXTENSION # define SQLITE_OMIT_LOAD_EXTENSION # endif @@ -10801,7 +11223,7 @@ SQLITE_API int sqlite3_deserialize( #ifdef __cplusplus } /* End of the 'extern "C"' block */ #endif -#endif /* SQLITE3_H */ +/* #endif for SQLITE3_H will be added by mksqlite3.tcl */ /******** Begin file sqlite3rtree.h *********/ /* @@ -11282,9 +11704,10 @@ SQLITE_API void sqlite3session_table_filter( ** is inserted while a session object is enabled, then later deleted while ** the same session object is disabled, no INSERT record will appear in the ** changeset, even though the delete took place while the session was disabled. -** Or, if one field of a row is updated while a session is disabled, and -** another field of the same row is updated while the session is enabled, the -** resulting changeset will contain an UPDATE change that updates both fields. +** Or, if one field of a row is updated while a session is enabled, and +** then another field of the same row is updated while the session is disabled, +** the resulting changeset will contain an UPDATE change that updates both +** fields. */ SQLITE_API int sqlite3session_changeset( sqlite3_session *pSession, /* Session object */ @@ -11356,8 +11779,9 @@ SQLITE_API sqlite3_int64 sqlite3session_changeset_size(sqlite3_session *pSession ** database zFrom the contents of the two compatible tables would be ** identical. ** -** It an error if database zFrom does not exist or does not contain the -** required compatible table. +** Unless the call to this function is a no-op as described above, it is an +** error if database zFrom does not exist or does not contain the required +** compatible table. ** ** If the operation is successful, SQLITE_OK is returned. Otherwise, an SQLite ** error code. In this case, if argument pzErrMsg is not NULL, *pzErrMsg @@ -11492,7 +11916,7 @@ SQLITE_API int sqlite3changeset_start_v2( ** The following flags may passed via the 4th parameter to ** [sqlite3changeset_start_v2] and [sqlite3changeset_start_v2_strm]: ** -**
          SQLITE_CHANGESETAPPLY_INVERT
          +**
          SQLITE_CHANGESETSTART_INVERT
          ** Invert the changeset while iterating through it. This is equivalent to ** inverting a changeset using sqlite3changeset_invert() before applying it. ** It is an error to specify this flag with a patchset. @@ -11807,19 +12231,6 @@ SQLITE_API int sqlite3changeset_concat( void **ppOut /* OUT: Buffer containing output changeset */ ); - -/* -** CAPI3REF: Upgrade the Schema of a Changeset/Patchset -*/ -SQLITE_API int sqlite3changeset_upgrade( - sqlite3 *db, - const char *zDb, - int nIn, const void *pIn, /* Input changeset */ - int *pnOut, void **ppOut /* OUT: Inverse of input */ -); - - - /* ** CAPI3REF: Changegroup Handle ** @@ -11981,6 +12392,30 @@ SQLITE_API int sqlite3changegroup_schema(sqlite3_changegroup*, sqlite3*, const c */ SQLITE_API int sqlite3changegroup_add(sqlite3_changegroup*, int nData, void *pData); +/* +** CAPI3REF: Add A Single Change To A Changegroup +** METHOD: sqlite3_changegroup +** +** This function adds the single change currently indicated by the iterator +** passed as the second argument to the changegroup object. The rules for +** adding the change are just as described for [sqlite3changegroup_add()]. +** +** If the change is successfully added to the changegroup, SQLITE_OK is +** returned. Otherwise, an SQLite error code is returned. +** +** The iterator must point to a valid entry when this function is called. +** If it does not, SQLITE_ERROR is returned and no change is added to the +** changegroup. Additionally, the iterator must not have been opened with +** the SQLITE_CHANGESETAPPLY_INVERT flag. In this case SQLITE_ERROR is also +** returned. +*/ +SQLITE_API int sqlite3changegroup_add_change( + sqlite3_changegroup*, + sqlite3_changeset_iter* +); + + + /* ** CAPI3REF: Obtain A Composite Changeset From A Changegroup ** METHOD: sqlite3_changegroup @@ -12025,14 +12460,32 @@ SQLITE_API void sqlite3changegroup_delete(sqlite3_changegroup*); ** update the "main" database attached to handle db with the changes found in ** the changeset passed via the second and third arguments. ** +** All changes made by these functions are enclosed in a savepoint transaction. +** If any other error (aside from a constraint failure when attempting to +** write to the target database) occurs, then the savepoint transaction is +** rolled back, restoring the target database to its original state, and an +** SQLite error code returned. Additionally, starting with version 3.51.0, +** an error code and error message that may be accessed using the +** [sqlite3_errcode()] and [sqlite3_errmsg()] APIs are left in the database +** handle. +** ** The fourth argument (xFilter) passed to these functions is the "filter -** callback". If it is not NULL, then for each table affected by at least one -** change in the changeset, the filter callback is invoked with -** the table name as the second argument, and a copy of the context pointer -** passed as the sixth argument as the first. If the "filter callback" -** returns zero, then no attempt is made to apply any changes to the table. -** Otherwise, if the return value is non-zero or the xFilter argument to -** is NULL, all changes related to the table are attempted. +** callback". This may be passed NULL, in which case all changes in the +** changeset are applied to the database. For sqlite3changeset_apply() and +** sqlite3_changeset_apply_v2(), if it is not NULL, then it is invoked once +** for each table affected by at least one change in the changeset. In this +** case the table name is passed as the second argument, and a copy of +** the context pointer passed as the sixth argument to apply() or apply_v2() +** as the first. If the "filter callback" returns zero, then no attempt is +** made to apply any changes to the table. Otherwise, if the return value is +** non-zero, all changes related to the table are attempted. +** +** For sqlite3_changeset_apply_v3(), the xFilter callback is invoked once +** per change. The second argument in this case is an sqlite3_changeset_iter +** that may be queried using the usual APIs for the details of the current +** change. If the "filter callback" returns zero in this case, then no attempt +** is made to apply the current change. If it returns non-zero, the change +** is applied. ** ** For each table that is not excluded by the filter callback, this function ** tests that the target database contains a compatible table. A table is @@ -12053,11 +12506,11 @@ SQLITE_API void sqlite3changegroup_delete(sqlite3_changegroup*); ** one such warning is issued for each table in the changeset. ** ** For each change for which there is a compatible table, an attempt is made -** to modify the table contents according to the UPDATE, INSERT or DELETE -** change. If a change cannot be applied cleanly, the conflict handler -** function passed as the fifth argument to sqlite3changeset_apply() may be -** invoked. A description of exactly when the conflict handler is invoked for -** each type of change is below. +** to modify the table contents according to each UPDATE, INSERT or DELETE +** change that is not excluded by a filter callback. If a change cannot be +** applied cleanly, the conflict handler function passed as the fifth argument +** to sqlite3changeset_apply() may be invoked. A description of exactly when +** the conflict handler is invoked for each type of change is below. ** ** Unlike the xFilter argument, xConflict may not be passed NULL. The results ** of passing anything other than a valid function pointer as the xConflict @@ -12153,12 +12606,6 @@ SQLITE_API void sqlite3changegroup_delete(sqlite3_changegroup*); ** This can be used to further customize the application's conflict ** resolution strategy. ** -** All changes made by these functions are enclosed in a savepoint transaction. -** If any other error (aside from a constraint failure when attempting to -** write to the target database) occurs, then the savepoint transaction is -** rolled back, restoring the target database to its original state, and an -** SQLite error code returned. -** ** If the output parameters (ppRebase) and (pnRebase) are non-NULL and ** the input is a changeset (not a patchset), then sqlite3changeset_apply_v2() ** may set (*ppRebase) to point to a "rebase" that may be used with the @@ -12208,6 +12655,23 @@ SQLITE_API int sqlite3changeset_apply_v2( void **ppRebase, int *pnRebase, /* OUT: Rebase data */ int flags /* SESSION_CHANGESETAPPLY_* flags */ ); +SQLITE_API int sqlite3changeset_apply_v3( + sqlite3 *db, /* Apply change to "main" db of this handle */ + int nChangeset, /* Size of changeset in bytes */ + void *pChangeset, /* Changeset blob */ + int(*xFilter)( + void *pCtx, /* Copy of sixth arg to _apply() */ + sqlite3_changeset_iter *p /* Handle describing change */ + ), + int(*xConflict)( + void *pCtx, /* Copy of sixth arg to _apply() */ + int eConflict, /* DATA, MISSING, CONFLICT, CONSTRAINT */ + sqlite3_changeset_iter *p /* Handle describing change and conflict */ + ), + void *pCtx, /* First argument passed to xConflict */ + void **ppRebase, int *pnRebase, /* OUT: Rebase data */ + int flags /* SESSION_CHANGESETAPPLY_* flags */ +); /* ** CAPI3REF: Flags for sqlite3changeset_apply_v2 @@ -12627,6 +13091,23 @@ SQLITE_API int sqlite3changeset_apply_v2_strm( void **ppRebase, int *pnRebase, int flags ); +SQLITE_API int sqlite3changeset_apply_v3_strm( + sqlite3 *db, /* Apply change to "main" db of this handle */ + int (*xInput)(void *pIn, void *pData, int *pnData), /* Input function */ + void *pIn, /* First arg for xInput */ + int(*xFilter)( + void *pCtx, /* Copy of sixth arg to _apply() */ + sqlite3_changeset_iter *p + ), + int(*xConflict)( + void *pCtx, /* Copy of sixth arg to _apply() */ + int eConflict, /* DATA, MISSING, CONFLICT, CONSTRAINT */ + sqlite3_changeset_iter *p /* Handle describing change and conflict */ + ), + void *pCtx, /* First argument passed to xConflict */ + void **ppRebase, int *pnRebase, + int flags +); SQLITE_API int sqlite3changeset_concat_strm( int (*xInputA)(void *pIn, void *pData, int *pnData), void *pInA, @@ -12785,8 +13266,8 @@ struct Fts5PhraseIter { ** EXTENSION API FUNCTIONS ** ** xUserData(pFts): -** Return a copy of the context pointer the extension function was -** registered with. +** Return a copy of the pUserData pointer passed to the xCreateFunction() +** API when the extension function was registered. ** ** xColumnTotalSize(pFts, iCol, pnToken): ** If parameter iCol is less than zero, set output variable *pnToken @@ -12968,6 +13449,10 @@ struct Fts5PhraseIter { ** (i.e. if it is a contentless table), then this API always iterates ** through an empty set (all calls to xPhraseFirst() set iCol to -1). ** +** In all cases, matches are visited in (column ASC, offset ASC) order. +** i.e. all those in column 0, sorted by offset, followed by those in +** column 1, etc. +** ** xPhraseNext() ** See xPhraseFirst above. ** @@ -13024,19 +13509,57 @@ struct Fts5PhraseIter { ** value returned by xInstCount(), SQLITE_RANGE is returned. Otherwise, ** output variable (*ppToken) is set to point to a buffer containing the ** matching document token, and (*pnToken) to the size of that buffer in -** bytes. This API is not available if the specified token matches a -** prefix query term. In that case both output variables are always set -** to 0. +** bytes. ** ** The output text is not a copy of the document text that was tokenized. ** It is the output of the tokenizer module. For tokendata=1 tables, this ** includes any embedded 0x00 and trailing data. ** +** This API may be slow in some cases if the token identified by parameters +** iIdx and iToken matched a prefix token in the query. In most cases, the +** first call to this API for each prefix token in the query is forced +** to scan the portion of the full-text index that matches the prefix +** token to collect the extra data required by this API. If the prefix +** token matches a large number of token instances in the document set, +** this may be a performance problem. +** +** If the user knows in advance that a query may use this API for a +** prefix token, FTS5 may be configured to collect all required data as part +** of the initial querying of the full-text index, avoiding the second scan +** entirely. This also causes prefix queries that do not use this API to +** run more slowly and use more memory. FTS5 may be configured in this way +** either on a per-table basis using the [FTS5 insttoken | 'insttoken'] +** option, or on a per-query basis using the +** [fts5_insttoken | fts5_insttoken()] user function. +** ** This API can be quite slow if used with an FTS5 table created with the ** "detail=none" or "detail=column" option. +** +** xColumnLocale(pFts5, iIdx, pzLocale, pnLocale) +** If parameter iCol is less than zero, or greater than or equal to the +** number of columns in the table, SQLITE_RANGE is returned. +** +** Otherwise, this function attempts to retrieve the locale associated +** with column iCol of the current row. Usually, there is no associated +** locale, and output parameters (*pzLocale) and (*pnLocale) are set +** to NULL and 0, respectively. However, if the fts5_locale() function +** was used to associate a locale with the value when it was inserted +** into the fts5 table, then (*pzLocale) is set to point to a nul-terminated +** buffer containing the name of the locale in utf-8 encoding. (*pnLocale) +** is set to the size in bytes of the buffer, not including the +** nul-terminator. +** +** If successful, SQLITE_OK is returned. Or, if an error occurs, an +** SQLite error code is returned. The final value of the output parameters +** is undefined in this case. +** +** xTokenize_v2: +** Tokenize text using the tokenizer belonging to the FTS5 table. This +** API is the same as the xTokenize() API, except that it allows a tokenizer +** locale to be specified. */ struct Fts5ExtensionApi { - int iVersion; /* Currently always set to 3 */ + int iVersion; /* Currently always set to 4 */ void *(*xUserData)(Fts5Context*); @@ -13078,6 +13601,15 @@ struct Fts5ExtensionApi { const char **ppToken, int *pnToken ); int (*xInstToken)(Fts5Context*, int iIdx, int iToken, const char**, int*); + + /* Below this point are iVersion>=4 only */ + int (*xColumnLocale)(Fts5Context*, int iCol, const char **pz, int *pn); + int (*xTokenize_v2)(Fts5Context*, + const char *pText, int nText, /* Text to tokenize */ + const char *pLocale, int nLocale, /* Locale to pass to tokenizer */ + void *pCtx, /* Context passed to xToken() */ + int (*xToken)(void*, int, const char*, int, int, int) /* Callback */ + ); }; /* @@ -13098,7 +13630,7 @@ struct Fts5ExtensionApi { ** A tokenizer instance is required to actually tokenize text. ** ** The first argument passed to this function is a copy of the (void*) -** pointer provided by the application when the fts5_tokenizer object +** pointer provided by the application when the fts5_tokenizer_v2 object ** was registered with FTS5 (the third argument to xCreateTokenizer()). ** The second and third arguments are an array of nul-terminated strings ** containing the tokenizer arguments, if any, specified following the @@ -13122,7 +13654,7 @@ struct Fts5ExtensionApi { ** argument passed to this function is a pointer to an Fts5Tokenizer object ** returned by an earlier call to xCreate(). ** -** The second argument indicates the reason that FTS5 is requesting +** The third argument indicates the reason that FTS5 is requesting ** tokenization of the supplied text. This is always one of the following ** four values: ** @@ -13146,6 +13678,13 @@ struct Fts5ExtensionApi { ** on a columnsize=0 database. ** ** +** The sixth and seventh arguments passed to xTokenize() - pLocale and +** nLocale - are a pointer to a buffer containing the locale to use for +** tokenization (e.g. "en_US") and its size in bytes, respectively. The +** pLocale buffer is not nul-terminated. pLocale may be passed NULL (in +** which case nLocale is always 0) to indicate that the tokenizer should +** use its default locale. +** ** For each token in the input string, the supplied callback xToken() must ** be invoked. The first argument to it should be a copy of the pointer ** passed as the second argument to xTokenize(). The third and fourth @@ -13169,6 +13708,30 @@ struct Fts5ExtensionApi { ** may abandon the tokenization and return any error code other than ** SQLITE_OK or SQLITE_DONE. ** +** If the tokenizer is registered using an fts5_tokenizer_v2 object, +** then the xTokenize() method has two additional arguments - pLocale +** and nLocale. These specify the locale that the tokenizer should use +** for the current request. If pLocale and nLocale are both 0, then the +** tokenizer should use its default locale. Otherwise, pLocale points to +** an nLocale byte buffer containing the name of the locale to use as utf-8 +** text. pLocale is not nul-terminated. +** +** FTS5_TOKENIZER +** +** There is also an fts5_tokenizer object. This is an older, deprecated, +** version of fts5_tokenizer_v2. It is similar except that: +** +**
            +**
          • There is no "iVersion" field, and +**
          • The xTokenize() method does not take a locale argument. +**
          +** +** Legacy fts5_tokenizer tokenizers must be registered using the +** legacy xCreateTokenizer() function, instead of xCreateTokenizer_v2(). +** +** Tokenizer implementations registered using either API may be retrieved +** using both xFindTokenizer() and xFindTokenizer_v2(). +** ** SYNONYM SUPPORT ** ** Custom tokenizers may also support synonyms. Consider a case in which a @@ -13277,6 +13840,33 @@ struct Fts5ExtensionApi { ** inefficient. */ typedef struct Fts5Tokenizer Fts5Tokenizer; +typedef struct fts5_tokenizer_v2 fts5_tokenizer_v2; +struct fts5_tokenizer_v2 { + int iVersion; /* Currently always 2 */ + + int (*xCreate)(void*, const char **azArg, int nArg, Fts5Tokenizer **ppOut); + void (*xDelete)(Fts5Tokenizer*); + int (*xTokenize)(Fts5Tokenizer*, + void *pCtx, + int flags, /* Mask of FTS5_TOKENIZE_* flags */ + const char *pText, int nText, + const char *pLocale, int nLocale, + int (*xToken)( + void *pCtx, /* Copy of 2nd argument to xTokenize() */ + int tflags, /* Mask of FTS5_TOKEN_* flags */ + const char *pToken, /* Pointer to buffer containing token */ + int nToken, /* Size of token in bytes */ + int iStart, /* Byte offset of token within input text */ + int iEnd /* Byte offset of end of token within input text */ + ) + ); +}; + +/* +** New code should use the fts5_tokenizer_v2 type to define tokenizer +** implementations. The following type is included for legacy applications +** that still use it. +*/ typedef struct fts5_tokenizer fts5_tokenizer; struct fts5_tokenizer { int (*xCreate)(void*, const char **azArg, int nArg, Fts5Tokenizer **ppOut); @@ -13296,6 +13886,7 @@ struct fts5_tokenizer { ); }; + /* Flags that may be passed as the third argument to xTokenize() */ #define FTS5_TOKENIZE_QUERY 0x0001 #define FTS5_TOKENIZE_PREFIX 0x0002 @@ -13315,7 +13906,7 @@ struct fts5_tokenizer { */ typedef struct fts5_api fts5_api; struct fts5_api { - int iVersion; /* Currently always set to 2 */ + int iVersion; /* Currently always set to 3 */ /* Create a new tokenizer */ int (*xCreateTokenizer)( @@ -13342,6 +13933,25 @@ struct fts5_api { fts5_extension_function xFunction, void (*xDestroy)(void*) ); + + /* APIs below this point are only available if iVersion>=3 */ + + /* Create a new tokenizer */ + int (*xCreateTokenizer_v2)( + fts5_api *pApi, + const char *zName, + void *pUserData, + fts5_tokenizer_v2 *pTokenizer, + void (*xDestroy)(void*) + ); + + /* Find an existing tokenizer */ + int (*xFindTokenizer_v2)( + fts5_api *pApi, + const char *zName, + void **ppUserData, + fts5_tokenizer_v2 **ppTokenizer + ); }; /* @@ -13355,3 +13965,4 @@ struct fts5_api { #endif /* _FTS5_H */ /******** End of fts5.h *********/ +#endif /* SQLITE3_H */ From 370544029e350be33afb50ff4600d5151a8d6134 Mon Sep 17 00:00:00 2001 From: Patrick Stephens Date: Sun, 3 May 2026 12:35:10 +0100 Subject: [PATCH 156/213] [upstream] lib: c-ares: upgrade to v1.34.6 Upstream-Ref: https://github.com/fluent/fluent-bit/commit/7caffd150ce81077d72b68ac26068baf78059050 Cherry-picked from Fluent Bit v4.2.4 --- source/cmake/libraries.cmake | 2 +- source/lib/c-ares-1.34.4/RELEASE-NOTES.md | 25 - .../m4/ax_cxx_compile_stdcxx_14.m4 | 34 - .../{c-ares-1.34.4 => c-ares-1.34.6}/AUTHORS | 0 .../CMakeLists.txt | 34 +- .../CONTRIBUTING.md | 0 .../DEVELOPER-NOTES.md | 0 .../INSTALL.md | 0 .../LICENSE.md | 0 .../Makefile.Watcom | 0 .../Makefile.am | 0 .../Makefile.dj | 0 .../Makefile.in | 5 +- .../Makefile.m32 | 4 +- .../Makefile.msvc | 0 .../Makefile.netware | 0 .../README.md | 0 .../README.msvc | 0 source/lib/c-ares-1.34.6/RELEASE-NOTES.md | 42 + .../SECURITY.md | 0 .../aclocal.m4 | 1 - .../aminclude_static.am | 2 +- .../buildconf | 0 .../buildconf.bat | 0 .../c-ares-config.cmake.in | 0 .../cmake/EnableWarnings.cmake | 0 .../config/compile | 0 .../config/config.guess | 0 .../config/config.sub | 0 .../config/depcomp | 0 .../config/install-sh | 0 .../config/ltmain.sh | 851 +- .../config/missing | 0 .../config/test-driver | 0 .../configure | 12064 +++++++++------- .../configure.ac | 25 +- .../docs/CMakeLists.txt | 0 .../docs/Makefile.am | 0 .../docs/Makefile.in | 5 +- .../docs/Makefile.inc | 0 .../docs/adig.1 | 0 .../docs/ahost.1 | 0 .../docs/ares_cancel.3 | 0 .../docs/ares_create_query.3 | 0 .../docs/ares_destroy.3 | 0 .../docs/ares_destroy_options.3 | 0 .../docs/ares_dns_class_fromstr.3 | 0 .../docs/ares_dns_class_t.3 | 0 .../docs/ares_dns_class_tostr.3 | 0 .../docs/ares_dns_datatype_t.3 | 0 .../docs/ares_dns_flags_t.3 | 0 .../docs/ares_dns_mapping.3 | 0 .../docs/ares_dns_opcode_t.3 | 0 .../docs/ares_dns_opcode_tostr.3 | 0 .../docs/ares_dns_opt_datatype_t.3 | 0 .../docs/ares_dns_opt_get_datatype.3 | 0 .../docs/ares_dns_opt_get_name.3 | 0 .../docs/ares_dns_parse.3 | 0 .../docs/ares_dns_rcode_t.3 | 0 .../docs/ares_dns_rcode_tostr.3 | 0 .../docs/ares_dns_rec_type_fromstr.3 | 0 .../docs/ares_dns_rec_type_t.3 | 0 .../docs/ares_dns_rec_type_tostr.3 | 0 .../docs/ares_dns_record.3 | 0 .../docs/ares_dns_record_create.3 | 0 .../docs/ares_dns_record_destroy.3 | 0 .../docs/ares_dns_record_duplicate.3 | 0 .../docs/ares_dns_record_get_flags.3 | 0 .../docs/ares_dns_record_get_id.3 | 0 .../docs/ares_dns_record_get_opcode.3 | 0 .../docs/ares_dns_record_get_rcode.3 | 0 .../docs/ares_dns_record_query_add.3 | 0 .../docs/ares_dns_record_query_cnt.3 | 0 .../docs/ares_dns_record_query_get.3 | 0 .../docs/ares_dns_record_query_set_name.3 | 0 .../docs/ares_dns_record_query_set_type.3 | 0 .../docs/ares_dns_record_rr_add.3 | 0 .../docs/ares_dns_record_rr_cnt.3 | 0 .../docs/ares_dns_record_rr_del.3 | 0 .../docs/ares_dns_record_rr_get.3 | 0 .../docs/ares_dns_record_rr_get_const.3 | 0 .../docs/ares_dns_record_set_id.3 | 0 .../docs/ares_dns_rr.3 | 0 .../docs/ares_dns_rr_add_abin.3 | 0 .../docs/ares_dns_rr_del_abin.3 | 0 .../docs/ares_dns_rr_del_opt_byid.3 | 0 .../docs/ares_dns_rr_get_abin.3 | 0 .../docs/ares_dns_rr_get_abin_cnt.3 | 0 .../docs/ares_dns_rr_get_addr.3 | 0 .../docs/ares_dns_rr_get_addr6.3 | 0 .../docs/ares_dns_rr_get_bin.3 | 0 .../docs/ares_dns_rr_get_class.3 | 0 .../docs/ares_dns_rr_get_keys.3 | 0 .../docs/ares_dns_rr_get_name.3 | 0 .../docs/ares_dns_rr_get_opt.3 | 0 .../docs/ares_dns_rr_get_opt_byid.3 | 0 .../docs/ares_dns_rr_get_opt_cnt.3 | 0 .../docs/ares_dns_rr_get_str.3 | 0 .../docs/ares_dns_rr_get_ttl.3 | 0 .../docs/ares_dns_rr_get_type.3 | 0 .../docs/ares_dns_rr_get_u16.3 | 0 .../docs/ares_dns_rr_get_u32.3 | 0 .../docs/ares_dns_rr_get_u8.3 | 0 .../docs/ares_dns_rr_key_datatype.3 | 0 .../docs/ares_dns_rr_key_t.3 | 0 .../docs/ares_dns_rr_key_to_rec_type.3 | 0 .../docs/ares_dns_rr_key_tostr.3 | 0 .../docs/ares_dns_rr_set_addr.3 | 0 .../docs/ares_dns_rr_set_addr6.3 | 0 .../docs/ares_dns_rr_set_bin.3 | 0 .../docs/ares_dns_rr_set_opt.3 | 0 .../docs/ares_dns_rr_set_str.3 | 0 .../docs/ares_dns_rr_set_u16.3 | 0 .../docs/ares_dns_rr_set_u32.3 | 0 .../docs/ares_dns_rr_set_u8.3 | 0 .../docs/ares_dns_section_t.3 | 0 .../docs/ares_dns_section_tostr.3 | 0 .../docs/ares_dns_write.3 | 0 .../docs/ares_dup.3 | 0 .../docs/ares_expand_name.3 | 0 .../docs/ares_expand_string.3 | 0 .../docs/ares_fds.3 | 0 .../docs/ares_free_data.3 | 0 .../docs/ares_free_hostent.3 | 0 .../docs/ares_free_string.3 | 0 .../docs/ares_freeaddrinfo.3 | 0 .../docs/ares_get_servers.3 | 0 .../docs/ares_get_servers_csv.3 | 0 .../docs/ares_get_servers_ports.3 | 0 .../docs/ares_getaddrinfo.3 | 0 .../docs/ares_gethostbyaddr.3 | 0 .../docs/ares_gethostbyname.3 | 0 .../docs/ares_gethostbyname_file.3 | 0 .../docs/ares_getnameinfo.3 | 0 .../docs/ares_getsock.3 | 0 .../docs/ares_inet_ntop.3 | 0 .../docs/ares_inet_pton.3 | 0 .../docs/ares_init.3 | 0 .../docs/ares_init_options.3 | 0 .../docs/ares_library_cleanup.3 | 0 .../docs/ares_library_init.3 | 0 .../docs/ares_library_init_android.3 | 0 .../docs/ares_library_initialized.3 | 0 .../docs/ares_mkquery.3 | 0 .../docs/ares_opt_param_t.3 | 0 .../docs/ares_parse_a_reply.3 | 0 .../docs/ares_parse_aaaa_reply.3 | 0 .../docs/ares_parse_caa_reply.3 | 0 .../docs/ares_parse_mx_reply.3 | 0 .../docs/ares_parse_naptr_reply.3 | 0 .../docs/ares_parse_ns_reply.3 | 0 .../docs/ares_parse_ptr_reply.3 | 0 .../docs/ares_parse_soa_reply.3 | 0 .../docs/ares_parse_srv_reply.3 | 0 .../docs/ares_parse_txt_reply.3 | 0 .../docs/ares_parse_uri_reply.3 | 0 .../docs/ares_process.3 | 0 .../docs/ares_process_fd.3 | 0 .../docs/ares_process_fds.3 | 0 .../docs/ares_process_pending_write.3 | 0 .../docs/ares_query.3 | 0 .../docs/ares_query_dnsrec.3 | 0 .../docs/ares_queue.3 | 0 .../docs/ares_queue_active_queries.3 | 0 .../docs/ares_queue_wait_empty.3 | 0 .../docs/ares_reinit.3 | 0 .../docs/ares_save_options.3 | 0 .../docs/ares_search.3 | 0 .../docs/ares_search_dnsrec.3 | 0 .../docs/ares_send.3 | 0 .../docs/ares_send_dnsrec.3 | 0 .../docs/ares_set_local_dev.3 | 0 .../docs/ares_set_local_ip4.3 | 0 .../docs/ares_set_local_ip6.3 | 0 .../docs/ares_set_pending_write_cb.3 | 0 .../docs/ares_set_server_state_callback.3 | 0 .../docs/ares_set_servers.3 | 0 .../docs/ares_set_servers_csv.3 | 0 .../docs/ares_set_servers_ports.3 | 0 .../docs/ares_set_servers_ports_csv.3 | 0 .../docs/ares_set_socket_callback.3 | 0 .../docs/ares_set_socket_configure_callback.3 | 0 .../docs/ares_set_socket_functions.3 | 0 .../docs/ares_set_socket_functions_ex.3 | 0 .../docs/ares_set_sortlist.3 | 0 .../docs/ares_strerror.3 | 0 .../docs/ares_svcb_param_t.3 | 0 .../docs/ares_threadsafety.3 | 0 .../docs/ares_timeout.3 | 0 .../docs/ares_tlsa_match_t.3 | 0 .../docs/ares_tlsa_selector_t.3 | 0 .../docs/ares_tlsa_usage_t.3 | 0 .../docs/ares_version.3 | 0 .../include/CMakeLists.txt | 0 .../include/Makefile.am | 0 .../include/Makefile.in | 5 +- .../include/ares.h | 0 .../include/ares_build.h | 0 .../include/ares_build.h.cmake | 0 .../include/ares_build.h.in | 0 .../include/ares_dns.h | 0 .../include/ares_dns_record.h | 2 +- .../include/ares_nameser.h | 0 .../include/ares_version.h | 6 +- .../libcares.pc.cmake | 0 .../libcares.pc.in | 0 .../m4/ares_check_user_namespace.m4 | 0 .../m4/ares_check_uts_namespace.m4 | 0 .../m4/ax_ac_append_to_file.m4 | 0 .../m4/ax_ac_print_to_file.m4 | 0 .../m4/ax_add_am_macro_static.m4 | 0 .../m4/ax_am_macros_static.m4 | 0 .../m4/ax_append_compile_flags.m4 | 0 .../m4/ax_append_flag.m4 | 0 .../m4/ax_append_link_flags.m4 | 0 .../m4/ax_check_compile_flag.m4 | 0 .../m4/ax_check_gnu_make.m4 | 0 .../m4/ax_check_link_flag.m4 | 0 .../m4/ax_code_coverage.m4 | 0 .../m4/ax_compiler_vendor.m4 | 0 .../m4/ax_cxx_compile_stdcxx.m4 | 0 .../m4/ax_file_escapes.m4 | 0 .../m4/ax_pthread.m4 | 2 +- .../m4/ax_require_defined.m4 | 0 .../m4/libtool.m4 | 229 +- .../m4/ltoptions.m4 | 4 +- .../m4/ltsugar.m4 | 2 +- .../m4/ltversion.m4 | 13 +- .../m4/lt~obsolete.m4 | 4 +- .../m4/pkg.m4 | 0 .../src/CMakeLists.txt | 0 .../src/Makefile.am | 0 .../src/Makefile.in | 5 +- .../src/lib/CMakeLists.txt | 8 + .../src/lib/Makefile.am | 0 .../src/lib/Makefile.in | 7 +- .../src/lib/Makefile.inc | 0 .../src/lib/ares_addrinfo2hostent.c | 169 +- .../src/lib/ares_addrinfo_localhost.c | 54 +- .../src/lib/ares_android.c | 0 .../src/lib/ares_android.h | 0 .../src/lib/ares_cancel.c | 0 .../src/lib/ares_close_sockets.c | 2 +- .../src/lib/ares_config.h.cmake | 3 + .../src/lib/ares_config.h.in | 6 + .../src/lib/ares_conn.c | 0 .../src/lib/ares_conn.h | 0 .../src/lib/ares_cookie.c | 8 +- .../src/lib/ares_data.c | 0 .../src/lib/ares_data.h | 0 .../src/lib/ares_destroy.c | 0 .../src/lib/ares_free_hostent.c | 7 +- .../src/lib/ares_free_string.c | 0 .../src/lib/ares_freeaddrinfo.c | 0 .../src/lib/ares_getaddrinfo.c | 39 +- .../src/lib/ares_getenv.c | 0 .../src/lib/ares_getenv.h | 0 .../src/lib/ares_gethostbyaddr.c | 2 +- .../src/lib/ares_gethostbyname.c | 12 +- .../src/lib/ares_getnameinfo.c | 0 .../src/lib/ares_hosts_file.c | 13 +- .../src/lib/ares_inet_net_pton.h | 0 .../src/lib/ares_init.c | 41 +- .../src/lib/ares_ipv6.h | 10 + .../src/lib/ares_library_init.c | 0 .../src/lib/ares_metrics.c | 2 +- .../src/lib/ares_options.c | 0 .../src/lib/ares_parse_into_addrinfo.c | 0 .../src/lib/ares_private.h | 38 +- .../src/lib/ares_process.c | 205 +- .../src/lib/ares_qcache.c | 20 +- .../src/lib/ares_query.c | 0 .../src/lib/ares_search.c | 0 .../src/lib/ares_send.c | 0 .../src/lib/ares_set_socket_functions.c | 4 +- .../src/lib/ares_setup.h | 0 .../src/lib/ares_socket.c | 0 .../src/lib/ares_socket.h | 0 .../src/lib/ares_sortaddrinfo.c | 0 .../src/lib/ares_strerror.c | 0 .../src/lib/ares_sysconfig.c | 0 .../src/lib/ares_sysconfig_files.c | 0 .../src/lib/ares_sysconfig_mac.c | 0 .../src/lib/ares_sysconfig_win.c | 98 +- .../src/lib/ares_timeout.c | 0 .../src/lib/ares_update_servers.c | 0 .../src/lib/ares_version.c | 0 .../src/lib/cares.rc | 2 +- .../src/lib/config-dos.h | 0 .../src/lib/config-win32.h | 8 +- .../src/lib/dsa/ares_array.c | 0 .../src/lib/dsa/ares_htable.c | 0 .../src/lib/dsa/ares_htable.h | 0 .../src/lib/dsa/ares_htable_asvp.c | 0 .../src/lib/dsa/ares_htable_dict.c | 0 .../src/lib/dsa/ares_htable_strvp.c | 0 .../src/lib/dsa/ares_htable_szvp.c | 0 .../src/lib/dsa/ares_htable_vpstr.c | 0 .../src/lib/dsa/ares_htable_vpvp.c | 0 .../src/lib/dsa/ares_llist.c | 0 .../src/lib/dsa/ares_slist.c | 0 .../src/lib/dsa/ares_slist.h | 0 .../src/lib/event/ares_event.h | 25 +- .../src/lib/event/ares_event_configchg.c | 20 +- .../src/lib/event/ares_event_epoll.c | 4 +- .../src/lib/event/ares_event_kqueue.c | 4 +- .../src/lib/event/ares_event_poll.c | 5 +- .../src/lib/event/ares_event_select.c | 9 +- .../src/lib/event/ares_event_thread.c | 24 + .../src/lib/event/ares_event_wake_pipe.c | 16 +- .../src/lib/event/ares_event_win32.c | 8 +- .../src/lib/event/ares_event_win32.h | 0 .../src/lib/include/ares_array.h | 0 .../src/lib/include/ares_buf.h | 0 .../src/lib/include/ares_htable_asvp.h | 0 .../src/lib/include/ares_htable_dict.h | 0 .../src/lib/include/ares_htable_strvp.h | 0 .../src/lib/include/ares_htable_szvp.h | 0 .../src/lib/include/ares_htable_vpstr.h | 0 .../src/lib/include/ares_htable_vpvp.h | 0 .../src/lib/include/ares_llist.h | 0 .../src/lib/include/ares_mem.h | 0 .../src/lib/include/ares_str.h | 0 .../src/lib/inet_net_pton.c | 0 .../src/lib/inet_ntop.c | 0 .../src/lib/legacy/ares_create_query.c | 0 .../src/lib/legacy/ares_expand_name.c | 0 .../src/lib/legacy/ares_expand_string.c | 0 .../src/lib/legacy/ares_fds.c | 0 .../src/lib/legacy/ares_getsock.c | 0 .../src/lib/legacy/ares_parse_a_reply.c | 1 + .../src/lib/legacy/ares_parse_aaaa_reply.c | 1 + .../src/lib/legacy/ares_parse_caa_reply.c | 0 .../src/lib/legacy/ares_parse_mx_reply.c | 0 .../src/lib/legacy/ares_parse_naptr_reply.c | 0 .../src/lib/legacy/ares_parse_ns_reply.c | 0 .../src/lib/legacy/ares_parse_ptr_reply.c | 0 .../src/lib/legacy/ares_parse_soa_reply.c | 0 .../src/lib/legacy/ares_parse_srv_reply.c | 0 .../src/lib/legacy/ares_parse_txt_reply.c | 0 .../src/lib/legacy/ares_parse_uri_reply.c | 0 .../src/lib/record/ares_dns_mapping.c | 0 .../src/lib/record/ares_dns_multistring.c | 0 .../src/lib/record/ares_dns_multistring.h | 0 .../src/lib/record/ares_dns_name.c | 0 .../src/lib/record/ares_dns_parse.c | 0 .../src/lib/record/ares_dns_private.h | 0 .../src/lib/record/ares_dns_record.c | 0 .../src/lib/record/ares_dns_write.c | 0 .../src/lib/str/ares_buf.c | 0 .../src/lib/str/ares_str.c | 0 .../src/lib/str/ares_strsplit.c | 0 .../src/lib/str/ares_strsplit.h | 0 .../src/lib/thirdparty/apple/dnsinfo.h | 0 .../src/lib/util/ares_iface_ips.c | 8 +- .../src/lib/util/ares_iface_ips.h | 0 .../src/lib/util/ares_math.c | 0 .../src/lib/util/ares_math.h | 22 +- .../src/lib/util/ares_rand.c | 8 +- .../src/lib/util/ares_rand.h | 0 .../src/lib/util/ares_threads.c | 0 .../src/lib/util/ares_threads.h | 0 .../src/lib/util/ares_time.h | 0 .../src/lib/util/ares_timeval.c | 0 .../src/lib/util/ares_uri.c | 2 +- .../src/lib/util/ares_uri.h | 2 +- .../src/lib/windows_port.c | 0 .../src/tools/CMakeLists.txt | 0 .../src/tools/Makefile.am | 0 .../src/tools/Makefile.in | 5 +- .../src/tools/Makefile.inc | 0 .../src/tools/adig.c | 0 .../src/tools/ahost.c | 0 .../src/tools/ares_getopt.c | 0 .../src/tools/ares_getopt.h | 0 .../test/CMakeLists.txt | 4 + .../test/Makefile.am | 0 .../test/Makefile.in | 5 +- .../test/Makefile.inc | 0 .../test/Makefile.m32 | 4 +- .../test/Makefile.msvc | 0 .../test/README.md | 0 .../test/ares-fuzz.c | 2 +- .../test/ares-test-ai.h | 0 .../test/ares-test-fuzz-name.c | 0 .../test/ares-test-fuzz.c | 0 .../test/ares-test-init.cc | 0 .../test/ares-test-internal.cc | 14 + .../test/ares-test-live.cc | 108 - .../test/ares-test-main.cc | 0 .../test/ares-test-misc.cc | 0 .../test/ares-test-mock-ai.cc | 302 + .../test/ares-test-mock-et.cc | 108 + .../test/ares-test-mock.cc | 194 +- .../test/ares-test-ns.cc | 0 .../test/ares-test-parse-a.cc | 0 .../test/ares-test-parse-aaaa.cc | 0 .../test/ares-test-parse-caa.cc | 0 .../test/ares-test-parse-mx.cc | 0 .../test/ares-test-parse-naptr.cc | 0 .../test/ares-test-parse-ns.cc | 0 .../test/ares-test-parse-ptr.cc | 0 .../test/ares-test-parse-soa-any.cc | 0 .../test/ares-test-parse-soa.cc | 0 .../test/ares-test-parse-srv.cc | 0 .../test/ares-test-parse-txt.cc | 0 .../test/ares-test-parse-uri.cc | 0 .../test/ares-test-parse.cc | 0 .../test/ares-test.cc | 35 +- .../test/ares-test.h | 23 + .../test/ares_queryloop.c | 0 .../test/dns-dump.cc | 0 .../test/dns-proto-test.cc | 0 .../test/dns-proto.cc | 0 .../test/dns-proto.h | 0 .../test/fuzzcheck.sh | 0 .../004a216d3cff18b0c5c6b68b807f1529 | Bin .../00539467ca159b36aea95e61f9729115 | Bin .../00e846db8f43f2f507cd1666ed5a753e | Bin .../0177b7566f08c013699eaea9a77abeb3 | Bin .../020a4fa317715bfdb236ed13751e6b65 | Bin .../0310f2e81bea31f4fe3f330872a877dd | Bin .../0449be67df1730b2d0887d412a9b7cc4 | Bin .../0449dd14f7aa94bf0d716bfe09b287a8 | Bin .../04c93cdf7208979aa4df80a3a0d5a2d8 | Bin .../0567e7171e08e75f3f91c4ca74c17adc | Bin .../05ba948578a397e9cbc6a7b3e78622fa | Bin .../060afe5ed25f3e2e86167e545f27edca | Bin .../06d47d3681493f1b1d41236f460d896f | Bin .../0724a810b0e131c2fddb6de9003b9064 | Bin .../0b5279148826f5b962bcf1896bdb4ede | Bin .../114048c0f6b10bdc67ce9166405d195e | Bin .../11b8464a0ef8735d202955c34c36b0c7 | Bin .../11cb626f1668c7b41954ce7d768fe528 | Bin .../14b133bf18125b75a1976fa63a1df6b7 | Bin .../153c6b3afa8faa03c8bc28f936a6d4cf | Bin .../182cad2a342ed7317b7c21a5d17020d1 | Bin .../1c61a61bb7057b52c5b15188345a5238 | Bin .../1dbe2cf62ed2e4fa1c3cb473f08710b5 | Bin .../21199be504fcfece5c7096ee0dbba507 | Bin .../21891480074b5635dbbe7137bdcabccd | Bin .../233aea42e15aa73e131eefabf16088c9 | Bin .../24660d4e7ac7aa21d600ea7a3d198bbb | Bin .../25589deb55c08429345f289d1c9b0254 | Bin .../2573bd823e4da11f727a17f8e1f35c26 | Bin .../276f12da56866273e76059ad0e7be97e | Bin .../29198a2e380cb19babec9e02116d213e | Bin .../2c94ba9434b1a1b9396fc5364f101363 | Bin .../2d578c357dc2f5e02dc55cddb30641d1 | Bin .../2dff6cc5a223e67fde9e5e79af456992 | Bin .../2f103b1f9477f2d8934bd84328d51c75 | Bin .../31cd3a8413de13d9624adbb1613784bf | Bin .../36415bdf1d180098fe6234b4186e69f3 | Bin .../3a04a80f0242e8dff0cd732e7c4767da | Bin .../44d0f973b7b0fb3e4a07770c943dcd5a | Bin .../50bc00daa0ddcd6cfb2b5d9f62c81f47 | Bin .../51ed2d1fb77b3078b54e94e85606b7df | Bin .../5c5e0e899cf2e7d053a9e45fb76f6e5a | Bin .../70152ed033f139443fbfb1b858bb3b1b | Bin .../7030ca2b24e5a7f9dd8f62096a48eb33 | Bin .../71eec1a0ef2d25bb9e2ef17f23be7e9e | Bin .../7a6b0177210ea4ef40b254daf99393c5 | Bin .../7f1567733711ffb61839621af0cbfa33 | Bin .../850c6d57c5bb7be8205fc2438d14d7e5 | Bin .../a5c8cd2784a5792b9e91c2d7895b3b34 | Bin .../a9135cdc7151d023300ff194bad90af9 | Bin .../af2597e8ac7dec1e8b4a47518312912a | Bin .../test/fuzzinput/answer_a | Bin .../test/fuzzinput/answer_aaaa | Bin .../b3f53ef826b831bb09dd25c7f5960249 | Bin .../cda0f8751f5c4993974c2b549d29bcc8 | Bin .../ce6c26c0e469339873d0e7f616ab0945 | Bin .../fuzzinput/clusterfuzz-5637790584012800 | Bin .../fuzzinput/clusterfuzz-5650695891451904 | Bin .../fuzzinput/clusterfuzz-5651369832218624 | Bin .../fuzzinput/clusterfuzz-5674462260756480 | Bin .../fuzzinput/clusterfuzz-5680630672654336 | Bin .../fuzzinput/clusterfuzz-5683497160671232 | Bin .../fuzzinput/clusterfuzz-5687310655422464 | Bin .../fuzzinput/clusterfuzz-5695341573177344 | Bin .../fuzzinput/clusterfuzz-5697835103682560 | Bin .../fuzzinput/clusterfuzz-5728518081609728 | Bin .../fuzzinput/clusterfuzz-5732960017317888 | Bin .../e4dd7e7c2dd4ed7c2e17a6af5d04f9c9 | Bin .../ed50ed8ee36230a5a69746ad830437e5 | Bin .../f1b900d50806021953321c3b604ee497 | Bin .../f6606f624be8c628328cea01d2cd07a9 | Bin .../f89f6c8176b564a7dd646f14305573ce | Bin .../f9ad508d2dbd08d3aaaabc7d1174677d | Bin .../test/fuzzinput/multi-indir | Bin .../test/fuzznames/name01 | 0 .../test/fuzznames/name02 | 0 .../test/fuzznames/name03 | 0 .../test/fuzznames/name04 | 0 .../test/fuzznames/name05 | 0 .../test/fuzznames/name06 | 0 .../test/fuzznames/name07 | 0 .../test/fuzznames/name08 | 0 .../test/fuzznames/name09 | 0 .../test/fuzznames/uri1 | 0 .../test/fuzznames/uri10 | 0 .../test/fuzznames/uri11 | 0 .../test/fuzznames/uri12 | 0 .../test/fuzznames/uri13 | 0 .../test/fuzznames/uri14 | 0 .../test/fuzznames/uri16 | 0 .../test/fuzznames/uri17 | 0 .../test/fuzznames/uri18 | 0 .../test/fuzznames/uri19 | 0 .../test/fuzznames/uri2 | 0 .../test/fuzznames/uri20 | 0 .../test/fuzznames/uri21 | 0 .../test/fuzznames/uri22 | 0 .../test/fuzznames/uri23 | 0 .../test/fuzznames/uri24 | 0 .../test/fuzznames/uri25 | 0 .../test/fuzznames/uri26 | 0 .../test/fuzznames/uri27 | 0 .../test/fuzznames/uri28 | 0 .../test/fuzznames/uri29 | 0 .../test/fuzznames/uri3 | 0 .../test/fuzznames/uri30 | 0 .../test/fuzznames/uri31 | 0 .../test/fuzznames/uri32 | 0 .../test/fuzznames/uri33 | 0 .../test/fuzznames/uri34 | 0 .../test/fuzznames/uri35 | 0 .../test/fuzznames/uri36 | 0 .../test/fuzznames/uri37 | 0 .../test/fuzznames/uri4 | 0 .../test/fuzznames/uri5 | 0 .../test/fuzznames/uri6 | 0 .../test/fuzznames/uri7 | 0 .../test/fuzznames/uri8 | 0 .../test/fuzznames/uri9 | 0 535 files changed, 9110 insertions(+), 5977 deletions(-) delete mode 100644 source/lib/c-ares-1.34.4/RELEASE-NOTES.md delete mode 100644 source/lib/c-ares-1.34.4/m4/ax_cxx_compile_stdcxx_14.m4 rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/AUTHORS (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/CMakeLists.txt (97%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/CONTRIBUTING.md (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/DEVELOPER-NOTES.md (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/INSTALL.md (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/LICENSE.md (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/Makefile.Watcom (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/Makefile.am (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/Makefile.dj (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/Makefile.in (99%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/Makefile.m32 (96%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/Makefile.msvc (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/Makefile.netware (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/README.md (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/README.msvc (100%) create mode 100644 source/lib/c-ares-1.34.6/RELEASE-NOTES.md rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/SECURITY.md (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/aclocal.m4 (99%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/aminclude_static.am (99%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/buildconf (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/buildconf.bat (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/c-ares-config.cmake.in (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/cmake/EnableWarnings.cmake (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/config/compile (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/config/config.guess (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/config/config.sub (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/config/depcomp (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/config/install-sh (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/config/ltmain.sh (94%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/config/missing (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/config/test-driver (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/configure (94%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/configure.ac (98%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/docs/CMakeLists.txt (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/docs/Makefile.am (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/docs/Makefile.in (99%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/docs/Makefile.inc (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/docs/adig.1 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/docs/ahost.1 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/docs/ares_cancel.3 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/docs/ares_create_query.3 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/docs/ares_destroy.3 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/docs/ares_destroy_options.3 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/docs/ares_dns_class_fromstr.3 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/docs/ares_dns_class_t.3 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/docs/ares_dns_class_tostr.3 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/docs/ares_dns_datatype_t.3 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/docs/ares_dns_flags_t.3 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/docs/ares_dns_mapping.3 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/docs/ares_dns_opcode_t.3 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/docs/ares_dns_opcode_tostr.3 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/docs/ares_dns_opt_datatype_t.3 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/docs/ares_dns_opt_get_datatype.3 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/docs/ares_dns_opt_get_name.3 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/docs/ares_dns_parse.3 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/docs/ares_dns_rcode_t.3 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/docs/ares_dns_rcode_tostr.3 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/docs/ares_dns_rec_type_fromstr.3 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/docs/ares_dns_rec_type_t.3 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/docs/ares_dns_rec_type_tostr.3 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/docs/ares_dns_record.3 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/docs/ares_dns_record_create.3 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/docs/ares_dns_record_destroy.3 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/docs/ares_dns_record_duplicate.3 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/docs/ares_dns_record_get_flags.3 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/docs/ares_dns_record_get_id.3 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/docs/ares_dns_record_get_opcode.3 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/docs/ares_dns_record_get_rcode.3 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/docs/ares_dns_record_query_add.3 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/docs/ares_dns_record_query_cnt.3 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/docs/ares_dns_record_query_get.3 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/docs/ares_dns_record_query_set_name.3 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/docs/ares_dns_record_query_set_type.3 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/docs/ares_dns_record_rr_add.3 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/docs/ares_dns_record_rr_cnt.3 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/docs/ares_dns_record_rr_del.3 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/docs/ares_dns_record_rr_get.3 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/docs/ares_dns_record_rr_get_const.3 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/docs/ares_dns_record_set_id.3 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/docs/ares_dns_rr.3 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/docs/ares_dns_rr_add_abin.3 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/docs/ares_dns_rr_del_abin.3 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/docs/ares_dns_rr_del_opt_byid.3 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/docs/ares_dns_rr_get_abin.3 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/docs/ares_dns_rr_get_abin_cnt.3 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/docs/ares_dns_rr_get_addr.3 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/docs/ares_dns_rr_get_addr6.3 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/docs/ares_dns_rr_get_bin.3 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/docs/ares_dns_rr_get_class.3 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/docs/ares_dns_rr_get_keys.3 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/docs/ares_dns_rr_get_name.3 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/docs/ares_dns_rr_get_opt.3 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/docs/ares_dns_rr_get_opt_byid.3 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/docs/ares_dns_rr_get_opt_cnt.3 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/docs/ares_dns_rr_get_str.3 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/docs/ares_dns_rr_get_ttl.3 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/docs/ares_dns_rr_get_type.3 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/docs/ares_dns_rr_get_u16.3 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/docs/ares_dns_rr_get_u32.3 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/docs/ares_dns_rr_get_u8.3 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/docs/ares_dns_rr_key_datatype.3 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/docs/ares_dns_rr_key_t.3 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/docs/ares_dns_rr_key_to_rec_type.3 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/docs/ares_dns_rr_key_tostr.3 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/docs/ares_dns_rr_set_addr.3 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/docs/ares_dns_rr_set_addr6.3 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/docs/ares_dns_rr_set_bin.3 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/docs/ares_dns_rr_set_opt.3 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/docs/ares_dns_rr_set_str.3 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/docs/ares_dns_rr_set_u16.3 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/docs/ares_dns_rr_set_u32.3 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/docs/ares_dns_rr_set_u8.3 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/docs/ares_dns_section_t.3 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/docs/ares_dns_section_tostr.3 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/docs/ares_dns_write.3 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/docs/ares_dup.3 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/docs/ares_expand_name.3 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/docs/ares_expand_string.3 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/docs/ares_fds.3 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/docs/ares_free_data.3 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/docs/ares_free_hostent.3 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/docs/ares_free_string.3 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/docs/ares_freeaddrinfo.3 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/docs/ares_get_servers.3 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/docs/ares_get_servers_csv.3 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/docs/ares_get_servers_ports.3 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/docs/ares_getaddrinfo.3 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/docs/ares_gethostbyaddr.3 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/docs/ares_gethostbyname.3 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/docs/ares_gethostbyname_file.3 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/docs/ares_getnameinfo.3 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/docs/ares_getsock.3 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/docs/ares_inet_ntop.3 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/docs/ares_inet_pton.3 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/docs/ares_init.3 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/docs/ares_init_options.3 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/docs/ares_library_cleanup.3 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/docs/ares_library_init.3 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/docs/ares_library_init_android.3 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/docs/ares_library_initialized.3 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/docs/ares_mkquery.3 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/docs/ares_opt_param_t.3 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/docs/ares_parse_a_reply.3 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/docs/ares_parse_aaaa_reply.3 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/docs/ares_parse_caa_reply.3 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/docs/ares_parse_mx_reply.3 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/docs/ares_parse_naptr_reply.3 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/docs/ares_parse_ns_reply.3 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/docs/ares_parse_ptr_reply.3 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/docs/ares_parse_soa_reply.3 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/docs/ares_parse_srv_reply.3 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/docs/ares_parse_txt_reply.3 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/docs/ares_parse_uri_reply.3 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/docs/ares_process.3 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/docs/ares_process_fd.3 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/docs/ares_process_fds.3 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/docs/ares_process_pending_write.3 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/docs/ares_query.3 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/docs/ares_query_dnsrec.3 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/docs/ares_queue.3 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/docs/ares_queue_active_queries.3 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/docs/ares_queue_wait_empty.3 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/docs/ares_reinit.3 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/docs/ares_save_options.3 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/docs/ares_search.3 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/docs/ares_search_dnsrec.3 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/docs/ares_send.3 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/docs/ares_send_dnsrec.3 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/docs/ares_set_local_dev.3 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/docs/ares_set_local_ip4.3 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/docs/ares_set_local_ip6.3 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/docs/ares_set_pending_write_cb.3 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/docs/ares_set_server_state_callback.3 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/docs/ares_set_servers.3 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/docs/ares_set_servers_csv.3 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/docs/ares_set_servers_ports.3 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/docs/ares_set_servers_ports_csv.3 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/docs/ares_set_socket_callback.3 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/docs/ares_set_socket_configure_callback.3 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/docs/ares_set_socket_functions.3 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/docs/ares_set_socket_functions_ex.3 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/docs/ares_set_sortlist.3 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/docs/ares_strerror.3 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/docs/ares_svcb_param_t.3 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/docs/ares_threadsafety.3 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/docs/ares_timeout.3 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/docs/ares_tlsa_match_t.3 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/docs/ares_tlsa_selector_t.3 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/docs/ares_tlsa_usage_t.3 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/docs/ares_version.3 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/include/CMakeLists.txt (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/include/Makefile.am (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/include/Makefile.in (99%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/include/ares.h (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/include/ares_build.h (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/include/ares_build.h.cmake (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/include/ares_build.h.in (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/include/ares_dns.h (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/include/ares_dns_record.h (99%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/include/ares_nameser.h (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/include/ares_version.h (93%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/libcares.pc.cmake (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/libcares.pc.in (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/m4/ares_check_user_namespace.m4 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/m4/ares_check_uts_namespace.m4 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/m4/ax_ac_append_to_file.m4 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/m4/ax_ac_print_to_file.m4 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/m4/ax_add_am_macro_static.m4 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/m4/ax_am_macros_static.m4 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/m4/ax_append_compile_flags.m4 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/m4/ax_append_flag.m4 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/m4/ax_append_link_flags.m4 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/m4/ax_check_compile_flag.m4 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/m4/ax_check_gnu_make.m4 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/m4/ax_check_link_flag.m4 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/m4/ax_code_coverage.m4 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/m4/ax_compiler_vendor.m4 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/m4/ax_cxx_compile_stdcxx.m4 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/m4/ax_file_escapes.m4 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/m4/ax_pthread.m4 (99%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/m4/ax_require_defined.m4 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/m4/libtool.m4 (97%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/m4/ltoptions.m4 (99%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/m4/ltsugar.m4 (98%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/m4/ltversion.m4 (66%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/m4/lt~obsolete.m4 (98%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/m4/pkg.m4 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/src/CMakeLists.txt (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/src/Makefile.am (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/src/Makefile.in (99%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/src/lib/CMakeLists.txt (92%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/src/lib/Makefile.am (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/src/lib/Makefile.in (99%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/src/lib/Makefile.inc (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/src/lib/ares_addrinfo2hostent.c (67%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/src/lib/ares_addrinfo_localhost.c (83%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/src/lib/ares_android.c (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/src/lib/ares_android.h (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/src/lib/ares_cancel.c (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/src/lib/ares_close_sockets.c (99%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/src/lib/ares_config.h.cmake (99%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/src/lib/ares_config.h.in (99%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/src/lib/ares_conn.c (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/src/lib/ares_conn.h (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/src/lib/ares_cookie.c (99%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/src/lib/ares_data.c (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/src/lib/ares_data.h (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/src/lib/ares_destroy.c (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/src/lib/ares_free_hostent.c (91%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/src/lib/ares_free_string.c (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/src/lib/ares_freeaddrinfo.c (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/src/lib/ares_getaddrinfo.c (96%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/src/lib/ares_getenv.c (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/src/lib/ares_getenv.h (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/src/lib/ares_gethostbyaddr.c (99%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/src/lib/ares_gethostbyname.c (96%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/src/lib/ares_getnameinfo.c (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/src/lib/ares_hosts_file.c (98%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/src/lib/ares_inet_net_pton.h (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/src/lib/ares_init.c (92%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/src/lib/ares_ipv6.h (89%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/src/lib/ares_library_init.c (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/src/lib/ares_metrics.c (99%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/src/lib/ares_options.c (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/src/lib/ares_parse_into_addrinfo.c (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/src/lib/ares_private.h (96%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/src/lib/ares_process.c (87%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/src/lib/ares_qcache.c (95%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/src/lib/ares_query.c (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/src/lib/ares_search.c (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/src/lib/ares_send.c (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/src/lib/ares_set_socket_functions.c (98%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/src/lib/ares_setup.h (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/src/lib/ares_socket.c (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/src/lib/ares_socket.h (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/src/lib/ares_sortaddrinfo.c (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/src/lib/ares_strerror.c (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/src/lib/ares_sysconfig.c (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/src/lib/ares_sysconfig_files.c (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/src/lib/ares_sysconfig_mac.c (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/src/lib/ares_sysconfig_win.c (88%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/src/lib/ares_timeout.c (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/src/lib/ares_update_servers.c (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/src/lib/ares_version.c (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/src/lib/cares.rc (97%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/src/lib/config-dos.h (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/src/lib/config-win32.h (98%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/src/lib/dsa/ares_array.c (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/src/lib/dsa/ares_htable.c (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/src/lib/dsa/ares_htable.h (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/src/lib/dsa/ares_htable_asvp.c (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/src/lib/dsa/ares_htable_dict.c (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/src/lib/dsa/ares_htable_strvp.c (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/src/lib/dsa/ares_htable_szvp.c (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/src/lib/dsa/ares_htable_vpstr.c (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/src/lib/dsa/ares_htable_vpvp.c (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/src/lib/dsa/ares_llist.c (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/src/lib/dsa/ares_slist.c (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/src/lib/dsa/ares_slist.h (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/src/lib/event/ares_event.h (97%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/src/lib/event/ares_event_configchg.c (97%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/src/lib/event/ares_event_epoll.c (99%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/src/lib/event/ares_event_kqueue.c (99%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/src/lib/event/ares_event_poll.c (98%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/src/lib/event/ares_event_select.c (98%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/src/lib/event/ares_event_thread.c (97%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/src/lib/event/ares_event_wake_pipe.c (96%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/src/lib/event/ares_event_win32.c (99%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/src/lib/event/ares_event_win32.h (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/src/lib/include/ares_array.h (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/src/lib/include/ares_buf.h (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/src/lib/include/ares_htable_asvp.h (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/src/lib/include/ares_htable_dict.h (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/src/lib/include/ares_htable_strvp.h (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/src/lib/include/ares_htable_szvp.h (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/src/lib/include/ares_htable_vpstr.h (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/src/lib/include/ares_htable_vpvp.h (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/src/lib/include/ares_llist.h (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/src/lib/include/ares_mem.h (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/src/lib/include/ares_str.h (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/src/lib/inet_net_pton.c (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/src/lib/inet_ntop.c (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/src/lib/legacy/ares_create_query.c (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/src/lib/legacy/ares_expand_name.c (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/src/lib/legacy/ares_expand_string.c (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/src/lib/legacy/ares_fds.c (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/src/lib/legacy/ares_getsock.c (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/src/lib/legacy/ares_parse_a_reply.c (99%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/src/lib/legacy/ares_parse_aaaa_reply.c (99%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/src/lib/legacy/ares_parse_caa_reply.c (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/src/lib/legacy/ares_parse_mx_reply.c (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/src/lib/legacy/ares_parse_naptr_reply.c (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/src/lib/legacy/ares_parse_ns_reply.c (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/src/lib/legacy/ares_parse_ptr_reply.c (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/src/lib/legacy/ares_parse_soa_reply.c (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/src/lib/legacy/ares_parse_srv_reply.c (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/src/lib/legacy/ares_parse_txt_reply.c (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/src/lib/legacy/ares_parse_uri_reply.c (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/src/lib/record/ares_dns_mapping.c (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/src/lib/record/ares_dns_multistring.c (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/src/lib/record/ares_dns_multistring.h (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/src/lib/record/ares_dns_name.c (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/src/lib/record/ares_dns_parse.c (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/src/lib/record/ares_dns_private.h (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/src/lib/record/ares_dns_record.c (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/src/lib/record/ares_dns_write.c (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/src/lib/str/ares_buf.c (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/src/lib/str/ares_str.c (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/src/lib/str/ares_strsplit.c (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/src/lib/str/ares_strsplit.h (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/src/lib/thirdparty/apple/dnsinfo.h (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/src/lib/util/ares_iface_ips.c (98%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/src/lib/util/ares_iface_ips.h (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/src/lib/util/ares_math.c (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/src/lib/util/ares_math.h (78%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/src/lib/util/ares_rand.c (98%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/src/lib/util/ares_rand.h (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/src/lib/util/ares_threads.c (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/src/lib/util/ares_threads.h (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/src/lib/util/ares_time.h (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/src/lib/util/ares_timeval.c (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/src/lib/util/ares_uri.c (99%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/src/lib/util/ares_uri.h (99%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/src/lib/windows_port.c (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/src/tools/CMakeLists.txt (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/src/tools/Makefile.am (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/src/tools/Makefile.in (99%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/src/tools/Makefile.inc (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/src/tools/adig.c (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/src/tools/ahost.c (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/src/tools/ares_getopt.c (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/src/tools/ares_getopt.h (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/test/CMakeLists.txt (97%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/test/Makefile.am (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/test/Makefile.in (99%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/test/Makefile.inc (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/test/Makefile.m32 (96%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/test/Makefile.msvc (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/test/README.md (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/test/ares-fuzz.c (99%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/test/ares-test-ai.h (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/test/ares-test-fuzz-name.c (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/test/ares-test-fuzz.c (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/test/ares-test-init.cc (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/test/ares-test-internal.cc (99%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/test/ares-test-live.cc (89%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/test/ares-test-main.cc (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/test/ares-test-misc.cc (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/test/ares-test-mock-ai.cc (76%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/test/ares-test-mock-et.cc (95%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/test/ares-test-mock.cc (93%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/test/ares-test-ns.cc (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/test/ares-test-parse-a.cc (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/test/ares-test-parse-aaaa.cc (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/test/ares-test-parse-caa.cc (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/test/ares-test-parse-mx.cc (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/test/ares-test-parse-naptr.cc (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/test/ares-test-parse-ns.cc (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/test/ares-test-parse-ptr.cc (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/test/ares-test-parse-soa-any.cc (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/test/ares-test-parse-soa.cc (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/test/ares-test-parse-srv.cc (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/test/ares-test-parse-txt.cc (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/test/ares-test-parse-uri.cc (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/test/ares-test-parse.cc (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/test/ares-test.cc (98%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/test/ares-test.h (97%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/test/ares_queryloop.c (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/test/dns-dump.cc (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/test/dns-proto-test.cc (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/test/dns-proto.cc (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/test/dns-proto.h (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/test/fuzzcheck.sh (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/test/fuzzinput/004a216d3cff18b0c5c6b68b807f1529 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/test/fuzzinput/00539467ca159b36aea95e61f9729115 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/test/fuzzinput/00e846db8f43f2f507cd1666ed5a753e (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/test/fuzzinput/0177b7566f08c013699eaea9a77abeb3 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/test/fuzzinput/020a4fa317715bfdb236ed13751e6b65 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/test/fuzzinput/0310f2e81bea31f4fe3f330872a877dd (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/test/fuzzinput/0449be67df1730b2d0887d412a9b7cc4 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/test/fuzzinput/0449dd14f7aa94bf0d716bfe09b287a8 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/test/fuzzinput/04c93cdf7208979aa4df80a3a0d5a2d8 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/test/fuzzinput/0567e7171e08e75f3f91c4ca74c17adc (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/test/fuzzinput/05ba948578a397e9cbc6a7b3e78622fa (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/test/fuzzinput/060afe5ed25f3e2e86167e545f27edca (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/test/fuzzinput/06d47d3681493f1b1d41236f460d896f (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/test/fuzzinput/0724a810b0e131c2fddb6de9003b9064 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/test/fuzzinput/0b5279148826f5b962bcf1896bdb4ede (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/test/fuzzinput/114048c0f6b10bdc67ce9166405d195e (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/test/fuzzinput/11b8464a0ef8735d202955c34c36b0c7 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/test/fuzzinput/11cb626f1668c7b41954ce7d768fe528 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/test/fuzzinput/14b133bf18125b75a1976fa63a1df6b7 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/test/fuzzinput/153c6b3afa8faa03c8bc28f936a6d4cf (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/test/fuzzinput/182cad2a342ed7317b7c21a5d17020d1 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/test/fuzzinput/1c61a61bb7057b52c5b15188345a5238 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/test/fuzzinput/1dbe2cf62ed2e4fa1c3cb473f08710b5 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/test/fuzzinput/21199be504fcfece5c7096ee0dbba507 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/test/fuzzinput/21891480074b5635dbbe7137bdcabccd (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/test/fuzzinput/233aea42e15aa73e131eefabf16088c9 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/test/fuzzinput/24660d4e7ac7aa21d600ea7a3d198bbb (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/test/fuzzinput/25589deb55c08429345f289d1c9b0254 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/test/fuzzinput/2573bd823e4da11f727a17f8e1f35c26 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/test/fuzzinput/276f12da56866273e76059ad0e7be97e (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/test/fuzzinput/29198a2e380cb19babec9e02116d213e (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/test/fuzzinput/2c94ba9434b1a1b9396fc5364f101363 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/test/fuzzinput/2d578c357dc2f5e02dc55cddb30641d1 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/test/fuzzinput/2dff6cc5a223e67fde9e5e79af456992 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/test/fuzzinput/2f103b1f9477f2d8934bd84328d51c75 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/test/fuzzinput/31cd3a8413de13d9624adbb1613784bf (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/test/fuzzinput/36415bdf1d180098fe6234b4186e69f3 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/test/fuzzinput/3a04a80f0242e8dff0cd732e7c4767da (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/test/fuzzinput/44d0f973b7b0fb3e4a07770c943dcd5a (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/test/fuzzinput/50bc00daa0ddcd6cfb2b5d9f62c81f47 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/test/fuzzinput/51ed2d1fb77b3078b54e94e85606b7df (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/test/fuzzinput/5c5e0e899cf2e7d053a9e45fb76f6e5a (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/test/fuzzinput/70152ed033f139443fbfb1b858bb3b1b (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/test/fuzzinput/7030ca2b24e5a7f9dd8f62096a48eb33 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/test/fuzzinput/71eec1a0ef2d25bb9e2ef17f23be7e9e (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/test/fuzzinput/7a6b0177210ea4ef40b254daf99393c5 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/test/fuzzinput/7f1567733711ffb61839621af0cbfa33 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/test/fuzzinput/850c6d57c5bb7be8205fc2438d14d7e5 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/test/fuzzinput/a5c8cd2784a5792b9e91c2d7895b3b34 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/test/fuzzinput/a9135cdc7151d023300ff194bad90af9 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/test/fuzzinput/af2597e8ac7dec1e8b4a47518312912a (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/test/fuzzinput/answer_a (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/test/fuzzinput/answer_aaaa (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/test/fuzzinput/b3f53ef826b831bb09dd25c7f5960249 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/test/fuzzinput/cda0f8751f5c4993974c2b549d29bcc8 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/test/fuzzinput/ce6c26c0e469339873d0e7f616ab0945 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/test/fuzzinput/clusterfuzz-5637790584012800 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/test/fuzzinput/clusterfuzz-5650695891451904 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/test/fuzzinput/clusterfuzz-5651369832218624 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/test/fuzzinput/clusterfuzz-5674462260756480 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/test/fuzzinput/clusterfuzz-5680630672654336 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/test/fuzzinput/clusterfuzz-5683497160671232 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/test/fuzzinput/clusterfuzz-5687310655422464 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/test/fuzzinput/clusterfuzz-5695341573177344 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/test/fuzzinput/clusterfuzz-5697835103682560 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/test/fuzzinput/clusterfuzz-5728518081609728 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/test/fuzzinput/clusterfuzz-5732960017317888 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/test/fuzzinput/e4dd7e7c2dd4ed7c2e17a6af5d04f9c9 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/test/fuzzinput/ed50ed8ee36230a5a69746ad830437e5 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/test/fuzzinput/f1b900d50806021953321c3b604ee497 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/test/fuzzinput/f6606f624be8c628328cea01d2cd07a9 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/test/fuzzinput/f89f6c8176b564a7dd646f14305573ce (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/test/fuzzinput/f9ad508d2dbd08d3aaaabc7d1174677d (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/test/fuzzinput/multi-indir (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/test/fuzznames/name01 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/test/fuzznames/name02 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/test/fuzznames/name03 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/test/fuzznames/name04 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/test/fuzznames/name05 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/test/fuzznames/name06 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/test/fuzznames/name07 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/test/fuzznames/name08 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/test/fuzznames/name09 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/test/fuzznames/uri1 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/test/fuzznames/uri10 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/test/fuzznames/uri11 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/test/fuzznames/uri12 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/test/fuzznames/uri13 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/test/fuzznames/uri14 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/test/fuzznames/uri16 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/test/fuzznames/uri17 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/test/fuzznames/uri18 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/test/fuzznames/uri19 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/test/fuzznames/uri2 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/test/fuzznames/uri20 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/test/fuzznames/uri21 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/test/fuzznames/uri22 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/test/fuzznames/uri23 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/test/fuzznames/uri24 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/test/fuzznames/uri25 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/test/fuzznames/uri26 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/test/fuzznames/uri27 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/test/fuzznames/uri28 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/test/fuzznames/uri29 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/test/fuzznames/uri3 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/test/fuzznames/uri30 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/test/fuzznames/uri31 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/test/fuzznames/uri32 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/test/fuzznames/uri33 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/test/fuzznames/uri34 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/test/fuzznames/uri35 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/test/fuzznames/uri36 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/test/fuzznames/uri37 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/test/fuzznames/uri4 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/test/fuzznames/uri5 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/test/fuzznames/uri6 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/test/fuzznames/uri7 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/test/fuzznames/uri8 (100%) rename source/lib/{c-ares-1.34.4 => c-ares-1.34.6}/test/fuzznames/uri9 (100%) diff --git a/source/cmake/libraries.cmake b/source/cmake/libraries.cmake index 2433106f..c73c81f1 100644 --- a/source/cmake/libraries.cmake +++ b/source/cmake/libraries.cmake @@ -20,7 +20,7 @@ set(FLB_PATH_LIB_ONIGMO "lib/onigmo") set(FLB_PATH_LIB_MPACK "lib/mpack-amalgamation-1.1.1") set(FLB_PATH_LIB_MINIZ "lib/miniz") set(FLB_PATH_LIB_TUTF8E "lib/tutf8e") -set(FLB_PATH_LIB_CARES "lib/c-ares-1.34.4") +set(FLB_PATH_LIB_CARES "lib/c-ares-1.34.6") set(FLB_PATH_LIB_SNAPPY "lib/snappy-fef67ac") set(FLB_PATH_LIB_RDKAFKA "lib/librdkafka-2.10.1") set(FLB_PATH_LIB_RING_BUFFER "lib/lwrb") diff --git a/source/lib/c-ares-1.34.4/RELEASE-NOTES.md b/source/lib/c-ares-1.34.4/RELEASE-NOTES.md deleted file mode 100644 index 19a204b3..00000000 --- a/source/lib/c-ares-1.34.4/RELEASE-NOTES.md +++ /dev/null @@ -1,25 +0,0 @@ -## c-ares version 1.34.4 - December 14 2024 - -This is a bugfix release. - -Changes: -* QNX Port: Port to QNX 8, add primary config reading support, add CI build. [PR #934](https://github.com/c-ares/c-ares/pull/934), [PR #937](https://github.com/c-ares/c-ares/pull/937), [PR #938](https://github.com/c-ares/c-ares/pull/938) - -Bugfixes: -* Empty TXT records were not being preserved. [PR #922](https://github.com/c-ares/c-ares/pull/922) -* docs: update deprecation notices for `ares_create_query()` and `ares_mkquery()`. [PR #910](https://github.com/c-ares/c-ares/pull/910) -* license: some files weren't properly updated. [PR #920](https://github.com/c-ares/c-ares/pull/920) -* Fix bind local device regression from 1.34.0. [PR #929](https://github.com/c-ares/c-ares/pull/929), [PR #931](https://github.com/c-ares/c-ares/pull/931), [PR #935](https://github.com/c-ares/c-ares/pull/935) -* CMake: set policy version to prevent deprecation warnings. [PR #932](https://github.com/c-ares/c-ares/pull/932) -* CMake: shared and static library names should be the same on unix platforms like autotools uses. [PR #933](https://github.com/c-ares/c-ares/pull/933) -* Update to latest autoconf archive macros for enhanced system compatibility. [PR #936](https://github.com/c-ares/c-ares/pull/936) - -Thanks go to these friendly people for their efforts and contributions for this -release: - -* Brad House (@bradh352) -* Daniel Stenberg (@bagder) -* Gregor Jasny (@gjasny) -* @marcovsz -* Nikolaos Chatzikonstantinou (@createyourpersonalaccount) -* @vlasovsoft1979 diff --git a/source/lib/c-ares-1.34.4/m4/ax_cxx_compile_stdcxx_14.m4 b/source/lib/c-ares-1.34.4/m4/ax_cxx_compile_stdcxx_14.m4 deleted file mode 100644 index 094db0d0..00000000 --- a/source/lib/c-ares-1.34.4/m4/ax_cxx_compile_stdcxx_14.m4 +++ /dev/null @@ -1,34 +0,0 @@ -# ============================================================================= -# https://www.gnu.org/software/autoconf-archive/ax_cxx_compile_stdcxx_14.html -# ============================================================================= -# -# SYNOPSIS -# -# AX_CXX_COMPILE_STDCXX_14([ext|noext], [mandatory|optional]) -# -# DESCRIPTION -# -# Check for baseline language coverage in the compiler for the C++14 -# standard; if necessary, add switches to CXX and CXXCPP to enable -# support. -# -# This macro is a convenience alias for calling the AX_CXX_COMPILE_STDCXX -# macro with the version set to C++14. The two optional arguments are -# forwarded literally as the second and third argument respectively. -# Please see the documentation for the AX_CXX_COMPILE_STDCXX macro for -# more information. If you want to use this macro, you also need to -# download the ax_cxx_compile_stdcxx.m4 file. -# -# LICENSE -# -# Copyright (c) 2015 Moritz Klammler -# -# Copying and distribution of this file, with or without modification, are -# permitted in any medium without royalty provided the copyright notice -# and this notice are preserved. This file is offered as-is, without any -# warranty. - -#serial 5 - -AX_REQUIRE_DEFINED([AX_CXX_COMPILE_STDCXX]) -AC_DEFUN([AX_CXX_COMPILE_STDCXX_14], [AX_CXX_COMPILE_STDCXX([14], [$1], [$2])]) diff --git a/source/lib/c-ares-1.34.4/AUTHORS b/source/lib/c-ares-1.34.6/AUTHORS similarity index 100% rename from source/lib/c-ares-1.34.4/AUTHORS rename to source/lib/c-ares-1.34.6/AUTHORS diff --git a/source/lib/c-ares-1.34.4/CMakeLists.txt b/source/lib/c-ares-1.34.6/CMakeLists.txt similarity index 97% rename from source/lib/c-ares-1.34.4/CMakeLists.txt rename to source/lib/c-ares-1.34.6/CMakeLists.txt index 139defd8..a2a9c193 100644 --- a/source/lib/c-ares-1.34.4/CMakeLists.txt +++ b/source/lib/c-ares-1.34.6/CMakeLists.txt @@ -12,7 +12,7 @@ INCLUDE (CheckCSourceCompiles) INCLUDE (CheckStructHasMember) INCLUDE (CheckLibraryExists) -PROJECT (c-ares LANGUAGES C VERSION "1.34.4" ) +PROJECT (c-ares LANGUAGES C VERSION "1.34.6" ) # Set this version before release SET (CARES_VERSION "${PROJECT_VERSION}") @@ -30,7 +30,7 @@ INCLUDE (GNUInstallDirs) # include this *AFTER* PROJECT(), otherwise paths are w # For example, a version of 4:0:2 would generate output such as: # libname.so -> libname.so.2 # libname.so.2 -> libname.so.2.2.0 -SET (CARES_LIB_VERSIONINFO "21:3:19") +SET (CARES_LIB_VERSIONINFO "21:5:19") OPTION (CARES_STATIC "Build as a static library" OFF) @@ -45,15 +45,6 @@ OPTION (CARES_THREADS "Build with thread-safety support" OPTION (CARES_COVERAGE "Build for code coverage" OFF) SET (CARES_RANDOM_FILE "/dev/urandom" CACHE STRING "Suitable File / Device Path for entropy, such as /dev/urandom") - -# Tests require a C++14 compiler -IF (CARES_BUILD_TESTS OR CARES_BUILD_CONTAINER_TESTS) - set(CMAKE_CXX_STANDARD 14) - set(CMAKE_CXX_STANDARD_REQUIRED TRUE) - set(CMAKE_CXX_EXTENSIONS FALSE) - enable_language(CXX) -ENDIF () - # Tests require static to be enabled on Windows to be able to access otherwise hidden symbols IF ((CARES_BUILD_TESTS OR CARES_BUILD_CONTAINER_TESTS) AND (NOT CARES_STATIC) AND WIN32) SET (CARES_STATIC ON) @@ -269,17 +260,19 @@ ELSEIF (CMAKE_SYSTEM_NAME STREQUAL "SunOS") LIST (APPEND SYSFLAGS -D__EXTENSIONS__ -D_REENTRANT -D_XOPEN_SOURCE=600) ELSEIF (CMAKE_SYSTEM_NAME STREQUAL "AIX") LIST (APPEND SYSFLAGS -D_ALL_SOURCE -D_XOPEN_SOURCE=700 -D_USE_IRS) -ELSEIF (CMAKE_SYSTEM_NAME STREQUAL "FreeBSD") +ELSEIF (CMAKE_SYSTEM_NAME STREQUAL "FreeBSD" OR CMAKE_SYSTEM_NAME STREQUAL "MidnightBSD") # Don't define _XOPEN_SOURCE on FreeBSD, it actually reduces visibility instead of increasing it ELSEIF (CMAKE_SYSTEM_NAME STREQUAL "QNX") LIST (APPEND SYSFLAGS -D_QNX_SOURCE) ELSEIF (WIN32) - LIST (APPEND SYSFLAGS -DWIN32_LEAN_AND_MEAN -D_CRT_SECURE_NO_DEPRECATE -D_CRT_NONSTDC_NO_DEPRECATE -D_WIN32_WINNT=0x0602) + LIST (APPEND SYSFLAGS -DWIN32_LEAN_AND_MEAN -D_CRT_SECURE_NO_DEPRECATE -D_CRT_NONSTDC_NO_DEPRECATE) + IF (NOT CMAKE_C_FLAGS MATCHES ".*-D_WIN32_WINNT=.*") + LIST (APPEND SYSFLAGS -D_WIN32_WINNT=0x0602) + ENDIF () ENDIF () ADD_DEFINITIONS(${SYSFLAGS}) - # Tell C-Ares about libraries to depend on IF (HAVE_LIBRESOLV) LIST (APPEND CARES_DEPENDENT_LIBS resolv) @@ -426,6 +419,7 @@ CHECK_SYMBOL_EXISTS (getservbyname_r "${CMAKE_EXTRA_INCLUDE_FILES}" HAVE_GETSERV CHECK_SYMBOL_EXISTS (gettimeofday "${CMAKE_EXTRA_INCLUDE_FILES}" HAVE_GETTIMEOFDAY) CHECK_SYMBOL_EXISTS (if_indextoname "${CMAKE_EXTRA_INCLUDE_FILES}" HAVE_IF_INDEXTONAME) CHECK_SYMBOL_EXISTS (if_nametoindex "${CMAKE_EXTRA_INCLUDE_FILES}" HAVE_IF_NAMETOINDEX) +CHECK_SYMBOL_EXISTS (GetBestRoute2 "${CMAKE_EXTRA_INCLUDE_FILES}" HAVE_GETBESTROUTE2) CHECK_SYMBOL_EXISTS (ConvertInterfaceIndexToLuid "${CMAKE_EXTRA_INCLUDE_FILES}" HAVE_CONVERTINTERFACEINDEXTOLUID) CHECK_SYMBOL_EXISTS (ConvertInterfaceLuidToNameA "${CMAKE_EXTRA_INCLUDE_FILES}" HAVE_CONVERTINTERFACELUIDTONAMEA) CHECK_SYMBOL_EXISTS (NotifyIpInterfaceChange "${CMAKE_EXTRA_INCLUDE_FILES}" HAVE_NOTIFYIPINTERFACECHANGE) @@ -503,11 +497,7 @@ IF (CARES_THREADS) CARES_EXTRAINCLUDE_IFSET (HAVE_PTHREAD_H pthread.h) CARES_EXTRAINCLUDE_IFSET (HAVE_PTHREAD_NP_H pthread_np.h) CHECK_SYMBOL_EXISTS (pthread_init "${CMAKE_EXTRA_INCLUDE_FILES}" HAVE_PTHREAD_INIT) - # Make sure libcares.pc.cmake knows about thread libraries on static builds - # The variable set by FIND_PACKAGE(Threads) has a -l prefix on it, we need - # to strip that first since CARES_DEPENDENT_LIBS doesn't expect that. - STRING (REPLACE "-l" "" CARES_THREAD_LIBRARY "${CMAKE_THREAD_LIBS_INIT}") - LIST (APPEND CARES_DEPENDENT_LIBS ${CARES_THREAD_LIBRARY}) + LIST (APPEND CARES_DEPENDENT_LIBS ${CMAKE_THREAD_LIBS_INIT}) ELSE () MESSAGE (WARNING "Threading support not found, disabling...") SET (CARES_THREADS OFF) @@ -631,6 +621,7 @@ IF (HAVE_GETADDRINFO) CMAKE_SYSTEM_NAME STREQUAL "NetBSD" OR CMAKE_SYSTEM_NAME STREQUAL "SunOS" OR CMAKE_SYSTEM_NAME STREQUAL "FreeBSD" OR + CMAKE_SYSTEM_NAME STREQUAL "MidnightBSD" OR CMAKE_SYSTEM_NAME STREQUAL "AIX" OR WIN32) SET (HAVE_GETADDRINFO_THREADSAFE 1) @@ -777,7 +768,10 @@ IF (CARES_INSTALL) # pkgconfig support for static builds FOREACH (LIB ${CARES_DEPENDENT_LIBS}) - SET (CARES_PRIVATE_LIBS "${CARES_PRIVATE_LIBS} -l${LIB}") + IF (NOT LIB MATCHES "^-") + SET (LIB "-l${LIB}") + ENDIF () + SET (CARES_PRIVATE_LIBS "${CARES_PRIVATE_LIBS} ${LIB}") ENDFOREACH () CONFIGURE_FILE("libcares.pc.cmake" "libcares.pc" @ONLY) diff --git a/source/lib/c-ares-1.34.4/CONTRIBUTING.md b/source/lib/c-ares-1.34.6/CONTRIBUTING.md similarity index 100% rename from source/lib/c-ares-1.34.4/CONTRIBUTING.md rename to source/lib/c-ares-1.34.6/CONTRIBUTING.md diff --git a/source/lib/c-ares-1.34.4/DEVELOPER-NOTES.md b/source/lib/c-ares-1.34.6/DEVELOPER-NOTES.md similarity index 100% rename from source/lib/c-ares-1.34.4/DEVELOPER-NOTES.md rename to source/lib/c-ares-1.34.6/DEVELOPER-NOTES.md diff --git a/source/lib/c-ares-1.34.4/INSTALL.md b/source/lib/c-ares-1.34.6/INSTALL.md similarity index 100% rename from source/lib/c-ares-1.34.4/INSTALL.md rename to source/lib/c-ares-1.34.6/INSTALL.md diff --git a/source/lib/c-ares-1.34.4/LICENSE.md b/source/lib/c-ares-1.34.6/LICENSE.md similarity index 100% rename from source/lib/c-ares-1.34.4/LICENSE.md rename to source/lib/c-ares-1.34.6/LICENSE.md diff --git a/source/lib/c-ares-1.34.4/Makefile.Watcom b/source/lib/c-ares-1.34.6/Makefile.Watcom similarity index 100% rename from source/lib/c-ares-1.34.4/Makefile.Watcom rename to source/lib/c-ares-1.34.6/Makefile.Watcom diff --git a/source/lib/c-ares-1.34.4/Makefile.am b/source/lib/c-ares-1.34.6/Makefile.am similarity index 100% rename from source/lib/c-ares-1.34.4/Makefile.am rename to source/lib/c-ares-1.34.6/Makefile.am diff --git a/source/lib/c-ares-1.34.4/Makefile.dj b/source/lib/c-ares-1.34.6/Makefile.dj similarity index 100% rename from source/lib/c-ares-1.34.4/Makefile.dj rename to source/lib/c-ares-1.34.6/Makefile.dj diff --git a/source/lib/c-ares-1.34.4/Makefile.in b/source/lib/c-ares-1.34.6/Makefile.in similarity index 99% rename from source/lib/c-ares-1.34.4/Makefile.in rename to source/lib/c-ares-1.34.6/Makefile.in index 2342125d..ba5253f8 100644 --- a/source/lib/c-ares-1.34.4/Makefile.in +++ b/source/lib/c-ares-1.34.6/Makefile.in @@ -133,7 +133,6 @@ am__aclocal_m4_deps = $(top_srcdir)/m4/ares_check_user_namespace.m4 \ $(top_srcdir)/m4/ax_code_coverage.m4 \ $(top_srcdir)/m4/ax_compiler_vendor.m4 \ $(top_srcdir)/m4/ax_cxx_compile_stdcxx.m4 \ - $(top_srcdir)/m4/ax_cxx_compile_stdcxx_14.m4 \ $(top_srcdir)/m4/ax_file_escapes.m4 \ $(top_srcdir)/m4/ax_pthread.m4 \ $(top_srcdir)/m4/ax_require_defined.m4 \ @@ -329,14 +328,18 @@ EGREP = @EGREP@ ETAGS = @ETAGS@ EXEEXT = @EXEEXT@ FGREP = @FGREP@ +FILECMD = @FILECMD@ GCOV = @GCOV@ GENHTML = @GENHTML@ GMOCK112_CFLAGS = @GMOCK112_CFLAGS@ GMOCK112_LIBS = @GMOCK112_LIBS@ +GMOCK117_CFLAGS = @GMOCK117_CFLAGS@ +GMOCK117_LIBS = @GMOCK117_LIBS@ GMOCK_CFLAGS = @GMOCK_CFLAGS@ GMOCK_LIBS = @GMOCK_LIBS@ GREP = @GREP@ HAVE_CXX14 = @HAVE_CXX14@ +HAVE_CXX17 = @HAVE_CXX17@ INSTALL = @INSTALL@ INSTALL_DATA = @INSTALL_DATA@ INSTALL_PROGRAM = @INSTALL_PROGRAM@ diff --git a/source/lib/c-ares-1.34.4/Makefile.m32 b/source/lib/c-ares-1.34.6/Makefile.m32 similarity index 96% rename from source/lib/c-ares-1.34.4/Makefile.m32 rename to source/lib/c-ares-1.34.6/Makefile.m32 index 7bd85165..427b0877 100644 --- a/source/lib/c-ares-1.34.4/Makefile.m32 +++ b/source/lib/c-ares-1.34.6/Makefile.m32 @@ -19,7 +19,9 @@ RANLIB = $(CROSSPREFIX)ranlib #RM = rm -f CP = cp -afv -CFLAGS = $(CARES_CFLAG_EXTRAS) -O2 -Wall -I./include -I./src/lib -I./src/lib/include -D_WIN32_WINNT=0x0602 +WIN32_WINNT ?= 0x0602 + +CFLAGS = $(CARES_CFLAG_EXTRAS) -O2 -Wall -I./include -I./src/lib -I./src/lib/include -D_WIN32_WINNT=$(WIN32_WINNT) CFLAGS += -DCARES_STATICLIB LDFLAGS = $(CARES_LDFLAG_EXTRAS) -s LIBS = -lws2_32 -liphlpapi diff --git a/source/lib/c-ares-1.34.4/Makefile.msvc b/source/lib/c-ares-1.34.6/Makefile.msvc similarity index 100% rename from source/lib/c-ares-1.34.4/Makefile.msvc rename to source/lib/c-ares-1.34.6/Makefile.msvc diff --git a/source/lib/c-ares-1.34.4/Makefile.netware b/source/lib/c-ares-1.34.6/Makefile.netware similarity index 100% rename from source/lib/c-ares-1.34.4/Makefile.netware rename to source/lib/c-ares-1.34.6/Makefile.netware diff --git a/source/lib/c-ares-1.34.4/README.md b/source/lib/c-ares-1.34.6/README.md similarity index 100% rename from source/lib/c-ares-1.34.4/README.md rename to source/lib/c-ares-1.34.6/README.md diff --git a/source/lib/c-ares-1.34.4/README.msvc b/source/lib/c-ares-1.34.6/README.msvc similarity index 100% rename from source/lib/c-ares-1.34.4/README.msvc rename to source/lib/c-ares-1.34.6/README.msvc diff --git a/source/lib/c-ares-1.34.6/RELEASE-NOTES.md b/source/lib/c-ares-1.34.6/RELEASE-NOTES.md new file mode 100644 index 00000000..ded75001 --- /dev/null +++ b/source/lib/c-ares-1.34.6/RELEASE-NOTES.md @@ -0,0 +1,42 @@ +## c-ares version 1.34.6 - December 8 2025 + +This is a security release. + +Security: +* CVE-2025-62408. A use-after-free bug has been uncovered in read_answers() that + was introduced in v1.32.3. Please see https://github.com/c-ares/c-ares/security/advisories/GHSA-jq53-42q6-pqr5 + +Changes: +* Ignore Windows IDN Search Domains until proper IDN support is added. [PR #1034](https://github.com/c-ares/c-ares/pull/1034) + +Bugfixes: +* Event Thread could stall when not notified of new queries on existing + connections that are in a bad state + [PR #1032](https://github.com/c-ares/c-ares/pull/1032) +* fix conversion of invalid service to port number in ares_getaddrinfo() + [PR #1029](https://github.com/c-ares/c-ares/pull/1029) +* fix memory leak in ares_uri + [PR #1012](https://github.com/c-ares/c-ares/pull/1012) +* Ignore ares_event_configchg_init failures + [PR #1009](https://github.com/c-ares/c-ares/pull/1009) +* Use XOR for random seed generation on fallback logic. + [PR #994](https://github.com/c-ares/c-ares/pull/994) +* Fix clang build on windows. + [PR #996](https://github.com/c-ares/c-ares/pull/996) +* Fix IPv6 link-local nameservers in /etc/resolv.conf + [PR #996](https://github.com/c-ares/c-ares/pull/997) +* Fix a few build issues on MidnightBSD. + [PR #983](https://github.com/c-ares/c-ares/pull/983) + +Thanks go to these friendly people for their efforts and contributions for this +release: + +* Brad House (@bradh352) +* (@F3lixTheCat) +* Lucas Holt (@laffer1) +* @oargon +* Pavel P (@pps83) +* Sean Harmer (@seanharmer) +* Uwe (@nixblik) + + diff --git a/source/lib/c-ares-1.34.4/SECURITY.md b/source/lib/c-ares-1.34.6/SECURITY.md similarity index 100% rename from source/lib/c-ares-1.34.4/SECURITY.md rename to source/lib/c-ares-1.34.6/SECURITY.md diff --git a/source/lib/c-ares-1.34.4/aclocal.m4 b/source/lib/c-ares-1.34.6/aclocal.m4 similarity index 99% rename from source/lib/c-ares-1.34.4/aclocal.m4 rename to source/lib/c-ares-1.34.6/aclocal.m4 index 04f8786c..a732fb0e 100644 --- a/source/lib/c-ares-1.34.4/aclocal.m4 +++ b/source/lib/c-ares-1.34.6/aclocal.m4 @@ -1236,7 +1236,6 @@ m4_include([m4/ax_check_link_flag.m4]) m4_include([m4/ax_code_coverage.m4]) m4_include([m4/ax_compiler_vendor.m4]) m4_include([m4/ax_cxx_compile_stdcxx.m4]) -m4_include([m4/ax_cxx_compile_stdcxx_14.m4]) m4_include([m4/ax_file_escapes.m4]) m4_include([m4/ax_pthread.m4]) m4_include([m4/ax_require_defined.m4]) diff --git a/source/lib/c-ares-1.34.4/aminclude_static.am b/source/lib/c-ares-1.34.6/aminclude_static.am similarity index 99% rename from source/lib/c-ares-1.34.4/aminclude_static.am rename to source/lib/c-ares-1.34.6/aminclude_static.am index ec7a86a4..07239563 100644 --- a/source/lib/c-ares-1.34.4/aminclude_static.am +++ b/source/lib/c-ares-1.34.6/aminclude_static.am @@ -1,6 +1,6 @@ # aminclude_static.am generated automatically by Autoconf -# from AX_AM_MACROS_STATIC on Sat Dec 14 15:15:44 UTC 2024 +# from AX_AM_MACROS_STATIC on Mon Dec 8 16:21:41 UTC 2025 # Code coverage diff --git a/source/lib/c-ares-1.34.4/buildconf b/source/lib/c-ares-1.34.6/buildconf similarity index 100% rename from source/lib/c-ares-1.34.4/buildconf rename to source/lib/c-ares-1.34.6/buildconf diff --git a/source/lib/c-ares-1.34.4/buildconf.bat b/source/lib/c-ares-1.34.6/buildconf.bat similarity index 100% rename from source/lib/c-ares-1.34.4/buildconf.bat rename to source/lib/c-ares-1.34.6/buildconf.bat diff --git a/source/lib/c-ares-1.34.4/c-ares-config.cmake.in b/source/lib/c-ares-1.34.6/c-ares-config.cmake.in similarity index 100% rename from source/lib/c-ares-1.34.4/c-ares-config.cmake.in rename to source/lib/c-ares-1.34.6/c-ares-config.cmake.in diff --git a/source/lib/c-ares-1.34.4/cmake/EnableWarnings.cmake b/source/lib/c-ares-1.34.6/cmake/EnableWarnings.cmake similarity index 100% rename from source/lib/c-ares-1.34.4/cmake/EnableWarnings.cmake rename to source/lib/c-ares-1.34.6/cmake/EnableWarnings.cmake diff --git a/source/lib/c-ares-1.34.4/config/compile b/source/lib/c-ares-1.34.6/config/compile similarity index 100% rename from source/lib/c-ares-1.34.4/config/compile rename to source/lib/c-ares-1.34.6/config/compile diff --git a/source/lib/c-ares-1.34.4/config/config.guess b/source/lib/c-ares-1.34.6/config/config.guess similarity index 100% rename from source/lib/c-ares-1.34.4/config/config.guess rename to source/lib/c-ares-1.34.6/config/config.guess diff --git a/source/lib/c-ares-1.34.4/config/config.sub b/source/lib/c-ares-1.34.6/config/config.sub similarity index 100% rename from source/lib/c-ares-1.34.4/config/config.sub rename to source/lib/c-ares-1.34.6/config/config.sub diff --git a/source/lib/c-ares-1.34.4/config/depcomp b/source/lib/c-ares-1.34.6/config/depcomp similarity index 100% rename from source/lib/c-ares-1.34.4/config/depcomp rename to source/lib/c-ares-1.34.6/config/depcomp diff --git a/source/lib/c-ares-1.34.4/config/install-sh b/source/lib/c-ares-1.34.6/config/install-sh similarity index 100% rename from source/lib/c-ares-1.34.4/config/install-sh rename to source/lib/c-ares-1.34.6/config/install-sh diff --git a/source/lib/c-ares-1.34.4/config/ltmain.sh b/source/lib/c-ares-1.34.6/config/ltmain.sh similarity index 94% rename from source/lib/c-ares-1.34.4/config/ltmain.sh rename to source/lib/c-ares-1.34.6/config/ltmain.sh index 540a92ab..977e5237 100755 --- a/source/lib/c-ares-1.34.4/config/ltmain.sh +++ b/source/lib/c-ares-1.34.6/config/ltmain.sh @@ -1,12 +1,12 @@ -#! /bin/sh +#! /usr/bin/env sh ## DO NOT EDIT - This file generated from ./build-aux/ltmain.in -## by inline-source v2014-01-03.01 +## by inline-source v2019-02-19.15 -# libtool (GNU libtool) 2.4.6 +# libtool (GNU libtool) 2.4.7 # Provide generalized library-building support services. # Written by Gordon Matzigkeit , 1996 -# Copyright (C) 1996-2015 Free Software Foundation, Inc. +# Copyright (C) 1996-2019, 2021-2022 Free Software Foundation, Inc. # This is free software; see the source for copying conditions. There is NO # warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. @@ -31,8 +31,8 @@ PROGRAM=libtool PACKAGE=libtool -VERSION="2.4.6 Debian-2.4.6-15build2" -package_revision=2.4.6 +VERSION="2.4.7 Debian-2.4.7-7build1" +package_revision=2.4.7 ## ------ ## @@ -64,34 +64,25 @@ package_revision=2.4.6 # libraries, which are installed to $pkgauxdir. # Set a version string for this script. -scriptversion=2015-01-20.17; # UTC +scriptversion=2019-02-19.15; # UTC # General shell script boiler plate, and helper functions. # Written by Gary V. Vaughan, 2004 -# Copyright (C) 2004-2015 Free Software Foundation, Inc. -# This is free software; see the source for copying conditions. There is NO -# warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 3 of the License, or -# (at your option) any later version. - -# As a special exception to the GNU General Public License, if you distribute -# this file as part of a program or library that is built using GNU Libtool, -# you may include this file under the same distribution terms that you use -# for the rest of that program. - -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNES FOR A PARTICULAR PURPOSE. See the GNU -# General Public License for more details. - -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . +# This is free software. There is NO warranty; not even for +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +# +# Copyright (C) 2004-2019, 2021 Bootstrap Authors +# +# This file is dual licensed under the terms of the MIT license +# , and GPL version 2 or later +# . You must apply one of +# these licenses when using or redistributing this software or any of +# the files within it. See the URLs above, or the file `LICENSE` +# included in the Bootstrap distribution for the full license texts. -# Please report bugs or propose patches to gary@gnu.org. +# Please report bugs or propose patches to: +# ## ------ ## @@ -139,9 +130,12 @@ do _G_safe_locale=\"$_G_var=C; \$_G_safe_locale\" fi" done - -# CDPATH. -(unset CDPATH) >/dev/null 2>&1 && unset CDPATH +# These NLS vars are set unconditionally (bootstrap issue #24). Unset those +# in case the environment reset is needed later and the $save_* variant is not +# defined (see the code above). +LC_ALL=C +LANGUAGE=C +export LANGUAGE LC_ALL # Make sure IFS has a sensible default sp=' ' @@ -159,6 +153,26 @@ if test "${PATH_SEPARATOR+set}" != set; then fi +# func_unset VAR +# -------------- +# Portably unset VAR. +# In some shells, an 'unset VAR' statement leaves a non-zero return +# status if VAR is already unset, which might be problematic if the +# statement is used at the end of a function (thus poisoning its return +# value) or when 'set -e' is active (causing even a spurious abort of +# the script in this case). +func_unset () +{ + { eval $1=; (eval unset $1) >/dev/null 2>&1 && eval unset $1 || : ; } +} + + +# Make sure CDPATH doesn't cause `cd` commands to output the target dir. +func_unset CDPATH + +# Make sure ${,E,F}GREP behave sanely. +func_unset GREP_OPTIONS + ## ------------------------- ## ## Locate command utilities. ## @@ -259,7 +273,7 @@ test -z "$SED" && { rm -f conftest.in conftest.tmp conftest.nl conftest.out } - func_path_progs "sed gsed" func_check_prog_sed $PATH:/usr/xpg4/bin + func_path_progs "sed gsed" func_check_prog_sed "$PATH:/usr/xpg4/bin" rm -f conftest.sed SED=$func_path_progs_result } @@ -295,7 +309,7 @@ test -z "$GREP" && { rm -f conftest.in conftest.tmp conftest.nl conftest.out } - func_path_progs "grep ggrep" func_check_prog_grep $PATH:/usr/xpg4/bin + func_path_progs "grep ggrep" func_check_prog_grep "$PATH:/usr/xpg4/bin" GREP=$func_path_progs_result } @@ -360,6 +374,35 @@ sed_double_backslash="\ s/\\([^$_G_bs]\\)$_G_bs2$_G_dollar/\\1$_G_bs2$_G_bs$_G_dollar/g s/\n//g" +# require_check_ifs_backslash +# --------------------------- +# Check if we can use backslash as IFS='\' separator, and set +# $check_ifs_backshlash_broken to ':' or 'false'. +require_check_ifs_backslash=func_require_check_ifs_backslash +func_require_check_ifs_backslash () +{ + _G_save_IFS=$IFS + IFS='\' + _G_check_ifs_backshlash='a\\b' + for _G_i in $_G_check_ifs_backshlash + do + case $_G_i in + a) + check_ifs_backshlash_broken=false + ;; + '') + break + ;; + *) + check_ifs_backshlash_broken=: + break + ;; + esac + done + IFS=$_G_save_IFS + require_check_ifs_backslash=: +} + ## ----------------- ## ## Global variables. ## @@ -529,27 +572,15 @@ func_require_term_colors () # --------------------- # Append VALUE onto the existing contents of VAR. - # We should try to minimise forks, especially on Windows where they are - # unreasonably slow, so skip the feature probes when bash or zsh are - # being used: - if test set = "${BASH_VERSION+set}${ZSH_VERSION+set}"; then - : ${_G_HAVE_ARITH_OP="yes"} - : ${_G_HAVE_XSI_OPS="yes"} - # The += operator was introduced in bash 3.1 - case $BASH_VERSION in - [12].* | 3.0 | 3.0*) ;; - *) - : ${_G_HAVE_PLUSEQ_OP="yes"} - ;; - esac - fi - # _G_HAVE_PLUSEQ_OP # Can be empty, in which case the shell is probed, "yes" if += is # useable or anything else if it does not work. - test -z "$_G_HAVE_PLUSEQ_OP" \ - && (eval 'x=a; x+=" b"; test "a b" = "$x"') 2>/dev/null \ - && _G_HAVE_PLUSEQ_OP=yes + if test -z "$_G_HAVE_PLUSEQ_OP" && \ + __PLUSEQ_TEST="a" && \ + __PLUSEQ_TEST+=" b" 2>/dev/null && \ + test "a b" = "$__PLUSEQ_TEST"; then + _G_HAVE_PLUSEQ_OP=yes + fi if test yes = "$_G_HAVE_PLUSEQ_OP" then @@ -580,16 +611,16 @@ if test yes = "$_G_HAVE_PLUSEQ_OP"; then { $debug_cmd - func_quote_for_eval "$2" - eval "$1+=\\ \$func_quote_for_eval_result" + func_quote_arg pretty "$2" + eval "$1+=\\ \$func_quote_arg_result" }' else func_append_quoted () { $debug_cmd - func_quote_for_eval "$2" - eval "$1=\$$1\\ \$func_quote_for_eval_result" + func_quote_arg pretty "$2" + eval "$1=\$$1\\ \$func_quote_arg_result" } fi @@ -1091,85 +1122,203 @@ func_relative_path () } -# func_quote_for_eval ARG... -# -------------------------- -# Aesthetically quote ARGs to be evaled later. -# This function returns two values: -# i) func_quote_for_eval_result -# double-quoted, suitable for a subsequent eval -# ii) func_quote_for_eval_unquoted_result -# has all characters that are still active within double -# quotes backslashified. -func_quote_for_eval () +# func_quote_portable EVAL ARG +# ---------------------------- +# Internal function to portably implement func_quote_arg. Note that we still +# keep attention to performance here so we as much as possible try to avoid +# calling sed binary (so far O(N) complexity as long as func_append is O(1)). +func_quote_portable () { $debug_cmd - func_quote_for_eval_unquoted_result= - func_quote_for_eval_result= - while test 0 -lt $#; do - case $1 in - *[\\\`\"\$]*) - _G_unquoted_arg=`printf '%s\n' "$1" |$SED "$sed_quote_subst"` ;; - *) - _G_unquoted_arg=$1 ;; - esac - if test -n "$func_quote_for_eval_unquoted_result"; then - func_append func_quote_for_eval_unquoted_result " $_G_unquoted_arg" - else - func_append func_quote_for_eval_unquoted_result "$_G_unquoted_arg" + $require_check_ifs_backslash + + func_quote_portable_result=$2 + + # one-time-loop (easy break) + while true + do + if $1; then + func_quote_portable_result=`$ECHO "$2" | $SED \ + -e "$sed_double_quote_subst" -e "$sed_double_backslash"` + break fi - case $_G_unquoted_arg in - # Double-quote args containing shell metacharacters to delay - # word splitting, command substitution and variable expansion - # for a subsequent eval. - # Many Bourne shells cannot handle close brackets correctly - # in scan sets, so we specify it separately. - *[\[\~\#\^\&\*\(\)\{\}\|\;\<\>\?\'\ \ ]*|*]*|"") - _G_quoted_arg=\"$_G_unquoted_arg\" + # Quote for eval. + case $func_quote_portable_result in + *[\\\`\"\$]*) + # Fallback to sed for $func_check_bs_ifs_broken=:, or when the string + # contains the shell wildcard characters. + case $check_ifs_backshlash_broken$func_quote_portable_result in + :*|*[\[\*\?]*) + func_quote_portable_result=`$ECHO "$func_quote_portable_result" \ + | $SED "$sed_quote_subst"` + break + ;; + esac + + func_quote_portable_old_IFS=$IFS + for _G_char in '\' '`' '"' '$' + do + # STATE($1) PREV($2) SEPARATOR($3) + set start "" "" + func_quote_portable_result=dummy"$_G_char$func_quote_portable_result$_G_char"dummy + IFS=$_G_char + for _G_part in $func_quote_portable_result + do + case $1 in + quote) + func_append func_quote_portable_result "$3$2" + set quote "$_G_part" "\\$_G_char" + ;; + start) + set first "" "" + func_quote_portable_result= + ;; + first) + set quote "$_G_part" "" + ;; + esac + done + done + IFS=$func_quote_portable_old_IFS ;; - *) - _G_quoted_arg=$_G_unquoted_arg - ;; + *) ;; esac - - if test -n "$func_quote_for_eval_result"; then - func_append func_quote_for_eval_result " $_G_quoted_arg" - else - func_append func_quote_for_eval_result "$_G_quoted_arg" - fi - shift + break done + + func_quote_portable_unquoted_result=$func_quote_portable_result + case $func_quote_portable_result in + # double-quote args containing shell metacharacters to delay + # word splitting, command substitution and variable expansion + # for a subsequent eval. + # many bourne shells cannot handle close brackets correctly + # in scan sets, so we specify it separately. + *[\[\~\#\^\&\*\(\)\{\}\|\;\<\>\?\'\ \ ]*|*]*|"") + func_quote_portable_result=\"$func_quote_portable_result\" + ;; + esac } -# func_quote_for_expand ARG -# ------------------------- -# Aesthetically quote ARG to be evaled later; same as above, -# but do not quote variable references. -func_quote_for_expand () -{ - $debug_cmd +# func_quotefast_eval ARG +# ----------------------- +# Quote one ARG (internal). This is equivalent to 'func_quote_arg eval ARG', +# but optimized for speed. Result is stored in $func_quotefast_eval. +if test xyes = `(x=; printf -v x %q yes; echo x"$x") 2>/dev/null`; then + printf -v _GL_test_printf_tilde %q '~' + if test '\~' = "$_GL_test_printf_tilde"; then + func_quotefast_eval () + { + printf -v func_quotefast_eval_result %q "$1" + } + else + # Broken older Bash implementations. Make those faster too if possible. + func_quotefast_eval () + { + case $1 in + '~'*) + func_quote_portable false "$1" + func_quotefast_eval_result=$func_quote_portable_result + ;; + *) + printf -v func_quotefast_eval_result %q "$1" + ;; + esac + } + fi +else + func_quotefast_eval () + { + func_quote_portable false "$1" + func_quotefast_eval_result=$func_quote_portable_result + } +fi - case $1 in - *[\\\`\"]*) - _G_arg=`$ECHO "$1" | $SED \ - -e "$sed_double_quote_subst" -e "$sed_double_backslash"` ;; - *) - _G_arg=$1 ;; + +# func_quote_arg MODEs ARG +# ------------------------ +# Quote one ARG to be evaled later. MODEs argument may contain zero or more +# specifiers listed below separated by ',' character. This function returns two +# values: +# i) func_quote_arg_result +# double-quoted (when needed), suitable for a subsequent eval +# ii) func_quote_arg_unquoted_result +# has all characters that are still active within double +# quotes backslashified. Available only if 'unquoted' is specified. +# +# Available modes: +# ---------------- +# 'eval' (default) +# - escape shell special characters +# 'expand' +# - the same as 'eval'; but do not quote variable references +# 'pretty' +# - request aesthetic output, i.e. '"a b"' instead of 'a\ b'. This might +# be used later in func_quote to get output like: 'echo "a b"' instead +# of 'echo a\ b'. This is slower than default on some shells. +# 'unquoted' +# - produce also $func_quote_arg_unquoted_result which does not contain +# wrapping double-quotes. +# +# Examples for 'func_quote_arg pretty,unquoted string': +# +# string | *_result | *_unquoted_result +# ------------+-----------------------+------------------- +# " | \" | \" +# a b | "a b" | a b +# "a b" | "\"a b\"" | \"a b\" +# * | "*" | * +# z="${x-$y}" | "z=\"\${x-\$y}\"" | z=\"\${x-\$y}\" +# +# Examples for 'func_quote_arg pretty,unquoted,expand string': +# +# string | *_result | *_unquoted_result +# --------------+---------------------+-------------------- +# z="${x-$y}" | "z=\"${x-$y}\"" | z=\"${x-$y}\" +func_quote_arg () +{ + _G_quote_expand=false + case ,$1, in + *,expand,*) + _G_quote_expand=: + ;; esac - case $_G_arg in - # Double-quote args containing shell metacharacters to delay - # word splitting and command substitution for a subsequent eval. - # Many Bourne shells cannot handle close brackets correctly - # in scan sets, so we specify it separately. - *[\[\~\#\^\&\*\(\)\{\}\|\;\<\>\?\'\ \ ]*|*]*|"") - _G_arg=\"$_G_arg\" + case ,$1, in + *,pretty,*|*,expand,*|*,unquoted,*) + func_quote_portable $_G_quote_expand "$2" + func_quote_arg_result=$func_quote_portable_result + func_quote_arg_unquoted_result=$func_quote_portable_unquoted_result + ;; + *) + # Faster quote-for-eval for some shells. + func_quotefast_eval "$2" + func_quote_arg_result=$func_quotefast_eval_result ;; esac +} + - func_quote_for_expand_result=$_G_arg +# func_quote MODEs ARGs... +# ------------------------ +# Quote all ARGs to be evaled later and join them into single command. See +# func_quote_arg's description for more info. +func_quote () +{ + $debug_cmd + _G_func_quote_mode=$1 ; shift + func_quote_result= + while test 0 -lt $#; do + func_quote_arg "$_G_func_quote_mode" "$1" + if test -n "$func_quote_result"; then + func_append func_quote_result " $func_quote_arg_result" + else + func_append func_quote_result "$func_quote_arg_result" + fi + shift + done } @@ -1215,8 +1364,8 @@ func_show_eval () _G_cmd=$1 _G_fail_exp=${2-':'} - func_quote_for_expand "$_G_cmd" - eval "func_notquiet $func_quote_for_expand_result" + func_quote_arg pretty,expand "$_G_cmd" + eval "func_notquiet $func_quote_arg_result" $opt_dry_run || { eval "$_G_cmd" @@ -1241,8 +1390,8 @@ func_show_eval_locale () _G_fail_exp=${2-':'} $opt_quiet || { - func_quote_for_expand "$_G_cmd" - eval "func_echo $func_quote_for_expand_result" + func_quote_arg expand,pretty "$_G_cmd" + eval "func_echo $func_quote_arg_result" } $opt_dry_run || { @@ -1369,30 +1518,26 @@ func_lt_ver () # End: #! /bin/sh -# Set a version string for this script. -scriptversion=2015-10-07.11; # UTC - # A portable, pluggable option parser for Bourne shell. # Written by Gary V. Vaughan, 2010 -# Copyright (C) 2010-2015 Free Software Foundation, Inc. -# This is free software; see the source for copying conditions. There is NO -# warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. - -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. +# This is free software. There is NO warranty; not even for +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +# +# Copyright (C) 2010-2019, 2021 Bootstrap Authors +# +# This file is dual licensed under the terms of the MIT license +# , and GPL version 2 or later +# . You must apply one of +# these licenses when using or redistributing this software or any of +# the files within it. See the URLs above, or the file `LICENSE` +# included in the Bootstrap distribution for the full license texts. -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . +# Please report bugs or propose patches to: +# -# Please report bugs or propose patches to gary@gnu.org. +# Set a version string for this script. +scriptversion=2019-02-19.15; # UTC ## ------ ## @@ -1415,7 +1560,7 @@ scriptversion=2015-10-07.11; # UTC # # In order for the '--version' option to work, you will need to have a # suitably formatted comment like the one at the top of this file -# starting with '# Written by ' and ending with '# warranty; '. +# starting with '# Written by ' and ending with '# Copyright'. # # For '-h' and '--help' to work, you will also need a one line # description of your script's purpose in a comment directly above the @@ -1427,7 +1572,7 @@ scriptversion=2015-10-07.11; # UTC # to display verbose messages only when your user has specified # '--verbose'. # -# After sourcing this file, you can plug processing for additional +# After sourcing this file, you can plug in processing for additional # options by amending the variables from the 'Configuration' section # below, and following the instructions in the 'Option parsing' # section further down. @@ -1476,8 +1621,8 @@ fatal_help="Try '\$progname --help' for more information." ## ------------------------- ## # This section contains functions for adding, removing, and running hooks -# to the main code. A hook is just a named list of of function, that can -# be run in order later on. +# in the main code. A hook is just a list of function names that can be +# run in order later on. # func_hookable FUNC_NAME # ----------------------- @@ -1510,7 +1655,8 @@ func_add_hook () # func_remove_hook FUNC_NAME HOOK_FUNC # ------------------------------------ -# Remove HOOK_FUNC from the list of functions called by FUNC_NAME. +# Remove HOOK_FUNC from the list of hook functions to be called by +# FUNC_NAME. func_remove_hook () { $debug_cmd @@ -1519,10 +1665,28 @@ func_remove_hook () } +# func_propagate_result FUNC_NAME_A FUNC_NAME_B +# --------------------------------------------- +# If the *_result variable of FUNC_NAME_A _is set_, assign its value to +# *_result variable of FUNC_NAME_B. +func_propagate_result () +{ + $debug_cmd + + func_propagate_result_result=: + if eval "test \"\${${1}_result+set}\" = set" + then + eval "${2}_result=\$${1}_result" + else + func_propagate_result_result=false + fi +} + + # func_run_hooks FUNC_NAME [ARG]... # --------------------------------- # Run all hook functions registered to FUNC_NAME. -# It is assumed that the list of hook functions contains nothing more +# It's assumed that the list of hook functions contains nothing more # than a whitespace-delimited list of legal shell function names, and # no effort is wasted trying to catch shell meta-characters or preserve # whitespace. @@ -1534,22 +1698,19 @@ func_run_hooks () case " $hookable_fns " in *" $1 "*) ;; - *) func_fatal_error "'$1' does not support hook funcions.n" ;; + *) func_fatal_error "'$1' does not support hook functions." ;; esac eval _G_hook_fns=\$$1_hooks; shift for _G_hook in $_G_hook_fns; do - if eval $_G_hook '"$@"'; then - # store returned options list back into positional - # parameters for next 'cmd' execution. - eval _G_hook_result=\$${_G_hook}_result - eval set dummy "$_G_hook_result"; shift - _G_rc_run_hooks=: + func_unset "${_G_hook}_result" + eval $_G_hook '${1+"$@"}' + func_propagate_result $_G_hook func_run_hooks + if $func_propagate_result_result; then + eval set dummy "$func_run_hooks_result"; shift fi done - - $_G_rc_run_hooks && func_run_hooks_result=$_G_hook_result } @@ -1559,14 +1720,16 @@ func_run_hooks () ## --------------- ## # In order to add your own option parsing hooks, you must accept the -# full positional parameter list in your hook function, you may remove/edit -# any options that you action, and then pass back the remaining unprocessed -# options in '_result', escaped suitably for -# 'eval'. In this case you also must return $EXIT_SUCCESS to let the -# hook's caller know that it should pay attention to -# '_result'. Returning $EXIT_FAILURE signalizes that -# arguments are left untouched by the hook and therefore caller will ignore the -# result variable. +# full positional parameter list from your hook function. You may remove +# or edit any options that you action, and then pass back the remaining +# unprocessed options in '_result', escaped +# suitably for 'eval'. +# +# The '_result' variable is automatically unset +# before your hook gets called; for best performance, only set the +# *_result variable when necessary (i.e. don't call the 'func_quote' +# function unnecessarily because it can be an expensive operation on some +# machines). # # Like this: # @@ -1578,11 +1741,8 @@ func_run_hooks () # usage_message=$usage_message' # -s, --silent don'\''t print informational messages # ' -# # No change in '$@' (ignored completely by this hook). There is -# # no need to do the equivalent (but slower) action: -# # func_quote_for_eval ${1+"$@"} -# # my_options_prep_result=$func_quote_for_eval_result -# false +# # No change in '$@' (ignored completely by this hook). Leave +# # my_options_prep_result variable intact. # } # func_add_hook func_options_prep my_options_prep # @@ -1593,7 +1753,7 @@ func_run_hooks () # # args_changed=false # -# # Note that for efficiency, we parse as many options as we can +# # Note that, for efficiency, we parse as many options as we can # # recognise in a loop before passing the remainder back to the # # caller on the first unrecognised argument we encounter. # while test $# -gt 0; do @@ -1610,18 +1770,17 @@ func_run_hooks () # args_changed=: # ;; # *) # Make sure the first unrecognised option "$_G_opt" -# # is added back to "$@", we could need that later -# # if $args_changed is true. +# # is added back to "$@" in case we need it later, +# # if $args_changed was set to 'true'. # set dummy "$_G_opt" ${1+"$@"}; shift; break ;; # esac # done # +# # Only call 'func_quote' here if we processed at least one argument. # if $args_changed; then -# func_quote_for_eval ${1+"$@"} -# my_silent_option_result=$func_quote_for_eval_result +# func_quote eval ${1+"$@"} +# my_silent_option_result=$func_quote_result # fi -# -# $args_changed # } # func_add_hook func_parse_options my_silent_option # @@ -1632,8 +1791,6 @@ func_run_hooks () # # $opt_silent && $opt_verbose && func_fatal_help "\ # '--silent' and '--verbose' options are mutually exclusive." -# -# false # } # func_add_hook func_validate_options my_option_validation # @@ -1649,13 +1806,8 @@ func_options_finish () { $debug_cmd - _G_func_options_finish_exit=false - if func_run_hooks func_options ${1+"$@"}; then - func_options_finish_result=$func_run_hooks_result - _G_func_options_finish_exit=: - fi - - $_G_func_options_finish_exit + func_run_hooks func_options ${1+"$@"} + func_propagate_result func_run_hooks func_options_finish } @@ -1668,28 +1820,27 @@ func_options () { $debug_cmd - _G_rc_options=false + _G_options_quoted=false for my_func in options_prep parse_options validate_options options_finish do - if eval func_$my_func '${1+"$@"}'; then - eval _G_res_var='$'"func_${my_func}_result" - eval set dummy "$_G_res_var" ; shift - _G_rc_options=: + func_unset func_${my_func}_result + func_unset func_run_hooks_result + eval func_$my_func '${1+"$@"}' + func_propagate_result func_$my_func func_options + if $func_propagate_result_result; then + eval set dummy "$func_options_result"; shift + _G_options_quoted=: fi done - # Save modified positional parameters for caller. As a top-level - # options-parser function we always need to set the 'func_options_result' - # variable (regardless the $_G_rc_options value). - if $_G_rc_options; then - func_options_result=$_G_res_var - else - func_quote_for_eval ${1+"$@"} - func_options_result=$func_quote_for_eval_result - fi - - $_G_rc_options + $_G_options_quoted || { + # As we (func_options) are top-level options-parser function and + # nobody quoted "$@" for us yet, we need to do it explicitly for + # caller. + func_quote eval ${1+"$@"} + func_options_result=$func_quote_result + } } @@ -1699,8 +1850,7 @@ func_options () # Note that when calling hook functions, we pass through the list of # positional parameters. If a hook function modifies that list, and # needs to propagate that back to rest of this script, then the complete -# modified list must be put in 'func_run_hooks_result' before -# returning $EXIT_SUCCESS (otherwise $EXIT_FAILURE is returned). +# modified list must be put in 'func_run_hooks_result' before returning. func_hookable func_options_prep func_options_prep () { @@ -1710,14 +1860,8 @@ func_options_prep () opt_verbose=false opt_warning_types= - _G_rc_options_prep=false - if func_run_hooks func_options_prep ${1+"$@"}; then - _G_rc_options_prep=: - # save modified positional parameters for caller - func_options_prep_result=$func_run_hooks_result - fi - - $_G_rc_options_prep + func_run_hooks func_options_prep ${1+"$@"} + func_propagate_result func_run_hooks func_options_prep } @@ -1729,27 +1873,32 @@ func_parse_options () { $debug_cmd - func_parse_options_result= - - _G_rc_parse_options=false + _G_parse_options_requote=false # this just eases exit handling while test $# -gt 0; do # Defer to hook functions for initial option parsing, so they # get priority in the event of reusing an option name. - if func_run_hooks func_parse_options ${1+"$@"}; then - eval set dummy "$func_run_hooks_result"; shift - _G_rc_parse_options=: + func_run_hooks func_parse_options ${1+"$@"} + func_propagate_result func_run_hooks func_parse_options + if $func_propagate_result_result; then + eval set dummy "$func_parse_options_result"; shift + # Even though we may have changed "$@", we passed the "$@" array + # down into the hook and it quoted it for us (because we are in + # this if-branch). No need to quote it again. + _G_parse_options_requote=false fi # Break out of the loop if we already parsed every option. test $# -gt 0 || break + # We expect that one of the options parsed in this function matches + # and thus we remove _G_opt from "$@" and need to re-quote. _G_match_parse_options=: _G_opt=$1 shift case $_G_opt in --debug|-x) debug_cmd='set -x' - func_echo "enabling shell trace mode" + func_echo "enabling shell trace mode" >&2 $debug_cmd ;; @@ -1760,7 +1909,7 @@ func_parse_options () --warnings|--warning|-W) if test $# = 0 && func_missing_arg $_G_opt; then - _G_rc_parse_options=: + _G_parse_options_requote=: break fi case " $warning_categories $1" in @@ -1815,7 +1964,7 @@ func_parse_options () shift ;; - --) _G_rc_parse_options=: ; break ;; + --) _G_parse_options_requote=: ; break ;; -*) func_fatal_help "unrecognised option: '$_G_opt'" ;; *) set dummy "$_G_opt" ${1+"$@"}; shift _G_match_parse_options=false @@ -1823,17 +1972,16 @@ func_parse_options () ;; esac - $_G_match_parse_options && _G_rc_parse_options=: + if $_G_match_parse_options; then + _G_parse_options_requote=: + fi done - - if $_G_rc_parse_options; then + if $_G_parse_options_requote; then # save modified positional parameters for caller - func_quote_for_eval ${1+"$@"} - func_parse_options_result=$func_quote_for_eval_result + func_quote eval ${1+"$@"} + func_parse_options_result=$func_quote_result fi - - $_G_rc_parse_options } @@ -1846,21 +1994,14 @@ func_validate_options () { $debug_cmd - _G_rc_validate_options=false - # Display all warnings if -W was not given. test -n "$opt_warning_types" || opt_warning_types=" $warning_categories" - if func_run_hooks func_validate_options ${1+"$@"}; then - # save modified positional parameters for caller - func_validate_options_result=$func_run_hooks_result - _G_rc_validate_options=: - fi + func_run_hooks func_validate_options ${1+"$@"} + func_propagate_result func_run_hooks func_validate_options # Bail if the options were screwed! $exit_cmd $EXIT_FAILURE - - $_G_rc_validate_options } @@ -1916,8 +2057,8 @@ func_missing_arg () # func_split_equals STRING # ------------------------ -# Set func_split_equals_lhs and func_split_equals_rhs shell variables after -# splitting STRING at the '=' sign. +# Set func_split_equals_lhs and func_split_equals_rhs shell variables +# after splitting STRING at the '=' sign. test -z "$_G_HAVE_XSI_OPS" \ && (eval 'x=a/b/c; test 5aa/bb/cc = "${#x}${x%%/*}${x%/*}${x#*/}${x##*/}"') 2>/dev/null \ @@ -1932,8 +2073,9 @@ then func_split_equals_lhs=${1%%=*} func_split_equals_rhs=${1#*=} - test "x$func_split_equals_lhs" = "x$1" \ - && func_split_equals_rhs= + if test "x$func_split_equals_lhs" = "x$1"; then + func_split_equals_rhs= + fi }' else # ...otherwise fall back to using expr, which is often a shell builtin. @@ -1943,7 +2085,7 @@ else func_split_equals_lhs=`expr "x$1" : 'x\([^=]*\)'` func_split_equals_rhs= - test "x$func_split_equals_lhs" = "x$1" \ + test "x$func_split_equals_lhs=" = "x$1" \ || func_split_equals_rhs=`expr "x$1" : 'x[^=]*=\(.*\)$'` } fi #func_split_equals @@ -1969,7 +2111,7 @@ else { $debug_cmd - func_split_short_opt_name=`expr "x$1" : 'x-\(.\)'` + func_split_short_opt_name=`expr "x$1" : 'x\(-.\)'` func_split_short_opt_arg=`expr "x$1" : 'x-.\(.*\)$'` } fi #func_split_short_opt @@ -2011,31 +2153,44 @@ func_usage_message () # func_version # ------------ # Echo version message to standard output and exit. +# The version message is extracted from the calling file's header +# comments, with leading '# ' stripped: +# 1. First display the progname and version +# 2. Followed by the header comment line matching /^# Written by / +# 3. Then a blank line followed by the first following line matching +# /^# Copyright / +# 4. Immediately followed by any lines between the previous matches, +# except lines preceding the intervening completely blank line. +# For example, see the header comments of this file. func_version () { $debug_cmd printf '%s\n' "$progname $scriptversion" $SED -n ' - /(C)/!b go - :more - /\./!{ - N - s|\n# | | - b more - } - :go - /^# Written by /,/# warranty; / { - s|^# || - s|^# *$|| - s|\((C)\)[ 0-9,-]*[ ,-]\([1-9][0-9]* \)|\1 \2| - p + /^# Written by /!b + s|^# ||; p; n + + :fwd2blnk + /./ { + n + b fwd2blnk } - /^# Written by / { - s|^# || - p + p; n + + :holdwrnt + s|^# || + s|^# *$|| + /^Copyright /!{ + /./H + n + b holdwrnt } - /^warranty; /q' < "$progpath" + + s|\((C)\)[ 0-9,-]*[ ,-]\([1-9][0-9]* \)|\1 \2| + G + s|\(\n\)\n*|\1|g + p; q' < "$progpath" exit $? } @@ -2045,12 +2200,12 @@ func_version () # mode: shell-script # sh-indentation: 2 # eval: (add-hook 'before-save-hook 'time-stamp) -# time-stamp-pattern: "10/scriptversion=%:y-%02m-%02d.%02H; # UTC" +# time-stamp-pattern: "30/scriptversion=%:y-%02m-%02d.%02H; # UTC" # time-stamp-time-zone: "UTC" # End: # Set a version string. -scriptversion='(GNU libtool) 2.4.6' +scriptversion='(GNU libtool) 2.4.7' # func_echo ARG... @@ -2141,7 +2296,7 @@ include the following information: compiler: $LTCC compiler flags: $LTCFLAGS linker: $LD (gnu? $with_gnu_ld) - version: $progname $scriptversion Debian-2.4.6-15build2 + version: $progname $scriptversion Debian-2.4.7-7build1 automake: `($AUTOMAKE --version) 2>/dev/null |$SED 1q` autoconf: `($AUTOCONF --version) 2>/dev/null |$SED 1q` @@ -2197,7 +2352,7 @@ fi # a configuration failure hint, and exit. func_fatal_configuration () { - func__fatal_error ${1+"$@"} \ + func_fatal_error ${1+"$@"} \ "See the $PACKAGE documentation for more information." \ "Fatal configuration error." } @@ -2345,6 +2500,8 @@ libtool_options_prep () _G_rc_lt_options_prep=: + _G_rc_lt_options_prep=: + # Shorthand for --mode=foo, only valid as the first argument case $1 in clean|clea|cle|cl) @@ -2375,11 +2532,9 @@ libtool_options_prep () if $_G_rc_lt_options_prep; then # Pass back the list of options. - func_quote_for_eval ${1+"$@"} - libtool_options_prep_result=$func_quote_for_eval_result + func_quote eval ${1+"$@"} + libtool_options_prep_result=$func_quote_result fi - - $_G_rc_lt_options_prep } func_add_hook func_options_prep libtool_options_prep @@ -2482,11 +2637,9 @@ libtool_parse_options () if $_G_rc_lt_parse_options; then # save modified positional parameters for caller - func_quote_for_eval ${1+"$@"} - libtool_parse_options_result=$func_quote_for_eval_result + func_quote eval ${1+"$@"} + libtool_parse_options_result=$func_quote_result fi - - $_G_rc_lt_parse_options } func_add_hook func_parse_options libtool_parse_options @@ -2543,8 +2696,8 @@ libtool_validate_options () } # Pass back the unparsed argument list - func_quote_for_eval ${1+"$@"} - libtool_validate_options_result=$func_quote_for_eval_result + func_quote eval ${1+"$@"} + libtool_validate_options_result=$func_quote_result } func_add_hook func_validate_options libtool_validate_options @@ -3510,8 +3663,8 @@ func_mode_compile () esac done - func_quote_for_eval "$libobj" - test "X$libobj" != "X$func_quote_for_eval_result" \ + func_quote_arg pretty "$libobj" + test "X$libobj" != "X$func_quote_arg_result" \ && $ECHO "X$libobj" | $GREP '[]~#^*{};<>?"'"'"' &()|`$[]' \ && func_warning "libobj name '$libobj' may not contain shell special characters." func_dirname_and_basename "$obj" "/" "" @@ -3584,8 +3737,8 @@ compiler." func_to_tool_file "$srcfile" func_convert_file_msys_to_w32 srcfile=$func_to_tool_file_result - func_quote_for_eval "$srcfile" - qsrcfile=$func_quote_for_eval_result + func_quote_arg pretty "$srcfile" + qsrcfile=$func_quote_arg_result # Only build a PIC object if we are building libtool libraries. if test yes = "$build_libtool_libs"; then @@ -3740,7 +3893,8 @@ This mode accepts the following additional options: -prefer-non-pic try to build non-PIC objects only -shared do not build a '.o' file suitable for static linking -static only build a '.o' file suitable for static linking - -Wc,FLAG pass FLAG directly to the compiler + -Wc,FLAG + -Xcompiler FLAG pass FLAG directly to the compiler COMPILE-COMMAND is a command to be used in creating a 'standard' object file from the given SOURCEFILE. @@ -3846,6 +4000,8 @@ The following components of LINK-COMMAND are treated specially: -weak LIBNAME declare that the target provides the LIBNAME interface -Wc,FLAG -Xcompiler FLAG pass linker-specific FLAG directly to the compiler + -Wa,FLAG + -Xassembler FLAG pass linker-specific FLAG directly to the assembler -Wl,FLAG -Xlinker FLAG pass linker-specific FLAG directly to the linker -XCClinker FLAG pass link-specific FLAG to the compiler driver (CC) @@ -4188,8 +4344,8 @@ func_mode_install () case $nonopt in *shtool*) :;; *) false;; esac then # Aesthetically quote it. - func_quote_for_eval "$nonopt" - install_prog="$func_quote_for_eval_result " + func_quote_arg pretty "$nonopt" + install_prog="$func_quote_arg_result " arg=$1 shift else @@ -4199,8 +4355,8 @@ func_mode_install () # The real first argument should be the name of the installation program. # Aesthetically quote it. - func_quote_for_eval "$arg" - func_append install_prog "$func_quote_for_eval_result" + func_quote_arg pretty "$arg" + func_append install_prog "$func_quote_arg_result" install_shared_prog=$install_prog case " $install_prog " in *[\\\ /]cp\ *) install_cp=: ;; @@ -4257,12 +4413,12 @@ func_mode_install () esac # Aesthetically quote the argument. - func_quote_for_eval "$arg" - func_append install_prog " $func_quote_for_eval_result" + func_quote_arg pretty "$arg" + func_append install_prog " $func_quote_arg_result" if test -n "$arg2"; then - func_quote_for_eval "$arg2" + func_quote_arg pretty "$arg2" fi - func_append install_shared_prog " $func_quote_for_eval_result" + func_append install_shared_prog " $func_quote_arg_result" done test -z "$install_prog" && \ @@ -4273,8 +4429,8 @@ func_mode_install () if test -n "$install_override_mode" && $no_mode; then if $install_cp; then :; else - func_quote_for_eval "$install_override_mode" - func_append install_shared_prog " -m $func_quote_for_eval_result" + func_quote_arg pretty "$install_override_mode" + func_append install_shared_prog " -m $func_quote_arg_result" fi fi @@ -4570,8 +4726,8 @@ func_mode_install () relink_command=`$ECHO "$relink_command" | $SED 's%@OUTPUT@%'"$outputname"'%g'` $opt_quiet || { - func_quote_for_expand "$relink_command" - eval "func_echo $func_quote_for_expand_result" + func_quote_arg expand,pretty "$relink_command" + eval "func_echo $func_quote_arg_result" } if eval "$relink_command"; then : else @@ -5350,7 +5506,8 @@ else if test \"\$libtool_execute_magic\" != \"$magic\"; then file=\"\$0\"" - qECHO=`$ECHO "$ECHO" | $SED "$sed_quote_subst"` + func_quote_arg pretty "$ECHO" + qECHO=$func_quote_arg_result $ECHO "\ # A function that is used when there is no print builtin or printf. @@ -5360,7 +5517,7 @@ func_fallback_echo () \$1 _LTECHO_EOF' } - ECHO=\"$qECHO\" + ECHO=$qECHO fi # Very basic option parsing. These options are (a) specific to @@ -6703,9 +6860,9 @@ func_mode_link () while test "$#" -gt 0; do arg=$1 shift - func_quote_for_eval "$arg" - qarg=$func_quote_for_eval_unquoted_result - func_append libtool_args " $func_quote_for_eval_result" + func_quote_arg pretty,unquoted "$arg" + qarg=$func_quote_arg_unquoted_result + func_append libtool_args " $func_quote_arg_result" # If the previous option needs an argument, assign it. if test -n "$prev"; then @@ -6941,6 +7098,13 @@ func_mode_link () prev= continue ;; + xassembler) + func_append compiler_flags " -Xassembler $qarg" + prev= + func_append compile_command " -Xassembler $qarg" + func_append finalize_command " -Xassembler $qarg" + continue + ;; xcclinker) func_append linker_flags " $qarg" func_append compiler_flags " $qarg" @@ -7111,7 +7275,7 @@ func_mode_link () # These systems don't actually have a C library (as such) test X-lc = "X$arg" && continue ;; - *-*-openbsd* | *-*-freebsd* | *-*-dragonfly* | *-*-bitrig*) + *-*-openbsd* | *-*-freebsd* | *-*-dragonfly* | *-*-bitrig* | *-*-midnightbsd*) # Do not include libc due to us having libc/libc_r. test X-lc = "X$arg" && continue ;; @@ -7131,7 +7295,7 @@ func_mode_link () esac elif test X-lc_r = "X$arg"; then case $host in - *-*-openbsd* | *-*-freebsd* | *-*-dragonfly* | *-*-bitrig*) + *-*-openbsd* | *-*-freebsd* | *-*-dragonfly* | *-*-bitrig* | *-*-midnightbsd*) # Do not include libc_r directly, use -pthread flag. continue ;; @@ -7161,8 +7325,20 @@ func_mode_link () prev=xcompiler continue ;; - - -mt|-mthreads|-kthread|-Kthread|-pthread|-pthreads|--thread-safe \ + # Solaris ld rejects as of 11.4. Refer to Oracle bug 22985199. + -pthread) + case $host in + *solaris2*) ;; + *) + case "$new_inherited_linker_flags " in + *" $arg "*) ;; + * ) func_append new_inherited_linker_flags " $arg" ;; + esac + ;; + esac + continue + ;; + -mt|-mthreads|-kthread|-Kthread|-pthreads|--thread-safe \ |-threads|-fopenmp|-openmp|-mp|-xopenmp|-omp|-qsmp=*) func_append compiler_flags " $arg" func_append compile_command " $arg" @@ -7303,9 +7479,9 @@ func_mode_link () save_ifs=$IFS; IFS=, for flag in $args; do IFS=$save_ifs - func_quote_for_eval "$flag" - func_append arg " $func_quote_for_eval_result" - func_append compiler_flags " $func_quote_for_eval_result" + func_quote_arg pretty "$flag" + func_append arg " $func_quote_arg_result" + func_append compiler_flags " $func_quote_arg_result" done IFS=$save_ifs func_stripname ' ' '' "$arg" @@ -7319,16 +7495,21 @@ func_mode_link () save_ifs=$IFS; IFS=, for flag in $args; do IFS=$save_ifs - func_quote_for_eval "$flag" - func_append arg " $wl$func_quote_for_eval_result" - func_append compiler_flags " $wl$func_quote_for_eval_result" - func_append linker_flags " $func_quote_for_eval_result" + func_quote_arg pretty "$flag" + func_append arg " $wl$func_quote_arg_result" + func_append compiler_flags " $wl$func_quote_arg_result" + func_append linker_flags " $func_quote_arg_result" done IFS=$save_ifs func_stripname ' ' '' "$arg" arg=$func_stripname_result ;; + -Xassembler) + prev=xassembler + continue + ;; + -Xcompiler) prev=xcompiler continue @@ -7346,8 +7527,8 @@ func_mode_link () # -msg_* for osf cc -msg_*) - func_quote_for_eval "$arg" - arg=$func_quote_for_eval_result + func_quote_arg pretty "$arg" + arg=$func_quote_arg_result ;; # Flags to be passed through unchanged, with rationale: @@ -7370,12 +7551,13 @@ func_mode_link () # -fuse-ld=* Linker select flags for GCC # -static-* direct GCC to link specific libraries statically # -fcilkplus Cilk Plus language extension features for C/C++ + # -Wa,* Pass flags directly to the assembler -64|-mips[0-9]|-r[0-9][0-9]*|-xarch=*|-xtarget=*|+DA*|+DD*|-q*|-m*| \ -t[45]*|-txscale*|-p|-pg|--coverage|-fprofile-*|-F*|@*|-tp=*|--sysroot=*| \ -O*|-g*|-flto*|-fwhopr*|-fuse-linker-plugin|-fstack-protector*|-stdlib=*| \ - -specs=*|-fsanitize=*|-fuse-ld=*|-static-*|-fcilkplus) - func_quote_for_eval "$arg" - arg=$func_quote_for_eval_result + -specs=*|-fsanitize=*|-fuse-ld=*|-static-*|-fcilkplus|-Wa,*) + func_quote_arg pretty "$arg" + arg=$func_quote_arg_result func_append compile_command " $arg" func_append finalize_command " $arg" func_append compiler_flags " $arg" @@ -7396,15 +7578,15 @@ func_mode_link () continue else # Otherwise treat like 'Some other compiler flag' below - func_quote_for_eval "$arg" - arg=$func_quote_for_eval_result + func_quote_arg pretty "$arg" + arg=$func_quote_arg_result fi ;; # Some other compiler flag. -* | +*) - func_quote_for_eval "$arg" - arg=$func_quote_for_eval_result + func_quote_arg pretty "$arg" + arg=$func_quote_arg_result ;; *.$objext) @@ -7524,8 +7706,8 @@ func_mode_link () *) # Unknown arguments in both finalize_command and compile_command need # to be aesthetically quoted because they are evaled later. - func_quote_for_eval "$arg" - arg=$func_quote_for_eval_result + func_quote_arg pretty "$arg" + arg=$func_quote_arg_result ;; esac # arg @@ -8733,7 +8915,7 @@ func_mode_link () test CXX = "$tagname" && { case $host_os in linux*) - case `$CC -V 2>&1 | sed 5q` in + case `$CC -V 2>&1 | $SED 5q` in *Sun\ C*) # Sun C++ 5.9 func_suncc_cstd_abi @@ -8906,7 +9088,7 @@ func_mode_link () # case $version_type in # correct linux to gnu/linux during the next big refactor - darwin|freebsd-elf|linux|osf|windows|none) + darwin|freebsd-elf|linux|midnightbsd-elf|osf|windows|none) func_arith $number_major + $number_minor current=$func_arith_result age=$number_minor @@ -9000,7 +9182,7 @@ func_mode_link () versuffix=.$current.$revision ;; - freebsd-elf) + freebsd-elf | midnightbsd-elf) func_arith $current - $age major=.$func_arith_result versuffix=$major.$age.$revision @@ -9226,7 +9408,7 @@ func_mode_link () *-*-netbsd*) # Don't link with libc until the a.out ld.so is fixed. ;; - *-*-openbsd* | *-*-freebsd* | *-*-dragonfly*) + *-*-openbsd* | *-*-freebsd* | *-*-dragonfly* | *-*-midnightbsd*) # Do not include libc due to us having libc/libc_r. ;; *-*-sco3.2v5* | *-*-sco5v6*) @@ -10037,8 +10219,8 @@ EOF for cmd in $concat_cmds; do IFS=$save_ifs $opt_quiet || { - func_quote_for_expand "$cmd" - eval "func_echo $func_quote_for_expand_result" + func_quote_arg expand,pretty "$cmd" + eval "func_echo $func_quote_arg_result" } $opt_dry_run || eval "$cmd" || { lt_exit=$? @@ -10131,8 +10313,8 @@ EOF eval cmd=\"$cmd\" IFS=$save_ifs $opt_quiet || { - func_quote_for_expand "$cmd" - eval "func_echo $func_quote_for_expand_result" + func_quote_arg expand,pretty "$cmd" + eval "func_echo $func_quote_arg_result" } $opt_dry_run || eval "$cmd" || { lt_exit=$? @@ -10606,12 +10788,13 @@ EOF elif eval var_value=\$$var; test -z "$var_value"; then relink_command="$var=; export $var; $relink_command" else - func_quote_for_eval "$var_value" - relink_command="$var=$func_quote_for_eval_result; export $var; $relink_command" + func_quote_arg pretty "$var_value" + relink_command="$var=$func_quote_arg_result; export $var; $relink_command" fi done - relink_command="(cd `pwd`; $relink_command)" - relink_command=`$ECHO "$relink_command" | $SED "$sed_quote_subst"` + func_quote eval cd "`pwd`" + func_quote_arg pretty,unquoted "($func_quote_result; $relink_command)" + relink_command=$func_quote_arg_unquoted_result fi # Only actually do things if not in dry run mode. @@ -10851,13 +11034,15 @@ EOF elif eval var_value=\$$var; test -z "$var_value"; then relink_command="$var=; export $var; $relink_command" else - func_quote_for_eval "$var_value" - relink_command="$var=$func_quote_for_eval_result; export $var; $relink_command" + func_quote_arg pretty,unquoted "$var_value" + relink_command="$var=$func_quote_arg_unquoted_result; export $var; $relink_command" fi done # Quote the link command for shipping. - relink_command="(cd `pwd`; $SHELL \"$progpath\" $preserve_args --mode=relink $libtool_args @inst_prefix_dir@)" - relink_command=`$ECHO "$relink_command" | $SED "$sed_quote_subst"` + func_quote eval cd "`pwd`" + relink_command="($func_quote_result; $SHELL \"$progpath\" $preserve_args --mode=relink $libtool_args @inst_prefix_dir@)" + func_quote_arg pretty,unquoted "$relink_command" + relink_command=$func_quote_arg_unquoted_result if test yes = "$hardcode_automatic"; then relink_command= fi diff --git a/source/lib/c-ares-1.34.4/config/missing b/source/lib/c-ares-1.34.6/config/missing similarity index 100% rename from source/lib/c-ares-1.34.4/config/missing rename to source/lib/c-ares-1.34.6/config/missing diff --git a/source/lib/c-ares-1.34.4/config/test-driver b/source/lib/c-ares-1.34.6/config/test-driver similarity index 100% rename from source/lib/c-ares-1.34.4/config/test-driver rename to source/lib/c-ares-1.34.6/config/test-driver diff --git a/source/lib/c-ares-1.34.4/configure b/source/lib/c-ares-1.34.6/configure similarity index 94% rename from source/lib/c-ares-1.34.4/configure rename to source/lib/c-ares-1.34.6/configure index d02f117d..f91e2c95 100755 --- a/source/lib/c-ares-1.34.4/configure +++ b/source/lib/c-ares-1.34.6/configure @@ -1,6 +1,6 @@ #! /bin/sh # Guess values for system-dependent variables and create Makefiles. -# Generated by GNU Autoconf 2.71 for c-ares 1.34.4. +# Generated by GNU Autoconf 2.71 for c-ares 1.34.6. # # Report bugs to . # @@ -621,8 +621,8 @@ MAKEFLAGS= # Identity of this package. PACKAGE_NAME='c-ares' PACKAGE_TARNAME='c-ares' -PACKAGE_VERSION='1.34.4' -PACKAGE_STRING='c-ares 1.34.4' +PACKAGE_VERSION='1.34.6' +PACKAGE_STRING='c-ares 1.34.6' PACKAGE_BUGREPORT='c-ares mailing list: http://lists.haxx.se/listinfo/c-ares' PACKAGE_URL='' @@ -669,6 +669,8 @@ AM_CPPFLAGS AM_CFLAGS BUILD_TESTS_FALSE BUILD_TESTS_TRUE +GMOCK117_LIBS +GMOCK117_CFLAGS GMOCK112_LIBS GMOCK112_CFLAGS GMOCK_LIBS @@ -715,6 +717,7 @@ MANIFEST_TOOL RANLIB ac_ct_AR AR +FILECMD LN_S NM ac_ct_DUMPBIN @@ -778,6 +781,7 @@ am__isrc INSTALL_DATA INSTALL_SCRIPT INSTALL_PROGRAM +HAVE_CXX17 HAVE_CXX14 ac_ct_CXX CXXFLAGS @@ -875,7 +879,9 @@ PKG_CONFIG_LIBDIR GMOCK_CFLAGS GMOCK_LIBS GMOCK112_CFLAGS -GMOCK112_LIBS' +GMOCK112_LIBS +GMOCK117_CFLAGS +GMOCK117_LIBS' # Initialize some variables set by options. @@ -1424,7 +1430,7 @@ if test "$ac_init_help" = "long"; then # Omit some internal or obsolete options to make the list less imposing. # This message is too long to be a string in the A/UX 3.1 sh. cat <<_ACEOF -\`configure' configures c-ares 1.34.4 to adapt to many kinds of systems. +\`configure' configures c-ares 1.34.6 to adapt to many kinds of systems. Usage: $0 [OPTION]... [VAR=VALUE]... @@ -1495,7 +1501,7 @@ fi if test -n "$ac_init_help"; then case $ac_init_help in - short | recursive ) echo "Configuration of c-ares 1.34.4:";; + short | recursive ) echo "Configuration of c-ares 1.34.6:";; esac cat <<\_ACEOF @@ -1569,6 +1575,10 @@ Some influential environment variables: C compiler flags for GMOCK112, overriding pkg-config GMOCK112_LIBS linker flags for GMOCK112, overriding pkg-config + GMOCK117_CFLAGS + C compiler flags for GMOCK117, overriding pkg-config + GMOCK117_LIBS + linker flags for GMOCK117, overriding pkg-config Use these variables to override the choices made by `configure' or to help it to find libraries and programs with nonstandard names/locations. @@ -1637,7 +1647,7 @@ fi test -n "$ac_init_help" && exit $ac_status if $ac_init_version; then cat <<\_ACEOF -c-ares configure 1.34.4 +c-ares configure 1.34.6 generated by GNU Autoconf 2.71 Copyright (C) 2021 Free Software Foundation, Inc. @@ -2261,7 +2271,7 @@ cat >config.log <<_ACEOF This file contains any messages produced by compilers while running configure, to aid debugging if configure makes a mistake. -It was created by c-ares $as_me 1.34.4, which was +It was created by c-ares $as_me 1.34.6, which was generated by GNU Autoconf 2.71. Invocation command line was $ $0$ac_configure_args_raw @@ -3235,7 +3245,7 @@ ac_compiler_gnu=$ac_cv_c_compiler_gnu -CARES_VERSION_INFO="21:3:19" +CARES_VERSION_INFO="21:5:19" @@ -4755,11 +4765,11 @@ if test x$ac_prog_cxx_stdcxx = xno then : { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for $CXX option to enable C++11 features" >&5 printf %s "checking for $CXX option to enable C++11 features... " >&6; } -if test ${ac_cv_prog_cxx_11+y} +if test ${ac_cv_prog_cxx_cxx11+y} then : printf %s "(cached) " >&6 else $as_nop - ac_cv_prog_cxx_11=no + ac_cv_prog_cxx_cxx11=no ac_save_CXX=$CXX cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ @@ -4801,11 +4811,11 @@ if test x$ac_prog_cxx_stdcxx = xno then : { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for $CXX option to enable C++98 features" >&5 printf %s "checking for $CXX option to enable C++98 features... " >&6; } -if test ${ac_cv_prog_cxx_98+y} +if test ${ac_cv_prog_cxx_cxx98+y} then : printf %s "(cached) " >&6 else $as_nop - ac_cv_prog_cxx_98=no + ac_cv_prog_cxx_cxx98=no ac_save_CXX=$CXX cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ @@ -5354,217 +5364,1098 @@ printf "%s\n" "#define HAVE_CXX14 1" >>confdefs.h fi -am__api_version='1.16' + ax_cxx_compile_alternatives="17 1z" ax_cxx_compile_cxx17_required=false + ac_ext=cpp +ac_cpp='$CXXCPP $CPPFLAGS' +ac_compile='$CXX -c $CXXFLAGS $CPPFLAGS conftest.$ac_ext >&5' +ac_link='$CXX -o conftest$ac_exeext $CXXFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5' +ac_compiler_gnu=$ac_cv_cxx_compiler_gnu + ac_success=no - # Find a good install program. We prefer a C program (faster), -# so one script is as good as another. But avoid the broken or -# incompatible versions: -# SysV /etc/install, /usr/sbin/install -# SunOS /usr/etc/install -# IRIX /sbin/install -# AIX /bin/install -# AmigaOS /C/install, which installs bootblocks on floppy discs -# AIX 4 /usr/bin/installbsd, which doesn't work without a -g flag -# AFS /usr/afsws/bin/install, which mishandles nonexistent args -# SVR4 /usr/ucb/install, which tries to use the nonexistent group "staff" -# OS/2's system install, which has a completely different semantic -# ./install, which can be erroneously created by make from ./install.sh. -# Reject install programs that cannot install multiple files. -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for a BSD-compatible install" >&5 -printf %s "checking for a BSD-compatible install... " >&6; } -if test -z "$INSTALL"; then -if test ${ac_cv_path_install+y} + + + + if test x$ac_success = xno; then + for alternative in ${ax_cxx_compile_alternatives}; do + for switch in -std=c++${alternative} +std=c++${alternative} "-h std=c++${alternative}" MSVC; do + if test x"$switch" = xMSVC; then + switch=-std:c++${alternative} + cachevar=`printf "%s\n" "ax_cv_cxx_compile_cxx17_${switch}_MSVC" | $as_tr_sh` + else + cachevar=`printf "%s\n" "ax_cv_cxx_compile_cxx17_$switch" | $as_tr_sh` + fi + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether $CXX supports C++17 features with $switch" >&5 +printf %s "checking whether $CXX supports C++17 features with $switch... " >&6; } +if eval test \${$cachevar+y} then : printf %s "(cached) " >&6 else $as_nop - as_save_IFS=$IFS; IFS=$PATH_SEPARATOR -for as_dir in $PATH -do - IFS=$as_save_IFS - case $as_dir in #((( - '') as_dir=./ ;; - */) ;; - *) as_dir=$as_dir/ ;; - esac - # Account for fact that we put trailing slashes in our PATH walk. -case $as_dir in #(( - ./ | /[cC]/* | \ - /etc/* | /usr/sbin/* | /usr/etc/* | /sbin/* | /usr/afsws/bin/* | \ - ?:[\\/]os2[\\/]install[\\/]* | ?:[\\/]OS2[\\/]INSTALL[\\/]* | \ - /usr/ucb/* ) ;; - *) - # OSF1 and SCO ODT 3.0 have their own names for install. - # Don't use installbsd from OSF since it installs stuff as root - # by default. - for ac_prog in ginstall scoinst install; do - for ac_exec_ext in '' $ac_executable_extensions; do - if as_fn_executable_p "$as_dir$ac_prog$ac_exec_ext"; then - if test $ac_prog = install && - grep dspmsg "$as_dir$ac_prog$ac_exec_ext" >/dev/null 2>&1; then - # AIX install. It has an incompatible calling convention. - : - elif test $ac_prog = install && - grep pwplus "$as_dir$ac_prog$ac_exec_ext" >/dev/null 2>&1; then - # program-specific install script used by HP pwplus--don't use. - : - else - rm -rf conftest.one conftest.two conftest.dir - echo one > conftest.one - echo two > conftest.two - mkdir conftest.dir - if "$as_dir$ac_prog$ac_exec_ext" -c conftest.one conftest.two "`pwd`/conftest.dir/" && - test -s conftest.one && test -s conftest.two && - test -s conftest.dir/conftest.one && - test -s conftest.dir/conftest.two - then - ac_cv_path_install="$as_dir$ac_prog$ac_exec_ext -c" - break 3 - fi - fi - fi - done - done - ;; -esac + ac_save_CXX="$CXX" + CXX="$CXX $switch" + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ - done -IFS=$as_save_IFS -rm -rf conftest.one conftest.two conftest.dir +// If the compiler admits that it is not ready for C++11, why torture it? +// Hopefully, this will speed up the test. -fi - if test ${ac_cv_path_install+y}; then - INSTALL=$ac_cv_path_install - else - # As a last resort, use the slow shell script. Don't cache a - # value for INSTALL within a source directory, because that will - # break other packages using the cache if that directory is - # removed, or if the value is a relative name. - INSTALL=$ac_install_sh - fi -fi -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $INSTALL" >&5 -printf "%s\n" "$INSTALL" >&6; } +#ifndef __cplusplus -# Use test -z because SunOS4 sh mishandles braces in ${var-val}. -# It thinks the first close brace ends the variable substitution. -test -z "$INSTALL_PROGRAM" && INSTALL_PROGRAM='${INSTALL}' +#error "This is not a C++ compiler" -test -z "$INSTALL_SCRIPT" && INSTALL_SCRIPT='${INSTALL}' +// MSVC always sets __cplusplus to 199711L in older versions; newer versions +// only set it correctly if /Zc:__cplusplus is specified as well as a +// /std:c++NN switch: +// +// https://devblogs.microsoft.com/cppblog/msvc-now-correctly-reports-__cplusplus/ +// +// The value __cplusplus ought to have is available in _MSVC_LANG since +// Visual Studio 2015 Update 3: +// +// https://learn.microsoft.com/en-us/cpp/preprocessor/predefined-macros +// +// This was also the first MSVC version to support C++14 so we can't use the +// value of either __cplusplus or _MSVC_LANG to quickly rule out MSVC having +// C++11 or C++14 support, but we can check _MSVC_LANG for C++17 and later. +#elif __cplusplus < 201103L && !defined _MSC_VER -test -z "$INSTALL_DATA" && INSTALL_DATA='${INSTALL} -m 644' +#error "This is not a C++11 compiler" -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether build environment is sane" >&5 -printf %s "checking whether build environment is sane... " >&6; } -# Reject unsafe characters in $srcdir or the absolute working directory -# name. Accept space and tab only in the latter. -am_lf=' -' -case `pwd` in - *[\\\"\#\$\&\'\`$am_lf]*) - as_fn_error $? "unsafe absolute working directory name" "$LINENO" 5;; -esac -case $srcdir in - *[\\\"\#\$\&\'\`$am_lf\ \ ]*) - as_fn_error $? "unsafe srcdir value: '$srcdir'" "$LINENO" 5;; -esac +#else -# Do 'set' in a subshell so we don't clobber the current shell's -# arguments. Must try -L first in case configure is actually a -# symlink; some systems play weird games with the mod time of symlinks -# (eg FreeBSD returns the mod time of the symlink's containing -# directory). -if ( - am_has_slept=no - for am_try in 1 2; do - echo "timestamp, slept: $am_has_slept" > conftest.file - set X `ls -Lt "$srcdir/configure" conftest.file 2> /dev/null` - if test "$*" = "X"; then - # -L didn't work. - set X `ls -t "$srcdir/configure" conftest.file` - fi - if test "$*" != "X $srcdir/configure conftest.file" \ - && test "$*" != "X conftest.file $srcdir/configure"; then +namespace cxx11 +{ - # If neither matched, then we have a broken ls. This can happen - # if, for instance, CONFIG_SHELL is bash and it inherits a - # broken ls alias from the environment. This has actually - # happened. Such a system could not be considered "sane". - as_fn_error $? "ls -t appears to fail. Make sure there is not a broken - alias in your environment" "$LINENO" 5 - fi - if test "$2" = conftest.file || test $am_try -eq 2; then - break - fi - # Just in case. - sleep 1 - am_has_slept=yes - done - test "$2" = conftest.file - ) -then - # Ok. - : -else - as_fn_error $? "newly created file is older than distributed files! -Check your system clock" "$LINENO" 5 -fi -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: yes" >&5 -printf "%s\n" "yes" >&6; } -# If we didn't sleep, we still need to ensure time stamps of config.status and -# generated files are strictly newer. -am_sleep_pid= -if grep 'slept: no' conftest.file >/dev/null 2>&1; then - ( sleep 1 ) & - am_sleep_pid=$! -fi + namespace test_static_assert + { -rm -f conftest.file + template + struct check + { + static_assert(sizeof(int) <= sizeof(T), "not big enough"); + }; -test "$program_prefix" != NONE && - program_transform_name="s&^&$program_prefix&;$program_transform_name" -# Use a double $ so make ignores it. -test "$program_suffix" != NONE && - program_transform_name="s&\$&$program_suffix&;$program_transform_name" -# Double any \ or $. -# By default was `s,x,x', remove it if useless. -ac_script='s/[\\$]/&&/g;s/;s,x,x,$//' -program_transform_name=`printf "%s\n" "$program_transform_name" | sed "$ac_script"` + } + namespace test_final_override + { - if test x"${MISSING+set}" != xset; then - MISSING="\${SHELL} '$am_aux_dir/missing'" -fi -# Use eval to expand $SHELL -if eval "$MISSING --is-lightweight"; then - am_missing_run="$MISSING " -else - am_missing_run= - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: WARNING: 'missing' script is too old or missing" >&5 -printf "%s\n" "$as_me: WARNING: 'missing' script is too old or missing" >&2;} -fi + struct Base + { + virtual ~Base() {} + virtual void f() {} + }; -if test x"${install_sh+set}" != xset; then - case $am_aux_dir in - *\ * | *\ *) - install_sh="\${SHELL} '$am_aux_dir/install-sh'" ;; - *) - install_sh="\${SHELL} $am_aux_dir/install-sh" - esac -fi + struct Derived : public Base + { + virtual ~Derived() override {} + virtual void f() override {} + }; -# Installed binaries are usually stripped using 'strip' when the user -# run "make install-strip". However 'strip' might not be the right -# tool to use in cross-compilation environments, therefore Automake -# will honor the 'STRIP' environment variable to overrule this program. -if test "$cross_compiling" != no; then - if test -n "$ac_tool_prefix"; then - # Extract the first word of "${ac_tool_prefix}strip", so it can be a program name with args. -set dummy ${ac_tool_prefix}strip; ac_word=$2 -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 -printf %s "checking for $ac_word... " >&6; } -if test ${ac_cv_prog_STRIP+y} + } + + namespace test_double_right_angle_brackets + { + + template < typename T > + struct check {}; + + typedef check single_type; + typedef check> double_type; + typedef check>> triple_type; + typedef check>>> quadruple_type; + + } + + namespace test_decltype + { + + int + f() + { + int a = 1; + decltype(a) b = 2; + return a + b; + } + + } + + namespace test_type_deduction + { + + template < typename T1, typename T2 > + struct is_same + { + static const bool value = false; + }; + + template < typename T > + struct is_same + { + static const bool value = true; + }; + + template < typename T1, typename T2 > + auto + add(T1 a1, T2 a2) -> decltype(a1 + a2) + { + return a1 + a2; + } + + int + test(const int c, volatile int v) + { + static_assert(is_same::value == true, ""); + static_assert(is_same::value == false, ""); + static_assert(is_same::value == false, ""); + auto ac = c; + auto av = v; + auto sumi = ac + av + 'x'; + auto sumf = ac + av + 1.0; + static_assert(is_same::value == true, ""); + static_assert(is_same::value == true, ""); + static_assert(is_same::value == true, ""); + static_assert(is_same::value == false, ""); + static_assert(is_same::value == true, ""); + return (sumf > 0.0) ? sumi : add(c, v); + } + + } + + namespace test_noexcept + { + + int f() { return 0; } + int g() noexcept { return 0; } + + static_assert(noexcept(f()) == false, ""); + static_assert(noexcept(g()) == true, ""); + + } + + namespace test_constexpr + { + + template < typename CharT > + unsigned long constexpr + strlen_c_r(const CharT *const s, const unsigned long acc) noexcept + { + return *s ? strlen_c_r(s + 1, acc + 1) : acc; + } + + template < typename CharT > + unsigned long constexpr + strlen_c(const CharT *const s) noexcept + { + return strlen_c_r(s, 0UL); + } + + static_assert(strlen_c("") == 0UL, ""); + static_assert(strlen_c("1") == 1UL, ""); + static_assert(strlen_c("example") == 7UL, ""); + static_assert(strlen_c("another\0example") == 7UL, ""); + + } + + namespace test_rvalue_references + { + + template < int N > + struct answer + { + static constexpr int value = N; + }; + + answer<1> f(int&) { return answer<1>(); } + answer<2> f(const int&) { return answer<2>(); } + answer<3> f(int&&) { return answer<3>(); } + + void + test() + { + int i = 0; + const int c = 0; + static_assert(decltype(f(i))::value == 1, ""); + static_assert(decltype(f(c))::value == 2, ""); + static_assert(decltype(f(0))::value == 3, ""); + } + + } + + namespace test_uniform_initialization + { + + struct test + { + static const int zero {}; + static const int one {1}; + }; + + static_assert(test::zero == 0, ""); + static_assert(test::one == 1, ""); + + } + + namespace test_lambdas + { + + void + test1() + { + auto lambda1 = [](){}; + auto lambda2 = lambda1; + lambda1(); + lambda2(); + } + + int + test2() + { + auto a = [](int i, int j){ return i + j; }(1, 2); + auto b = []() -> int { return '0'; }(); + auto c = [=](){ return a + b; }(); + auto d = [&](){ return c; }(); + auto e = [a, &b](int x) mutable { + const auto identity = [](int y){ return y; }; + for (auto i = 0; i < a; ++i) + a += b--; + return x + identity(a + b); + }(0); + return a + b + c + d + e; + } + + int + test3() + { + const auto nullary = [](){ return 0; }; + const auto unary = [](int x){ return x; }; + using nullary_t = decltype(nullary); + using unary_t = decltype(unary); + const auto higher1st = [](nullary_t f){ return f(); }; + const auto higher2nd = [unary](nullary_t f1){ + return [unary, f1](unary_t f2){ return f2(unary(f1())); }; + }; + return higher1st(nullary) + higher2nd(nullary)(unary); + } + + } + + namespace test_variadic_templates + { + + template + struct sum; + + template + struct sum + { + static constexpr auto value = N0 + sum::value; + }; + + template <> + struct sum<> + { + static constexpr auto value = 0; + }; + + static_assert(sum<>::value == 0, ""); + static_assert(sum<1>::value == 1, ""); + static_assert(sum<23>::value == 23, ""); + static_assert(sum<1, 2>::value == 3, ""); + static_assert(sum<5, 5, 11>::value == 21, ""); + static_assert(sum<2, 3, 5, 7, 11, 13>::value == 41, ""); + + } + + // http://stackoverflow.com/questions/13728184/template-aliases-and-sfinae + // Clang 3.1 fails with headers of libstd++ 4.8.3 when using std::function + // because of this. + namespace test_template_alias_sfinae + { + + struct foo {}; + + template + using member = typename T::member_type; + + template + void func(...) {} + + template + void func(member*) {} + + void test(); + + void test() { func(0); } + + } + +} // namespace cxx11 + +#endif // __cplusplus >= 201103L + + + + +// If the compiler admits that it is not ready for C++14, why torture it? +// Hopefully, this will speed up the test. + +#ifndef __cplusplus + +#error "This is not a C++ compiler" + +#elif __cplusplus < 201402L && !defined _MSC_VER + +#error "This is not a C++14 compiler" + +#else + +namespace cxx14 +{ + + namespace test_polymorphic_lambdas + { + + int + test() + { + const auto lambda = [](auto&&... args){ + const auto istiny = [](auto x){ + return (sizeof(x) == 1UL) ? 1 : 0; + }; + const int aretiny[] = { istiny(args)... }; + return aretiny[0]; + }; + return lambda(1, 1L, 1.0f, '1'); + } + + } + + namespace test_binary_literals + { + + constexpr auto ivii = 0b0000000000101010; + static_assert(ivii == 42, "wrong value"); + + } + + namespace test_generalized_constexpr + { + + template < typename CharT > + constexpr unsigned long + strlen_c(const CharT *const s) noexcept + { + auto length = 0UL; + for (auto p = s; *p; ++p) + ++length; + return length; + } + + static_assert(strlen_c("") == 0UL, ""); + static_assert(strlen_c("x") == 1UL, ""); + static_assert(strlen_c("test") == 4UL, ""); + static_assert(strlen_c("another\0test") == 7UL, ""); + + } + + namespace test_lambda_init_capture + { + + int + test() + { + auto x = 0; + const auto lambda1 = [a = x](int b){ return a + b; }; + const auto lambda2 = [a = lambda1(x)](){ return a; }; + return lambda2(); + } + + } + + namespace test_digit_separators + { + + constexpr auto ten_million = 100'000'000; + static_assert(ten_million == 100000000, ""); + + } + + namespace test_return_type_deduction + { + + auto f(int& x) { return x; } + decltype(auto) g(int& x) { return x; } + + template < typename T1, typename T2 > + struct is_same + { + static constexpr auto value = false; + }; + + template < typename T > + struct is_same + { + static constexpr auto value = true; + }; + + int + test() + { + auto x = 0; + static_assert(is_same::value, ""); + static_assert(is_same::value, ""); + return x; + } + + } + +} // namespace cxx14 + +#endif // __cplusplus >= 201402L + + + + +// If the compiler admits that it is not ready for C++17, why torture it? +// Hopefully, this will speed up the test. + +#ifndef __cplusplus + +#error "This is not a C++ compiler" + +#elif (defined _MSVC_LANG ? _MSVC_LANG : __cplusplus) < 201703L + +#error "This is not a C++17 compiler" + +#else + +#include +#include +#include + +namespace cxx17 +{ + + namespace test_constexpr_lambdas + { + + constexpr int foo = [](){return 42;}(); + + } + + namespace test::nested_namespace::definitions + { + + } + + namespace test_fold_expression + { + + template + int multiply(Args... args) + { + return (args * ... * 1); + } + + template + bool all(Args... args) + { + return (args && ...); + } + + } + + namespace test_extended_static_assert + { + + static_assert (true); + + } + + namespace test_auto_brace_init_list + { + + auto foo = {5}; + auto bar {5}; + + static_assert(std::is_same, decltype(foo)>::value); + static_assert(std::is_same::value); + } + + namespace test_typename_in_template_template_parameter + { + + template typename X> struct D; + + } + + namespace test_fallthrough_nodiscard_maybe_unused_attributes + { + + int f1() + { + return 42; + } + + [[nodiscard]] int f2() + { + [[maybe_unused]] auto unused = f1(); + + switch (f1()) + { + case 17: + f1(); + [[fallthrough]]; + case 42: + f1(); + } + return f1(); + } + + } + + namespace test_extended_aggregate_initialization + { + + struct base1 + { + int b1, b2 = 42; + }; + + struct base2 + { + base2() { + b3 = 42; + } + int b3; + }; + + struct derived : base1, base2 + { + int d; + }; + + derived d1 {{1, 2}, {}, 4}; // full initialization + derived d2 {{}, {}, 4}; // value-initialized bases + + } + + namespace test_general_range_based_for_loop + { + + struct iter + { + int i; + + int& operator* () + { + return i; + } + + const int& operator* () const + { + return i; + } + + iter& operator++() + { + ++i; + return *this; + } + }; + + struct sentinel + { + int i; + }; + + bool operator== (const iter& i, const sentinel& s) + { + return i.i == s.i; + } + + bool operator!= (const iter& i, const sentinel& s) + { + return !(i == s); + } + + struct range + { + iter begin() const + { + return {0}; + } + + sentinel end() const + { + return {5}; + } + }; + + void f() + { + range r {}; + + for (auto i : r) + { + [[maybe_unused]] auto v = i; + } + } + + } + + namespace test_lambda_capture_asterisk_this_by_value + { + + struct t + { + int i; + int foo() + { + return [*this]() + { + return i; + }(); + } + }; + + } + + namespace test_enum_class_construction + { + + enum class byte : unsigned char + {}; + + byte foo {42}; + + } + + namespace test_constexpr_if + { + + template + int f () + { + if constexpr(cond) + { + return 13; + } + else + { + return 42; + } + } + + } + + namespace test_selection_statement_with_initializer + { + + int f() + { + return 13; + } + + int f2() + { + if (auto i = f(); i > 0) + { + return 3; + } + + switch (auto i = f(); i + 4) + { + case 17: + return 2; + + default: + return 1; + } + } + + } + + namespace test_template_argument_deduction_for_class_templates + { + + template + struct pair + { + pair (T1 p1, T2 p2) + : m1 {p1}, + m2 {p2} + {} + + T1 m1; + T2 m2; + }; + + void f() + { + [[maybe_unused]] auto p = pair{13, 42u}; + } + + } + + namespace test_non_type_auto_template_parameters + { + + template + struct B + {}; + + B<5> b1; + B<'a'> b2; + + } + + namespace test_structured_bindings + { + + int arr[2] = { 1, 2 }; + std::pair pr = { 1, 2 }; + + auto f1() -> int(&)[2] + { + return arr; + } + + auto f2() -> std::pair& + { + return pr; + } + + struct S + { + int x1 : 2; + volatile double y1; + }; + + S f3() + { + return {}; + } + + auto [ x1, y1 ] = f1(); + auto& [ xr1, yr1 ] = f1(); + auto [ x2, y2 ] = f2(); + auto& [ xr2, yr2 ] = f2(); + const auto [ x3, y3 ] = f3(); + + } + + namespace test_exception_spec_type_system + { + + struct Good {}; + struct Bad {}; + + void g1() noexcept; + void g2(); + + template + Bad + f(T*, T*); + + template + Good + f(T1*, T2*); + + static_assert (std::is_same_v); + + } + + namespace test_inline_variables + { + + template void f(T) + {} + + template inline T g(T) + { + return T{}; + } + + template<> inline void f<>(int) + {} + + template<> int g<>(int) + { + return 5; + } + + } + +} // namespace cxx17 + +#endif // (defined _MSVC_LANG ? _MSVC_LANG : __cplusplus) < 201703L + + + +_ACEOF +if ac_fn_cxx_try_compile "$LINENO" +then : + eval $cachevar=yes +else $as_nop + eval $cachevar=no +fi +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext + CXX="$ac_save_CXX" +fi +eval ac_res=\$$cachevar + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_res" >&5 +printf "%s\n" "$ac_res" >&6; } + if eval test x\$$cachevar = xyes; then + CXX="$CXX $switch" + if test -n "$CXXCPP" ; then + CXXCPP="$CXXCPP $switch" + fi + ac_success=yes + break + fi + done + if test x$ac_success = xyes; then + break + fi + done + fi + ac_ext=c +ac_cpp='$CPP $CPPFLAGS' +ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5' +ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5' +ac_compiler_gnu=$ac_cv_c_compiler_gnu + + if test x$ax_cxx_compile_cxx17_required = xtrue; then + if test x$ac_success = xno; then + as_fn_error $? "*** A compiler with support for C++17 language features is required." "$LINENO" 5 + fi + fi + if test x$ac_success = xno; then + HAVE_CXX17=0 + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: No compiler with C++17 support was found" >&5 +printf "%s\n" "$as_me: No compiler with C++17 support was found" >&6;} + else + HAVE_CXX17=1 + +printf "%s\n" "#define HAVE_CXX17 1" >>confdefs.h + + fi + + +am__api_version='1.16' + + + # Find a good install program. We prefer a C program (faster), +# so one script is as good as another. But avoid the broken or +# incompatible versions: +# SysV /etc/install, /usr/sbin/install +# SunOS /usr/etc/install +# IRIX /sbin/install +# AIX /bin/install +# AmigaOS /C/install, which installs bootblocks on floppy discs +# AIX 4 /usr/bin/installbsd, which doesn't work without a -g flag +# AFS /usr/afsws/bin/install, which mishandles nonexistent args +# SVR4 /usr/ucb/install, which tries to use the nonexistent group "staff" +# OS/2's system install, which has a completely different semantic +# ./install, which can be erroneously created by make from ./install.sh. +# Reject install programs that cannot install multiple files. +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for a BSD-compatible install" >&5 +printf %s "checking for a BSD-compatible install... " >&6; } +if test -z "$INSTALL"; then +if test ${ac_cv_path_install+y} +then : + printf %s "(cached) " >&6 +else $as_nop + as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +for as_dir in $PATH +do + IFS=$as_save_IFS + case $as_dir in #((( + '') as_dir=./ ;; + */) ;; + *) as_dir=$as_dir/ ;; + esac + # Account for fact that we put trailing slashes in our PATH walk. +case $as_dir in #(( + ./ | /[cC]/* | \ + /etc/* | /usr/sbin/* | /usr/etc/* | /sbin/* | /usr/afsws/bin/* | \ + ?:[\\/]os2[\\/]install[\\/]* | ?:[\\/]OS2[\\/]INSTALL[\\/]* | \ + /usr/ucb/* ) ;; + *) + # OSF1 and SCO ODT 3.0 have their own names for install. + # Don't use installbsd from OSF since it installs stuff as root + # by default. + for ac_prog in ginstall scoinst install; do + for ac_exec_ext in '' $ac_executable_extensions; do + if as_fn_executable_p "$as_dir$ac_prog$ac_exec_ext"; then + if test $ac_prog = install && + grep dspmsg "$as_dir$ac_prog$ac_exec_ext" >/dev/null 2>&1; then + # AIX install. It has an incompatible calling convention. + : + elif test $ac_prog = install && + grep pwplus "$as_dir$ac_prog$ac_exec_ext" >/dev/null 2>&1; then + # program-specific install script used by HP pwplus--don't use. + : + else + rm -rf conftest.one conftest.two conftest.dir + echo one > conftest.one + echo two > conftest.two + mkdir conftest.dir + if "$as_dir$ac_prog$ac_exec_ext" -c conftest.one conftest.two "`pwd`/conftest.dir/" && + test -s conftest.one && test -s conftest.two && + test -s conftest.dir/conftest.one && + test -s conftest.dir/conftest.two + then + ac_cv_path_install="$as_dir$ac_prog$ac_exec_ext -c" + break 3 + fi + fi + fi + done + done + ;; +esac + + done +IFS=$as_save_IFS + +rm -rf conftest.one conftest.two conftest.dir + +fi + if test ${ac_cv_path_install+y}; then + INSTALL=$ac_cv_path_install + else + # As a last resort, use the slow shell script. Don't cache a + # value for INSTALL within a source directory, because that will + # break other packages using the cache if that directory is + # removed, or if the value is a relative name. + INSTALL=$ac_install_sh + fi +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $INSTALL" >&5 +printf "%s\n" "$INSTALL" >&6; } + +# Use test -z because SunOS4 sh mishandles braces in ${var-val}. +# It thinks the first close brace ends the variable substitution. +test -z "$INSTALL_PROGRAM" && INSTALL_PROGRAM='${INSTALL}' + +test -z "$INSTALL_SCRIPT" && INSTALL_SCRIPT='${INSTALL}' + +test -z "$INSTALL_DATA" && INSTALL_DATA='${INSTALL} -m 644' + +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether build environment is sane" >&5 +printf %s "checking whether build environment is sane... " >&6; } +# Reject unsafe characters in $srcdir or the absolute working directory +# name. Accept space and tab only in the latter. +am_lf=' +' +case `pwd` in + *[\\\"\#\$\&\'\`$am_lf]*) + as_fn_error $? "unsafe absolute working directory name" "$LINENO" 5;; +esac +case $srcdir in + *[\\\"\#\$\&\'\`$am_lf\ \ ]*) + as_fn_error $? "unsafe srcdir value: '$srcdir'" "$LINENO" 5;; +esac + +# Do 'set' in a subshell so we don't clobber the current shell's +# arguments. Must try -L first in case configure is actually a +# symlink; some systems play weird games with the mod time of symlinks +# (eg FreeBSD returns the mod time of the symlink's containing +# directory). +if ( + am_has_slept=no + for am_try in 1 2; do + echo "timestamp, slept: $am_has_slept" > conftest.file + set X `ls -Lt "$srcdir/configure" conftest.file 2> /dev/null` + if test "$*" = "X"; then + # -L didn't work. + set X `ls -t "$srcdir/configure" conftest.file` + fi + if test "$*" != "X $srcdir/configure conftest.file" \ + && test "$*" != "X conftest.file $srcdir/configure"; then + + # If neither matched, then we have a broken ls. This can happen + # if, for instance, CONFIG_SHELL is bash and it inherits a + # broken ls alias from the environment. This has actually + # happened. Such a system could not be considered "sane". + as_fn_error $? "ls -t appears to fail. Make sure there is not a broken + alias in your environment" "$LINENO" 5 + fi + if test "$2" = conftest.file || test $am_try -eq 2; then + break + fi + # Just in case. + sleep 1 + am_has_slept=yes + done + test "$2" = conftest.file + ) +then + # Ok. + : +else + as_fn_error $? "newly created file is older than distributed files! +Check your system clock" "$LINENO" 5 +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +printf "%s\n" "yes" >&6; } +# If we didn't sleep, we still need to ensure time stamps of config.status and +# generated files are strictly newer. +am_sleep_pid= +if grep 'slept: no' conftest.file >/dev/null 2>&1; then + ( sleep 1 ) & + am_sleep_pid=$! +fi + +rm -f conftest.file + +test "$program_prefix" != NONE && + program_transform_name="s&^&$program_prefix&;$program_transform_name" +# Use a double $ so make ignores it. +test "$program_suffix" != NONE && + program_transform_name="s&\$&$program_suffix&;$program_transform_name" +# Double any \ or $. +# By default was `s,x,x', remove it if useless. +ac_script='s/[\\$]/&&/g;s/;s,x,x,$//' +program_transform_name=`printf "%s\n" "$program_transform_name" | sed "$ac_script"` + + + if test x"${MISSING+set}" != xset; then + MISSING="\${SHELL} '$am_aux_dir/missing'" +fi +# Use eval to expand $SHELL +if eval "$MISSING --is-lightweight"; then + am_missing_run="$MISSING " +else + am_missing_run= + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: WARNING: 'missing' script is too old or missing" >&5 +printf "%s\n" "$as_me: WARNING: 'missing' script is too old or missing" >&2;} +fi + +if test x"${install_sh+set}" != xset; then + case $am_aux_dir in + *\ * | *\ *) + install_sh="\${SHELL} '$am_aux_dir/install-sh'" ;; + *) + install_sh="\${SHELL} $am_aux_dir/install-sh" + esac +fi + +# Installed binaries are usually stripped using 'strip' when the user +# run "make install-strip". However 'strip' might not be the right +# tool to use in cross-compilation environments, therefore Automake +# will honor the 'STRIP' environment variable to overrule this program. +if test "$cross_compiling" != no; then + if test -n "$ac_tool_prefix"; then + # Extract the first word of "${ac_tool_prefix}strip", so it can be a program name with args. +set dummy ${ac_tool_prefix}strip; ac_word=$2 +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 +printf %s "checking for $ac_word... " >&6; } +if test ${ac_cv_prog_STRIP+y} then : printf %s "(cached) " >&6 else $as_nop @@ -5927,7 +6818,7 @@ fi # Define the identity of the package. PACKAGE='c-ares' - VERSION='1.34.4' + VERSION='1.34.6' printf "%s\n" "#define PACKAGE \"$PACKAGE\"" >>confdefs.h @@ -6296,8 +7187,8 @@ esac -macro_version='2.4.6' -macro_revision='2.4.6' +macro_version='2.4.7' +macro_revision='2.4.7' @@ -6925,13 +7816,13 @@ else mingw*) lt_bad_file=conftest.nm/nofile ;; *) lt_bad_file=/dev/null ;; esac - case `"$tmp_nm" -B $lt_bad_file 2>&1 | sed '1q'` in + case `"$tmp_nm" -B $lt_bad_file 2>&1 | $SED '1q'` in *$lt_bad_file* | *'Invalid file or object type'*) lt_cv_path_NM="$tmp_nm -B" break 2 ;; *) - case `"$tmp_nm" -p /dev/null 2>&1 | sed '1q'` in + case `"$tmp_nm" -p /dev/null 2>&1 | $SED '1q'` in */dev/null*) lt_cv_path_NM="$tmp_nm -p" break 2 @@ -7069,7 +7960,7 @@ esac fi fi - case `$DUMPBIN -symbols -headers /dev/null 2>&1 | sed '1q'` in + case `$DUMPBIN -symbols -headers /dev/null 2>&1 | $SED '1q'` in *COFF*) DUMPBIN="$DUMPBIN -symbols -headers" ;; @@ -7173,7 +8064,7 @@ else $as_nop lt_cv_sys_max_cmd_len=8192; ;; - bitrig* | darwin* | dragonfly* | freebsd* | netbsd* | openbsd*) + bitrig* | darwin* | dragonfly* | freebsd* | midnightbsd* | netbsd* | openbsd*) # This has been around since 386BSD, at least. Likely further. if test -x /sbin/sysctl; then lt_cv_sys_max_cmd_len=`/sbin/sysctl -n kern.argmax` @@ -7216,7 +8107,7 @@ else $as_nop sysv5* | sco5v6* | sysv4.2uw2*) kargmax=`grep ARG_MAX /etc/conf/cf.d/stune 2>/dev/null` if test -n "$kargmax"; then - lt_cv_sys_max_cmd_len=`echo $kargmax | sed 's/.*[ ]//'` + lt_cv_sys_max_cmd_len=`echo $kargmax | $SED 's/.*[ ]//'` else lt_cv_sys_max_cmd_len=32768 fi @@ -7421,6 +8312,114 @@ esac +if test -n "$ac_tool_prefix"; then + # Extract the first word of "${ac_tool_prefix}file", so it can be a program name with args. +set dummy ${ac_tool_prefix}file; ac_word=$2 +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 +printf %s "checking for $ac_word... " >&6; } +if test ${ac_cv_prog_FILECMD+y} +then : + printf %s "(cached) " >&6 +else $as_nop + if test -n "$FILECMD"; then + ac_cv_prog_FILECMD="$FILECMD" # Let the user override the test. +else +as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +for as_dir in $PATH +do + IFS=$as_save_IFS + case $as_dir in #((( + '') as_dir=./ ;; + */) ;; + *) as_dir=$as_dir/ ;; + esac + for ac_exec_ext in '' $ac_executable_extensions; do + if as_fn_executable_p "$as_dir$ac_word$ac_exec_ext"; then + ac_cv_prog_FILECMD="${ac_tool_prefix}file" + printf "%s\n" "$as_me:${as_lineno-$LINENO}: found $as_dir$ac_word$ac_exec_ext" >&5 + break 2 + fi +done + done +IFS=$as_save_IFS + +fi +fi +FILECMD=$ac_cv_prog_FILECMD +if test -n "$FILECMD"; then + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $FILECMD" >&5 +printf "%s\n" "$FILECMD" >&6; } +else + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 +printf "%s\n" "no" >&6; } +fi + + +fi +if test -z "$ac_cv_prog_FILECMD"; then + ac_ct_FILECMD=$FILECMD + # Extract the first word of "file", so it can be a program name with args. +set dummy file; ac_word=$2 +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 +printf %s "checking for $ac_word... " >&6; } +if test ${ac_cv_prog_ac_ct_FILECMD+y} +then : + printf %s "(cached) " >&6 +else $as_nop + if test -n "$ac_ct_FILECMD"; then + ac_cv_prog_ac_ct_FILECMD="$ac_ct_FILECMD" # Let the user override the test. +else +as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +for as_dir in $PATH +do + IFS=$as_save_IFS + case $as_dir in #((( + '') as_dir=./ ;; + */) ;; + *) as_dir=$as_dir/ ;; + esac + for ac_exec_ext in '' $ac_executable_extensions; do + if as_fn_executable_p "$as_dir$ac_word$ac_exec_ext"; then + ac_cv_prog_ac_ct_FILECMD="file" + printf "%s\n" "$as_me:${as_lineno-$LINENO}: found $as_dir$ac_word$ac_exec_ext" >&5 + break 2 + fi +done + done +IFS=$as_save_IFS + +fi +fi +ac_ct_FILECMD=$ac_cv_prog_ac_ct_FILECMD +if test -n "$ac_ct_FILECMD"; then + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_ct_FILECMD" >&5 +printf "%s\n" "$ac_ct_FILECMD" >&6; } +else + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 +printf "%s\n" "no" >&6; } +fi + + if test "x$ac_ct_FILECMD" = x; then + FILECMD=":" + else + case $cross_compiling:$ac_tool_warned in +yes:) +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: WARNING: using cross tools not prefixed with host triplet" >&5 +printf "%s\n" "$as_me: WARNING: using cross tools not prefixed with host triplet" >&2;} +ac_tool_warned=yes ;; +esac + FILECMD=$ac_ct_FILECMD + fi +else + FILECMD="$ac_cv_prog_FILECMD" +fi + + + + + + + if test -n "$ac_tool_prefix"; then # Extract the first word of "${ac_tool_prefix}objdump", so it can be a program name with args. set dummy ${ac_tool_prefix}objdump; ac_word=$2 @@ -7561,7 +8560,7 @@ beos*) bsdi[45]*) lt_cv_deplibs_check_method='file_magic ELF [0-9][0-9]*-bit [ML]SB (shared object|dynamic lib)' - lt_cv_file_magic_cmd='/usr/bin/file -L' + lt_cv_file_magic_cmd='$FILECMD -L' lt_cv_file_magic_test_file=/shlib/libc.so ;; @@ -7595,14 +8594,14 @@ darwin* | rhapsody*) lt_cv_deplibs_check_method=pass_all ;; -freebsd* | dragonfly*) +freebsd* | dragonfly* | midnightbsd*) if echo __ELF__ | $CC -E - | $GREP __ELF__ > /dev/null; then case $host_cpu in i*86 ) # Not sure whether the presence of OpenBSD here was a mistake. # Let's accept both of them until this is cleared up. lt_cv_deplibs_check_method='file_magic (FreeBSD|OpenBSD|DragonFly)/i[3-9]86 (compact )?demand paged shared library' - lt_cv_file_magic_cmd=/usr/bin/file + lt_cv_file_magic_cmd=$FILECMD lt_cv_file_magic_test_file=`echo /usr/lib/libc.so.*` ;; esac @@ -7616,7 +8615,7 @@ haiku*) ;; hpux10.20* | hpux11*) - lt_cv_file_magic_cmd=/usr/bin/file + lt_cv_file_magic_cmd=$FILECMD case $host_cpu in ia64*) lt_cv_deplibs_check_method='file_magic (s[0-9][0-9][0-9]|ELF-[0-9][0-9]) shared object file - IA64' @@ -7663,7 +8662,7 @@ netbsd* | netbsdelf*-gnu) newos6*) lt_cv_deplibs_check_method='file_magic ELF [0-9][0-9]*-bit [ML]SB (executable|dynamic lib)' - lt_cv_file_magic_cmd=/usr/bin/file + lt_cv_file_magic_cmd=$FILECMD lt_cv_file_magic_test_file=/usr/lib/libnls.so ;; @@ -8033,13 +9032,29 @@ esac fi : ${AR=ar} -: ${AR_FLAGS=cr} +# Use ARFLAGS variable as AR's operation code to sync the variable naming with +# Automake. If both AR_FLAGS and ARFLAGS are specified, AR_FLAGS should have +# higher priority because thats what people were doing historically (setting +# ARFLAGS for automake and AR_FLAGS for libtool). FIXME: Make the AR_FLAGS +# variable obsoleted/removed. + +test ${AR_FLAGS+y} || AR_FLAGS=${ARFLAGS-cr} +lt_ar_flags=$AR_FLAGS + + + + + + +# Make AR_FLAGS overridable by 'make ARFLAGS='. Don't try to run-time override +# by AR_FLAGS because that was never working and AR_FLAGS is about to die. + @@ -8456,7 +9471,7 @@ esac if test "$lt_cv_nm_interface" = "MS dumpbin"; then # Gets list of data symbols to import. - lt_cv_sys_global_symbol_to_import="sed -n -e 's/^I .* \(.*\)$/\1/p'" + lt_cv_sys_global_symbol_to_import="$SED -n -e 's/^I .* \(.*\)$/\1/p'" # Adjust the below global symbol transforms to fixup imported variables. lt_cdecl_hook=" -e 's/^I .* \(.*\)$/extern __declspec(dllimport) char \1;/p'" lt_c_name_hook=" -e 's/^I .* \(.*\)$/ {\"\1\", (void *) 0},/p'" @@ -8474,20 +9489,20 @@ fi # Transform an extracted symbol line into a proper C declaration. # Some systems (esp. on ia64) link data and code symbols differently, # so use this general approach. -lt_cv_sys_global_symbol_to_cdecl="sed -n"\ +lt_cv_sys_global_symbol_to_cdecl="$SED -n"\ $lt_cdecl_hook\ " -e 's/^T .* \(.*\)$/extern int \1();/p'"\ " -e 's/^$symcode$symcode* .* \(.*\)$/extern char \1;/p'" # Transform an extracted symbol line into symbol name and symbol address -lt_cv_sys_global_symbol_to_c_name_address="sed -n"\ +lt_cv_sys_global_symbol_to_c_name_address="$SED -n"\ $lt_c_name_hook\ " -e 's/^: \(.*\) .*$/ {\"\1\", (void *) 0},/p'"\ " -e 's/^$symcode$symcode* .* \(.*\)$/ {\"\1\", (void *) \&\1},/p'" # Transform an extracted symbol line into symbol name with lib prefix and # symbol address. -lt_cv_sys_global_symbol_to_c_name_address_lib_prefix="sed -n"\ +lt_cv_sys_global_symbol_to_c_name_address_lib_prefix="$SED -n"\ $lt_c_name_lib_hook\ " -e 's/^: \(.*\) .*$/ {\"\1\", (void *) 0},/p'"\ " -e 's/^$symcode$symcode* .* \(lib.*\)$/ {\"\1\", (void *) \&\1},/p'"\ @@ -8511,7 +9526,7 @@ for ac_symprfx in "" "_"; do if test "$lt_cv_nm_interface" = "MS dumpbin"; then # Fake it for dumpbin and say T for any non-static function, # D for any global variable and I for any imported variable. - # Also find C++ and __fastcall symbols from MSVC++, + # Also find C++ and __fastcall symbols from MSVC++ or ICC, # which start with @ or ?. lt_cv_sys_global_symbol_pipe="$AWK '"\ " {last_section=section; section=\$ 3};"\ @@ -8529,9 +9544,9 @@ for ac_symprfx in "" "_"; do " s[1]~prfx {split(s[1],t,\"@\"); print f,t[1],substr(t[1],length(prfx))}"\ " ' prfx=^$ac_symprfx" else - lt_cv_sys_global_symbol_pipe="sed -n -e 's/^.*[ ]\($symcode$symcode*\)[ ][ ]*$ac_symprfx$sympat$opt_cr$/$symxfrm/p'" + lt_cv_sys_global_symbol_pipe="$SED -n -e 's/^.*[ ]\($symcode$symcode*\)[ ][ ]*$ac_symprfx$sympat$opt_cr$/$symxfrm/p'" fi - lt_cv_sys_global_symbol_pipe="$lt_cv_sys_global_symbol_pipe | sed '/ __gnu_lto/d'" + lt_cv_sys_global_symbol_pipe="$lt_cv_sys_global_symbol_pipe | $SED '/ __gnu_lto/d'" # Check to see that the pipe works correctly. pipe_works=no @@ -8731,7 +9746,7 @@ case $with_sysroot in #( fi ;; #( /*) - lt_sysroot=`echo "$with_sysroot" | sed -e "$sed_quote_subst"` + lt_sysroot=`echo "$with_sysroot" | $SED -e "$sed_quote_subst"` ;; #( no|'') ;; #( @@ -8856,7 +9871,7 @@ ia64-*-hpux*) ac_status=$? printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 test $ac_status = 0; }; then - case `/usr/bin/file conftest.$ac_objext` in + case `$FILECMD conftest.$ac_objext` in *ELF-32*) HPUX_IA64_MODE=32 ;; @@ -8877,7 +9892,7 @@ ia64-*-hpux*) printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 test $ac_status = 0; }; then if test yes = "$lt_cv_prog_gnu_ld"; then - case `/usr/bin/file conftest.$ac_objext` in + case `$FILECMD conftest.$ac_objext` in *32-bit*) LD="${LD-ld} -melf32bsmip" ;; @@ -8889,7 +9904,7 @@ ia64-*-hpux*) ;; esac else - case `/usr/bin/file conftest.$ac_objext` in + case `$FILECMD conftest.$ac_objext` in *32-bit*) LD="${LD-ld} -32" ;; @@ -8915,7 +9930,7 @@ mips64*-*linux*) printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 test $ac_status = 0; }; then emul=elf - case `/usr/bin/file conftest.$ac_objext` in + case `$FILECMD conftest.$ac_objext` in *32-bit*) emul="${emul}32" ;; @@ -8923,7 +9938,7 @@ mips64*-*linux*) emul="${emul}64" ;; esac - case `/usr/bin/file conftest.$ac_objext` in + case `$FILECMD conftest.$ac_objext` in *MSB*) emul="${emul}btsmip" ;; @@ -8931,7 +9946,7 @@ mips64*-*linux*) emul="${emul}ltsmip" ;; esac - case `/usr/bin/file conftest.$ac_objext` in + case `$FILECMD conftest.$ac_objext` in *N32*) emul="${emul}n32" ;; @@ -8955,14 +9970,14 @@ s390*-*linux*|s390*-*tpf*|sparc*-*linux*) ac_status=$? printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 test $ac_status = 0; }; then - case `/usr/bin/file conftest.o` in + case `$FILECMD conftest.o` in *32-bit*) case $host in x86_64-*kfreebsd*-gnu) LD="${LD-ld} -m elf_i386_fbsd" ;; x86_64-*linux*) - case `/usr/bin/file conftest.o` in + case `$FILECMD conftest.o` in *x86-64*) LD="${LD-ld} -m elf32_x86_64" ;; @@ -9070,7 +10085,7 @@ printf "%s\n" "$lt_cv_cc_needs_belf" >&6; } ac_status=$? printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 test $ac_status = 0; }; then - case `/usr/bin/file conftest.o` in + case `$FILECMD conftest.o` in *64-bit*) case $lt_cv_prog_gnu_ld in yes*) @@ -9853,8 +10868,8 @@ int forced_loaded() { return 2;} _LT_EOF echo "$LTCC $LTCFLAGS -c -o conftest.o conftest.c" >&5 $LTCC $LTCFLAGS -c -o conftest.o conftest.c 2>&5 - echo "$AR cr libconftest.a conftest.o" >&5 - $AR cr libconftest.a conftest.o 2>&5 + echo "$AR $AR_FLAGS libconftest.a conftest.o" >&5 + $AR $AR_FLAGS libconftest.a conftest.o 2>&5 echo "$RANLIB libconftest.a" >&5 $RANLIB libconftest.a 2>&5 cat > conftest.c << _LT_EOF @@ -9881,17 +10896,12 @@ printf "%s\n" "$lt_cv_ld_force_load" >&6; } _lt_dar_allow_undefined='$wl-undefined ${wl}suppress' ;; darwin1.*) _lt_dar_allow_undefined='$wl-flat_namespace $wl-undefined ${wl}suppress' ;; - darwin*) # darwin 5.x on - # if running on 10.5 or later, the deployment target defaults - # to the OS version, if on x86, and 10.4, the deployment - # target defaults to 10.4. Don't you love it? - case ${MACOSX_DEPLOYMENT_TARGET-10.0},$host in - 10.0,*86*-darwin8*|10.0,*-darwin[912]*) - _lt_dar_allow_undefined='$wl-undefined ${wl}dynamic_lookup' ;; - 10.[012][,.]*) - _lt_dar_allow_undefined='$wl-flat_namespace $wl-undefined ${wl}suppress' ;; - 10.*|11.*) - _lt_dar_allow_undefined='$wl-undefined ${wl}dynamic_lookup' ;; + darwin*) + case $MACOSX_DEPLOYMENT_TARGET,$host in + 10.[012],*|,*powerpc*-darwin[5-8]*) + _lt_dar_allow_undefined='$wl-flat_namespace $wl-undefined ${wl}suppress' ;; + *) + _lt_dar_allow_undefined='$wl-undefined ${wl}dynamic_lookup' ;; esac ;; esac @@ -10588,8 +11598,8 @@ esac ofile=libtool can_build_shared=yes -# All known linkers require a '.a' archive for static linking (except MSVC, -# which needs '.lib'). +# All known linkers require a '.a' archive for static linking (except MSVC and +# ICC, which need '.lib'). libext=a with_gnu_ld=$lt_cv_prog_gnu_ld @@ -11107,7 +12117,7 @@ lt_prog_compiler_static= lt_prog_compiler_static='-qstaticlink' ;; *) - case `$CC -V 2>&1 | sed 5q` in + case `$CC -V 2>&1 | $SED 5q` in *Sun\ Ceres\ Fortran* | *Sun*Fortran*\ [1-7].* | *Sun*Fortran*\ 8.[0-3]*) # Sun Fortran 8.3 passes all unrecognized flags to the linker lt_prog_compiler_pic='-KPIC' @@ -11530,15 +12540,15 @@ printf %s "checking whether the $compiler linker ($LD) supports shared libraries case $host_os in cygwin* | mingw* | pw32* | cegcc*) - # FIXME: the MSVC++ port hasn't been tested in a loooong time + # FIXME: the MSVC++ and ICC port hasn't been tested in a loooong time # When not using gcc, we currently assume that we are using - # Microsoft Visual C++. + # Microsoft Visual C++ or Intel C++ Compiler. if test yes != "$GCC"; then with_gnu_ld=no fi ;; interix*) - # we just hope/assume this is gcc and not c89 (= MSVC++) + # we just hope/assume this is gcc and not c89 (= MSVC++ or ICC) with_gnu_ld=yes ;; openbsd* | bitrig*) @@ -11593,7 +12603,7 @@ printf %s "checking whether the $compiler linker ($LD) supports shared libraries whole_archive_flag_spec= fi supports_anon_versioning=no - case `$LD -v | $SED -e 's/(^)\+)\s\+//' 2>&1` in + case `$LD -v | $SED -e 's/([^)]\+)\s\+//' 2>&1` in *GNU\ gold*) supports_anon_versioning=yes ;; *\ [01].* | *\ 2.[0-9].* | *\ 2.10.*) ;; # catch versions < 2.11 *\ 2.11.93.0.2\ *) supports_anon_versioning=yes ;; # RH7.3 ... @@ -11705,6 +12715,7 @@ _LT_EOF emximp -o $lib $output_objdir/$libname.def' old_archive_From_new_cmds='emximp -o $output_objdir/${libname}_dll.a $output_objdir/$libname.def' enable_shared_with_static_runtimes=yes + file_list_spec='@' ;; interix[3-9]*) @@ -11719,7 +12730,7 @@ _LT_EOF # 256 KiB-aligned image base between 0x50000000 and 0x6FFC0000 at link # time. Moving up from 0x10000000 also allows more sbrk(2) space. archive_cmds='$CC -shared $pic_flag $libobjs $deplibs $compiler_flags $wl-h,$soname $wl--image-base,`expr ${RANDOM-$$} % 4096 / 2 \* 262144 + 1342177280` -o $lib' - archive_expsym_cmds='sed "s|^|_|" $export_symbols >$output_objdir/$soname.expsym~$CC -shared $pic_flag $libobjs $deplibs $compiler_flags $wl-h,$soname $wl--retain-symbols-file,$output_objdir/$soname.expsym $wl--image-base,`expr ${RANDOM-$$} % 4096 / 2 \* 262144 + 1342177280` -o $lib' + archive_expsym_cmds='$SED "s|^|_|" $export_symbols >$output_objdir/$soname.expsym~$CC -shared $pic_flag $libobjs $deplibs $compiler_flags $wl-h,$soname $wl--retain-symbols-file,$output_objdir/$soname.expsym $wl--image-base,`expr ${RANDOM-$$} % 4096 / 2 \* 262144 + 1342177280` -o $lib' ;; gnu* | linux* | tpf* | k*bsd*-gnu | kopensolaris*-gnu) @@ -11762,7 +12773,7 @@ _LT_EOF compiler_needs_object=yes ;; esac - case `$CC -V 2>&1 | sed 5q` in + case `$CC -V 2>&1 | $SED 5q` in *Sun\ C*) # Sun C 5.9 whole_archive_flag_spec='$wl--whole-archive`new_convenience=; for conv in $convenience\"\"; do test -z \"$conv\" || new_convenience=\"$new_convenience,$conv\"; done; func_echo_all \"$new_convenience\"` $wl--no-whole-archive' compiler_needs_object=yes @@ -11774,13 +12785,14 @@ _LT_EOF if test yes = "$supports_anon_versioning"; then archive_expsym_cmds='echo "{ global:" > $output_objdir/$libname.ver~ - cat $export_symbols | sed -e "s/\(.*\)/\1;/" >> $output_objdir/$libname.ver~ + cat $export_symbols | $SED -e "s/\(.*\)/\1;/" >> $output_objdir/$libname.ver~ echo "local: *; };" >> $output_objdir/$libname.ver~ $CC '"$tmp_sharedflag""$tmp_addflag"' $libobjs $deplibs $compiler_flags $wl-soname $wl$soname $wl-version-script $wl$output_objdir/$libname.ver -o $lib' fi case $cc_basename in tcc*) + hardcode_libdir_flag_spec='$wl-rpath $wl$libdir' export_dynamic_flag_spec='-rdynamic' ;; xlf* | bgf* | bgxlf* | mpixlf*) @@ -11790,7 +12802,7 @@ _LT_EOF archive_cmds='$LD -shared $libobjs $deplibs $linker_flags -soname $soname -o $lib' if test yes = "$supports_anon_versioning"; then archive_expsym_cmds='echo "{ global:" > $output_objdir/$libname.ver~ - cat $export_symbols | sed -e "s/\(.*\)/\1;/" >> $output_objdir/$libname.ver~ + cat $export_symbols | $SED -e "s/\(.*\)/\1;/" >> $output_objdir/$libname.ver~ echo "local: *; };" >> $output_objdir/$libname.ver~ $LD -shared $libobjs $deplibs $linker_flags -soname $soname -version-script $output_objdir/$libname.ver -o $lib' fi @@ -11922,7 +12934,7 @@ _LT_EOF if $NM -V 2>&1 | $GREP 'GNU' > /dev/null; then export_symbols_cmds='$NM -Bpg $libobjs $convenience | awk '\''{ if (((\$ 2 == "T") || (\$ 2 == "D") || (\$ 2 == "B") || (\$ 2 == "W")) && (substr(\$ 3,1,1) != ".")) { if (\$ 2 == "W") { print \$ 3 " weak" } else { print \$ 3 } } }'\'' | sort -u > $export_symbols' else - export_symbols_cmds='`func_echo_all $NM | $SED -e '\''s/B\([^B]*\)$/P\1/'\''` -PCpgl $libobjs $convenience | awk '\''{ if (((\$ 2 == "T") || (\$ 2 == "D") || (\$ 2 == "B") || (\$ 2 == "W") || (\$ 2 == "V") || (\$ 2 == "Z")) && (substr(\$ 1,1,1) != ".")) { if ((\$ 2 == "W") || (\$ 2 == "V") || (\$ 2 == "Z")) { print \$ 1 " weak" } else { print \$ 1 } } }'\'' | sort -u > $export_symbols' + export_symbols_cmds='`func_echo_all $NM | $SED -e '\''s/B\([^B]*\)$/P\1/'\''` -PCpgl $libobjs $convenience | awk '\''{ if (((\$ 2 == "T") || (\$ 2 == "D") || (\$ 2 == "B") || (\$ 2 == "L") || (\$ 2 == "W") || (\$ 2 == "V") || (\$ 2 == "Z")) && (substr(\$ 1,1,1) != ".")) { if ((\$ 2 == "W") || (\$ 2 == "V") || (\$ 2 == "Z")) { print \$ 1 " weak" } else { print \$ 1 } } }'\'' | sort -u > $export_symbols' fi aix_use_runtimelinking=no @@ -12193,12 +13205,12 @@ fi cygwin* | mingw* | pw32* | cegcc*) # When not using gcc, we currently assume that we are using - # Microsoft Visual C++. + # Microsoft Visual C++ or Intel C++ Compiler. # hardcode_libdir_flag_spec is actually meaningless, as there is # no search path for DLLs. case $cc_basename in - cl*) - # Native MSVC + cl* | icl*) + # Native MSVC or ICC hardcode_libdir_flag_spec=' ' allow_undefined_flag=unsupported always_export_symbols=yes @@ -12239,7 +13251,7 @@ fi fi' ;; *) - # Assume MSVC wrapper + # Assume MSVC and ICC wrapper hardcode_libdir_flag_spec=' ' allow_undefined_flag=unsupported # Tell ltmain to make .lib files, not .a files. @@ -12280,8 +13292,8 @@ fi output_verbose_link_cmd=func_echo_all archive_cmds="\$CC -dynamiclib \$allow_undefined_flag -o \$lib \$libobjs \$deplibs \$compiler_flags -install_name \$rpath/\$soname \$verstring $_lt_dar_single_mod$_lt_dsymutil" module_cmds="\$CC \$allow_undefined_flag -o \$lib -bundle \$libobjs \$deplibs \$compiler_flags$_lt_dsymutil" - archive_expsym_cmds="sed 's|^|_|' < \$export_symbols > \$output_objdir/\$libname-symbols.expsym~\$CC -dynamiclib \$allow_undefined_flag -o \$lib \$libobjs \$deplibs \$compiler_flags -install_name \$rpath/\$soname \$verstring $_lt_dar_single_mod$_lt_dar_export_syms$_lt_dsymutil" - module_expsym_cmds="sed -e 's|^|_|' < \$export_symbols > \$output_objdir/\$libname-symbols.expsym~\$CC \$allow_undefined_flag -o \$lib -bundle \$libobjs \$deplibs \$compiler_flags$_lt_dar_export_syms$_lt_dsymutil" + archive_expsym_cmds="$SED 's|^|_|' < \$export_symbols > \$output_objdir/\$libname-symbols.expsym~\$CC -dynamiclib \$allow_undefined_flag -o \$lib \$libobjs \$deplibs \$compiler_flags -install_name \$rpath/\$soname \$verstring $_lt_dar_single_mod$_lt_dar_export_syms$_lt_dsymutil" + module_expsym_cmds="$SED -e 's|^|_|' < \$export_symbols > \$output_objdir/\$libname-symbols.expsym~\$CC \$allow_undefined_flag -o \$lib -bundle \$libobjs \$deplibs \$compiler_flags$_lt_dar_export_syms$_lt_dsymutil" else ld_shlibs=no @@ -12315,7 +13327,7 @@ fi ;; # FreeBSD 3 and greater uses gcc -shared to do shared libraries. - freebsd* | dragonfly*) + freebsd* | dragonfly* | midnightbsd*) archive_cmds='$CC -shared $pic_flag -o $lib $libobjs $deplibs $compiler_flags' hardcode_libdir_flag_spec='-R$libdir' hardcode_direct=yes @@ -12496,6 +13508,7 @@ printf "%s\n" "$lt_cv_irix_exported_symbol" >&6; } # Fabrice Bellard et al's Tiny C Compiler ld_shlibs=yes archive_cmds='$CC -shared $pic_flag -o $lib $libobjs $deplibs $compiler_flags' + hardcode_libdir_flag_spec='$wl-rpath $wl$libdir' ;; esac ;; @@ -12567,6 +13580,7 @@ printf "%s\n" "$lt_cv_irix_exported_symbol" >&6; } emximp -o $lib $output_objdir/$libname.def' old_archive_From_new_cmds='emximp -o $output_objdir/${libname}_dll.a $output_objdir/$libname.def' enable_shared_with_static_runtimes=yes + file_list_spec='@' ;; osf3*) @@ -13259,7 +14273,7 @@ cygwin* | mingw* | pw32* | cegcc*) case $host_os in cygwin*) # Cygwin DLLs use 'cyg' prefix rather than 'lib' - soname_spec='`echo $libname | sed -e 's/^lib/cyg/'``echo $release | $SED -e 's/[.]/-/g'`$versuffix$shared_ext' + soname_spec='`echo $libname | $SED -e 's/^lib/cyg/'``echo $release | $SED -e 's/[.]/-/g'`$versuffix$shared_ext' sys_lib_search_path_spec="$sys_lib_search_path_spec /usr/lib/w32api" ;; @@ -13269,14 +14283,14 @@ cygwin* | mingw* | pw32* | cegcc*) ;; pw32*) # pw32 DLLs use 'pw' prefix rather than 'lib' - library_names_spec='`echo $libname | sed -e 's/^lib/pw/'``echo $release | $SED -e 's/[.]/-/g'`$versuffix$shared_ext' + library_names_spec='`echo $libname | $SED -e 's/^lib/pw/'``echo $release | $SED -e 's/[.]/-/g'`$versuffix$shared_ext' ;; esac dynamic_linker='Win32 ld.exe' ;; - *,cl*) - # Native MSVC + *,cl* | *,icl*) + # Native MSVC or ICC libname_spec='$name' soname_spec='$libname`echo $release | $SED -e 's/[.]/-/g'`$versuffix$shared_ext' library_names_spec='$libname.dll.lib' @@ -13295,7 +14309,7 @@ cygwin* | mingw* | pw32* | cegcc*) done IFS=$lt_save_ifs # Convert to MSYS style. - sys_lib_search_path_spec=`$ECHO "$sys_lib_search_path_spec" | sed -e 's|\\\\|/|g' -e 's| \\([a-zA-Z]\\):| /\\1|g' -e 's|^ ||'` + sys_lib_search_path_spec=`$ECHO "$sys_lib_search_path_spec" | $SED -e 's|\\\\|/|g' -e 's| \\([a-zA-Z]\\):| /\\1|g' -e 's|^ ||'` ;; cygwin*) # Convert to unix form, then to dos form, then back to unix form @@ -13332,7 +14346,7 @@ cygwin* | mingw* | pw32* | cegcc*) ;; *) - # Assume MSVC wrapper + # Assume MSVC and ICC wrapper library_names_spec='$libname`echo $release | $SED -e 's/[.]/-/g'`$versuffix$shared_ext $libname.lib' dynamic_linker='Win32 ld.exe' ;; @@ -13365,7 +14379,7 @@ dgux*) shlibpath_var=LD_LIBRARY_PATH ;; -freebsd* | dragonfly*) +freebsd* | dragonfly* | midnightbsd*) # DragonFly does not have aout. When/if they implement a new # versioning mechanism, adjust this. if test -x /usr/bin/objformat; then @@ -14530,30 +15544,41 @@ striplib= old_striplib= { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether stripping libraries is possible" >&5 printf %s "checking whether stripping libraries is possible... " >&6; } -if test -n "$STRIP" && $STRIP -V 2>&1 | $GREP "GNU strip" >/dev/null; then - test -z "$old_striplib" && old_striplib="$STRIP --strip-debug" - test -z "$striplib" && striplib="$STRIP --strip-unneeded" - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: yes" >&5 -printf "%s\n" "yes" >&6; } +if test -z "$STRIP"; then + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 +printf "%s\n" "no" >&6; } else -# FIXME - insert some real tests, host_os isn't really good enough - case $host_os in - darwin*) - if test -n "$STRIP"; then + if $STRIP -V 2>&1 | $GREP "GNU strip" >/dev/null; then + old_striplib="$STRIP --strip-debug" + striplib="$STRIP --strip-unneeded" + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +printf "%s\n" "yes" >&6; } + else + case $host_os in + darwin*) + # FIXME - insert some real tests, host_os isn't really good enough striplib="$STRIP -x" old_striplib="$STRIP -S" { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: yes" >&5 printf "%s\n" "yes" >&6; } - else - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 + ;; + freebsd*) + if $STRIP -V 2>&1 | $GREP "elftoolchain" >/dev/null; then + old_striplib="$STRIP --strip-debug" + striplib="$STRIP --strip-unneeded" + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +printf "%s\n" "yes" >&6; } + else + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 printf "%s\n" "no" >&6; } - fi - ;; - *) - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 + fi + ;; + *) + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 printf "%s\n" "no" >&6; } - ;; - esac + ;; + esac + fi fi @@ -15323,8 +16348,8 @@ fi cygwin* | mingw* | pw32* | cegcc*) case $GXX,$cc_basename in - ,cl* | no,cl*) - # Native MSVC + ,cl* | no,cl* | ,icl* | no,icl*) + # Native MSVC or ICC # hardcode_libdir_flag_spec is actually meaningless, as there is # no search path for DLLs. hardcode_libdir_flag_spec_CXX=' ' @@ -15415,11 +16440,11 @@ fi output_verbose_link_cmd=func_echo_all archive_cmds_CXX="\$CC -dynamiclib \$allow_undefined_flag -o \$lib \$libobjs \$deplibs \$compiler_flags -install_name \$rpath/\$soname \$verstring $_lt_dar_single_mod$_lt_dsymutil" module_cmds_CXX="\$CC \$allow_undefined_flag -o \$lib -bundle \$libobjs \$deplibs \$compiler_flags$_lt_dsymutil" - archive_expsym_cmds_CXX="sed 's|^|_|' < \$export_symbols > \$output_objdir/\$libname-symbols.expsym~\$CC -dynamiclib \$allow_undefined_flag -o \$lib \$libobjs \$deplibs \$compiler_flags -install_name \$rpath/\$soname \$verstring $_lt_dar_single_mod$_lt_dar_export_syms$_lt_dsymutil" - module_expsym_cmds_CXX="sed -e 's|^|_|' < \$export_symbols > \$output_objdir/\$libname-symbols.expsym~\$CC \$allow_undefined_flag -o \$lib -bundle \$libobjs \$deplibs \$compiler_flags$_lt_dar_export_syms$_lt_dsymutil" + archive_expsym_cmds_CXX="$SED 's|^|_|' < \$export_symbols > \$output_objdir/\$libname-symbols.expsym~\$CC -dynamiclib \$allow_undefined_flag -o \$lib \$libobjs \$deplibs \$compiler_flags -install_name \$rpath/\$soname \$verstring $_lt_dar_single_mod$_lt_dar_export_syms$_lt_dsymutil" + module_expsym_cmds_CXX="$SED -e 's|^|_|' < \$export_symbols > \$output_objdir/\$libname-symbols.expsym~\$CC \$allow_undefined_flag -o \$lib -bundle \$libobjs \$deplibs \$compiler_flags$_lt_dar_export_syms$_lt_dsymutil" if test yes != "$lt_cv_apple_cc_single_mod"; then archive_cmds_CXX="\$CC -r -keep_private_externs -nostdlib -o \$lib-master.o \$libobjs~\$CC -dynamiclib \$allow_undefined_flag -o \$lib \$lib-master.o \$deplibs \$compiler_flags -install_name \$rpath/\$soname \$verstring$_lt_dsymutil" - archive_expsym_cmds_CXX="sed 's|^|_|' < \$export_symbols > \$output_objdir/\$libname-symbols.expsym~\$CC -r -keep_private_externs -nostdlib -o \$lib-master.o \$libobjs~\$CC -dynamiclib \$allow_undefined_flag -o \$lib \$lib-master.o \$deplibs \$compiler_flags -install_name \$rpath/\$soname \$verstring$_lt_dar_export_syms$_lt_dsymutil" + archive_expsym_cmds_CXX="$SED 's|^|_|' < \$export_symbols > \$output_objdir/\$libname-symbols.expsym~\$CC -r -keep_private_externs -nostdlib -o \$lib-master.o \$libobjs~\$CC -dynamiclib \$allow_undefined_flag -o \$lib \$lib-master.o \$deplibs \$compiler_flags -install_name \$rpath/\$soname \$verstring$_lt_dar_export_syms$_lt_dsymutil" fi else @@ -15454,6 +16479,7 @@ fi emximp -o $lib $output_objdir/$libname.def' old_archive_From_new_cmds_CXX='emximp -o $output_objdir/${libname}_dll.a $output_objdir/$libname.def' enable_shared_with_static_runtimes_CXX=yes + file_list_spec_CXX='@' ;; dgux*) @@ -15484,7 +16510,7 @@ fi archive_cmds_need_lc_CXX=no ;; - freebsd* | dragonfly*) + freebsd* | dragonfly* | midnightbsd*) # FreeBSD 3 and later use GNU C++ and GNU ld with standard ELF # conventions ld_shlibs_CXX=yes @@ -15621,7 +16647,7 @@ fi # 256 KiB-aligned image base between 0x50000000 and 0x6FFC0000 at link # time. Moving up from 0x10000000 also allows more sbrk(2) space. archive_cmds_CXX='$CC -shared $pic_flag $libobjs $deplibs $compiler_flags $wl-h,$soname $wl--image-base,`expr ${RANDOM-$$} % 4096 / 2 \* 262144 + 1342177280` -o $lib' - archive_expsym_cmds_CXX='sed "s|^|_|" $export_symbols >$output_objdir/$soname.expsym~$CC -shared $pic_flag $libobjs $deplibs $compiler_flags $wl-h,$soname $wl--retain-symbols-file,$output_objdir/$soname.expsym $wl--image-base,`expr ${RANDOM-$$} % 4096 / 2 \* 262144 + 1342177280` -o $lib' + archive_expsym_cmds_CXX='$SED "s|^|_|" $export_symbols >$output_objdir/$soname.expsym~$CC -shared $pic_flag $libobjs $deplibs $compiler_flags $wl-h,$soname $wl--retain-symbols-file,$output_objdir/$soname.expsym $wl--image-base,`expr ${RANDOM-$$} % 4096 / 2 \* 262144 + 1342177280` -o $lib' ;; irix5* | irix6*) case $cc_basename in @@ -15761,13 +16787,13 @@ fi archive_cmds_CXX='$CC -qmkshrobj $libobjs $deplibs $compiler_flags $wl-soname $wl$soname -o $lib' if test yes = "$supports_anon_versioning"; then archive_expsym_cmds_CXX='echo "{ global:" > $output_objdir/$libname.ver~ - cat $export_symbols | sed -e "s/\(.*\)/\1;/" >> $output_objdir/$libname.ver~ + cat $export_symbols | $SED -e "s/\(.*\)/\1;/" >> $output_objdir/$libname.ver~ echo "local: *; };" >> $output_objdir/$libname.ver~ $CC -qmkshrobj $libobjs $deplibs $compiler_flags $wl-soname $wl$soname $wl-version-script $wl$output_objdir/$libname.ver -o $lib' fi ;; *) - case `$CC -V 2>&1 | sed 5q` in + case `$CC -V 2>&1 | $SED 5q` in *Sun\ C*) # Sun C++ 5.9 no_undefined_flag_CXX=' -zdefs' @@ -16424,7 +17450,7 @@ lt_prog_compiler_static_CXX= ;; esac ;; - freebsd* | dragonfly*) + freebsd* | dragonfly* | midnightbsd*) # FreeBSD uses GNU C++ ;; hpux9* | hpux10* | hpux11*) @@ -16507,7 +17533,7 @@ lt_prog_compiler_static_CXX= lt_prog_compiler_static_CXX='-qstaticlink' ;; *) - case `$CC -V 2>&1 | sed 5q` in + case `$CC -V 2>&1 | $SED 5q` in *Sun\ C*) # Sun C++ 5.9 lt_prog_compiler_pic_CXX='-KPIC' @@ -16894,7 +17920,7 @@ printf %s "checking whether the $compiler linker ($LD) supports shared libraries if $NM -V 2>&1 | $GREP 'GNU' > /dev/null; then export_symbols_cmds_CXX='$NM -Bpg $libobjs $convenience | awk '\''{ if (((\$ 2 == "T") || (\$ 2 == "D") || (\$ 2 == "B") || (\$ 2 == "W")) && (substr(\$ 3,1,1) != ".")) { if (\$ 2 == "W") { print \$ 3 " weak" } else { print \$ 3 } } }'\'' | sort -u > $export_symbols' else - export_symbols_cmds_CXX='`func_echo_all $NM | $SED -e '\''s/B\([^B]*\)$/P\1/'\''` -PCpgl $libobjs $convenience | awk '\''{ if (((\$ 2 == "T") || (\$ 2 == "D") || (\$ 2 == "B") || (\$ 2 == "W") || (\$ 2 == "V") || (\$ 2 == "Z")) && (substr(\$ 1,1,1) != ".")) { if ((\$ 2 == "W") || (\$ 2 == "V") || (\$ 2 == "Z")) { print \$ 1 " weak" } else { print \$ 1 } } }'\'' | sort -u > $export_symbols' + export_symbols_cmds_CXX='`func_echo_all $NM | $SED -e '\''s/B\([^B]*\)$/P\1/'\''` -PCpgl $libobjs $convenience | awk '\''{ if (((\$ 2 == "T") || (\$ 2 == "D") || (\$ 2 == "B") || (\$ 2 == "L") || (\$ 2 == "W") || (\$ 2 == "V") || (\$ 2 == "Z")) && (substr(\$ 1,1,1) != ".")) { if ((\$ 2 == "W") || (\$ 2 == "V") || (\$ 2 == "Z")) { print \$ 1 " weak" } else { print \$ 1 } } }'\'' | sort -u > $export_symbols' fi ;; pw32*) @@ -16902,7 +17928,7 @@ printf %s "checking whether the $compiler linker ($LD) supports shared libraries ;; cygwin* | mingw* | cegcc*) case $cc_basename in - cl*) + cl* | icl*) exclude_expsyms_CXX='_NULL_IMPORT_DESCRIPTOR|_IMPORT_DESCRIPTOR_.*' ;; *) @@ -17253,7 +18279,7 @@ cygwin* | mingw* | pw32* | cegcc*) case $host_os in cygwin*) # Cygwin DLLs use 'cyg' prefix rather than 'lib' - soname_spec='`echo $libname | sed -e 's/^lib/cyg/'``echo $release | $SED -e 's/[.]/-/g'`$versuffix$shared_ext' + soname_spec='`echo $libname | $SED -e 's/^lib/cyg/'``echo $release | $SED -e 's/[.]/-/g'`$versuffix$shared_ext' ;; mingw* | cegcc*) @@ -17262,14 +18288,14 @@ cygwin* | mingw* | pw32* | cegcc*) ;; pw32*) # pw32 DLLs use 'pw' prefix rather than 'lib' - library_names_spec='`echo $libname | sed -e 's/^lib/pw/'``echo $release | $SED -e 's/[.]/-/g'`$versuffix$shared_ext' + library_names_spec='`echo $libname | $SED -e 's/^lib/pw/'``echo $release | $SED -e 's/[.]/-/g'`$versuffix$shared_ext' ;; esac dynamic_linker='Win32 ld.exe' ;; - *,cl*) - # Native MSVC + *,cl* | *,icl*) + # Native MSVC or ICC libname_spec='$name' soname_spec='$libname`echo $release | $SED -e 's/[.]/-/g'`$versuffix$shared_ext' library_names_spec='$libname.dll.lib' @@ -17288,7 +18314,7 @@ cygwin* | mingw* | pw32* | cegcc*) done IFS=$lt_save_ifs # Convert to MSYS style. - sys_lib_search_path_spec=`$ECHO "$sys_lib_search_path_spec" | sed -e 's|\\\\|/|g' -e 's| \\([a-zA-Z]\\):| /\\1|g' -e 's|^ ||'` + sys_lib_search_path_spec=`$ECHO "$sys_lib_search_path_spec" | $SED -e 's|\\\\|/|g' -e 's| \\([a-zA-Z]\\):| /\\1|g' -e 's|^ ||'` ;; cygwin*) # Convert to unix form, then to dos form, then back to unix form @@ -17325,7 +18351,7 @@ cygwin* | mingw* | pw32* | cegcc*) ;; *) - # Assume MSVC wrapper + # Assume MSVC and ICC wrapper library_names_spec='$libname`echo $release | $SED -e 's/[.]/-/g'`$versuffix$shared_ext $libname.lib' dynamic_linker='Win32 ld.exe' ;; @@ -17357,7 +18383,7 @@ dgux*) shlibpath_var=LD_LIBRARY_PATH ;; -freebsd* | dragonfly*) +freebsd* | dragonfly* | midnightbsd*) # DragonFly does not have aout. When/if they implement a new # versioning mechanism, adjust this. if test -x /usr/bin/objformat; then @@ -18006,7 +19032,219 @@ do esac for ac_exec_ext in '' $ac_executable_extensions; do if as_fn_executable_p "$as_dir$ac_word$ac_exec_ext"; then - ac_cv_prog_CC="${ac_tool_prefix}gcc" + ac_cv_prog_CC="${ac_tool_prefix}gcc" + printf "%s\n" "$as_me:${as_lineno-$LINENO}: found $as_dir$ac_word$ac_exec_ext" >&5 + break 2 + fi +done + done +IFS=$as_save_IFS + +fi +fi +CC=$ac_cv_prog_CC +if test -n "$CC"; then + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $CC" >&5 +printf "%s\n" "$CC" >&6; } +else + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 +printf "%s\n" "no" >&6; } +fi + + +fi +if test -z "$ac_cv_prog_CC"; then + ac_ct_CC=$CC + # Extract the first word of "gcc", so it can be a program name with args. +set dummy gcc; ac_word=$2 +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 +printf %s "checking for $ac_word... " >&6; } +if test ${ac_cv_prog_ac_ct_CC+y} +then : + printf %s "(cached) " >&6 +else $as_nop + if test -n "$ac_ct_CC"; then + ac_cv_prog_ac_ct_CC="$ac_ct_CC" # Let the user override the test. +else +as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +for as_dir in $PATH +do + IFS=$as_save_IFS + case $as_dir in #((( + '') as_dir=./ ;; + */) ;; + *) as_dir=$as_dir/ ;; + esac + for ac_exec_ext in '' $ac_executable_extensions; do + if as_fn_executable_p "$as_dir$ac_word$ac_exec_ext"; then + ac_cv_prog_ac_ct_CC="gcc" + printf "%s\n" "$as_me:${as_lineno-$LINENO}: found $as_dir$ac_word$ac_exec_ext" >&5 + break 2 + fi +done + done +IFS=$as_save_IFS + +fi +fi +ac_ct_CC=$ac_cv_prog_ac_ct_CC +if test -n "$ac_ct_CC"; then + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_ct_CC" >&5 +printf "%s\n" "$ac_ct_CC" >&6; } +else + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 +printf "%s\n" "no" >&6; } +fi + + if test "x$ac_ct_CC" = x; then + CC="" + else + case $cross_compiling:$ac_tool_warned in +yes:) +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: WARNING: using cross tools not prefixed with host triplet" >&5 +printf "%s\n" "$as_me: WARNING: using cross tools not prefixed with host triplet" >&2;} +ac_tool_warned=yes ;; +esac + CC=$ac_ct_CC + fi +else + CC="$ac_cv_prog_CC" +fi + +if test -z "$CC"; then + if test -n "$ac_tool_prefix"; then + # Extract the first word of "${ac_tool_prefix}cc", so it can be a program name with args. +set dummy ${ac_tool_prefix}cc; ac_word=$2 +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 +printf %s "checking for $ac_word... " >&6; } +if test ${ac_cv_prog_CC+y} +then : + printf %s "(cached) " >&6 +else $as_nop + if test -n "$CC"; then + ac_cv_prog_CC="$CC" # Let the user override the test. +else +as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +for as_dir in $PATH +do + IFS=$as_save_IFS + case $as_dir in #((( + '') as_dir=./ ;; + */) ;; + *) as_dir=$as_dir/ ;; + esac + for ac_exec_ext in '' $ac_executable_extensions; do + if as_fn_executable_p "$as_dir$ac_word$ac_exec_ext"; then + ac_cv_prog_CC="${ac_tool_prefix}cc" + printf "%s\n" "$as_me:${as_lineno-$LINENO}: found $as_dir$ac_word$ac_exec_ext" >&5 + break 2 + fi +done + done +IFS=$as_save_IFS + +fi +fi +CC=$ac_cv_prog_CC +if test -n "$CC"; then + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $CC" >&5 +printf "%s\n" "$CC" >&6; } +else + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 +printf "%s\n" "no" >&6; } +fi + + + fi +fi +if test -z "$CC"; then + # Extract the first word of "cc", so it can be a program name with args. +set dummy cc; ac_word=$2 +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 +printf %s "checking for $ac_word... " >&6; } +if test ${ac_cv_prog_CC+y} +then : + printf %s "(cached) " >&6 +else $as_nop + if test -n "$CC"; then + ac_cv_prog_CC="$CC" # Let the user override the test. +else + ac_prog_rejected=no +as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +for as_dir in $PATH +do + IFS=$as_save_IFS + case $as_dir in #((( + '') as_dir=./ ;; + */) ;; + *) as_dir=$as_dir/ ;; + esac + for ac_exec_ext in '' $ac_executable_extensions; do + if as_fn_executable_p "$as_dir$ac_word$ac_exec_ext"; then + if test "$as_dir$ac_word$ac_exec_ext" = "/usr/ucb/cc"; then + ac_prog_rejected=yes + continue + fi + ac_cv_prog_CC="cc" + printf "%s\n" "$as_me:${as_lineno-$LINENO}: found $as_dir$ac_word$ac_exec_ext" >&5 + break 2 + fi +done + done +IFS=$as_save_IFS + +if test $ac_prog_rejected = yes; then + # We found a bogon in the path, so make sure we never use it. + set dummy $ac_cv_prog_CC + shift + if test $# != 0; then + # We chose a different compiler from the bogus one. + # However, it has the same basename, so the bogon will be chosen + # first if we set CC to just the basename; use the full file name. + shift + ac_cv_prog_CC="$as_dir$ac_word${1+' '}$@" + fi +fi +fi +fi +CC=$ac_cv_prog_CC +if test -n "$CC"; then + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $CC" >&5 +printf "%s\n" "$CC" >&6; } +else + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 +printf "%s\n" "no" >&6; } +fi + + +fi +if test -z "$CC"; then + if test -n "$ac_tool_prefix"; then + for ac_prog in cl.exe + do + # Extract the first word of "$ac_tool_prefix$ac_prog", so it can be a program name with args. +set dummy $ac_tool_prefix$ac_prog; ac_word=$2 +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 +printf %s "checking for $ac_word... " >&6; } +if test ${ac_cv_prog_CC+y} +then : + printf %s "(cached) " >&6 +else $as_nop + if test -n "$CC"; then + ac_cv_prog_CC="$CC" # Let the user override the test. +else +as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +for as_dir in $PATH +do + IFS=$as_save_IFS + case $as_dir in #((( + '') as_dir=./ ;; + */) ;; + *) as_dir=$as_dir/ ;; + esac + for ac_exec_ext in '' $ac_executable_extensions; do + if as_fn_executable_p "$as_dir$ac_word$ac_exec_ext"; then + ac_cv_prog_CC="$ac_tool_prefix$ac_prog" printf "%s\n" "$as_me:${as_lineno-$LINENO}: found $as_dir$ac_word$ac_exec_ext" >&5 break 2 fi @@ -18026,11 +19264,15 @@ printf "%s\n" "no" >&6; } fi + test -n "$CC" && break + done fi -if test -z "$ac_cv_prog_CC"; then +if test -z "$CC"; then ac_ct_CC=$CC - # Extract the first word of "gcc", so it can be a program name with args. -set dummy gcc; ac_word=$2 + for ac_prog in cl.exe +do + # Extract the first word of "$ac_prog", so it can be a program name with args. +set dummy $ac_prog; ac_word=$2 { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 printf %s "checking for $ac_word... " >&6; } if test ${ac_cv_prog_ac_ct_CC+y} @@ -18051,7 +19293,7 @@ do esac for ac_exec_ext in '' $ac_executable_extensions; do if as_fn_executable_p "$as_dir$ac_word$ac_exec_ext"; then - ac_cv_prog_ac_ct_CC="gcc" + ac_cv_prog_ac_ct_CC="$ac_prog" printf "%s\n" "$as_me:${as_lineno-$LINENO}: found $as_dir$ac_word$ac_exec_ext" >&5 break 2 fi @@ -18070,6 +19312,10 @@ else printf "%s\n" "no" >&6; } fi + + test -n "$ac_ct_CC" && break +done + if test "x$ac_ct_CC" = x; then CC="" else @@ -18081,14 +19327,13 @@ ac_tool_warned=yes ;; esac CC=$ac_ct_CC fi -else - CC="$ac_cv_prog_CC" fi +fi if test -z "$CC"; then - if test -n "$ac_tool_prefix"; then - # Extract the first word of "${ac_tool_prefix}cc", so it can be a program name with args. -set dummy ${ac_tool_prefix}cc; ac_word=$2 + if test -n "$ac_tool_prefix"; then + # Extract the first word of "${ac_tool_prefix}clang", so it can be a program name with args. +set dummy ${ac_tool_prefix}clang; ac_word=$2 { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 printf %s "checking for $ac_word... " >&6; } if test ${ac_cv_prog_CC+y} @@ -18109,7 +19354,7 @@ do esac for ac_exec_ext in '' $ac_executable_extensions; do if as_fn_executable_p "$as_dir$ac_word$ac_exec_ext"; then - ac_cv_prog_CC="${ac_tool_prefix}cc" + ac_cv_prog_CC="${ac_tool_prefix}clang" printf "%s\n" "$as_me:${as_lineno-$LINENO}: found $as_dir$ac_word$ac_exec_ext" >&5 break 2 fi @@ -18129,241 +19374,443 @@ printf "%s\n" "no" >&6; } fi - fi fi -if test -z "$CC"; then - # Extract the first word of "cc", so it can be a program name with args. -set dummy cc; ac_word=$2 +if test -z "$ac_cv_prog_CC"; then + ac_ct_CC=$CC + # Extract the first word of "clang", so it can be a program name with args. +set dummy clang; ac_word=$2 { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 printf %s "checking for $ac_word... " >&6; } -if test ${ac_cv_prog_CC+y} +if test ${ac_cv_prog_ac_ct_CC+y} +then : + printf %s "(cached) " >&6 +else $as_nop + if test -n "$ac_ct_CC"; then + ac_cv_prog_ac_ct_CC="$ac_ct_CC" # Let the user override the test. +else +as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +for as_dir in $PATH +do + IFS=$as_save_IFS + case $as_dir in #((( + '') as_dir=./ ;; + */) ;; + *) as_dir=$as_dir/ ;; + esac + for ac_exec_ext in '' $ac_executable_extensions; do + if as_fn_executable_p "$as_dir$ac_word$ac_exec_ext"; then + ac_cv_prog_ac_ct_CC="clang" + printf "%s\n" "$as_me:${as_lineno-$LINENO}: found $as_dir$ac_word$ac_exec_ext" >&5 + break 2 + fi +done + done +IFS=$as_save_IFS + +fi +fi +ac_ct_CC=$ac_cv_prog_ac_ct_CC +if test -n "$ac_ct_CC"; then + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_ct_CC" >&5 +printf "%s\n" "$ac_ct_CC" >&6; } +else + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 +printf "%s\n" "no" >&6; } +fi + + if test "x$ac_ct_CC" = x; then + CC="" + else + case $cross_compiling:$ac_tool_warned in +yes:) +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: WARNING: using cross tools not prefixed with host triplet" >&5 +printf "%s\n" "$as_me: WARNING: using cross tools not prefixed with host triplet" >&2;} +ac_tool_warned=yes ;; +esac + CC=$ac_ct_CC + fi +else + CC="$ac_cv_prog_CC" +fi + +fi + + +test -z "$CC" && { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 +printf "%s\n" "$as_me: error: in \`$ac_pwd':" >&2;} +as_fn_error $? "no acceptable C compiler found in \$PATH +See \`config.log' for more details" "$LINENO" 5; } + +# Provide some information about the compiler. +printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for C compiler version" >&5 +set X $ac_compile +ac_compiler=$2 +for ac_option in --version -v -V -qversion -version; do + { { ac_try="$ac_compiler $ac_option >&5" +case "(($ac_try" in + *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;; + *) ac_try_echo=$ac_try;; +esac +eval ac_try_echo="\"\$as_me:${as_lineno-$LINENO}: $ac_try_echo\"" +printf "%s\n" "$ac_try_echo"; } >&5 + (eval "$ac_compiler $ac_option >&5") 2>conftest.err + ac_status=$? + if test -s conftest.err; then + sed '10a\ +... rest of stderr output deleted ... + 10q' conftest.err >conftest.er1 + cat conftest.er1 >&5 + fi + rm -f conftest.er1 conftest.err + printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; } +done + +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether the compiler supports GNU C" >&5 +printf %s "checking whether the compiler supports GNU C... " >&6; } +if test ${ac_cv_c_compiler_gnu+y} +then : + printf %s "(cached) " >&6 +else $as_nop + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +int +main (void) +{ +#ifndef __GNUC__ + choke me +#endif + + ; + return 0; +} +_ACEOF +if ac_fn_c_try_compile "$LINENO" +then : + ac_compiler_gnu=yes +else $as_nop + ac_compiler_gnu=no +fi +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext +ac_cv_c_compiler_gnu=$ac_compiler_gnu + +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_c_compiler_gnu" >&5 +printf "%s\n" "$ac_cv_c_compiler_gnu" >&6; } +ac_compiler_gnu=$ac_cv_c_compiler_gnu + +if test $ac_compiler_gnu = yes; then + GCC=yes +else + GCC= +fi +ac_test_CFLAGS=${CFLAGS+y} +ac_save_CFLAGS=$CFLAGS +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether $CC accepts -g" >&5 +printf %s "checking whether $CC accepts -g... " >&6; } +if test ${ac_cv_prog_cc_g+y} +then : + printf %s "(cached) " >&6 +else $as_nop + ac_save_c_werror_flag=$ac_c_werror_flag + ac_c_werror_flag=yes + ac_cv_prog_cc_g=no + CFLAGS="-g" + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +int +main (void) +{ + + ; + return 0; +} +_ACEOF +if ac_fn_c_try_compile "$LINENO" +then : + ac_cv_prog_cc_g=yes +else $as_nop + CFLAGS="" + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +int +main (void) +{ + + ; + return 0; +} +_ACEOF +if ac_fn_c_try_compile "$LINENO" +then : + +else $as_nop + ac_c_werror_flag=$ac_save_c_werror_flag + CFLAGS="-g" + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +int +main (void) +{ + + ; + return 0; +} +_ACEOF +if ac_fn_c_try_compile "$LINENO" +then : + ac_cv_prog_cc_g=yes +fi +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext +fi +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext +fi +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext + ac_c_werror_flag=$ac_save_c_werror_flag +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_prog_cc_g" >&5 +printf "%s\n" "$ac_cv_prog_cc_g" >&6; } +if test $ac_test_CFLAGS; then + CFLAGS=$ac_save_CFLAGS +elif test $ac_cv_prog_cc_g = yes; then + if test "$GCC" = yes; then + CFLAGS="-g -O2" + else + CFLAGS="-g" + fi +else + if test "$GCC" = yes; then + CFLAGS="-O2" + else + CFLAGS= + fi +fi +ac_prog_cc_stdc=no +if test x$ac_prog_cc_stdc = xno +then : + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for $CC option to enable C11 features" >&5 +printf %s "checking for $CC option to enable C11 features... " >&6; } +if test ${ac_cv_prog_cc_c11+y} then : printf %s "(cached) " >&6 else $as_nop - if test -n "$CC"; then - ac_cv_prog_CC="$CC" # Let the user override the test. -else - ac_prog_rejected=no -as_save_IFS=$IFS; IFS=$PATH_SEPARATOR -for as_dir in $PATH + ac_cv_prog_cc_c11=no +ac_save_CC=$CC +cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ +$ac_c_conftest_c11_program +_ACEOF +for ac_arg in '' -std=gnu11 do - IFS=$as_save_IFS - case $as_dir in #((( - '') as_dir=./ ;; - */) ;; - *) as_dir=$as_dir/ ;; - esac - for ac_exec_ext in '' $ac_executable_extensions; do - if as_fn_executable_p "$as_dir$ac_word$ac_exec_ext"; then - if test "$as_dir$ac_word$ac_exec_ext" = "/usr/ucb/cc"; then - ac_prog_rejected=yes - continue - fi - ac_cv_prog_CC="cc" - printf "%s\n" "$as_me:${as_lineno-$LINENO}: found $as_dir$ac_word$ac_exec_ext" >&5 - break 2 - fi -done - done -IFS=$as_save_IFS - -if test $ac_prog_rejected = yes; then - # We found a bogon in the path, so make sure we never use it. - set dummy $ac_cv_prog_CC - shift - if test $# != 0; then - # We chose a different compiler from the bogus one. - # However, it has the same basename, so the bogon will be chosen - # first if we set CC to just the basename; use the full file name. - shift - ac_cv_prog_CC="$as_dir$ac_word${1+' '}$@" - fi + CC="$ac_save_CC $ac_arg" + if ac_fn_c_try_compile "$LINENO" +then : + ac_cv_prog_cc_c11=$ac_arg fi +rm -f core conftest.err conftest.$ac_objext conftest.beam + test "x$ac_cv_prog_cc_c11" != "xno" && break +done +rm -f conftest.$ac_ext +CC=$ac_save_CC fi + +if test "x$ac_cv_prog_cc_c11" = xno +then : + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: unsupported" >&5 +printf "%s\n" "unsupported" >&6; } +else $as_nop + if test "x$ac_cv_prog_cc_c11" = x +then : + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: none needed" >&5 +printf "%s\n" "none needed" >&6; } +else $as_nop + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_prog_cc_c11" >&5 +printf "%s\n" "$ac_cv_prog_cc_c11" >&6; } + CC="$CC $ac_cv_prog_cc_c11" fi -CC=$ac_cv_prog_CC -if test -n "$CC"; then - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $CC" >&5 -printf "%s\n" "$CC" >&6; } -else - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 -printf "%s\n" "no" >&6; } + ac_cv_prog_cc_stdc=$ac_cv_prog_cc_c11 + ac_prog_cc_stdc=c11 fi - - fi -if test -z "$CC"; then - if test -n "$ac_tool_prefix"; then - for ac_prog in cl.exe - do - # Extract the first word of "$ac_tool_prefix$ac_prog", so it can be a program name with args. -set dummy $ac_tool_prefix$ac_prog; ac_word=$2 -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 -printf %s "checking for $ac_word... " >&6; } -if test ${ac_cv_prog_CC+y} +if test x$ac_prog_cc_stdc = xno +then : + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for $CC option to enable C99 features" >&5 +printf %s "checking for $CC option to enable C99 features... " >&6; } +if test ${ac_cv_prog_cc_c99+y} then : printf %s "(cached) " >&6 else $as_nop - if test -n "$CC"; then - ac_cv_prog_CC="$CC" # Let the user override the test. -else -as_save_IFS=$IFS; IFS=$PATH_SEPARATOR -for as_dir in $PATH + ac_cv_prog_cc_c99=no +ac_save_CC=$CC +cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ +$ac_c_conftest_c99_program +_ACEOF +for ac_arg in '' -std=gnu99 -std=c99 -c99 -qlanglvl=extc1x -qlanglvl=extc99 -AC99 -D_STDC_C99= do - IFS=$as_save_IFS - case $as_dir in #((( - '') as_dir=./ ;; - */) ;; - *) as_dir=$as_dir/ ;; - esac - for ac_exec_ext in '' $ac_executable_extensions; do - if as_fn_executable_p "$as_dir$ac_word$ac_exec_ext"; then - ac_cv_prog_CC="$ac_tool_prefix$ac_prog" - printf "%s\n" "$as_me:${as_lineno-$LINENO}: found $as_dir$ac_word$ac_exec_ext" >&5 - break 2 - fi + CC="$ac_save_CC $ac_arg" + if ac_fn_c_try_compile "$LINENO" +then : + ac_cv_prog_cc_c99=$ac_arg +fi +rm -f core conftest.err conftest.$ac_objext conftest.beam + test "x$ac_cv_prog_cc_c99" != "xno" && break done - done -IFS=$as_save_IFS - +rm -f conftest.$ac_ext +CC=$ac_save_CC fi + +if test "x$ac_cv_prog_cc_c99" = xno +then : + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: unsupported" >&5 +printf "%s\n" "unsupported" >&6; } +else $as_nop + if test "x$ac_cv_prog_cc_c99" = x +then : + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: none needed" >&5 +printf "%s\n" "none needed" >&6; } +else $as_nop + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_prog_cc_c99" >&5 +printf "%s\n" "$ac_cv_prog_cc_c99" >&6; } + CC="$CC $ac_cv_prog_cc_c99" fi -CC=$ac_cv_prog_CC -if test -n "$CC"; then - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $CC" >&5 -printf "%s\n" "$CC" >&6; } -else - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 -printf "%s\n" "no" >&6; } + ac_cv_prog_cc_stdc=$ac_cv_prog_cc_c99 + ac_prog_cc_stdc=c99 fi - - - test -n "$CC" && break - done fi -if test -z "$CC"; then - ac_ct_CC=$CC - for ac_prog in cl.exe -do - # Extract the first word of "$ac_prog", so it can be a program name with args. -set dummy $ac_prog; ac_word=$2 -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 -printf %s "checking for $ac_word... " >&6; } -if test ${ac_cv_prog_ac_ct_CC+y} +if test x$ac_prog_cc_stdc = xno +then : + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for $CC option to enable C89 features" >&5 +printf %s "checking for $CC option to enable C89 features... " >&6; } +if test ${ac_cv_prog_cc_c89+y} then : printf %s "(cached) " >&6 else $as_nop - if test -n "$ac_ct_CC"; then - ac_cv_prog_ac_ct_CC="$ac_ct_CC" # Let the user override the test. -else -as_save_IFS=$IFS; IFS=$PATH_SEPARATOR -for as_dir in $PATH + ac_cv_prog_cc_c89=no +ac_save_CC=$CC +cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ +$ac_c_conftest_c89_program +_ACEOF +for ac_arg in '' -qlanglvl=extc89 -qlanglvl=ansi -std -Ae "-Aa -D_HPUX_SOURCE" "-Xc -D__EXTENSIONS__" do - IFS=$as_save_IFS - case $as_dir in #((( - '') as_dir=./ ;; - */) ;; - *) as_dir=$as_dir/ ;; - esac - for ac_exec_ext in '' $ac_executable_extensions; do - if as_fn_executable_p "$as_dir$ac_word$ac_exec_ext"; then - ac_cv_prog_ac_ct_CC="$ac_prog" - printf "%s\n" "$as_me:${as_lineno-$LINENO}: found $as_dir$ac_word$ac_exec_ext" >&5 - break 2 - fi + CC="$ac_save_CC $ac_arg" + if ac_fn_c_try_compile "$LINENO" +then : + ac_cv_prog_cc_c89=$ac_arg +fi +rm -f core conftest.err conftest.$ac_objext conftest.beam + test "x$ac_cv_prog_cc_c89" != "xno" && break done - done -IFS=$as_save_IFS +rm -f conftest.$ac_ext +CC=$ac_save_CC +fi +if test "x$ac_cv_prog_cc_c89" = xno +then : + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: unsupported" >&5 +printf "%s\n" "unsupported" >&6; } +else $as_nop + if test "x$ac_cv_prog_cc_c89" = x +then : + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: none needed" >&5 +printf "%s\n" "none needed" >&6; } +else $as_nop + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_prog_cc_c89" >&5 +printf "%s\n" "$ac_cv_prog_cc_c89" >&6; } + CC="$CC $ac_cv_prog_cc_c89" fi + ac_cv_prog_cc_stdc=$ac_cv_prog_cc_c89 + ac_prog_cc_stdc=c89 fi -ac_ct_CC=$ac_cv_prog_ac_ct_CC -if test -n "$ac_ct_CC"; then - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_ct_CC" >&5 -printf "%s\n" "$ac_ct_CC" >&6; } -else - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 -printf "%s\n" "no" >&6; } fi +ac_ext=c +ac_cpp='$CPP $CPPFLAGS' +ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5' +ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5' +ac_compiler_gnu=$ac_cv_c_compiler_gnu - test -n "$ac_ct_CC" && break -done - - if test "x$ac_ct_CC" = x; then - CC="" - else - case $cross_compiling:$ac_tool_warned in -yes:) -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: WARNING: using cross tools not prefixed with host triplet" >&5 -printf "%s\n" "$as_me: WARNING: using cross tools not prefixed with host triplet" >&2;} -ac_tool_warned=yes ;; -esac - CC=$ac_ct_CC - fi -fi - -fi -if test -z "$CC"; then - if test -n "$ac_tool_prefix"; then - # Extract the first word of "${ac_tool_prefix}clang", so it can be a program name with args. -set dummy ${ac_tool_prefix}clang; ac_word=$2 -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 -printf %s "checking for $ac_word... " >&6; } -if test ${ac_cv_prog_CC+y} + + ac_ext=c +ac_cpp='$CPP $CPPFLAGS' +ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5' +ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5' +ac_compiler_gnu=$ac_cv_c_compiler_gnu +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether $CC understands -c and -o together" >&5 +printf %s "checking whether $CC understands -c and -o together... " >&6; } +if test ${am_cv_prog_cc_c_o+y} then : printf %s "(cached) " >&6 else $as_nop - if test -n "$CC"; then - ac_cv_prog_CC="$CC" # Let the user override the test. -else -as_save_IFS=$IFS; IFS=$PATH_SEPARATOR -for as_dir in $PATH -do - IFS=$as_save_IFS - case $as_dir in #((( - '') as_dir=./ ;; - */) ;; - *) as_dir=$as_dir/ ;; - esac - for ac_exec_ext in '' $ac_executable_extensions; do - if as_fn_executable_p "$as_dir$ac_word$ac_exec_ext"; then - ac_cv_prog_CC="${ac_tool_prefix}clang" - printf "%s\n" "$as_me:${as_lineno-$LINENO}: found $as_dir$ac_word$ac_exec_ext" >&5 - break 2 - fi -done - done -IFS=$as_save_IFS + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ +int +main (void) +{ + + ; + return 0; +} +_ACEOF + # Make sure it works both with $CC and with simple cc. + # Following AC_PROG_CC_C_O, we do the test twice because some + # compilers refuse to overwrite an existing .o file with -o, + # though they will create one. + am_cv_prog_cc_c_o=yes + for am_i in 1 2; do + if { echo "$as_me:$LINENO: $CC -c conftest.$ac_ext -o conftest2.$ac_objext" >&5 + ($CC -c conftest.$ac_ext -o conftest2.$ac_objext) >&5 2>&5 + ac_status=$? + echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); } \ + && test -f conftest2.$ac_objext; then + : OK + else + am_cv_prog_cc_c_o=no + break + fi + done + rm -f core conftest* + unset am_i fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $am_cv_prog_cc_c_o" >&5 +printf "%s\n" "$am_cv_prog_cc_c_o" >&6; } +if test "$am_cv_prog_cc_c_o" != yes; then + # Losing compiler, so override with the script. + # FIXME: It is wrong to rewrite CC. + # But if we don't then we get into trouble of one sort or another. + # A longer-term fix would be to have automake use am__CC in this case, + # and then we could set am__CC="\$(top_srcdir)/compile \$(CC)" + CC="$am_aux_dir/compile $CC" fi -CC=$ac_cv_prog_CC -if test -n "$CC"; then - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $CC" >&5 -printf "%s\n" "$CC" >&6; } -else - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 -printf "%s\n" "no" >&6; } -fi +ac_ext=c +ac_cpp='$CPP $CPPFLAGS' +ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5' +ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5' +ac_compiler_gnu=$ac_cv_c_compiler_gnu -fi -if test -z "$ac_cv_prog_CC"; then - ac_ct_CC=$CC - # Extract the first word of "clang", so it can be a program name with args. -set dummy clang; ac_word=$2 -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 -printf %s "checking for $ac_word... " >&6; } -if test ${ac_cv_prog_ac_ct_CC+y} + +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for egrep" >&5 +printf %s "checking for egrep... " >&6; } +if test ${ac_cv_path_EGREP+y} then : printf %s "(cached) " >&6 else $as_nop - if test -n "$ac_ct_CC"; then - ac_cv_prog_ac_ct_CC="$ac_ct_CC" # Let the user override the test. -else -as_save_IFS=$IFS; IFS=$PATH_SEPARATOR -for as_dir in $PATH + if echo a | $GREP -E '(a|b)' >/dev/null 2>&1 + then ac_cv_path_EGREP="$GREP -E" + else + if test -z "$EGREP"; then + ac_path_EGREP_found=false + # Loop through the user's path and test for each of PROGNAME-LIST + as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +for as_dir in $PATH$PATH_SEPARATOR/usr/xpg4/bin do IFS=$as_save_IFS case $as_dir in #((( @@ -18371,89 +19818,113 @@ do */) ;; *) as_dir=$as_dir/ ;; esac + for ac_prog in egrep + do for ac_exec_ext in '' $ac_executable_extensions; do - if as_fn_executable_p "$as_dir$ac_word$ac_exec_ext"; then - ac_cv_prog_ac_ct_CC="clang" - printf "%s\n" "$as_me:${as_lineno-$LINENO}: found $as_dir$ac_word$ac_exec_ext" >&5 - break 2 - fi -done + ac_path_EGREP="$as_dir$ac_prog$ac_exec_ext" + as_fn_executable_p "$ac_path_EGREP" || continue +# Check for GNU ac_path_EGREP and select it if it is found. + # Check for GNU $ac_path_EGREP +case `"$ac_path_EGREP" --version 2>&1` in +*GNU*) + ac_cv_path_EGREP="$ac_path_EGREP" ac_path_EGREP_found=:;; +*) + ac_count=0 + printf %s 0123456789 >"conftest.in" + while : + do + cat "conftest.in" "conftest.in" >"conftest.tmp" + mv "conftest.tmp" "conftest.in" + cp "conftest.in" "conftest.nl" + printf "%s\n" 'EGREP' >> "conftest.nl" + "$ac_path_EGREP" 'EGREP$' < "conftest.nl" >"conftest.out" 2>/dev/null || break + diff "conftest.out" "conftest.nl" >/dev/null 2>&1 || break + as_fn_arith $ac_count + 1 && ac_count=$as_val + if test $ac_count -gt ${ac_path_EGREP_max-0}; then + # Best one so far, save it but keep looking for a better one + ac_cv_path_EGREP="$ac_path_EGREP" + ac_path_EGREP_max=$ac_count + fi + # 10*(2^10) chars as input seems more than enough + test $ac_count -gt 10 && break done -IFS=$as_save_IFS - -fi -fi -ac_ct_CC=$ac_cv_prog_ac_ct_CC -if test -n "$ac_ct_CC"; then - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_ct_CC" >&5 -printf "%s\n" "$ac_ct_CC" >&6; } -else - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 -printf "%s\n" "no" >&6; } -fi - - if test "x$ac_ct_CC" = x; then - CC="" - else - case $cross_compiling:$ac_tool_warned in -yes:) -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: WARNING: using cross tools not prefixed with host triplet" >&5 -printf "%s\n" "$as_me: WARNING: using cross tools not prefixed with host triplet" >&2;} -ac_tool_warned=yes ;; + rm -f conftest.in conftest.tmp conftest.nl conftest.out;; esac - CC=$ac_ct_CC + + $ac_path_EGREP_found && break 3 + done + done + done +IFS=$as_save_IFS + if test -z "$ac_cv_path_EGREP"; then + as_fn_error $? "no acceptable egrep could be found in $PATH$PATH_SEPARATOR/usr/xpg4/bin" "$LINENO" 5 fi else - CC="$ac_cv_prog_CC" + ac_cv_path_EGREP=$EGREP fi + fi fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_path_EGREP" >&5 +printf "%s\n" "$ac_cv_path_EGREP" >&6; } + EGREP="$ac_cv_path_EGREP" -test -z "$CC" && { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 -printf "%s\n" "$as_me: error: in \`$ac_pwd':" >&2;} -as_fn_error $? "no acceptable C compiler found in \$PATH -See \`config.log' for more details" "$LINENO" 5; } -# Provide some information about the compiler. -printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for C compiler version" >&5 -set X $ac_compile -ac_compiler=$2 -for ac_option in --version -v -V -qversion -version; do - { { ac_try="$ac_compiler $ac_option >&5" -case "(($ac_try" in - *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;; - *) ac_try_echo=$ac_try;; -esac -eval ac_try_echo="\"\$as_me:${as_lineno-$LINENO}: $ac_try_echo\"" -printf "%s\n" "$ac_try_echo"; } >&5 - (eval "$ac_compiler $ac_option >&5") 2>conftest.err - ac_status=$? - if test -s conftest.err; then - sed '10a\ -... rest of stderr output deleted ... - 10q' conftest.err >conftest.er1 - cat conftest.er1 >&5 - fi - rm -f conftest.er1 conftest.err - printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 - test $ac_status = 0; } -done -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether the compiler supports GNU C" >&5 -printf %s "checking whether the compiler supports GNU C... " >&6; } -if test ${ac_cv_c_compiler_gnu+y} + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for C compiler vendor" >&5 +printf %s "checking for C compiler vendor... " >&6; } +if test ${ax_cv_c_compiler_vendor+y} then : printf %s "(cached) " >&6 else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext + + vendors=" + intel: __ICC,__ECC,__INTEL_COMPILER + ibm: __xlc__,__xlC__,__IBMC__,__IBMCPP__,__ibmxl__ + pathscale: __PATHCC__,__PATHSCALE__ + clang: __clang__ + cray: _CRAYC + fujitsu: __FUJITSU + sdcc: SDCC,__SDCC + sx: _SX + nvhpc: __NVCOMPILER + portland: __PGI + gnu: __GNUC__ + sun: __SUNPRO_C,__SUNPRO_CC,__SUNPRO_F90,__SUNPRO_F95 + hp: __HP_cc,__HP_aCC + dec: __DECC,__DECCXX,__DECC_VER,__DECCXX_VER + borland: __BORLANDC__,__CODEGEARC__,__TURBOC__ + comeau: __COMO__ + kai: __KCC + lcc: __LCC__ + sgi: __sgi,sgi + microsoft: _MSC_VER + metrowerks: __MWERKS__ + watcom: __WATCOMC__ + tcc: __TINYC__ + unknown: UNKNOWN + " + for ventest in $vendors; do + case $ventest in + *:) + vendor=$ventest + continue + ;; + *) + vencpp="defined("`echo $ventest | sed 's/,/) || defined(/g'`")" + ;; + esac + + cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ int main (void) { -#ifndef __GNUC__ - choke me + +#if !($vencpp) + thisisanerror; #endif ; @@ -18462,724 +19933,691 @@ main (void) _ACEOF if ac_fn_c_try_compile "$LINENO" then : - ac_compiler_gnu=yes -else $as_nop - ac_compiler_gnu=no + break fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext -ac_cv_c_compiler_gnu=$ac_compiler_gnu + done + + ax_cv_c_compiler_vendor=`echo $vendor | cut -d: -f1` fi -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_c_compiler_gnu" >&5 -printf "%s\n" "$ac_cv_c_compiler_gnu" >&6; } -ac_compiler_gnu=$ac_cv_c_compiler_gnu +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ax_cv_c_compiler_vendor" >&5 +printf "%s\n" "$ax_cv_c_compiler_vendor" >&6; } -if test $ac_compiler_gnu = yes; then - GCC=yes -else - GCC= + +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether this is native windows" >&5 +printf %s "checking whether this is native windows... " >&6; } +ac_cv_native_windows=no +ac_cv_windows=no +case $host_os in + mingw*) + ac_cv_native_windows=yes + ac_cv_windows=yes + ;; + cygwin*) + ac_cv_windows=yes + ;; +esac +if test "$ax_cv_c_compiler_vendor" = "microsoft" ; then + ac_cv_native_windows=yes + ac_cv_windows=yes fi -ac_test_CFLAGS=${CFLAGS+y} -ac_save_CFLAGS=$CFLAGS -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether $CC accepts -g" >&5 -printf %s "checking whether $CC accepts -g... " >&6; } -if test ${ac_cv_prog_cc_g+y} -then : - printf %s "(cached) " >&6 -else $as_nop - ac_save_c_werror_flag=$ac_c_werror_flag - ac_c_werror_flag=yes - ac_cv_prog_cc_g=no - CFLAGS="-g" - cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_native_windows" >&5 +printf "%s\n" "$ac_cv_native_windows" >&6; } -int -main (void) -{ - ; - return 0; -} -_ACEOF -if ac_fn_c_try_compile "$LINENO" +# Check whether --enable-shared was given. +if test ${enable_shared+y} then : - ac_cv_prog_cc_g=yes + enableval=$enable_shared; p=${PACKAGE-default} + case $enableval in + yes) enable_shared=yes ;; + no) enable_shared=no ;; + *) + enable_shared=no + # Look at the argument we got. We use all the common list separators. + lt_save_ifs=$IFS; IFS=$IFS$PATH_SEPARATOR, + for pkg in $enableval; do + IFS=$lt_save_ifs + if test "X$pkg" = "X$p"; then + enable_shared=yes + fi + done + IFS=$lt_save_ifs + ;; + esac else $as_nop - CFLAGS="" - cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ + enable_shared=yes +fi -int -main (void) -{ - ; - return 0; -} -_ACEOF -if ac_fn_c_try_compile "$LINENO" -then : -else $as_nop - ac_c_werror_flag=$ac_save_c_werror_flag - CFLAGS="-g" - cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ -int -main (void) -{ - ; - return 0; -} -_ACEOF -if ac_fn_c_try_compile "$LINENO" -then : - ac_cv_prog_cc_g=yes -fi -rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext -fi -rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext -fi -rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext - ac_c_werror_flag=$ac_save_c_werror_flag -fi -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_prog_cc_g" >&5 -printf "%s\n" "$ac_cv_prog_cc_g" >&6; } -if test $ac_test_CFLAGS; then - CFLAGS=$ac_save_CFLAGS -elif test $ac_cv_prog_cc_g = yes; then - if test "$GCC" = yes; then - CFLAGS="-g -O2" - else - CFLAGS="-g" - fi -else - if test "$GCC" = yes; then - CFLAGS="-O2" - else - CFLAGS= - fi -fi -ac_prog_cc_stdc=no -if test x$ac_prog_cc_stdc = xno + + +if test "x$ac_cv_windows" = "xyes" then : - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for $CC option to enable C11 features" >&5 -printf %s "checking for $CC option to enable C11 features... " >&6; } -if test ${ac_cv_prog_cc_c11+y} + # Check whether --enable-static was given. +if test ${enable_static+y} then : - printf %s "(cached) " >&6 + enableval=$enable_static; p=${PACKAGE-default} + case $enableval in + yes) enable_static=yes ;; + no) enable_static=no ;; + *) + enable_static=no + # Look at the argument we got. We use all the common list separators. + lt_save_ifs=$IFS; IFS=$IFS$PATH_SEPARATOR, + for pkg in $enableval; do + IFS=$lt_save_ifs + if test "X$pkg" = "X$p"; then + enable_static=yes + fi + done + IFS=$lt_save_ifs + ;; + esac else $as_nop - ac_cv_prog_cc_c11=no -ac_save_CC=$CC -cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ -$ac_c_conftest_c11_program -_ACEOF -for ac_arg in '' -std=gnu11 -do - CC="$ac_save_CC $ac_arg" - if ac_fn_c_try_compile "$LINENO" -then : - ac_cv_prog_cc_c11=$ac_arg -fi -rm -f core conftest.err conftest.$ac_objext conftest.beam - test "x$ac_cv_prog_cc_c11" != "xno" && break -done -rm -f conftest.$ac_ext -CC=$ac_save_CC + enable_static=no fi -if test "x$ac_cv_prog_cc_c11" = xno -then : - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: unsupported" >&5 -printf "%s\n" "unsupported" >&6; } -else $as_nop - if test "x$ac_cv_prog_cc_c11" = x -then : - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: none needed" >&5 -printf "%s\n" "none needed" >&6; } + + + + + else $as_nop - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_prog_cc_c11" >&5 -printf "%s\n" "$ac_cv_prog_cc_c11" >&6; } - CC="$CC $ac_cv_prog_cc_c11" -fi - ac_cv_prog_cc_stdc=$ac_cv_prog_cc_c11 - ac_prog_cc_stdc=c11 -fi -fi -if test x$ac_prog_cc_stdc = xno -then : - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for $CC option to enable C99 features" >&5 -printf %s "checking for $CC option to enable C99 features... " >&6; } -if test ${ac_cv_prog_cc_c99+y} + # Check whether --enable-static was given. +if test ${enable_static+y} then : - printf %s "(cached) " >&6 + enableval=$enable_static; p=${PACKAGE-default} + case $enableval in + yes) enable_static=yes ;; + no) enable_static=no ;; + *) + enable_static=no + # Look at the argument we got. We use all the common list separators. + lt_save_ifs=$IFS; IFS=$IFS$PATH_SEPARATOR, + for pkg in $enableval; do + IFS=$lt_save_ifs + if test "X$pkg" = "X$p"; then + enable_static=yes + fi + done + IFS=$lt_save_ifs + ;; + esac else $as_nop - ac_cv_prog_cc_c99=no -ac_save_CC=$CC -cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ -$ac_c_conftest_c99_program -_ACEOF -for ac_arg in '' -std=gnu99 -std=c99 -c99 -qlanglvl=extc1x -qlanglvl=extc99 -AC99 -D_STDC_C99= -do - CC="$ac_save_CC $ac_arg" - if ac_fn_c_try_compile "$LINENO" -then : - ac_cv_prog_cc_c99=$ac_arg + enable_static=yes fi -rm -f core conftest.err conftest.$ac_objext conftest.beam - test "x$ac_cv_prog_cc_c99" != "xno" && break -done -rm -f conftest.$ac_ext -CC=$ac_save_CC + + + + + + fi -if test "x$ac_cv_prog_cc_c99" = xno -then : - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: unsupported" >&5 -printf "%s\n" "unsupported" >&6; } -else $as_nop - if test "x$ac_cv_prog_cc_c99" = x +# Check whether --enable-warnings was given. +if test ${enable_warnings+y} then : - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: none needed" >&5 -printf "%s\n" "none needed" >&6; } + enableval=$enable_warnings; enable_warnings=${enableval} else $as_nop - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_prog_cc_c99" >&5 -printf "%s\n" "$ac_cv_prog_cc_c99" >&6; } - CC="$CC $ac_cv_prog_cc_c99" -fi - ac_cv_prog_cc_stdc=$ac_cv_prog_cc_c99 - ac_prog_cc_stdc=c99 -fi + enable_warnings=yes fi -if test x$ac_prog_cc_stdc = xno -then : - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for $CC option to enable C89 features" >&5 -printf %s "checking for $CC option to enable C89 features... " >&6; } -if test ${ac_cv_prog_cc_c89+y} + + +# Check whether --enable-symbol-hiding was given. +if test ${enable_symbol_hiding+y} then : - printf %s "(cached) " >&6 + enableval=$enable_symbol_hiding; + symbol_hiding="$enableval" + if test "$symbol_hiding" = "no" -a "x$enable_shared" = "xyes" ; then + case $host_os in + cygwin* | mingw* | pw32* | cegcc*) + as_fn_error $? "Cannot disable symbol hiding on windows" "$LINENO" 5 + ;; + esac + fi + else $as_nop - ac_cv_prog_cc_c89=no -ac_save_CC=$CC -cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ -$ac_c_conftest_c89_program -_ACEOF -for ac_arg in '' -qlanglvl=extc89 -qlanglvl=ansi -std -Ae "-Aa -D_HPUX_SOURCE" "-Xc -D__EXTENSIONS__" -do - CC="$ac_save_CC $ac_arg" - if ac_fn_c_try_compile "$LINENO" -then : - ac_cv_prog_cc_c89=$ac_arg -fi -rm -f core conftest.err conftest.$ac_objext conftest.beam - test "x$ac_cv_prog_cc_c89" != "xno" && break -done -rm -f conftest.$ac_ext -CC=$ac_save_CC + + if test "x$enable_shared" = "xyes" ; then + symbol_hiding="maybe" + else + symbol_hiding="no" + fi + + fi -if test "x$ac_cv_prog_cc_c89" = xno + +# Check whether --enable-tests was given. +if test ${enable_tests+y} then : - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: unsupported" >&5 -printf "%s\n" "unsupported" >&6; } + enableval=$enable_tests; build_tests="$enableval" else $as_nop - if test "x$ac_cv_prog_cc_c89" = x + if test "x$HAVE_CXX14" = "x1" && test "x$cross_compiling" = "xno" ; then + build_tests="maybe" + else + build_tests="no" + fi + + +fi + + +# Check whether --enable-cares-threads was given. +if test ${enable_cares_threads+y} then : - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: none needed" >&5 -printf "%s\n" "none needed" >&6; } + enableval=$enable_cares_threads; CARES_THREADS=${enableval} else $as_nop - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_prog_cc_c89" >&5 -printf "%s\n" "$ac_cv_prog_cc_c89" >&6; } - CC="$CC $ac_cv_prog_cc_c89" -fi - ac_cv_prog_cc_stdc=$ac_cv_prog_cc_c89 - ac_prog_cc_stdc=c89 -fi + CARES_THREADS=yes fi -ac_ext=c -ac_cpp='$CPP $CPPFLAGS' -ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5' -ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5' -ac_compiler_gnu=$ac_cv_c_compiler_gnu - ac_ext=c -ac_cpp='$CPP $CPPFLAGS' -ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5' -ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5' -ac_compiler_gnu=$ac_cv_c_compiler_gnu -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether $CC understands -c and -o together" >&5 -printf %s "checking whether $CC understands -c and -o together... " >&6; } -if test ${am_cv_prog_cc_c_o+y} +# Check whether --with-random was given. +if test ${with_random+y} then : - printf %s "(cached) " >&6 + withval=$with_random; CARES_RANDOM_FILE="$withval" else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ + CARES_RANDOM_FILE="/dev/urandom" -int -main (void) -{ +fi + +if test -n "$CARES_RANDOM_FILE" && test X"$CARES_RANDOM_FILE" != Xno ; then + + +printf "%s\n" "#define CARES_RANDOM_FILE \"$CARES_RANDOM_FILE\"" >>confdefs.h - ; - return 0; -} -_ACEOF - # Make sure it works both with $CC and with simple cc. - # Following AC_PROG_CC_C_O, we do the test twice because some - # compilers refuse to overwrite an existing .o file with -o, - # though they will create one. - am_cv_prog_cc_c_o=yes - for am_i in 1 2; do - if { echo "$as_me:$LINENO: $CC -c conftest.$ac_ext -o conftest2.$ac_objext" >&5 - ($CC -c conftest.$ac_ext -o conftest2.$ac_objext) >&5 2>&5 - ac_status=$? - echo "$as_me:$LINENO: \$? = $ac_status" >&5 - (exit $ac_status); } \ - && test -f conftest2.$ac_objext; then - : OK - else - am_cv_prog_cc_c_o=no - break - fi - done - rm -f core conftest* - unset am_i fi -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $am_cv_prog_cc_c_o" >&5 -printf "%s\n" "$am_cv_prog_cc_c_o" >&6; } -if test "$am_cv_prog_cc_c_o" != yes; then - # Losing compiler, so override with the script. - # FIXME: It is wrong to rewrite CC. - # But if we don't then we get into trouble of one sort or another. - # A longer-term fix would be to have automake use am__CC in this case, - # and then we could set am__CC="\$(top_srcdir)/compile \$(CC)" - CC="$am_aux_dir/compile $CC" + + +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether to enable maintainer-specific portions of Makefiles" >&5 +printf %s "checking whether to enable maintainer-specific portions of Makefiles... " >&6; } + # Check whether --enable-maintainer-mode was given. +if test ${enable_maintainer_mode+y} +then : + enableval=$enable_maintainer_mode; USE_MAINTAINER_MODE=$enableval +else $as_nop + USE_MAINTAINER_MODE=no +fi + + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $USE_MAINTAINER_MODE" >&5 +printf "%s\n" "$USE_MAINTAINER_MODE" >&6; } + if test $USE_MAINTAINER_MODE = yes; then + MAINTAINER_MODE_TRUE= + MAINTAINER_MODE_FALSE='#' +else + MAINTAINER_MODE_TRUE='#' + MAINTAINER_MODE_FALSE= fi -ac_ext=c -ac_cpp='$CPP $CPPFLAGS' -ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5' -ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5' -ac_compiler_gnu=$ac_cv_c_compiler_gnu + MAINT=$MAINTAINER_MODE_TRUE -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for egrep" >&5 -printf %s "checking for egrep... " >&6; } -if test ${ac_cv_path_EGREP+y} +# Check whether --enable-silent-rules was given. +if test ${enable_silent_rules+y} then : - printf %s "(cached) " >&6 -else $as_nop - if echo a | $GREP -E '(a|b)' >/dev/null 2>&1 - then ac_cv_path_EGREP="$GREP -E" - else - if test -z "$EGREP"; then - ac_path_EGREP_found=false - # Loop through the user's path and test for each of PROGNAME-LIST - as_save_IFS=$IFS; IFS=$PATH_SEPARATOR -for as_dir in $PATH$PATH_SEPARATOR/usr/xpg4/bin -do - IFS=$as_save_IFS - case $as_dir in #((( - '') as_dir=./ ;; - */) ;; - *) as_dir=$as_dir/ ;; - esac - for ac_prog in egrep - do - for ac_exec_ext in '' $ac_executable_extensions; do - ac_path_EGREP="$as_dir$ac_prog$ac_exec_ext" - as_fn_executable_p "$ac_path_EGREP" || continue -# Check for GNU ac_path_EGREP and select it if it is found. - # Check for GNU $ac_path_EGREP -case `"$ac_path_EGREP" --version 2>&1` in -*GNU*) - ac_cv_path_EGREP="$ac_path_EGREP" ac_path_EGREP_found=:;; -*) - ac_count=0 - printf %s 0123456789 >"conftest.in" - while : - do - cat "conftest.in" "conftest.in" >"conftest.tmp" - mv "conftest.tmp" "conftest.in" - cp "conftest.in" "conftest.nl" - printf "%s\n" 'EGREP' >> "conftest.nl" - "$ac_path_EGREP" 'EGREP$' < "conftest.nl" >"conftest.out" 2>/dev/null || break - diff "conftest.out" "conftest.nl" >/dev/null 2>&1 || break - as_fn_arith $ac_count + 1 && ac_count=$as_val - if test $ac_count -gt ${ac_path_EGREP_max-0}; then - # Best one so far, save it but keep looking for a better one - ac_cv_path_EGREP="$ac_path_EGREP" - ac_path_EGREP_max=$ac_count - fi - # 10*(2^10) chars as input seems more than enough - test $ac_count -gt 10 && break - done - rm -f conftest.in conftest.tmp conftest.nl conftest.out;; -esac + enableval=$enable_silent_rules; +fi - $ac_path_EGREP_found && break 3 - done - done - done -IFS=$as_save_IFS - if test -z "$ac_cv_path_EGREP"; then - as_fn_error $? "no acceptable egrep could be found in $PATH$PATH_SEPARATOR/usr/xpg4/bin" "$LINENO" 5 - fi +case $enable_silent_rules in # ((( + yes) AM_DEFAULT_VERBOSITY=0;; + no) AM_DEFAULT_VERBOSITY=1;; + *) AM_DEFAULT_VERBOSITY=0;; +esac +am_make=${MAKE-make} +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether $am_make supports nested variables" >&5 +printf %s "checking whether $am_make supports nested variables... " >&6; } +if test ${am_cv_make_support_nested_variables+y} +then : + printf %s "(cached) " >&6 +else $as_nop + if printf "%s\n" 'TRUE=$(BAR$(V)) +BAR0=false +BAR1=true +V=1 +am__doit: + @$(TRUE) +.PHONY: am__doit' | $am_make -f - >/dev/null 2>&1; then + am_cv_make_support_nested_variables=yes else - ac_cv_path_EGREP=$EGREP + am_cv_make_support_nested_variables=no fi - - fi fi -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_path_EGREP" >&5 -printf "%s\n" "$ac_cv_path_EGREP" >&6; } - EGREP="$ac_cv_path_EGREP" +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $am_cv_make_support_nested_variables" >&5 +printf "%s\n" "$am_cv_make_support_nested_variables" >&6; } +if test $am_cv_make_support_nested_variables = yes; then + AM_V='$(V)' + AM_DEFAULT_V='$(AM_DEFAULT_VERBOSITY)' +else + AM_V=$AM_DEFAULT_VERBOSITY + AM_DEFAULT_V=$AM_DEFAULT_VERBOSITY +fi +AM_BACKSLASH='\' - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for C compiler vendor" >&5 -printf %s "checking for C compiler vendor... " >&6; } -if test ${ax_cv_c_compiler_vendor+y} -then : - printf %s "(cached) " >&6 -else $as_nop - vendors=" - intel: __ICC,__ECC,__INTEL_COMPILER - ibm: __xlc__,__xlC__,__IBMC__,__IBMCPP__,__ibmxl__ - pathscale: __PATHCC__,__PATHSCALE__ - clang: __clang__ - cray: _CRAYC - fujitsu: __FUJITSU - sdcc: SDCC,__SDCC - sx: _SX - nvhpc: __NVCOMPILER - portland: __PGI - gnu: __GNUC__ - sun: __SUNPRO_C,__SUNPRO_CC,__SUNPRO_F90,__SUNPRO_F95 - hp: __HP_cc,__HP_aCC - dec: __DECC,__DECCXX,__DECC_VER,__DECCXX_VER - borland: __BORLANDC__,__CODEGEARC__,__TURBOC__ - comeau: __COMO__ - kai: __KCC - lcc: __LCC__ - sgi: __sgi,sgi - microsoft: _MSC_VER - metrowerks: __MWERKS__ - watcom: __WATCOMC__ - tcc: __TINYC__ - unknown: UNKNOWN - " - for ventest in $vendors; do - case $ventest in - *:) - vendor=$ventest - continue - ;; - *) - vencpp="defined("`echo $ventest | sed 's/,/) || defined(/g'`")" - ;; - esac - cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ -int -main (void) -{ -#if !($vencpp) - thisisanerror; -#endif - ; - return 0; -} -_ACEOF -if ac_fn_c_try_compile "$LINENO" -then : - break -fi -rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext - done - ax_cv_c_compiler_vendor=`echo $vendor | cut -d: -f1` -fi -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ax_cv_c_compiler_vendor" >&5 -printf "%s\n" "$ax_cv_c_compiler_vendor" >&6; } -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether this is native windows" >&5 -printf %s "checking whether this is native windows... " >&6; } -ac_cv_native_windows=no -ac_cv_windows=no -case $host_os in - mingw*) - ac_cv_native_windows=yes - ac_cv_windows=yes - ;; - cygwin*) - ac_cv_windows=yes - ;; -esac -if test "$ax_cv_c_compiler_vendor" = "microsoft" ; then - ac_cv_native_windows=yes - ac_cv_windows=yes + + + + + + + + # allow to override gcov location + +# Check whether --with-gcov was given. +if test ${with_gcov+y} +then : + withval=$with_gcov; _AX_CODE_COVERAGE_GCOV_PROG_WITH=$with_gcov +else $as_nop + _AX_CODE_COVERAGE_GCOV_PROG_WITH=gcov fi -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_native_windows" >&5 -printf "%s\n" "$ac_cv_native_windows" >&6; } -# Check whether --enable-shared was given. -if test ${enable_shared+y} + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether to build with code coverage support" >&5 +printf %s "checking whether to build with code coverage support... " >&6; } + # Check whether --enable-code-coverage was given. +if test ${enable_code_coverage+y} then : - enableval=$enable_shared; p=${PACKAGE-default} - case $enableval in - yes) enable_shared=yes ;; - no) enable_shared=no ;; - *) - enable_shared=no - # Look at the argument we got. We use all the common list separators. - lt_save_ifs=$IFS; IFS=$IFS$PATH_SEPARATOR, - for pkg in $enableval; do - IFS=$lt_save_ifs - if test "X$pkg" = "X$p"; then - enable_shared=yes - fi - done - IFS=$lt_save_ifs - ;; - esac + enableval=$enable_code_coverage; else $as_nop - enable_shared=yes + enable_code_coverage=no fi + if test "x$enable_code_coverage" = xyes; then + CODE_COVERAGE_ENABLED_TRUE= + CODE_COVERAGE_ENABLED_FALSE='#' +else + CODE_COVERAGE_ENABLED_TRUE='#' + CODE_COVERAGE_ENABLED_FALSE= +fi + CODE_COVERAGE_ENABLED=$enable_code_coverage + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $enable_code_coverage" >&5 +printf "%s\n" "$enable_code_coverage" >&6; } + if test "x$enable_code_coverage" = xyes +then : -if test "x$ac_cv_windows" = "xyes" + for ac_prog in gawk mawk nawk awk +do + # Extract the first word of "$ac_prog", so it can be a program name with args. +set dummy $ac_prog; ac_word=$2 +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 +printf %s "checking for $ac_word... " >&6; } +if test ${ac_cv_prog_AWK+y} then : - # Check whether --enable-static was given. -if test ${enable_static+y} + printf %s "(cached) " >&6 +else $as_nop + if test -n "$AWK"; then + ac_cv_prog_AWK="$AWK" # Let the user override the test. +else +as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +for as_dir in $PATH +do + IFS=$as_save_IFS + case $as_dir in #((( + '') as_dir=./ ;; + */) ;; + *) as_dir=$as_dir/ ;; + esac + for ac_exec_ext in '' $ac_executable_extensions; do + if as_fn_executable_p "$as_dir$ac_word$ac_exec_ext"; then + ac_cv_prog_AWK="$ac_prog" + printf "%s\n" "$as_me:${as_lineno-$LINENO}: found $as_dir$ac_word$ac_exec_ext" >&5 + break 2 + fi +done + done +IFS=$as_save_IFS + +fi +fi +AWK=$ac_cv_prog_AWK +if test -n "$AWK"; then + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $AWK" >&5 +printf "%s\n" "$AWK" >&6; } +else + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 +printf "%s\n" "no" >&6; } +fi + + + test -n "$AWK" && break +done + + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for GNU make" >&5 +printf %s "checking for GNU make... " >&6; } +if test ${_cv_gnu_make_command+y} then : - enableval=$enable_static; p=${PACKAGE-default} - case $enableval in - yes) enable_static=yes ;; - no) enable_static=no ;; - *) - enable_static=no - # Look at the argument we got. We use all the common list separators. - lt_save_ifs=$IFS; IFS=$IFS$PATH_SEPARATOR, - for pkg in $enableval; do - IFS=$lt_save_ifs - if test "X$pkg" = "X$p"; then - enable_static=yes - fi - done - IFS=$lt_save_ifs - ;; - esac + printf %s "(cached) " >&6 else $as_nop - enable_static=no + _cv_gnu_make_command="" ; + for a in "$MAKE" make gmake gnumake ; do + if test -z "$a" ; then continue ; fi ; + if "$a" --version 2> /dev/null | grep GNU 2>&1 > /dev/null ; then + _cv_gnu_make_command=$a ; + AX_CHECK_GNU_MAKE_HEADLINE=$("$a" --version 2> /dev/null | grep "GNU Make") + ax_check_gnu_make_version=$(echo ${AX_CHECK_GNU_MAKE_HEADLINE} | ${AWK} -F " " '{ print $(NF); }') + break ; + fi + done ; +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $_cv_gnu_make_command" >&5 +printf "%s\n" "$_cv_gnu_make_command" >&6; } + if test "x$_cv_gnu_make_command" = x"" +then : + ifGNUmake="#" +else $as_nop + ifGNUmake="" fi - - - - - - + if test "x$_cv_gnu_make_command" = x"" +then : + ifnGNUmake="" else $as_nop - # Check whether --enable-static was given. -if test ${enable_static+y} + ifnGNUmake="#" +fi + if test "x$_cv_gnu_make_command" = x"" then : - enableval=$enable_static; p=${PACKAGE-default} - case $enableval in - yes) enable_static=yes ;; - no) enable_static=no ;; - *) - enable_static=no - # Look at the argument we got. We use all the common list separators. - lt_save_ifs=$IFS; IFS=$IFS$PATH_SEPARATOR, - for pkg in $enableval; do - IFS=$lt_save_ifs - if test "X$pkg" = "X$p"; then - enable_static=yes - fi - done - IFS=$lt_save_ifs - ;; - esac + { ax_cv_gnu_make_command=; unset ax_cv_gnu_make_command;} else $as_nop - enable_static=yes + ax_cv_gnu_make_command=${_cv_gnu_make_command} +fi + if test "x$_cv_gnu_make_command" = x"" +then : + as_fn_error $? "not using GNU make that is needed for coverage" "$LINENO" 5 fi - - -fi - -# Check whether --enable-warnings was given. -if test ${enable_warnings+y} + # check for gcov + if test -n "$ac_tool_prefix"; then + # Extract the first word of "${ac_tool_prefix}$_AX_CODE_COVERAGE_GCOV_PROG_WITH", so it can be a program name with args. +set dummy ${ac_tool_prefix}$_AX_CODE_COVERAGE_GCOV_PROG_WITH; ac_word=$2 +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 +printf %s "checking for $ac_word... " >&6; } +if test ${ac_cv_prog_GCOV+y} then : - enableval=$enable_warnings; enable_warnings=${enableval} + printf %s "(cached) " >&6 else $as_nop - enable_warnings=yes + if test -n "$GCOV"; then + ac_cv_prog_GCOV="$GCOV" # Let the user override the test. +else +as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +for as_dir in $PATH +do + IFS=$as_save_IFS + case $as_dir in #((( + '') as_dir=./ ;; + */) ;; + *) as_dir=$as_dir/ ;; + esac + for ac_exec_ext in '' $ac_executable_extensions; do + if as_fn_executable_p "$as_dir$ac_word$ac_exec_ext"; then + ac_cv_prog_GCOV="${ac_tool_prefix}$_AX_CODE_COVERAGE_GCOV_PROG_WITH" + printf "%s\n" "$as_me:${as_lineno-$LINENO}: found $as_dir$ac_word$ac_exec_ext" >&5 + break 2 + fi +done + done +IFS=$as_save_IFS + +fi +fi +GCOV=$ac_cv_prog_GCOV +if test -n "$GCOV"; then + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $GCOV" >&5 +printf "%s\n" "$GCOV" >&6; } +else + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 +printf "%s\n" "no" >&6; } fi -# Check whether --enable-symbol-hiding was given. -if test ${enable_symbol_hiding+y} +fi +if test -z "$ac_cv_prog_GCOV"; then + ac_ct_GCOV=$GCOV + # Extract the first word of "$_AX_CODE_COVERAGE_GCOV_PROG_WITH", so it can be a program name with args. +set dummy $_AX_CODE_COVERAGE_GCOV_PROG_WITH; ac_word=$2 +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 +printf %s "checking for $ac_word... " >&6; } +if test ${ac_cv_prog_ac_ct_GCOV+y} then : - enableval=$enable_symbol_hiding; - symbol_hiding="$enableval" - if test "$symbol_hiding" = "no" -a "x$enable_shared" = "xyes" ; then - case $host_os in - cygwin* | mingw* | pw32* | cegcc*) - as_fn_error $? "Cannot disable symbol hiding on windows" "$LINENO" 5 - ;; - esac - fi - + printf %s "(cached) " >&6 else $as_nop + if test -n "$ac_ct_GCOV"; then + ac_cv_prog_ac_ct_GCOV="$ac_ct_GCOV" # Let the user override the test. +else +as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +for as_dir in $PATH +do + IFS=$as_save_IFS + case $as_dir in #((( + '') as_dir=./ ;; + */) ;; + *) as_dir=$as_dir/ ;; + esac + for ac_exec_ext in '' $ac_executable_extensions; do + if as_fn_executable_p "$as_dir$ac_word$ac_exec_ext"; then + ac_cv_prog_ac_ct_GCOV="$_AX_CODE_COVERAGE_GCOV_PROG_WITH" + printf "%s\n" "$as_me:${as_lineno-$LINENO}: found $as_dir$ac_word$ac_exec_ext" >&5 + break 2 + fi +done + done +IFS=$as_save_IFS - if test "x$enable_shared" = "xyes" ; then - symbol_hiding="maybe" - else - symbol_hiding="no" - fi +fi +fi +ac_ct_GCOV=$ac_cv_prog_ac_ct_GCOV +if test -n "$ac_ct_GCOV"; then + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_ct_GCOV" >&5 +printf "%s\n" "$ac_ct_GCOV" >&6; } +else + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 +printf "%s\n" "no" >&6; } +fi + if test "x$ac_ct_GCOV" = x; then + GCOV=":" + else + case $cross_compiling:$ac_tool_warned in +yes:) +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: WARNING: using cross tools not prefixed with host triplet" >&5 +printf "%s\n" "$as_me: WARNING: using cross tools not prefixed with host triplet" >&2;} +ac_tool_warned=yes ;; +esac + GCOV=$ac_ct_GCOV + fi +else + GCOV="$ac_cv_prog_GCOV" +fi + if test "X$GCOV" = "X:" +then : + as_fn_error $? "gcov is needed to do coverage" "$LINENO" 5 fi -# Check whether --enable-tests was given. -if test ${enable_tests+y} + if test "$GCC" = "no" then : - enableval=$enable_tests; build_tests="$enableval" -else $as_nop - if test "x$HAVE_CXX14" = "x1" && test "x$cross_compiling" = "xno" ; then - build_tests="maybe" - else - build_tests="no" - fi + as_fn_error $? "not compiling with gcc, which is required for gcov code coverage" "$LINENO" 5 fi - -# Check whether --enable-cares-threads was given. -if test ${enable_cares_threads+y} + # Extract the first word of "lcov", so it can be a program name with args. +set dummy lcov; ac_word=$2 +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 +printf %s "checking for $ac_word... " >&6; } +if test ${ac_cv_prog_LCOV+y} then : - enableval=$enable_cares_threads; CARES_THREADS=${enableval} + printf %s "(cached) " >&6 else $as_nop - CARES_THREADS=yes -fi + if test -n "$LCOV"; then + ac_cv_prog_LCOV="$LCOV" # Let the user override the test. +else +as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +for as_dir in $PATH +do + IFS=$as_save_IFS + case $as_dir in #((( + '') as_dir=./ ;; + */) ;; + *) as_dir=$as_dir/ ;; + esac + for ac_exec_ext in '' $ac_executable_extensions; do + if as_fn_executable_p "$as_dir$ac_word$ac_exec_ext"; then + ac_cv_prog_LCOV="lcov" + printf "%s\n" "$as_me:${as_lineno-$LINENO}: found $as_dir$ac_word$ac_exec_ext" >&5 + break 2 + fi +done + done +IFS=$as_save_IFS +fi +fi +LCOV=$ac_cv_prog_LCOV +if test -n "$LCOV"; then + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $LCOV" >&5 +printf "%s\n" "$LCOV" >&6; } +else + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 +printf "%s\n" "no" >&6; } +fi -# Check whether --with-random was given. -if test ${with_random+y} + # Extract the first word of "genhtml", so it can be a program name with args. +set dummy genhtml; ac_word=$2 +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 +printf %s "checking for $ac_word... " >&6; } +if test ${ac_cv_prog_GENHTML+y} then : - withval=$with_random; CARES_RANDOM_FILE="$withval" + printf %s "(cached) " >&6 else $as_nop - CARES_RANDOM_FILE="/dev/urandom" + if test -n "$GENHTML"; then + ac_cv_prog_GENHTML="$GENHTML" # Let the user override the test. +else +as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +for as_dir in $PATH +do + IFS=$as_save_IFS + case $as_dir in #((( + '') as_dir=./ ;; + */) ;; + *) as_dir=$as_dir/ ;; + esac + for ac_exec_ext in '' $ac_executable_extensions; do + if as_fn_executable_p "$as_dir$ac_word$ac_exec_ext"; then + ac_cv_prog_GENHTML="genhtml" + printf "%s\n" "$as_me:${as_lineno-$LINENO}: found $as_dir$ac_word$ac_exec_ext" >&5 + break 2 + fi +done + done +IFS=$as_save_IFS fi - -if test -n "$CARES_RANDOM_FILE" && test X"$CARES_RANDOM_FILE" != Xno ; then - - -printf "%s\n" "#define CARES_RANDOM_FILE \"$CARES_RANDOM_FILE\"" >>confdefs.h - +fi +GENHTML=$ac_cv_prog_GENHTML +if test -n "$GENHTML"; then + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $GENHTML" >&5 +printf "%s\n" "$GENHTML" >&6; } +else + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 +printf "%s\n" "no" >&6; } fi -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether to enable maintainer-specific portions of Makefiles" >&5 -printf %s "checking whether to enable maintainer-specific portions of Makefiles... " >&6; } - # Check whether --enable-maintainer-mode was given. -if test ${enable_maintainer_mode+y} + + if test x"$LCOV" = x then : - enableval=$enable_maintainer_mode; USE_MAINTAINER_MODE=$enableval -else $as_nop - USE_MAINTAINER_MODE=no -fi - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $USE_MAINTAINER_MODE" >&5 -printf "%s\n" "$USE_MAINTAINER_MODE" >&6; } - if test $USE_MAINTAINER_MODE = yes; then - MAINTAINER_MODE_TRUE= - MAINTAINER_MODE_FALSE='#' -else - MAINTAINER_MODE_TRUE='#' - MAINTAINER_MODE_FALSE= + as_fn_error $? "To enable code coverage reporting you must have lcov installed" "$LINENO" 5 + fi - MAINT=$MAINTAINER_MODE_TRUE + if test x"$GENHTML" = x +then : + as_fn_error $? "Could not find genhtml from the lcov package" "$LINENO" 5 -# Check whether --enable-silent-rules was given. -if test ${enable_silent_rules+y} -then : - enableval=$enable_silent_rules; fi -case $enable_silent_rules in # ((( - yes) AM_DEFAULT_VERBOSITY=0;; - no) AM_DEFAULT_VERBOSITY=1;; - *) AM_DEFAULT_VERBOSITY=0;; -esac -am_make=${MAKE-make} -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether $am_make supports nested variables" >&5 -printf %s "checking whether $am_make supports nested variables... " >&6; } -if test ${am_cv_make_support_nested_variables+y} + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for _gcov_init in -lgcov" >&5 +printf %s "checking for _gcov_init in -lgcov... " >&6; } +if test ${ac_cv_lib_gcov__gcov_init+y} then : printf %s "(cached) " >&6 else $as_nop - if printf "%s\n" 'TRUE=$(BAR$(V)) -BAR0=false -BAR1=true -V=1 -am__doit: - @$(TRUE) -.PHONY: am__doit' | $am_make -f - >/dev/null 2>&1; then - am_cv_make_support_nested_variables=yes -else - am_cv_make_support_nested_variables=no + ac_check_lib_save_LIBS=$LIBS +LIBS="-lgcov $LIBS" +cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +/* Override any GCC internal prototype to avoid an error. + Use char because int might match the return type of a GCC + builtin and then its argument prototype would still apply. */ +char _gcov_init (); +int +main (void) +{ +return _gcov_init (); + ; + return 0; +} +_ACEOF +if ac_fn_c_try_link "$LINENO" +then : + ac_cv_lib_gcov__gcov_init=yes +else $as_nop + ac_cv_lib_gcov__gcov_init=no fi +rm -f core conftest.err conftest.$ac_objext conftest.beam \ + conftest$ac_exeext conftest.$ac_ext +LIBS=$ac_check_lib_save_LIBS fi -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $am_cv_make_support_nested_variables" >&5 -printf "%s\n" "$am_cv_make_support_nested_variables" >&6; } -if test $am_cv_make_support_nested_variables = yes; then - AM_V='$(V)' - AM_DEFAULT_V='$(AM_DEFAULT_VERBOSITY)' -else - AM_V=$AM_DEFAULT_VERBOSITY - AM_DEFAULT_V=$AM_DEFAULT_VERBOSITY +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_gcov__gcov_init" >&5 +printf "%s\n" "$ac_cv_lib_gcov__gcov_init" >&6; } +if test "x$ac_cv_lib_gcov__gcov_init" = xyes +then : + CODE_COVERAGE_LIBS="-lgcov" +else $as_nop + CODE_COVERAGE_LIBS="" fi -AM_BACKSLASH='\' + CODE_COVERAGE_CPPFLAGS="-DNDEBUG" + CODE_COVERAGE_CFLAGS="-O0 -g -fprofile-arcs -ftest-coverage" + CODE_COVERAGE_CXXFLAGS="-O0 -g -fprofile-arcs -ftest-coverage" + +fi @@ -19191,449 +20629,609 @@ AM_BACKSLASH='\' +# Check whether --enable-largefile was given. +if test ${enable_largefile+y} +then : + enableval=$enable_largefile; +fi - # allow to override gcov location +if test "$enable_largefile" != no; then -# Check whether --with-gcov was given. -if test ${with_gcov+y} + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for special C compiler options needed for large files" >&5 +printf %s "checking for special C compiler options needed for large files... " >&6; } +if test ${ac_cv_sys_largefile_CC+y} +then : + printf %s "(cached) " >&6 +else $as_nop + ac_cv_sys_largefile_CC=no + if test "$GCC" != yes; then + ac_save_CC=$CC + while :; do + # IRIX 6.2 and later do not support large files by default, + # so use the C compiler's -n32 option if that helps. + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ +#include + /* Check that off_t can represent 2**63 - 1 correctly. + We can't simply define LARGE_OFF_T to be 9223372036854775807, + since some C++ compilers masquerading as C compilers + incorrectly reject 9223372036854775807. */ +#define LARGE_OFF_T (((off_t) 1 << 31 << 31) - 1 + ((off_t) 1 << 31 << 31)) + int off_t_is_large[(LARGE_OFF_T % 2147483629 == 721 + && LARGE_OFF_T % 2147483647 == 1) + ? 1 : -1]; +int +main (void) +{ + + ; + return 0; +} +_ACEOF + if ac_fn_c_try_compile "$LINENO" +then : + break +fi +rm -f core conftest.err conftest.$ac_objext conftest.beam + CC="$CC -n32" + if ac_fn_c_try_compile "$LINENO" +then : + ac_cv_sys_largefile_CC=' -n32'; break +fi +rm -f core conftest.err conftest.$ac_objext conftest.beam + break + done + CC=$ac_save_CC + rm -f conftest.$ac_ext + fi +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_sys_largefile_CC" >&5 +printf "%s\n" "$ac_cv_sys_largefile_CC" >&6; } + if test "$ac_cv_sys_largefile_CC" != no; then + CC=$CC$ac_cv_sys_largefile_CC + fi + + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for _FILE_OFFSET_BITS value needed for large files" >&5 +printf %s "checking for _FILE_OFFSET_BITS value needed for large files... " >&6; } +if test ${ac_cv_sys_file_offset_bits+y} +then : + printf %s "(cached) " >&6 +else $as_nop + while :; do + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ +#include + /* Check that off_t can represent 2**63 - 1 correctly. + We can't simply define LARGE_OFF_T to be 9223372036854775807, + since some C++ compilers masquerading as C compilers + incorrectly reject 9223372036854775807. */ +#define LARGE_OFF_T (((off_t) 1 << 31 << 31) - 1 + ((off_t) 1 << 31 << 31)) + int off_t_is_large[(LARGE_OFF_T % 2147483629 == 721 + && LARGE_OFF_T % 2147483647 == 1) + ? 1 : -1]; +int +main (void) +{ + + ; + return 0; +} +_ACEOF +if ac_fn_c_try_compile "$LINENO" +then : + ac_cv_sys_file_offset_bits=no; break +fi +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ +#define _FILE_OFFSET_BITS 64 +#include + /* Check that off_t can represent 2**63 - 1 correctly. + We can't simply define LARGE_OFF_T to be 9223372036854775807, + since some C++ compilers masquerading as C compilers + incorrectly reject 9223372036854775807. */ +#define LARGE_OFF_T (((off_t) 1 << 31 << 31) - 1 + ((off_t) 1 << 31 << 31)) + int off_t_is_large[(LARGE_OFF_T % 2147483629 == 721 + && LARGE_OFF_T % 2147483647 == 1) + ? 1 : -1]; +int +main (void) +{ + + ; + return 0; +} +_ACEOF +if ac_fn_c_try_compile "$LINENO" +then : + ac_cv_sys_file_offset_bits=64; break +fi +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext + ac_cv_sys_file_offset_bits=unknown + break +done +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_sys_file_offset_bits" >&5 +printf "%s\n" "$ac_cv_sys_file_offset_bits" >&6; } +case $ac_cv_sys_file_offset_bits in #( + no | unknown) ;; + *) +printf "%s\n" "#define _FILE_OFFSET_BITS $ac_cv_sys_file_offset_bits" >>confdefs.h +;; +esac +rm -rf conftest* + if test $ac_cv_sys_file_offset_bits = unknown; then + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for _LARGE_FILES value needed for large files" >&5 +printf %s "checking for _LARGE_FILES value needed for large files... " >&6; } +if test ${ac_cv_sys_large_files+y} +then : + printf %s "(cached) " >&6 +else $as_nop + while :; do + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ +#include + /* Check that off_t can represent 2**63 - 1 correctly. + We can't simply define LARGE_OFF_T to be 9223372036854775807, + since some C++ compilers masquerading as C compilers + incorrectly reject 9223372036854775807. */ +#define LARGE_OFF_T (((off_t) 1 << 31 << 31) - 1 + ((off_t) 1 << 31 << 31)) + int off_t_is_large[(LARGE_OFF_T % 2147483629 == 721 + && LARGE_OFF_T % 2147483647 == 1) + ? 1 : -1]; +int +main (void) +{ + + ; + return 0; +} +_ACEOF +if ac_fn_c_try_compile "$LINENO" then : - withval=$with_gcov; _AX_CODE_COVERAGE_GCOV_PROG_WITH=$with_gcov -else $as_nop - _AX_CODE_COVERAGE_GCOV_PROG_WITH=gcov + ac_cv_sys_large_files=no; break fi +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ +#define _LARGE_FILES 1 +#include + /* Check that off_t can represent 2**63 - 1 correctly. + We can't simply define LARGE_OFF_T to be 9223372036854775807, + since some C++ compilers masquerading as C compilers + incorrectly reject 9223372036854775807. */ +#define LARGE_OFF_T (((off_t) 1 << 31 << 31) - 1 + ((off_t) 1 << 31 << 31)) + int off_t_is_large[(LARGE_OFF_T % 2147483629 == 721 + && LARGE_OFF_T % 2147483647 == 1) + ? 1 : -1]; +int +main (void) +{ - - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether to build with code coverage support" >&5 -printf %s "checking whether to build with code coverage support... " >&6; } - # Check whether --enable-code-coverage was given. -if test ${enable_code_coverage+y} + ; + return 0; +} +_ACEOF +if ac_fn_c_try_compile "$LINENO" then : - enableval=$enable_code_coverage; -else $as_nop - enable_code_coverage=no + ac_cv_sys_large_files=1; break fi - - - if test "x$enable_code_coverage" = xyes; then - CODE_COVERAGE_ENABLED_TRUE= - CODE_COVERAGE_ENABLED_FALSE='#' -else - CODE_COVERAGE_ENABLED_TRUE='#' - CODE_COVERAGE_ENABLED_FALSE= +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext + ac_cv_sys_large_files=unknown + break +done +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_sys_large_files" >&5 +printf "%s\n" "$ac_cv_sys_large_files" >&6; } +case $ac_cv_sys_large_files in #( + no | unknown) ;; + *) +printf "%s\n" "#define _LARGE_FILES $ac_cv_sys_large_files" >>confdefs.h +;; +esac +rm -rf conftest* + fi fi - CODE_COVERAGE_ENABLED=$enable_code_coverage - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $enable_code_coverage" >&5 -printf "%s\n" "$enable_code_coverage" >&6; } +case $host_os in + solaris*) - if test "x$enable_code_coverage" = xyes -then : +printf "%s\n" "#define ETC_INET 1" >>confdefs.h + ;; +esac - for ac_prog in gawk mawk nawk awk -do - # Extract the first word of "$ac_prog", so it can be a program name with args. -set dummy $ac_prog; ac_word=$2 -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 -printf %s "checking for $ac_word... " >&6; } -if test ${ac_cv_prog_AWK+y} -then : - printf %s "(cached) " >&6 -else $as_nop - if test -n "$AWK"; then - ac_cv_prog_AWK="$AWK" # Let the user override the test. -else -as_save_IFS=$IFS; IFS=$PATH_SEPARATOR -for as_dir in $PATH -do - IFS=$as_save_IFS - case $as_dir in #((( - '') as_dir=./ ;; - */) ;; - *) as_dir=$as_dir/ ;; - esac - for ac_exec_ext in '' $ac_executable_extensions; do - if as_fn_executable_p "$as_dir$ac_word$ac_exec_ext"; then - ac_cv_prog_AWK="$ac_prog" - printf "%s\n" "$as_me:${as_lineno-$LINENO}: found $as_dir$ac_word$ac_exec_ext" >&5 - break 2 - fi -done - done -IFS=$as_save_IFS +case $host_os in + solaris2*) + if test "x$GCC" = 'xyes'; then -fi -fi -AWK=$ac_cv_prog_AWK -if test -n "$AWK"; then - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $AWK" >&5 -printf "%s\n" "$AWK" >&6; } -else - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 -printf "%s\n" "no" >&6; } -fi - test -n "$AWK" && break -done - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for GNU make" >&5 -printf %s "checking for GNU make... " >&6; } -if test ${_cv_gnu_make_command+y} +for flag in -mimpure-text; do + as_CACHEVAR=`printf "%s\n" "ax_cv_check_ldflags__$flag" | $as_tr_sh` +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether the linker accepts $flag" >&5 +printf %s "checking whether the linker accepts $flag... " >&6; } +if eval test \${$as_CACHEVAR+y} then : printf %s "(cached) " >&6 else $as_nop - _cv_gnu_make_command="" ; - for a in "$MAKE" make gmake gnumake ; do - if test -z "$a" ; then continue ; fi ; - if "$a" --version 2> /dev/null | grep GNU 2>&1 > /dev/null ; then - _cv_gnu_make_command=$a ; - AX_CHECK_GNU_MAKE_HEADLINE=$("$a" --version 2> /dev/null | grep "GNU Make") - ax_check_gnu_make_version=$(echo ${AX_CHECK_GNU_MAKE_HEADLINE} | ${AWK} -F " " '{ print $(NF); }') - break ; - fi - done ; -fi -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $_cv_gnu_make_command" >&5 -printf "%s\n" "$_cv_gnu_make_command" >&6; } - if test "x$_cv_gnu_make_command" = x"" -then : - ifGNUmake="#" -else $as_nop - ifGNUmake="" -fi - if test "x$_cv_gnu_make_command" = x"" -then : - ifnGNUmake="" -else $as_nop - ifnGNUmake="#" -fi - if test "x$_cv_gnu_make_command" = x"" -then : - { ax_cv_gnu_make_command=; unset ax_cv_gnu_make_command;} -else $as_nop - ax_cv_gnu_make_command=${_cv_gnu_make_command} -fi - if test "x$_cv_gnu_make_command" = x"" -then : - as_fn_error $? "not using GNU make that is needed for coverage" "$LINENO" 5 -fi - + ax_check_save_flags=$LDFLAGS + LDFLAGS="$LDFLAGS $flag" + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ +int +main (void) +{ - # check for gcov - if test -n "$ac_tool_prefix"; then - # Extract the first word of "${ac_tool_prefix}$_AX_CODE_COVERAGE_GCOV_PROG_WITH", so it can be a program name with args. -set dummy ${ac_tool_prefix}$_AX_CODE_COVERAGE_GCOV_PROG_WITH; ac_word=$2 -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 -printf %s "checking for $ac_word... " >&6; } -if test ${ac_cv_prog_GCOV+y} + ; + return 0; +} +_ACEOF +if ac_fn_c_try_link "$LINENO" then : - printf %s "(cached) " >&6 + eval "$as_CACHEVAR=yes" else $as_nop - if test -n "$GCOV"; then - ac_cv_prog_GCOV="$GCOV" # Let the user override the test. -else -as_save_IFS=$IFS; IFS=$PATH_SEPARATOR -for as_dir in $PATH -do - IFS=$as_save_IFS - case $as_dir in #((( - '') as_dir=./ ;; - */) ;; - *) as_dir=$as_dir/ ;; - esac - for ac_exec_ext in '' $ac_executable_extensions; do - if as_fn_executable_p "$as_dir$ac_word$ac_exec_ext"; then - ac_cv_prog_GCOV="${ac_tool_prefix}$_AX_CODE_COVERAGE_GCOV_PROG_WITH" - printf "%s\n" "$as_me:${as_lineno-$LINENO}: found $as_dir$ac_word$ac_exec_ext" >&5 - break 2 - fi -done - done -IFS=$as_save_IFS - -fi + eval "$as_CACHEVAR=no" fi -GCOV=$ac_cv_prog_GCOV -if test -n "$GCOV"; then - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $GCOV" >&5 -printf "%s\n" "$GCOV" >&6; } -else - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 -printf "%s\n" "no" >&6; } +rm -f core conftest.err conftest.$ac_objext conftest.beam \ + conftest$ac_exeext conftest.$ac_ext + LDFLAGS=$ax_check_save_flags fi +eval ac_res=\$$as_CACHEVAR + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_res" >&5 +printf "%s\n" "$ac_res" >&6; } +if eval test \"x\$"$as_CACHEVAR"\" = x"yes" +then : - -fi -if test -z "$ac_cv_prog_GCOV"; then - ac_ct_GCOV=$GCOV - # Extract the first word of "$_AX_CODE_COVERAGE_GCOV_PROG_WITH", so it can be a program name with args. -set dummy $_AX_CODE_COVERAGE_GCOV_PROG_WITH; ac_word=$2 -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 -printf %s "checking for $ac_word... " >&6; } -if test ${ac_cv_prog_ac_ct_GCOV+y} +if test ${LDFLAGS+y} then : - printf %s "(cached) " >&6 -else $as_nop - if test -n "$ac_ct_GCOV"; then - ac_cv_prog_ac_ct_GCOV="$ac_ct_GCOV" # Let the user override the test. -else -as_save_IFS=$IFS; IFS=$PATH_SEPARATOR -for as_dir in $PATH -do - IFS=$as_save_IFS - case $as_dir in #((( - '') as_dir=./ ;; - */) ;; - *) as_dir=$as_dir/ ;; - esac - for ac_exec_ext in '' $ac_executable_extensions; do - if as_fn_executable_p "$as_dir$ac_word$ac_exec_ext"; then - ac_cv_prog_ac_ct_GCOV="$_AX_CODE_COVERAGE_GCOV_PROG_WITH" - printf "%s\n" "$as_me:${as_lineno-$LINENO}: found $as_dir$ac_word$ac_exec_ext" >&5 - break 2 - fi -done - done -IFS=$as_save_IFS + + case " $LDFLAGS " in #( + *" $flag "*) : + { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: : LDFLAGS already contains \$flag"; } >&5 + (: LDFLAGS already contains $flag) 2>&5 + ac_status=$? + printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; } ;; #( + *) : + + as_fn_append LDFLAGS " $flag" + { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: : LDFLAGS=\"\$LDFLAGS\""; } >&5 + (: LDFLAGS="$LDFLAGS") 2>&5 + ac_status=$? + printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; } + ;; +esac + +else $as_nop + + LDFLAGS=$flag + { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: : LDFLAGS=\"\$LDFLAGS\""; } >&5 + (: LDFLAGS="$LDFLAGS") 2>&5 + ac_status=$? + printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; } fi -fi -ac_ct_GCOV=$ac_cv_prog_ac_ct_GCOV -if test -n "$ac_ct_GCOV"; then - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_ct_GCOV" >&5 -printf "%s\n" "$ac_ct_GCOV" >&6; } -else - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 -printf "%s\n" "no" >&6; } + +else $as_nop + : fi - if test "x$ac_ct_GCOV" = x; then - GCOV=":" - else - case $cross_compiling:$ac_tool_warned in -yes:) -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: WARNING: using cross tools not prefixed with host triplet" >&5 -printf "%s\n" "$as_me: WARNING: using cross tools not prefixed with host triplet" >&2;} -ac_tool_warned=yes ;; +done + + fi + ;; + *) + ;; esac - GCOV=$ac_ct_GCOV - fi + +cares_use_no_undefined=no +case $host_os in + cygwin* | mingw* | pw32* | cegcc* | os2* | aix*) + cares_use_no_undefined=yes + ;; + *) + ;; +esac + if test "$cares_use_no_undefined" = 'yes'; then + CARES_USE_NO_UNDEFINED_TRUE= + CARES_USE_NO_UNDEFINED_FALSE='#' else - GCOV="$ac_cv_prog_GCOV" + CARES_USE_NO_UNDEFINED_TRUE='#' + CARES_USE_NO_UNDEFINED_FALSE= fi - if test "X$GCOV" = "X:" -then : - as_fn_error $? "gcov is needed to do coverage" "$LINENO" 5 + + +if test "$ac_cv_native_windows" = "yes" ; then + AM_CPPFLAGS="$AM_CPPFLAGS -D_WIN32_WINNT=0x0602 -DWIN32_LEAN_AND_MEAN" fi +if test "$ac_cv_native_windows" = "yes" -a "x$enable_shared" = "xyes" -a "x$enable_static" = "xyes" ; then + as_fn_error $? "Windows cannot build both static and shared simultaneously, specify --disable-shared or --disable-static" "$LINENO" 5 +fi - if test "$GCC" = "no" +if test "x$enable_shared" = "xno" -a "x$enable_static" = "xyes" ; then + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether we need CARES_STATICLIB definition" >&5 +printf %s "checking whether we need CARES_STATICLIB definition... " >&6; } + if test "$ac_cv_native_windows" = "yes" ; then + +if test ${AM_CPPFLAGS+y} then : - as_fn_error $? "not compiling with gcc, which is required for gcov code coverage" "$LINENO" 5 + case " $AM_CPPFLAGS " in #( + *" -DCARES_STATICLIB "*) : + { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: : AM_CPPFLAGS already contains -DCARES_STATICLIB"; } >&5 + (: AM_CPPFLAGS already contains -DCARES_STATICLIB) 2>&5 + ac_status=$? + printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; } ;; #( + *) : -fi + as_fn_append AM_CPPFLAGS " -DCARES_STATICLIB" + { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: : AM_CPPFLAGS=\"\$AM_CPPFLAGS\""; } >&5 + (: AM_CPPFLAGS="$AM_CPPFLAGS") 2>&5 + ac_status=$? + printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; } + ;; +esac - # Extract the first word of "lcov", so it can be a program name with args. -set dummy lcov; ac_word=$2 -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 -printf %s "checking for $ac_word... " >&6; } -if test ${ac_cv_prog_LCOV+y} -then : - printf %s "(cached) " >&6 else $as_nop - if test -n "$LCOV"; then - ac_cv_prog_LCOV="$LCOV" # Let the user override the test. -else -as_save_IFS=$IFS; IFS=$PATH_SEPARATOR -for as_dir in $PATH -do - IFS=$as_save_IFS - case $as_dir in #((( - '') as_dir=./ ;; - */) ;; - *) as_dir=$as_dir/ ;; - esac - for ac_exec_ext in '' $ac_executable_extensions; do - if as_fn_executable_p "$as_dir$ac_word$ac_exec_ext"; then - ac_cv_prog_LCOV="lcov" - printf "%s\n" "$as_me:${as_lineno-$LINENO}: found $as_dir$ac_word$ac_exec_ext" >&5 - break 2 - fi -done - done -IFS=$as_save_IFS + + AM_CPPFLAGS=-DCARES_STATICLIB + { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: : AM_CPPFLAGS=\"\$AM_CPPFLAGS\""; } >&5 + (: AM_CPPFLAGS="$AM_CPPFLAGS") 2>&5 + ac_status=$? + printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; } fi -fi -LCOV=$ac_cv_prog_LCOV -if test -n "$LCOV"; then - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $LCOV" >&5 -printf "%s\n" "$LCOV" >&6; } -else - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 + + PKGCONFIG_CFLAGS="-DCARES_STATICLIB" + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +printf "%s\n" "yes" >&6; } + else + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 printf "%s\n" "no" >&6; } + fi fi +CARES_SYMBOL_HIDING_CFLAG="" +if test "$symbol_hiding" != "no" ; then + compiler_supports_symbol_hiding="no" + if test "$ac_cv_windows" = "yes" ; then + compiler_supports_symbol_hiding="yes" + else + case "$ax_cv_c_compiler_vendor" in + clang|gnu|intel) - # Extract the first word of "genhtml", so it can be a program name with args. -set dummy genhtml; ac_word=$2 -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 -printf %s "checking for $ac_word... " >&6; } -if test ${ac_cv_prog_GENHTML+y} + + + +for flag in -fvisibility=hidden; do + as_CACHEVAR=`printf "%s\n" "ax_cv_check_cflags__$flag" | $as_tr_sh` +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether the C compiler accepts $flag" >&5 +printf %s "checking whether the C compiler accepts $flag... " >&6; } +if eval test \${$as_CACHEVAR+y} then : printf %s "(cached) " >&6 else $as_nop - if test -n "$GENHTML"; then - ac_cv_prog_GENHTML="$GENHTML" # Let the user override the test. -else -as_save_IFS=$IFS; IFS=$PATH_SEPARATOR -for as_dir in $PATH -do - IFS=$as_save_IFS - case $as_dir in #((( - '') as_dir=./ ;; - */) ;; - *) as_dir=$as_dir/ ;; - esac - for ac_exec_ext in '' $ac_executable_extensions; do - if as_fn_executable_p "$as_dir$ac_word$ac_exec_ext"; then - ac_cv_prog_GENHTML="genhtml" - printf "%s\n" "$as_me:${as_lineno-$LINENO}: found $as_dir$ac_word$ac_exec_ext" >&5 - break 2 + + ax_check_save_flags=$CFLAGS + if test x"$GCC" = xyes ; then + add_gnu_werror="-Werror" fi -done - done -IFS=$as_save_IFS + CFLAGS="$CFLAGS $flag $add_gnu_werror" + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ +int +main (void) +{ + + ; + return 0; +} +_ACEOF +if ac_fn_c_try_compile "$LINENO" +then : + eval "$as_CACHEVAR=yes" +else $as_nop + eval "$as_CACHEVAR=no" fi +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext + CFLAGS=$ax_check_save_flags fi -GENHTML=$ac_cv_prog_GENHTML -if test -n "$GENHTML"; then - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $GENHTML" >&5 -printf "%s\n" "$GENHTML" >&6; } -else - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 -printf "%s\n" "no" >&6; } -fi +eval ac_res=\$$as_CACHEVAR + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_res" >&5 +printf "%s\n" "$ac_res" >&6; } +if eval test \"x\$"$as_CACHEVAR"\" = x"yes" +then : +if test ${CARES_SYMBOL_HIDING_CFLAG+y} +then : + case " $CARES_SYMBOL_HIDING_CFLAG " in #( + *" $flag "*) : + { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: : CARES_SYMBOL_HIDING_CFLAG already contains \$flag"; } >&5 + (: CARES_SYMBOL_HIDING_CFLAG already contains $flag) 2>&5 + ac_status=$? + printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; } ;; #( + *) : - if test x"$LCOV" = x -then : + as_fn_append CARES_SYMBOL_HIDING_CFLAG " $flag" + { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: : CARES_SYMBOL_HIDING_CFLAG=\"\$CARES_SYMBOL_HIDING_CFLAG\""; } >&5 + (: CARES_SYMBOL_HIDING_CFLAG="$CARES_SYMBOL_HIDING_CFLAG") 2>&5 + ac_status=$? + printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; } + ;; +esac - as_fn_error $? "To enable code coverage reporting you must have lcov installed" "$LINENO" 5 +else $as_nop + + CARES_SYMBOL_HIDING_CFLAG=$flag + { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: : CARES_SYMBOL_HIDING_CFLAG=\"\$CARES_SYMBOL_HIDING_CFLAG\""; } >&5 + (: CARES_SYMBOL_HIDING_CFLAG="$CARES_SYMBOL_HIDING_CFLAG") 2>&5 + ac_status=$? + printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; } + +fi +else $as_nop + : fi - if test x"$GENHTML" = x -then : +done - as_fn_error $? "Could not find genhtml from the lcov package" "$LINENO" 5 + if test "x$CARES_SYMBOL_HIDING_CFLAG" != "x" ; then + compiler_supports_symbol_hiding="yes" + fi + ;; + sun) -fi - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for _gcov_init in -lgcov" >&5 -printf %s "checking for _gcov_init in -lgcov... " >&6; } -if test ${ac_cv_lib_gcov__gcov_init+y} + + +for flag in -xldscope=hidden; do + as_CACHEVAR=`printf "%s\n" "ax_cv_check_cflags__$flag" | $as_tr_sh` +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether the C compiler accepts $flag" >&5 +printf %s "checking whether the C compiler accepts $flag... " >&6; } +if eval test \${$as_CACHEVAR+y} then : printf %s "(cached) " >&6 else $as_nop - ac_check_lib_save_LIBS=$LIBS -LIBS="-lgcov $LIBS" -cat confdefs.h - <<_ACEOF >conftest.$ac_ext + + ax_check_save_flags=$CFLAGS + if test x"$GCC" = xyes ; then + add_gnu_werror="-Werror" + fi + CFLAGS="$CFLAGS $flag $add_gnu_werror" + cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ -/* Override any GCC internal prototype to avoid an error. - Use char because int might match the return type of a GCC - builtin and then its argument prototype would still apply. */ -char _gcov_init (); int main (void) { -return _gcov_init (); + ; return 0; } _ACEOF -if ac_fn_c_try_link "$LINENO" +if ac_fn_c_try_compile "$LINENO" then : - ac_cv_lib_gcov__gcov_init=yes + eval "$as_CACHEVAR=yes" else $as_nop - ac_cv_lib_gcov__gcov_init=no + eval "$as_CACHEVAR=no" fi -rm -f core conftest.err conftest.$ac_objext conftest.beam \ - conftest$ac_exeext conftest.$ac_ext -LIBS=$ac_check_lib_save_LIBS +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext + CFLAGS=$ax_check_save_flags fi -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_gcov__gcov_init" >&5 -printf "%s\n" "$ac_cv_lib_gcov__gcov_init" >&6; } -if test "x$ac_cv_lib_gcov__gcov_init" = xyes +eval ac_res=\$$as_CACHEVAR + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_res" >&5 +printf "%s\n" "$ac_res" >&6; } +if eval test \"x\$"$as_CACHEVAR"\" = x"yes" then : - CODE_COVERAGE_LIBS="-lgcov" -else $as_nop - CODE_COVERAGE_LIBS="" -fi - - - CODE_COVERAGE_CPPFLAGS="-DNDEBUG" - CODE_COVERAGE_CFLAGS="-O0 -g -fprofile-arcs -ftest-coverage" - CODE_COVERAGE_CXXFLAGS="-O0 -g -fprofile-arcs -ftest-coverage" - +if test ${CARES_SYMBOL_HIDING_CFLAG+y} +then : + case " $CARES_SYMBOL_HIDING_CFLAG " in #( + *" $flag "*) : + { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: : CARES_SYMBOL_HIDING_CFLAG already contains \$flag"; } >&5 + (: CARES_SYMBOL_HIDING_CFLAG already contains $flag) 2>&5 + ac_status=$? + printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; } ;; #( + *) : + as_fn_append CARES_SYMBOL_HIDING_CFLAG " $flag" + { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: : CARES_SYMBOL_HIDING_CFLAG=\"\$CARES_SYMBOL_HIDING_CFLAG\""; } >&5 + (: CARES_SYMBOL_HIDING_CFLAG="$CARES_SYMBOL_HIDING_CFLAG") 2>&5 + ac_status=$? + printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; } + ;; +esac +else $as_nop + CARES_SYMBOL_HIDING_CFLAG=$flag + { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: : CARES_SYMBOL_HIDING_CFLAG=\"\$CARES_SYMBOL_HIDING_CFLAG\""; } >&5 + (: CARES_SYMBOL_HIDING_CFLAG="$CARES_SYMBOL_HIDING_CFLAG") 2>&5 + ac_status=$? + printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; } fi +else $as_nop + : +fi +done + if test "x$CARES_SYMBOL_HIDING_CFLAG" != "x" ; then + compiler_supports_symbol_hiding="yes" + fi + ;; + esac + fi + if test "$compiler_supports_symbol_hiding" = "no" ; then + if test "$symbol_hiding" = "yes" ; then + as_fn_error $? "Compiler does not support symbol hiding" "$LINENO" 5 + else + symbol_hiding="no" + fi + else +printf "%s\n" "#define CARES_SYMBOL_HIDING 1 " >>confdefs.h + symbol_hiding="yes" + fi +fi + if test "x$symbol_hiding" = "xyes"; then + CARES_SYMBOL_HIDING_TRUE= + CARES_SYMBOL_HIDING_FALSE='#' +else + CARES_SYMBOL_HIDING_TRUE='#' + CARES_SYMBOL_HIDING_FALSE= +fi +if test "$enable_warnings" = "yes"; then -# Check whether --enable-largefile was given. -if test ${enable_largefile+y} -then : - enableval=$enable_largefile; -fi -if test "$enable_largefile" != no; then - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for special C compiler options needed for large files" >&5 -printf %s "checking for special C compiler options needed for large files... " >&6; } -if test ${ac_cv_sys_largefile_CC+y} +for flag in -Wall -Wextra -Waggregate-return -Wcast-align -Wcast-qual -Wconversion -Wdeclaration-after-statement -Wdouble-promotion -Wfloat-equal -Wformat-security -Winit-self -Wjump-misses-init -Wlogical-op -Wmissing-braces -Wmissing-declarations -Wmissing-format-attribute -Wmissing-include-dirs -Wmissing-prototypes -Wnested-externs -Wno-coverage-mismatch -Wold-style-definition -Wpacked -Wpedantic -Wpointer-arith -Wredundant-decls -Wshadow -Wsign-conversion -Wstrict-overflow -Wstrict-prototypes -Wtrampolines -Wundef -Wunreachable-code -Wunused -Wvariadic-macros -Wvla -Wwrite-strings -Werror=implicit-int -Werror=implicit-function-declaration -Werror=partial-availability -Wno-long-long ; do + as_CACHEVAR=`printf "%s\n" "ax_cv_check_cflags_-Werror_$flag" | $as_tr_sh` +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether the C compiler accepts $flag" >&5 +printf %s "checking whether the C compiler accepts $flag... " >&6; } +if eval test \${$as_CACHEVAR+y} then : printf %s "(cached) " >&6 else $as_nop - ac_cv_sys_largefile_CC=no - if test "$GCC" != yes; then - ac_save_CC=$CC - while :; do - # IRIX 6.2 and later do not support large files by default, - # so use the C compiler's -n32 option if that helps. - cat confdefs.h - <<_ACEOF >conftest.$ac_ext + + ax_check_save_flags=$CFLAGS + if test x"$GCC" = xyes ; then + add_gnu_werror="-Werror" + fi + CFLAGS="$CFLAGS -Werror $flag $add_gnu_werror" + cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ -#include - /* Check that off_t can represent 2**63 - 1 correctly. - We can't simply define LARGE_OFF_T to be 9223372036854775807, - since some C++ compilers masquerading as C compilers - incorrectly reject 9223372036854775807. */ -#define LARGE_OFF_T (((off_t) 1 << 31 << 31) - 1 + ((off_t) 1 << 31 << 31)) - int off_t_is_large[(LARGE_OFF_T % 2147483629 == 721 - && LARGE_OFF_T % 2147483647 == 1) - ? 1 : -1]; + int main (void) { @@ -19642,47 +21240,84 @@ main (void) return 0; } _ACEOF - if ac_fn_c_try_compile "$LINENO" +if ac_fn_c_try_compile "$LINENO" then : - break + eval "$as_CACHEVAR=yes" +else $as_nop + eval "$as_CACHEVAR=no" fi -rm -f core conftest.err conftest.$ac_objext conftest.beam - CC="$CC -n32" - if ac_fn_c_try_compile "$LINENO" +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext + CFLAGS=$ax_check_save_flags +fi +eval ac_res=\$$as_CACHEVAR + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_res" >&5 +printf "%s\n" "$ac_res" >&6; } +if eval test \"x\$"$as_CACHEVAR"\" = x"yes" then : - ac_cv_sys_largefile_CC=' -n32'; break + +if test ${AM_CFLAGS+y} +then : + + case " $AM_CFLAGS " in #( + *" $flag "*) : + { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: : AM_CFLAGS already contains \$flag"; } >&5 + (: AM_CFLAGS already contains $flag) 2>&5 + ac_status=$? + printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; } ;; #( + *) : + + as_fn_append AM_CFLAGS " $flag" + { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: : AM_CFLAGS=\"\$AM_CFLAGS\""; } >&5 + (: AM_CFLAGS="$AM_CFLAGS") 2>&5 + ac_status=$? + printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; } + ;; +esac + +else $as_nop + + AM_CFLAGS=$flag + { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: : AM_CFLAGS=\"\$AM_CFLAGS\""; } >&5 + (: AM_CFLAGS="$AM_CFLAGS") 2>&5 + ac_status=$? + printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; } + fi -rm -f core conftest.err conftest.$ac_objext conftest.beam - break - done - CC=$ac_save_CC - rm -f conftest.$ac_ext - fi + +else $as_nop + : fi -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_sys_largefile_CC" >&5 -printf "%s\n" "$ac_cv_sys_largefile_CC" >&6; } - if test "$ac_cv_sys_largefile_CC" != no; then - CC=$CC$ac_cv_sys_largefile_CC - fi - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for _FILE_OFFSET_BITS value needed for large files" >&5 -printf %s "checking for _FILE_OFFSET_BITS value needed for large files... " >&6; } -if test ${ac_cv_sys_file_offset_bits+y} +done + +fi + +case $host_os in + *qnx*|*android*) + + + + +for flag in -std=c99; do + as_CACHEVAR=`printf "%s\n" "ax_cv_check_cflags_-Werror_$flag" | $as_tr_sh` +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether the C compiler accepts $flag" >&5 +printf %s "checking whether the C compiler accepts $flag... " >&6; } +if eval test \${$as_CACHEVAR+y} then : printf %s "(cached) " >&6 else $as_nop - while :; do + + ax_check_save_flags=$CFLAGS + if test x"$GCC" = xyes ; then + add_gnu_werror="-Werror" + fi + CFLAGS="$CFLAGS -Werror $flag $add_gnu_werror" cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ -#include - /* Check that off_t can represent 2**63 - 1 correctly. - We can't simply define LARGE_OFF_T to be 9223372036854775807, - since some C++ compilers masquerading as C compilers - incorrectly reject 9223372036854775807. */ -#define LARGE_OFF_T (((off_t) 1 << 31 << 31) - 1 + ((off_t) 1 << 31 << 31)) - int off_t_is_large[(LARGE_OFF_T % 2147483629 == 721 - && LARGE_OFF_T % 2147483647 == 1) - ? 1 : -1]; +/* end confdefs.h. */ + int main (void) { @@ -19693,66 +21328,80 @@ main (void) _ACEOF if ac_fn_c_try_compile "$LINENO" then : - ac_cv_sys_file_offset_bits=no; break + eval "$as_CACHEVAR=yes" +else $as_nop + eval "$as_CACHEVAR=no" fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext - cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ -#define _FILE_OFFSET_BITS 64 -#include - /* Check that off_t can represent 2**63 - 1 correctly. - We can't simply define LARGE_OFF_T to be 9223372036854775807, - since some C++ compilers masquerading as C compilers - incorrectly reject 9223372036854775807. */ -#define LARGE_OFF_T (((off_t) 1 << 31 << 31) - 1 + ((off_t) 1 << 31 << 31)) - int off_t_is_large[(LARGE_OFF_T % 2147483629 == 721 - && LARGE_OFF_T % 2147483647 == 1) - ? 1 : -1]; -int -main (void) -{ + CFLAGS=$ax_check_save_flags +fi +eval ac_res=\$$as_CACHEVAR + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_res" >&5 +printf "%s\n" "$ac_res" >&6; } +if eval test \"x\$"$as_CACHEVAR"\" = x"yes" +then : - ; - return 0; -} -_ACEOF -if ac_fn_c_try_compile "$LINENO" +if test ${AM_CFLAGS+y} then : - ac_cv_sys_file_offset_bits=64; break + + case " $AM_CFLAGS " in #( + *" $flag "*) : + { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: : AM_CFLAGS already contains \$flag"; } >&5 + (: AM_CFLAGS already contains $flag) 2>&5 + ac_status=$? + printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; } ;; #( + *) : + + as_fn_append AM_CFLAGS " $flag" + { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: : AM_CFLAGS=\"\$AM_CFLAGS\""; } >&5 + (: AM_CFLAGS="$AM_CFLAGS") 2>&5 + ac_status=$? + printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; } + ;; +esac + +else $as_nop + + AM_CFLAGS=$flag + { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: : AM_CFLAGS=\"\$AM_CFLAGS\""; } >&5 + (: AM_CFLAGS="$AM_CFLAGS") 2>&5 + ac_status=$? + printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; } + fi -rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext - ac_cv_sys_file_offset_bits=unknown - break -done + +else $as_nop + : fi -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_sys_file_offset_bits" >&5 -printf "%s\n" "$ac_cv_sys_file_offset_bits" >&6; } -case $ac_cv_sys_file_offset_bits in #( - no | unknown) ;; + +done + + ;; *) -printf "%s\n" "#define _FILE_OFFSET_BITS $ac_cv_sys_file_offset_bits" >>confdefs.h -;; -esac -rm -rf conftest* - if test $ac_cv_sys_file_offset_bits = unknown; then - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for _LARGE_FILES value needed for large files" >&5 -printf %s "checking for _LARGE_FILES value needed for large files... " >&6; } -if test ${ac_cv_sys_large_files+y} + + + + +for flag in -std=c90; do + as_CACHEVAR=`printf "%s\n" "ax_cv_check_cflags_-Werror_$flag" | $as_tr_sh` +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether the C compiler accepts $flag" >&5 +printf %s "checking whether the C compiler accepts $flag... " >&6; } +if eval test \${$as_CACHEVAR+y} then : printf %s "(cached) " >&6 else $as_nop - while :; do + + ax_check_save_flags=$CFLAGS + if test x"$GCC" = xyes ; then + add_gnu_werror="-Werror" + fi + CFLAGS="$CFLAGS -Werror $flag $add_gnu_werror" cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ -#include - /* Check that off_t can represent 2**63 - 1 correctly. - We can't simply define LARGE_OFF_T to be 9223372036854775807, - since some C++ compilers masquerading as C compilers - incorrectly reject 9223372036854775807. */ -#define LARGE_OFF_T (((off_t) 1 << 31 << 31) - 1 + ((off_t) 1 << 31 << 31)) - int off_t_is_large[(LARGE_OFF_T % 2147483629 == 721 - && LARGE_OFF_T % 2147483647 == 1) - ? 1 : -1]; + int main (void) { @@ -19763,21 +21412,83 @@ main (void) _ACEOF if ac_fn_c_try_compile "$LINENO" then : - ac_cv_sys_large_files=no; break + eval "$as_CACHEVAR=yes" +else $as_nop + eval "$as_CACHEVAR=no" fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext + CFLAGS=$ax_check_save_flags +fi +eval ac_res=\$$as_CACHEVAR + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_res" >&5 +printf "%s\n" "$ac_res" >&6; } +if eval test \"x\$"$as_CACHEVAR"\" = x"yes" +then : + +if test ${AM_CFLAGS+y} +then : + + case " $AM_CFLAGS " in #( + *" $flag "*) : + { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: : AM_CFLAGS already contains \$flag"; } >&5 + (: AM_CFLAGS already contains $flag) 2>&5 + ac_status=$? + printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; } ;; #( + *) : + + as_fn_append AM_CFLAGS " $flag" + { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: : AM_CFLAGS=\"\$AM_CFLAGS\""; } >&5 + (: AM_CFLAGS="$AM_CFLAGS") 2>&5 + ac_status=$? + printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; } + ;; +esac + +else $as_nop + + AM_CFLAGS=$flag + { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: : AM_CFLAGS=\"\$AM_CFLAGS\""; } >&5 + (: AM_CFLAGS="$AM_CFLAGS") 2>&5 + ac_status=$? + printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; } + +fi + +else $as_nop + : +fi + +done + + ;; +esac + +case $host_os in + *qnx*) + + + + +for flag in -D_QNX_SOURCE; do + as_CACHEVAR=`printf "%s\n" "ax_cv_check_cflags_-Werror_$flag" | $as_tr_sh` +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether the C compiler accepts $flag" >&5 +printf %s "checking whether the C compiler accepts $flag... " >&6; } +if eval test \${$as_CACHEVAR+y} +then : + printf %s "(cached) " >&6 +else $as_nop + + ax_check_save_flags=$CFLAGS + if test x"$GCC" = xyes ; then + add_gnu_werror="-Werror" + fi + CFLAGS="$CFLAGS -Werror $flag $add_gnu_werror" cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ -#define _LARGE_FILES 1 -#include - /* Check that off_t can represent 2**63 - 1 correctly. - We can't simply define LARGE_OFF_T to be 9223372036854775807, - since some C++ compilers masquerading as C compilers - incorrectly reject 9223372036854775807. */ -#define LARGE_OFF_T (((off_t) 1 << 31 << 31) - 1 + ((off_t) 1 << 31 << 31)) - int off_t_is_large[(LARGE_OFF_T % 2147483629 == 721 - && LARGE_OFF_T % 2147483647 == 1) - ? 1 : -1]; + int main (void) { @@ -19788,52 +21499,79 @@ main (void) _ACEOF if ac_fn_c_try_compile "$LINENO" then : - ac_cv_sys_large_files=1; break + eval "$as_CACHEVAR=yes" +else $as_nop + eval "$as_CACHEVAR=no" fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext - ac_cv_sys_large_files=unknown - break -done + CFLAGS=$ax_check_save_flags fi -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_sys_large_files" >&5 -printf "%s\n" "$ac_cv_sys_large_files" >&6; } -case $ac_cv_sys_large_files in #( - no | unknown) ;; - *) -printf "%s\n" "#define _LARGE_FILES $ac_cv_sys_large_files" >>confdefs.h -;; +eval ac_res=\$$as_CACHEVAR + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_res" >&5 +printf "%s\n" "$ac_res" >&6; } +if eval test \"x\$"$as_CACHEVAR"\" = x"yes" +then : + +if test ${AM_CPPFLAGS+y} +then : + + case " $AM_CPPFLAGS " in #( + *" $flag "*) : + { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: : AM_CPPFLAGS already contains \$flag"; } >&5 + (: AM_CPPFLAGS already contains $flag) 2>&5 + ac_status=$? + printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; } ;; #( + *) : + + as_fn_append AM_CPPFLAGS " $flag" + { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: : AM_CPPFLAGS=\"\$AM_CPPFLAGS\""; } >&5 + (: AM_CPPFLAGS="$AM_CPPFLAGS") 2>&5 + ac_status=$? + printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; } + ;; esac -rm -rf conftest* - fi -fi +else $as_nop + + AM_CPPFLAGS=$flag + { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: : AM_CPPFLAGS=\"\$AM_CPPFLAGS\""; } >&5 + (: AM_CPPFLAGS="$AM_CPPFLAGS") 2>&5 + ac_status=$? + printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; } -case $host_os in - solaris*) +fi -printf "%s\n" "#define ETC_INET 1" >>confdefs.h +else $as_nop + : +fi + +done - ;; + ;; esac -case $host_os in - solaris2*) - if test "x$GCC" = 'xyes'; then +if test "$ax_cv_c_compiler_vendor" = "intel"; then -for flag in -mimpure-text; do - as_CACHEVAR=`printf "%s\n" "ax_cv_check_ldflags__$flag" | $as_tr_sh` -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether the linker accepts $flag" >&5 -printf %s "checking whether the linker accepts $flag... " >&6; } +for flag in -shared-intel; do + as_CACHEVAR=`printf "%s\n" "ax_cv_check_cflags__$flag" | $as_tr_sh` +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether the C compiler accepts $flag" >&5 +printf %s "checking whether the C compiler accepts $flag... " >&6; } if eval test \${$as_CACHEVAR+y} then : printf %s "(cached) " >&6 else $as_nop - ax_check_save_flags=$LDFLAGS - LDFLAGS="$LDFLAGS $flag" + ax_check_save_flags=$CFLAGS + if test x"$GCC" = xyes ; then + add_gnu_werror="-Werror" + fi + CFLAGS="$CFLAGS $flag $add_gnu_werror" cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ @@ -19845,15 +21583,14 @@ main (void) return 0; } _ACEOF -if ac_fn_c_try_link "$LINENO" +if ac_fn_c_try_compile "$LINENO" then : eval "$as_CACHEVAR=yes" else $as_nop eval "$as_CACHEVAR=no" fi -rm -f core conftest.err conftest.$ac_objext conftest.beam \ - conftest$ac_exeext conftest.$ac_ext - LDFLAGS=$ax_check_save_flags +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext + CFLAGS=$ax_check_save_flags fi eval ac_res=\$$as_CACHEVAR { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_res" >&5 @@ -19861,21 +21598,21 @@ printf "%s\n" "$ac_res" >&6; } if eval test \"x\$"$as_CACHEVAR"\" = x"yes" then : -if test ${LDFLAGS+y} +if test ${AM_CFLAGS+y} then : - case " $LDFLAGS " in #( + case " $AM_CFLAGS " in #( *" $flag "*) : - { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: : LDFLAGS already contains \$flag"; } >&5 - (: LDFLAGS already contains $flag) 2>&5 + { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: : AM_CFLAGS already contains \$flag"; } >&5 + (: AM_CFLAGS already contains $flag) 2>&5 ac_status=$? printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 test $ac_status = 0; } ;; #( *) : - as_fn_append LDFLAGS " $flag" - { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: : LDFLAGS=\"\$LDFLAGS\""; } >&5 - (: LDFLAGS="$LDFLAGS") 2>&5 + as_fn_append AM_CFLAGS " $flag" + { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: : AM_CFLAGS=\"\$AM_CFLAGS\""; } >&5 + (: AM_CFLAGS="$AM_CFLAGS") 2>&5 ac_status=$? printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 test $ac_status = 0; } @@ -19884,9 +21621,9 @@ esac else $as_nop - LDFLAGS=$flag - { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: : LDFLAGS=\"\$LDFLAGS\""; } >&5 - (: LDFLAGS="$LDFLAGS") 2>&5 + AM_CFLAGS=$flag + { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: : AM_CFLAGS=\"\$AM_CFLAGS\""; } >&5 + (: AM_CFLAGS="$AM_CFLAGS") 2>&5 ac_status=$? printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 test $ac_status = 0; } @@ -19899,110 +21636,292 @@ fi done - fi - ;; - *) - ;; -esac +fi -cares_use_no_undefined=no -case $host_os in - cygwin* | mingw* | pw32* | cegcc* | os2* | aix*) - cares_use_no_undefined=yes - ;; - *) - ;; -esac - if test "$cares_use_no_undefined" = 'yes'; then - CARES_USE_NO_UNDEFINED_TRUE= - CARES_USE_NO_UNDEFINED_FALSE='#' +if test "$ac_cv_native_windows" = "yes" ; then + + ac_ext=c +ac_cpp='$CPP $CPPFLAGS' +ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5' +ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5' +ac_compiler_gnu=$ac_cv_c_compiler_gnu +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking how to run the C preprocessor" >&5 +printf %s "checking how to run the C preprocessor... " >&6; } +# On Suns, sometimes $CPP names a directory. +if test -n "$CPP" && test -d "$CPP"; then + CPP= +fi +if test -z "$CPP"; then + if test ${ac_cv_prog_CPP+y} +then : + printf %s "(cached) " >&6 +else $as_nop + # Double quotes because $CC needs to be expanded + for CPP in "$CC -E" "$CC -E -traditional-cpp" cpp /lib/cpp + do + ac_preproc_ok=false +for ac_c_preproc_warn_flag in '' yes +do + # Use a header file that comes with gcc, so configuring glibc + # with a fresh cross-compiler works. + # On the NeXT, cc -E runs the code through the compiler's parser, + # not just through cpp. "Syntax error" is here to catch this case. + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ +#include + Syntax error +_ACEOF +if ac_fn_c_try_cpp "$LINENO" +then : + +else $as_nop + # Broken: fails on valid input. +continue +fi +rm -f conftest.err conftest.i conftest.$ac_ext + + # OK, works on sane cases. Now check whether nonexistent headers + # can be detected and how. + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ +#include +_ACEOF +if ac_fn_c_try_cpp "$LINENO" +then : + # Broken: success on invalid input. +continue +else $as_nop + # Passes both tests. +ac_preproc_ok=: +break +fi +rm -f conftest.err conftest.i conftest.$ac_ext + +done +# Because of `break', _AC_PREPROC_IFELSE's cleaning code was skipped. +rm -f conftest.i conftest.err conftest.$ac_ext +if $ac_preproc_ok +then : + break +fi + + done + ac_cv_prog_CPP=$CPP + +fi + CPP=$ac_cv_prog_CPP else - CARES_USE_NO_UNDEFINED_TRUE='#' - CARES_USE_NO_UNDEFINED_FALSE= + ac_cv_prog_CPP=$CPP +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $CPP" >&5 +printf "%s\n" "$CPP" >&6; } +ac_preproc_ok=false +for ac_c_preproc_warn_flag in '' yes +do + # Use a header file that comes with gcc, so configuring glibc + # with a fresh cross-compiler works. + # On the NeXT, cc -E runs the code through the compiler's parser, + # not just through cpp. "Syntax error" is here to catch this case. + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ +#include + Syntax error +_ACEOF +if ac_fn_c_try_cpp "$LINENO" +then : + +else $as_nop + # Broken: fails on valid input. +continue fi +rm -f conftest.err conftest.i conftest.$ac_ext + # OK, works on sane cases. Now check whether nonexistent headers + # can be detected and how. + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ +#include +_ACEOF +if ac_fn_c_try_cpp "$LINENO" +then : + # Broken: success on invalid input. +continue +else $as_nop + # Passes both tests. +ac_preproc_ok=: +break +fi +rm -f conftest.err conftest.i conftest.$ac_ext +done +# Because of `break', _AC_PREPROC_IFELSE's cleaning code was skipped. +rm -f conftest.i conftest.err conftest.$ac_ext +if $ac_preproc_ok +then : -if test "$ac_cv_native_windows" = "yes" ; then - AM_CPPFLAGS="$AM_CPPFLAGS -D_WIN32_WINNT=0x0602 -DWIN32_LEAN_AND_MEAN" +else $as_nop + { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 +printf "%s\n" "$as_me: error: in \`$ac_pwd':" >&2;} +as_fn_error $? "C preprocessor \"$CPP\" fails sanity check +See \`config.log' for more details" "$LINENO" 5; } fi -if test "$ac_cv_native_windows" = "yes" -a "x$enable_shared" = "xyes" -a "x$enable_static" = "xyes" ; then - as_fn_error $? "Windows cannot build both static and shared simultaneously, specify --disable-shared or --disable-static" "$LINENO" 5 +ac_ext=c +ac_cpp='$CPP $CPPFLAGS' +ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5' +ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5' +ac_compiler_gnu=$ac_cv_c_compiler_gnu + + +ac_fn_c_check_header_preproc "$LINENO" "windows.h" "ac_cv_header_windows_h" +if test "x$ac_cv_header_windows_h" = xyes +then : + printf "%s\n" "#define HAVE_WINDOWS_H 1" >>confdefs.h + +fi +ac_fn_c_check_header_preproc "$LINENO" "winsock2.h" "ac_cv_header_winsock2_h" +if test "x$ac_cv_header_winsock2_h" = xyes +then : + printf "%s\n" "#define HAVE_WINSOCK2_H 1" >>confdefs.h + +fi +ac_fn_c_check_header_preproc "$LINENO" "ws2tcpip.h" "ac_cv_header_ws2tcpip_h" +if test "x$ac_cv_header_ws2tcpip_h" = xyes +then : + printf "%s\n" "#define HAVE_WS2TCPIP_H 1" >>confdefs.h + +fi +ac_fn_c_check_header_preproc "$LINENO" "iphlpapi.h" "ac_cv_header_iphlpapi_h" +if test "x$ac_cv_header_iphlpapi_h" = xyes +then : + printf "%s\n" "#define HAVE_IPHLPAPI_H 1" >>confdefs.h + +fi +ac_fn_c_check_header_preproc "$LINENO" "netioapi.h" "ac_cv_header_netioapi_h" +if test "x$ac_cv_header_netioapi_h" = xyes +then : + printf "%s\n" "#define HAVE_NETIOAPI_H 1" >>confdefs.h + +fi +ac_fn_c_check_header_preproc "$LINENO" "ws2ipdef.h" "ac_cv_header_ws2ipdef_h" +if test "x$ac_cv_header_ws2ipdef_h" = xyes +then : + printf "%s\n" "#define HAVE_WS2IPDEF_H 1" >>confdefs.h + +fi +ac_fn_c_check_header_preproc "$LINENO" "winternl.h" "ac_cv_header_winternl_h" +if test "x$ac_cv_header_winternl_h" = xyes +then : + printf "%s\n" "#define HAVE_WINTERNL_H 1" >>confdefs.h + fi +ac_fn_c_check_header_preproc "$LINENO" "ntdef.h" "ac_cv_header_ntdef_h" +if test "x$ac_cv_header_ntdef_h" = xyes +then : + printf "%s\n" "#define HAVE_NTDEF_H 1" >>confdefs.h + +fi +ac_fn_c_check_header_preproc "$LINENO" "ntstatus.h" "ac_cv_header_ntstatus_h" +if test "x$ac_cv_header_ntstatus_h" = xyes +then : + printf "%s\n" "#define HAVE_NTSTATUS_H 1" >>confdefs.h + +fi +ac_fn_c_check_header_preproc "$LINENO" "mswsock.h" "ac_cv_header_mswsock_h" +if test "x$ac_cv_header_mswsock_h" = xyes +then : + printf "%s\n" "#define HAVE_MSWSOCK_H 1" >>confdefs.h -if test "x$enable_shared" = "xno" -a "x$enable_static" = "xyes" ; then - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether we need CARES_STATICLIB definition" >&5 -printf %s "checking whether we need CARES_STATICLIB definition... " >&6; } - if test "$ac_cv_native_windows" = "yes" ; then +fi -if test ${AM_CPPFLAGS+y} -then : - case " $AM_CPPFLAGS " in #( - *" -DCARES_STATICLIB "*) : - { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: : AM_CPPFLAGS already contains -DCARES_STATICLIB"; } >&5 - (: AM_CPPFLAGS already contains -DCARES_STATICLIB) 2>&5 - ac_status=$? - printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 - test $ac_status = 0; } ;; #( - *) : + if test "$ac_cv_header_winsock2_h" = "yes"; then + LIBS="$LIBS -lws2_32 -liphlpapi" + fi +fi - as_fn_append AM_CPPFLAGS " -DCARES_STATICLIB" - { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: : AM_CPPFLAGS=\"\$AM_CPPFLAGS\""; } >&5 - (: AM_CPPFLAGS="$AM_CPPFLAGS") 2>&5 - ac_status=$? - printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 - test $ac_status = 0; } - ;; -esac +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for library containing getservbyport" >&5 +printf %s "checking for library containing getservbyport... " >&6; } +if test ${ac_cv_search_getservbyport+y} +then : + printf %s "(cached) " >&6 else $as_nop + ac_func_search_save_LIBS=$LIBS +cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ - AM_CPPFLAGS=-DCARES_STATICLIB - { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: : AM_CPPFLAGS=\"\$AM_CPPFLAGS\""; } >&5 - (: AM_CPPFLAGS="$AM_CPPFLAGS") 2>&5 - ac_status=$? - printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 - test $ac_status = 0; } +/* Override any GCC internal prototype to avoid an error. + Use char because int might match the return type of a GCC + builtin and then its argument prototype would still apply. */ +char getservbyport (); +int +main (void) +{ +return getservbyport (); + ; + return 0; +} +_ACEOF +for ac_lib in '' nsl socket resolv +do + if test -z "$ac_lib"; then + ac_res="none required" + else + ac_res=-l$ac_lib + LIBS="-l$ac_lib $ac_func_search_save_LIBS" + fi + if ac_fn_c_try_link "$LINENO" +then : + ac_cv_search_getservbyport=$ac_res +fi +rm -f core conftest.err conftest.$ac_objext conftest.beam \ + conftest$ac_exeext + if test ${ac_cv_search_getservbyport+y} +then : + break +fi +done +if test ${ac_cv_search_getservbyport+y} +then : +else $as_nop + ac_cv_search_getservbyport=no fi +rm conftest.$ac_ext +LIBS=$ac_func_search_save_LIBS +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_search_getservbyport" >&5 +printf "%s\n" "$ac_cv_search_getservbyport" >&6; } +ac_res=$ac_cv_search_getservbyport +if test "$ac_res" != no +then : + test "$ac_res" = "none required" || LIBS="$ac_res $LIBS" - PKGCONFIG_CFLAGS="-DCARES_STATICLIB" - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: yes" >&5 -printf "%s\n" "yes" >&6; } - else - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 -printf "%s\n" "no" >&6; } - fi fi -CARES_SYMBOL_HIDING_CFLAG="" -if test "$symbol_hiding" != "no" ; then - compiler_supports_symbol_hiding="no" - if test "$ac_cv_windows" = "yes" ; then - compiler_supports_symbol_hiding="yes" - else - case "$ax_cv_c_compiler_vendor" in - clang|gnu|intel) + +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking if libxnet is required" >&5 +printf %s "checking if libxnet is required... " >&6; } +need_xnet=no +case $host_os in + hpux*) + XNET_LIBS="" -for flag in -fvisibility=hidden; do - as_CACHEVAR=`printf "%s\n" "ax_cv_check_cflags__$flag" | $as_tr_sh` -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether the C compiler accepts $flag" >&5 -printf %s "checking whether the C compiler accepts $flag... " >&6; } +for flag in -lxnet; do + as_CACHEVAR=`printf "%s\n" "ax_cv_check_ldflags__$flag" | $as_tr_sh` +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether the linker accepts $flag" >&5 +printf %s "checking whether the linker accepts $flag... " >&6; } if eval test \${$as_CACHEVAR+y} then : printf %s "(cached) " >&6 else $as_nop - ax_check_save_flags=$CFLAGS - if test x"$GCC" = xyes ; then - add_gnu_werror="-Werror" - fi - CFLAGS="$CFLAGS $flag $add_gnu_werror" + ax_check_save_flags=$LDFLAGS + LDFLAGS="$LDFLAGS $flag" cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ @@ -20014,14 +21933,15 @@ main (void) return 0; } _ACEOF -if ac_fn_c_try_compile "$LINENO" +if ac_fn_c_try_link "$LINENO" then : eval "$as_CACHEVAR=yes" else $as_nop eval "$as_CACHEVAR=no" fi -rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext - CFLAGS=$ax_check_save_flags +rm -f core conftest.err conftest.$ac_objext conftest.beam \ + conftest$ac_exeext conftest.$ac_ext + LDFLAGS=$ax_check_save_flags fi eval ac_res=\$$as_CACHEVAR { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_res" >&5 @@ -20029,21 +21949,21 @@ printf "%s\n" "$ac_res" >&6; } if eval test \"x\$"$as_CACHEVAR"\" = x"yes" then : -if test ${CARES_SYMBOL_HIDING_CFLAG+y} +if test ${XNET_LIBS+y} then : - case " $CARES_SYMBOL_HIDING_CFLAG " in #( + case " $XNET_LIBS " in #( *" $flag "*) : - { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: : CARES_SYMBOL_HIDING_CFLAG already contains \$flag"; } >&5 - (: CARES_SYMBOL_HIDING_CFLAG already contains $flag) 2>&5 + { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: : XNET_LIBS already contains \$flag"; } >&5 + (: XNET_LIBS already contains $flag) 2>&5 ac_status=$? printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 test $ac_status = 0; } ;; #( *) : - as_fn_append CARES_SYMBOL_HIDING_CFLAG " $flag" - { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: : CARES_SYMBOL_HIDING_CFLAG=\"\$CARES_SYMBOL_HIDING_CFLAG\""; } >&5 - (: CARES_SYMBOL_HIDING_CFLAG="$CARES_SYMBOL_HIDING_CFLAG") 2>&5 + as_fn_append XNET_LIBS " $flag" + { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: : XNET_LIBS=\"\$XNET_LIBS\""; } >&5 + (: XNET_LIBS="$XNET_LIBS") 2>&5 ac_status=$? printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 test $ac_status = 0; } @@ -20052,9 +21972,9 @@ esac else $as_nop - CARES_SYMBOL_HIDING_CFLAG=$flag - { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: : CARES_SYMBOL_HIDING_CFLAG=\"\$CARES_SYMBOL_HIDING_CFLAG\""; } >&5 - (: CARES_SYMBOL_HIDING_CFLAG="$CARES_SYMBOL_HIDING_CFLAG") 2>&5 + XNET_LIBS=$flag + { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: : XNET_LIBS=\"\$XNET_LIBS\""; } >&5 + (: XNET_LIBS="$XNET_LIBS") 2>&5 ac_status=$? printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 test $ac_status = 0; } @@ -20067,915 +21987,1342 @@ fi done - if test "x$CARES_SYMBOL_HIDING_CFLAG" != "x" ; then - compiler_supports_symbol_hiding="yes" - fi - ;; - sun) - - + if test "x$XNET_LIBS" != "x" ; then + LIBS="$LIBS $XNET_LIBS" + need_xnet=yes + fi + ;; +esac +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $need_xnet" >&5 +printf "%s\n" "$need_xnet" >&6; } +if test "x$host_vendor" = "xibm" -a "x$host_os" = "xopenedition" +then : -for flag in -xldscope=hidden; do - as_CACHEVAR=`printf "%s\n" "ax_cv_check_cflags__$flag" | $as_tr_sh` -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether the C compiler accepts $flag" >&5 -printf %s "checking whether the C compiler accepts $flag... " >&6; } -if eval test \${$as_CACHEVAR+y} + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for library containing res_init" >&5 +printf %s "checking for library containing res_init... " >&6; } +if test ${ac_cv_search_res_init+y} then : printf %s "(cached) " >&6 else $as_nop - - ax_check_save_flags=$CFLAGS - if test x"$GCC" = xyes ; then - add_gnu_werror="-Werror" - fi - CFLAGS="$CFLAGS $flag $add_gnu_werror" - cat confdefs.h - <<_ACEOF >conftest.$ac_ext + ac_func_search_save_LIBS=$LIBS +cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ +/* Override any GCC internal prototype to avoid an error. + Use char because int might match the return type of a GCC + builtin and then its argument prototype would still apply. */ +char res_init (); int main (void) { - +return res_init (); ; return 0; } _ACEOF -if ac_fn_c_try_compile "$LINENO" +for ac_lib in '' resolv +do + if test -z "$ac_lib"; then + ac_res="none required" + else + ac_res=-l$ac_lib + LIBS="-l$ac_lib $ac_func_search_save_LIBS" + fi + if ac_fn_c_try_link "$LINENO" then : - eval "$as_CACHEVAR=yes" -else $as_nop - eval "$as_CACHEVAR=no" + ac_cv_search_res_init=$ac_res fi -rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext - CFLAGS=$ax_check_save_flags +rm -f core conftest.err conftest.$ac_objext conftest.beam \ + conftest$ac_exeext + if test ${ac_cv_search_res_init+y} +then : + break fi -eval ac_res=\$$as_CACHEVAR - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_res" >&5 -printf "%s\n" "$ac_res" >&6; } -if eval test \"x\$"$as_CACHEVAR"\" = x"yes" +done +if test ${ac_cv_search_res_init+y} then : -if test ${CARES_SYMBOL_HIDING_CFLAG+y} +else $as_nop + ac_cv_search_res_init=no +fi +rm conftest.$ac_ext +LIBS=$ac_func_search_save_LIBS +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_search_res_init" >&5 +printf "%s\n" "$ac_cv_search_res_init" >&6; } +ac_res=$ac_cv_search_res_init +if test "$ac_res" != no then : + test "$ac_res" = "none required" || LIBS="$ac_res $LIBS" - case " $CARES_SYMBOL_HIDING_CFLAG " in #( - *" $flag "*) : - { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: : CARES_SYMBOL_HIDING_CFLAG already contains \$flag"; } >&5 - (: CARES_SYMBOL_HIDING_CFLAG already contains $flag) 2>&5 - ac_status=$? - printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 - test $ac_status = 0; } ;; #( - *) : - as_fn_append CARES_SYMBOL_HIDING_CFLAG " $flag" - { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: : CARES_SYMBOL_HIDING_CFLAG=\"\$CARES_SYMBOL_HIDING_CFLAG\""; } >&5 - (: CARES_SYMBOL_HIDING_CFLAG="$CARES_SYMBOL_HIDING_CFLAG") 2>&5 - ac_status=$? - printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 - test $ac_status = 0; } - ;; -esac +printf "%s\n" "#define CARES_USE_LIBRESOLV 1" >>confdefs.h + else $as_nop - CARES_SYMBOL_HIDING_CFLAG=$flag - { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: : CARES_SYMBOL_HIDING_CFLAG=\"\$CARES_SYMBOL_HIDING_CFLAG\""; } >&5 - (: CARES_SYMBOL_HIDING_CFLAG="$CARES_SYMBOL_HIDING_CFLAG") 2>&5 - ac_status=$? - printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 - test $ac_status = 0; } + as_fn_error $? "Unable to find libresolv which is required for z/OS" "$LINENO" 5 + +fi + fi -else $as_nop - : -fi -done +if test "x$host_vendor" = "xapple" +then : - if test "x$CARES_SYMBOL_HIDING_CFLAG" != "x" ; then - compiler_supports_symbol_hiding="yes" - fi - ;; - esac - fi - if test "$compiler_supports_symbol_hiding" = "no" ; then - if test "$symbol_hiding" = "yes" ; then - as_fn_error $? "Compiler does not support symbol hiding" "$LINENO" 5 - else - symbol_hiding="no" - fi - else + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for iOS minimum version 10 or later" >&5 +printf %s "checking for iOS minimum version 10 or later... " >&6; } + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ -printf "%s\n" "#define CARES_SYMBOL_HIDING 1 " >>confdefs.h - symbol_hiding="yes" - fi -fi - if test "x$symbol_hiding" = "xyes"; then - CARES_SYMBOL_HIDING_TRUE= - CARES_SYMBOL_HIDING_FALSE='#' -else - CARES_SYMBOL_HIDING_TRUE='#' - CARES_SYMBOL_HIDING_FALSE= -fi +#include +#include +#include +int +main (void) +{ +#if TARGET_OS_IPHONE == 0 || __IPHONE_OS_VERSION_MIN_REQUIRED < 100000 +#error Not iOS 10 or later +#endif +return 0; + ; + return 0; +} -if test "$enable_warnings" = "yes"; then +_ACEOF +if ac_fn_c_try_compile "$LINENO" +then : + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +printf "%s\n" "yes" >&6; } + ac_cv_ios_10="yes" +else $as_nop + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 +printf "%s\n" "no" >&6; } -for flag in -Wall -Wextra -Waggregate-return -Wcast-align -Wcast-qual -Wconversion -Wdeclaration-after-statement -Wdouble-promotion -Wfloat-equal -Wformat-security -Winit-self -Wjump-misses-init -Wlogical-op -Wmissing-braces -Wmissing-declarations -Wmissing-format-attribute -Wmissing-include-dirs -Wmissing-prototypes -Wnested-externs -Wno-coverage-mismatch -Wold-style-definition -Wpacked -Wpedantic -Wpointer-arith -Wredundant-decls -Wshadow -Wsign-conversion -Wstrict-overflow -Wstrict-prototypes -Wtrampolines -Wundef -Wunreachable-code -Wunused -Wvariadic-macros -Wvla -Wwrite-strings -Werror=implicit-int -Werror=implicit-function-declaration -Werror=partial-availability -Wno-long-long ; do - as_CACHEVAR=`printf "%s\n" "ax_cv_check_cflags_-Werror_$flag" | $as_tr_sh` -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether the C compiler accepts $flag" >&5 -printf %s "checking whether the C compiler accepts $flag... " >&6; } -if eval test \${$as_CACHEVAR+y} +fi +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext + +fi + +if test "x$host_vendor" = "xapple" then : - printf %s "(cached) " >&6 -else $as_nop - ax_check_save_flags=$CFLAGS - if test x"$GCC" = xyes ; then - add_gnu_werror="-Werror" - fi - CFLAGS="$CFLAGS -Werror $flag $add_gnu_werror" + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for macOS minimum version 10.12 or later" >&5 +printf %s "checking for macOS minimum version 10.12 or later... " >&6; } cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ + +#include +#include +#include + int main (void) { +#ifndef MAC_OS_X_VERSION_10_12 +# define MAC_OS_X_VERSION_10_12 101200 +#endif +#if MAC_OS_X_VERSION_MIN_REQUIRED < MAC_OS_X_VERSION_10_12 +#error Not macOS 10.12 or later +#endif +return 0; + ; return 0; } + _ACEOF if ac_fn_c_try_compile "$LINENO" then : - eval "$as_CACHEVAR=yes" + + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +printf "%s\n" "yes" >&6; } + ac_cv_macos_10_12="yes" + else $as_nop - eval "$as_CACHEVAR=no" + + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 +printf "%s\n" "no" >&6; } + fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext - CFLAGS=$ax_check_save_flags + fi -eval ac_res=\$$as_CACHEVAR - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_res" >&5 -printf "%s\n" "$ac_res" >&6; } -if eval test \"x\$"$as_CACHEVAR"\" = x"yes" + +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether to use libgcc" >&5 +printf %s "checking whether to use libgcc... " >&6; } +# Check whether --enable-libgcc was given. +if test ${enable_libgcc+y} then : + enableval=$enable_libgcc; case "$enableval" in + yes) + LIBS="$LIBS -lgcc" + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +printf "%s\n" "yes" >&6; } + ;; + *) + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 +printf "%s\n" "no" >&6; } + ;; + esac +else $as_nop + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 +printf "%s\n" "no" >&6; } -if test ${AM_CFLAGS+y} +fi + + +ac_fn_c_check_header_compile "$LINENO" "malloc.h" "ac_cv_header_malloc_h" " +#ifdef HAVE_SYS_TYPES_H +#include +#endif +#ifdef HAVE_SYS_TIME_H +#include +#endif +#ifdef HAVE_ARPA_NAMESER_H +#include +#endif + +#ifdef HAVE_SYS_SOCKET_H +#include +#endif +#ifdef HAVE_NETINET_IN_H +#include +#endif + + +" +if test "x$ac_cv_header_malloc_h" = xyes then : + printf "%s\n" "#define HAVE_MALLOC_H 1" >>confdefs.h - case " $AM_CFLAGS " in #( - *" $flag "*) : - { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: : AM_CFLAGS already contains \$flag"; } >&5 - (: AM_CFLAGS already contains $flag) 2>&5 - ac_status=$? - printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 - test $ac_status = 0; } ;; #( - *) : +fi +ac_fn_c_check_header_compile "$LINENO" "memory.h" "ac_cv_header_memory_h" " +#ifdef HAVE_SYS_TYPES_H +#include +#endif +#ifdef HAVE_SYS_TIME_H +#include +#endif +#ifdef HAVE_ARPA_NAMESER_H +#include +#endif - as_fn_append AM_CFLAGS " $flag" - { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: : AM_CFLAGS=\"\$AM_CFLAGS\""; } >&5 - (: AM_CFLAGS="$AM_CFLAGS") 2>&5 - ac_status=$? - printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 - test $ac_status = 0; } - ;; -esac +#ifdef HAVE_SYS_SOCKET_H +#include +#endif +#ifdef HAVE_NETINET_IN_H +#include +#endif -else $as_nop - AM_CFLAGS=$flag - { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: : AM_CFLAGS=\"\$AM_CFLAGS\""; } >&5 - (: AM_CFLAGS="$AM_CFLAGS") 2>&5 - ac_status=$? - printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 - test $ac_status = 0; } +" +if test "x$ac_cv_header_memory_h" = xyes +then : + printf "%s\n" "#define HAVE_MEMORY_H 1" >>confdefs.h fi +ac_fn_c_check_header_compile "$LINENO" "AvailabilityMacros.h" "ac_cv_header_AvailabilityMacros_h" " +#ifdef HAVE_SYS_TYPES_H +#include +#endif +#ifdef HAVE_SYS_TIME_H +#include +#endif +#ifdef HAVE_ARPA_NAMESER_H +#include +#endif + +#ifdef HAVE_SYS_SOCKET_H +#include +#endif +#ifdef HAVE_NETINET_IN_H +#include +#endif + + +" +if test "x$ac_cv_header_AvailabilityMacros_h" = xyes +then : + printf "%s\n" "#define HAVE_AVAILABILITYMACROS_H 1" >>confdefs.h + +fi +ac_fn_c_check_header_compile "$LINENO" "sys/types.h" "ac_cv_header_sys_types_h" " +#ifdef HAVE_SYS_TYPES_H +#include +#endif +#ifdef HAVE_SYS_TIME_H +#include +#endif +#ifdef HAVE_ARPA_NAMESER_H +#include +#endif + +#ifdef HAVE_SYS_SOCKET_H +#include +#endif +#ifdef HAVE_NETINET_IN_H +#include +#endif + + +" +if test "x$ac_cv_header_sys_types_h" = xyes +then : + printf "%s\n" "#define HAVE_SYS_TYPES_H 1" >>confdefs.h -else $as_nop - : fi +ac_fn_c_check_header_compile "$LINENO" "sys/time.h" "ac_cv_header_sys_time_h" " +#ifdef HAVE_SYS_TYPES_H +#include +#endif +#ifdef HAVE_SYS_TIME_H +#include +#endif +#ifdef HAVE_ARPA_NAMESER_H +#include +#endif -done +#ifdef HAVE_SYS_SOCKET_H +#include +#endif +#ifdef HAVE_NETINET_IN_H +#include +#endif -fi -case $host_os in - *qnx*|*android*) +" +if test "x$ac_cv_header_sys_time_h" = xyes +then : + printf "%s\n" "#define HAVE_SYS_TIME_H 1" >>confdefs.h +fi +ac_fn_c_check_header_compile "$LINENO" "sys/select.h" "ac_cv_header_sys_select_h" " +#ifdef HAVE_SYS_TYPES_H +#include +#endif +#ifdef HAVE_SYS_TIME_H +#include +#endif +#ifdef HAVE_ARPA_NAMESER_H +#include +#endif +#ifdef HAVE_SYS_SOCKET_H +#include +#endif +#ifdef HAVE_NETINET_IN_H +#include +#endif -for flag in -std=c99; do - as_CACHEVAR=`printf "%s\n" "ax_cv_check_cflags_-Werror_$flag" | $as_tr_sh` -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether the C compiler accepts $flag" >&5 -printf %s "checking whether the C compiler accepts $flag... " >&6; } -if eval test \${$as_CACHEVAR+y} +" +if test "x$ac_cv_header_sys_select_h" = xyes then : - printf %s "(cached) " >&6 -else $as_nop + printf "%s\n" "#define HAVE_SYS_SELECT_H 1" >>confdefs.h - ax_check_save_flags=$CFLAGS - if test x"$GCC" = xyes ; then - add_gnu_werror="-Werror" - fi - CFLAGS="$CFLAGS -Werror $flag $add_gnu_werror" - cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ +fi +ac_fn_c_check_header_compile "$LINENO" "sys/socket.h" "ac_cv_header_sys_socket_h" " +#ifdef HAVE_SYS_TYPES_H +#include +#endif +#ifdef HAVE_SYS_TIME_H +#include +#endif +#ifdef HAVE_ARPA_NAMESER_H +#include +#endif -int -main (void) -{ +#ifdef HAVE_SYS_SOCKET_H +#include +#endif +#ifdef HAVE_NETINET_IN_H +#include +#endif - ; - return 0; -} -_ACEOF -if ac_fn_c_try_compile "$LINENO" -then : - eval "$as_CACHEVAR=yes" -else $as_nop - eval "$as_CACHEVAR=no" -fi -rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext - CFLAGS=$ax_check_save_flags -fi -eval ac_res=\$$as_CACHEVAR - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_res" >&5 -printf "%s\n" "$ac_res" >&6; } -if eval test \"x\$"$as_CACHEVAR"\" = x"yes" -then : -if test ${AM_CFLAGS+y} +" +if test "x$ac_cv_header_sys_socket_h" = xyes then : + printf "%s\n" "#define HAVE_SYS_SOCKET_H 1" >>confdefs.h - case " $AM_CFLAGS " in #( - *" $flag "*) : - { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: : AM_CFLAGS already contains \$flag"; } >&5 - (: AM_CFLAGS already contains $flag) 2>&5 - ac_status=$? - printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 - test $ac_status = 0; } ;; #( - *) : +fi +ac_fn_c_check_header_compile "$LINENO" "sys/filio.h" "ac_cv_header_sys_filio_h" " +#ifdef HAVE_SYS_TYPES_H +#include +#endif +#ifdef HAVE_SYS_TIME_H +#include +#endif +#ifdef HAVE_ARPA_NAMESER_H +#include +#endif - as_fn_append AM_CFLAGS " $flag" - { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: : AM_CFLAGS=\"\$AM_CFLAGS\""; } >&5 - (: AM_CFLAGS="$AM_CFLAGS") 2>&5 - ac_status=$? - printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 - test $ac_status = 0; } - ;; -esac +#ifdef HAVE_SYS_SOCKET_H +#include +#endif +#ifdef HAVE_NETINET_IN_H +#include +#endif -else $as_nop - AM_CFLAGS=$flag - { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: : AM_CFLAGS=\"\$AM_CFLAGS\""; } >&5 - (: AM_CFLAGS="$AM_CFLAGS") 2>&5 - ac_status=$? - printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 - test $ac_status = 0; } +" +if test "x$ac_cv_header_sys_filio_h" = xyes +then : + printf "%s\n" "#define HAVE_SYS_FILIO_H 1" >>confdefs.h fi +ac_fn_c_check_header_compile "$LINENO" "sys/ioctl.h" "ac_cv_header_sys_ioctl_h" " +#ifdef HAVE_SYS_TYPES_H +#include +#endif +#ifdef HAVE_SYS_TIME_H +#include +#endif +#ifdef HAVE_ARPA_NAMESER_H +#include +#endif -else $as_nop - : -fi +#ifdef HAVE_SYS_SOCKET_H +#include +#endif +#ifdef HAVE_NETINET_IN_H +#include +#endif -done - ;; - *) +" +if test "x$ac_cv_header_sys_ioctl_h" = xyes +then : + printf "%s\n" "#define HAVE_SYS_IOCTL_H 1" >>confdefs.h +fi +ac_fn_c_check_header_compile "$LINENO" "sys/param.h" "ac_cv_header_sys_param_h" " +#ifdef HAVE_SYS_TYPES_H +#include +#endif +#ifdef HAVE_SYS_TIME_H +#include +#endif +#ifdef HAVE_ARPA_NAMESER_H +#include +#endif +#ifdef HAVE_SYS_SOCKET_H +#include +#endif +#ifdef HAVE_NETINET_IN_H +#include +#endif -for flag in -std=c90; do - as_CACHEVAR=`printf "%s\n" "ax_cv_check_cflags_-Werror_$flag" | $as_tr_sh` -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether the C compiler accepts $flag" >&5 -printf %s "checking whether the C compiler accepts $flag... " >&6; } -if eval test \${$as_CACHEVAR+y} +" +if test "x$ac_cv_header_sys_param_h" = xyes then : - printf %s "(cached) " >&6 -else $as_nop + printf "%s\n" "#define HAVE_SYS_PARAM_H 1" >>confdefs.h - ax_check_save_flags=$CFLAGS - if test x"$GCC" = xyes ; then - add_gnu_werror="-Werror" - fi - CFLAGS="$CFLAGS -Werror $flag $add_gnu_werror" - cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ +fi +ac_fn_c_check_header_compile "$LINENO" "sys/uio.h" "ac_cv_header_sys_uio_h" " +#ifdef HAVE_SYS_TYPES_H +#include +#endif +#ifdef HAVE_SYS_TIME_H +#include +#endif +#ifdef HAVE_ARPA_NAMESER_H +#include +#endif -int -main (void) -{ +#ifdef HAVE_SYS_SOCKET_H +#include +#endif +#ifdef HAVE_NETINET_IN_H +#include +#endif - ; - return 0; -} -_ACEOF -if ac_fn_c_try_compile "$LINENO" -then : - eval "$as_CACHEVAR=yes" -else $as_nop - eval "$as_CACHEVAR=no" -fi -rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext - CFLAGS=$ax_check_save_flags -fi -eval ac_res=\$$as_CACHEVAR - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_res" >&5 -printf "%s\n" "$ac_res" >&6; } -if eval test \"x\$"$as_CACHEVAR"\" = x"yes" -then : -if test ${AM_CFLAGS+y} +" +if test "x$ac_cv_header_sys_uio_h" = xyes then : + printf "%s\n" "#define HAVE_SYS_UIO_H 1" >>confdefs.h - case " $AM_CFLAGS " in #( - *" $flag "*) : - { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: : AM_CFLAGS already contains \$flag"; } >&5 - (: AM_CFLAGS already contains $flag) 2>&5 - ac_status=$? - printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 - test $ac_status = 0; } ;; #( - *) : +fi +ac_fn_c_check_header_compile "$LINENO" "sys/random.h" "ac_cv_header_sys_random_h" " +#ifdef HAVE_SYS_TYPES_H +#include +#endif +#ifdef HAVE_SYS_TIME_H +#include +#endif +#ifdef HAVE_ARPA_NAMESER_H +#include +#endif - as_fn_append AM_CFLAGS " $flag" - { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: : AM_CFLAGS=\"\$AM_CFLAGS\""; } >&5 - (: AM_CFLAGS="$AM_CFLAGS") 2>&5 - ac_status=$? - printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 - test $ac_status = 0; } - ;; -esac +#ifdef HAVE_SYS_SOCKET_H +#include +#endif +#ifdef HAVE_NETINET_IN_H +#include +#endif -else $as_nop - AM_CFLAGS=$flag - { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: : AM_CFLAGS=\"\$AM_CFLAGS\""; } >&5 - (: AM_CFLAGS="$AM_CFLAGS") 2>&5 - ac_status=$? - printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 - test $ac_status = 0; } +" +if test "x$ac_cv_header_sys_random_h" = xyes +then : + printf "%s\n" "#define HAVE_SYS_RANDOM_H 1" >>confdefs.h fi +ac_fn_c_check_header_compile "$LINENO" "sys/event.h" "ac_cv_header_sys_event_h" " +#ifdef HAVE_SYS_TYPES_H +#include +#endif +#ifdef HAVE_SYS_TIME_H +#include +#endif +#ifdef HAVE_ARPA_NAMESER_H +#include +#endif + +#ifdef HAVE_SYS_SOCKET_H +#include +#endif +#ifdef HAVE_NETINET_IN_H +#include +#endif + + +" +if test "x$ac_cv_header_sys_event_h" = xyes +then : + printf "%s\n" "#define HAVE_SYS_EVENT_H 1" >>confdefs.h -else $as_nop - : fi +ac_fn_c_check_header_compile "$LINENO" "sys/epoll.h" "ac_cv_header_sys_epoll_h" " +#ifdef HAVE_SYS_TYPES_H +#include +#endif +#ifdef HAVE_SYS_TIME_H +#include +#endif +#ifdef HAVE_ARPA_NAMESER_H +#include +#endif -done +#ifdef HAVE_SYS_SOCKET_H +#include +#endif +#ifdef HAVE_NETINET_IN_H +#include +#endif - ;; -esac -case $host_os in - *qnx*) +" +if test "x$ac_cv_header_sys_epoll_h" = xyes +then : + printf "%s\n" "#define HAVE_SYS_EPOLL_H 1" >>confdefs.h +fi +ac_fn_c_check_header_compile "$LINENO" "assert.h" "ac_cv_header_assert_h" " +#ifdef HAVE_SYS_TYPES_H +#include +#endif +#ifdef HAVE_SYS_TIME_H +#include +#endif +#ifdef HAVE_ARPA_NAMESER_H +#include +#endif +#ifdef HAVE_SYS_SOCKET_H +#include +#endif +#ifdef HAVE_NETINET_IN_H +#include +#endif -for flag in -D_QNX_SOURCE; do - as_CACHEVAR=`printf "%s\n" "ax_cv_check_cflags_-Werror_$flag" | $as_tr_sh` -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether the C compiler accepts $flag" >&5 -printf %s "checking whether the C compiler accepts $flag... " >&6; } -if eval test \${$as_CACHEVAR+y} +" +if test "x$ac_cv_header_assert_h" = xyes then : - printf %s "(cached) " >&6 -else $as_nop + printf "%s\n" "#define HAVE_ASSERT_H 1" >>confdefs.h - ax_check_save_flags=$CFLAGS - if test x"$GCC" = xyes ; then - add_gnu_werror="-Werror" - fi - CFLAGS="$CFLAGS -Werror $flag $add_gnu_werror" - cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ +fi +ac_fn_c_check_header_compile "$LINENO" "iphlpapi.h" "ac_cv_header_iphlpapi_h" " +#ifdef HAVE_SYS_TYPES_H +#include +#endif +#ifdef HAVE_SYS_TIME_H +#include +#endif +#ifdef HAVE_ARPA_NAMESER_H +#include +#endif -int -main (void) -{ +#ifdef HAVE_SYS_SOCKET_H +#include +#endif +#ifdef HAVE_NETINET_IN_H +#include +#endif - ; - return 0; -} -_ACEOF -if ac_fn_c_try_compile "$LINENO" + +" +if test "x$ac_cv_header_iphlpapi_h" = xyes then : - eval "$as_CACHEVAR=yes" -else $as_nop - eval "$as_CACHEVAR=no" -fi -rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext - CFLAGS=$ax_check_save_flags + printf "%s\n" "#define HAVE_IPHLPAPI_H 1" >>confdefs.h + fi -eval ac_res=\$$as_CACHEVAR - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_res" >&5 -printf "%s\n" "$ac_res" >&6; } -if eval test \"x\$"$as_CACHEVAR"\" = x"yes" -then : +ac_fn_c_check_header_compile "$LINENO" "netioapi.h" "ac_cv_header_netioapi_h" " +#ifdef HAVE_SYS_TYPES_H +#include +#endif +#ifdef HAVE_SYS_TIME_H +#include +#endif +#ifdef HAVE_ARPA_NAMESER_H +#include +#endif -if test ${AM_CPPFLAGS+y} +#ifdef HAVE_SYS_SOCKET_H +#include +#endif +#ifdef HAVE_NETINET_IN_H +#include +#endif + + +" +if test "x$ac_cv_header_netioapi_h" = xyes then : + printf "%s\n" "#define HAVE_NETIOAPI_H 1" >>confdefs.h - case " $AM_CPPFLAGS " in #( - *" $flag "*) : - { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: : AM_CPPFLAGS already contains \$flag"; } >&5 - (: AM_CPPFLAGS already contains $flag) 2>&5 - ac_status=$? - printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 - test $ac_status = 0; } ;; #( - *) : +fi +ac_fn_c_check_header_compile "$LINENO" "netdb.h" "ac_cv_header_netdb_h" " +#ifdef HAVE_SYS_TYPES_H +#include +#endif +#ifdef HAVE_SYS_TIME_H +#include +#endif +#ifdef HAVE_ARPA_NAMESER_H +#include +#endif - as_fn_append AM_CPPFLAGS " $flag" - { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: : AM_CPPFLAGS=\"\$AM_CPPFLAGS\""; } >&5 - (: AM_CPPFLAGS="$AM_CPPFLAGS") 2>&5 - ac_status=$? - printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 - test $ac_status = 0; } - ;; -esac +#ifdef HAVE_SYS_SOCKET_H +#include +#endif +#ifdef HAVE_NETINET_IN_H +#include +#endif -else $as_nop - AM_CPPFLAGS=$flag - { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: : AM_CPPFLAGS=\"\$AM_CPPFLAGS\""; } >&5 - (: AM_CPPFLAGS="$AM_CPPFLAGS") 2>&5 - ac_status=$? - printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 - test $ac_status = 0; } +" +if test "x$ac_cv_header_netdb_h" = xyes +then : + printf "%s\n" "#define HAVE_NETDB_H 1" >>confdefs.h fi +ac_fn_c_check_header_compile "$LINENO" "netinet/in.h" "ac_cv_header_netinet_in_h" " +#ifdef HAVE_SYS_TYPES_H +#include +#endif +#ifdef HAVE_SYS_TIME_H +#include +#endif +#ifdef HAVE_ARPA_NAMESER_H +#include +#endif + +#ifdef HAVE_SYS_SOCKET_H +#include +#endif +#ifdef HAVE_NETINET_IN_H +#include +#endif + + +" +if test "x$ac_cv_header_netinet_in_h" = xyes +then : + printf "%s\n" "#define HAVE_NETINET_IN_H 1" >>confdefs.h -else $as_nop - : fi +ac_fn_c_check_header_compile "$LINENO" "netinet6/in6.h" "ac_cv_header_netinet6_in6_h" " +#ifdef HAVE_SYS_TYPES_H +#include +#endif +#ifdef HAVE_SYS_TIME_H +#include +#endif +#ifdef HAVE_ARPA_NAMESER_H +#include +#endif -done +#ifdef HAVE_SYS_SOCKET_H +#include +#endif +#ifdef HAVE_NETINET_IN_H +#include +#endif - ;; -esac -if test "$ax_cv_c_compiler_vendor" = "intel"; then +" +if test "x$ac_cv_header_netinet6_in6_h" = xyes +then : + printf "%s\n" "#define HAVE_NETINET6_IN6_H 1" >>confdefs.h +fi +ac_fn_c_check_header_compile "$LINENO" "netinet/tcp.h" "ac_cv_header_netinet_tcp_h" " +#ifdef HAVE_SYS_TYPES_H +#include +#endif +#ifdef HAVE_SYS_TIME_H +#include +#endif +#ifdef HAVE_ARPA_NAMESER_H +#include +#endif +#ifdef HAVE_SYS_SOCKET_H +#include +#endif +#ifdef HAVE_NETINET_IN_H +#include +#endif -for flag in -shared-intel; do - as_CACHEVAR=`printf "%s\n" "ax_cv_check_cflags__$flag" | $as_tr_sh` -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether the C compiler accepts $flag" >&5 -printf %s "checking whether the C compiler accepts $flag... " >&6; } -if eval test \${$as_CACHEVAR+y} +" +if test "x$ac_cv_header_netinet_tcp_h" = xyes then : - printf %s "(cached) " >&6 -else $as_nop + printf "%s\n" "#define HAVE_NETINET_TCP_H 1" >>confdefs.h - ax_check_save_flags=$CFLAGS - if test x"$GCC" = xyes ; then - add_gnu_werror="-Werror" - fi - CFLAGS="$CFLAGS $flag $add_gnu_werror" - cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ +fi +ac_fn_c_check_header_compile "$LINENO" "net/if.h" "ac_cv_header_net_if_h" " +#ifdef HAVE_SYS_TYPES_H +#include +#endif +#ifdef HAVE_SYS_TIME_H +#include +#endif +#ifdef HAVE_ARPA_NAMESER_H +#include +#endif -int -main (void) -{ +#ifdef HAVE_SYS_SOCKET_H +#include +#endif +#ifdef HAVE_NETINET_IN_H +#include +#endif - ; - return 0; -} -_ACEOF -if ac_fn_c_try_compile "$LINENO" -then : - eval "$as_CACHEVAR=yes" -else $as_nop - eval "$as_CACHEVAR=no" -fi -rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext - CFLAGS=$ax_check_save_flags -fi -eval ac_res=\$$as_CACHEVAR - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_res" >&5 -printf "%s\n" "$ac_res" >&6; } -if eval test \"x\$"$as_CACHEVAR"\" = x"yes" -then : -if test ${AM_CFLAGS+y} +" +if test "x$ac_cv_header_net_if_h" = xyes then : + printf "%s\n" "#define HAVE_NET_IF_H 1" >>confdefs.h - case " $AM_CFLAGS " in #( - *" $flag "*) : - { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: : AM_CFLAGS already contains \$flag"; } >&5 - (: AM_CFLAGS already contains $flag) 2>&5 - ac_status=$? - printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 - test $ac_status = 0; } ;; #( - *) : +fi +ac_fn_c_check_header_compile "$LINENO" "ifaddrs.h" "ac_cv_header_ifaddrs_h" " +#ifdef HAVE_SYS_TYPES_H +#include +#endif +#ifdef HAVE_SYS_TIME_H +#include +#endif +#ifdef HAVE_ARPA_NAMESER_H +#include +#endif - as_fn_append AM_CFLAGS " $flag" - { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: : AM_CFLAGS=\"\$AM_CFLAGS\""; } >&5 - (: AM_CFLAGS="$AM_CFLAGS") 2>&5 - ac_status=$? - printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 - test $ac_status = 0; } - ;; -esac +#ifdef HAVE_SYS_SOCKET_H +#include +#endif +#ifdef HAVE_NETINET_IN_H +#include +#endif -else $as_nop - AM_CFLAGS=$flag - { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: : AM_CFLAGS=\"\$AM_CFLAGS\""; } >&5 - (: AM_CFLAGS="$AM_CFLAGS") 2>&5 - ac_status=$? - printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 - test $ac_status = 0; } +" +if test "x$ac_cv_header_ifaddrs_h" = xyes +then : + printf "%s\n" "#define HAVE_IFADDRS_H 1" >>confdefs.h fi +ac_fn_c_check_header_compile "$LINENO" "fcntl.h" "ac_cv_header_fcntl_h" " +#ifdef HAVE_SYS_TYPES_H +#include +#endif +#ifdef HAVE_SYS_TIME_H +#include +#endif +#ifdef HAVE_ARPA_NAMESER_H +#include +#endif -else $as_nop - : -fi +#ifdef HAVE_SYS_SOCKET_H +#include +#endif +#ifdef HAVE_NETINET_IN_H +#include +#endif -done + +" +if test "x$ac_cv_header_fcntl_h" = xyes +then : + printf "%s\n" "#define HAVE_FCNTL_H 1" >>confdefs.h fi +ac_fn_c_check_header_compile "$LINENO" "errno.h" "ac_cv_header_errno_h" " +#ifdef HAVE_SYS_TYPES_H +#include +#endif +#ifdef HAVE_SYS_TIME_H +#include +#endif +#ifdef HAVE_ARPA_NAMESER_H +#include +#endif -if test "$ac_cv_native_windows" = "yes" ; then +#ifdef HAVE_SYS_SOCKET_H +#include +#endif +#ifdef HAVE_NETINET_IN_H +#include +#endif - ac_ext=c -ac_cpp='$CPP $CPPFLAGS' -ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5' -ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5' -ac_compiler_gnu=$ac_cv_c_compiler_gnu -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking how to run the C preprocessor" >&5 -printf %s "checking how to run the C preprocessor... " >&6; } -# On Suns, sometimes $CPP names a directory. -if test -n "$CPP" && test -d "$CPP"; then - CPP= -fi -if test -z "$CPP"; then - if test ${ac_cv_prog_CPP+y} -then : - printf %s "(cached) " >&6 -else $as_nop - # Double quotes because $CC needs to be expanded - for CPP in "$CC -E" "$CC -E -traditional-cpp" cpp /lib/cpp - do - ac_preproc_ok=false -for ac_c_preproc_warn_flag in '' yes -do - # Use a header file that comes with gcc, so configuring glibc - # with a fresh cross-compiler works. - # On the NeXT, cc -E runs the code through the compiler's parser, - # not just through cpp. "Syntax error" is here to catch this case. - cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ -#include - Syntax error -_ACEOF -if ac_fn_c_try_cpp "$LINENO" + +" +if test "x$ac_cv_header_errno_h" = xyes then : + printf "%s\n" "#define HAVE_ERRNO_H 1" >>confdefs.h -else $as_nop - # Broken: fails on valid input. -continue fi -rm -f conftest.err conftest.i conftest.$ac_ext +ac_fn_c_check_header_compile "$LINENO" "socket.h" "ac_cv_header_socket_h" " +#ifdef HAVE_SYS_TYPES_H +#include +#endif +#ifdef HAVE_SYS_TIME_H +#include +#endif +#ifdef HAVE_ARPA_NAMESER_H +#include +#endif - # OK, works on sane cases. Now check whether nonexistent headers - # can be detected and how. - cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ -#include -_ACEOF -if ac_fn_c_try_cpp "$LINENO" +#ifdef HAVE_SYS_SOCKET_H +#include +#endif +#ifdef HAVE_NETINET_IN_H +#include +#endif + + +" +if test "x$ac_cv_header_socket_h" = xyes then : - # Broken: success on invalid input. -continue -else $as_nop - # Passes both tests. -ac_preproc_ok=: -break + printf "%s\n" "#define HAVE_SOCKET_H 1" >>confdefs.h + fi -rm -f conftest.err conftest.i conftest.$ac_ext +ac_fn_c_check_header_compile "$LINENO" "strings.h" "ac_cv_header_strings_h" " +#ifdef HAVE_SYS_TYPES_H +#include +#endif +#ifdef HAVE_SYS_TIME_H +#include +#endif +#ifdef HAVE_ARPA_NAMESER_H +#include +#endif -done -# Because of `break', _AC_PREPROC_IFELSE's cleaning code was skipped. -rm -f conftest.i conftest.err conftest.$ac_ext -if $ac_preproc_ok -then : - break -fi +#ifdef HAVE_SYS_SOCKET_H +#include +#endif +#ifdef HAVE_NETINET_IN_H +#include +#endif - done - ac_cv_prog_CPP=$CPP -fi - CPP=$ac_cv_prog_CPP -else - ac_cv_prog_CPP=$CPP -fi -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $CPP" >&5 -printf "%s\n" "$CPP" >&6; } -ac_preproc_ok=false -for ac_c_preproc_warn_flag in '' yes -do - # Use a header file that comes with gcc, so configuring glibc - # with a fresh cross-compiler works. - # On the NeXT, cc -E runs the code through the compiler's parser, - # not just through cpp. "Syntax error" is here to catch this case. - cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ -#include - Syntax error -_ACEOF -if ac_fn_c_try_cpp "$LINENO" +" +if test "x$ac_cv_header_strings_h" = xyes then : + printf "%s\n" "#define HAVE_STRINGS_H 1" >>confdefs.h -else $as_nop - # Broken: fails on valid input. -continue fi -rm -f conftest.err conftest.i conftest.$ac_ext +ac_fn_c_check_header_compile "$LINENO" "stdbool.h" "ac_cv_header_stdbool_h" " +#ifdef HAVE_SYS_TYPES_H +#include +#endif +#ifdef HAVE_SYS_TIME_H +#include +#endif +#ifdef HAVE_ARPA_NAMESER_H +#include +#endif - # OK, works on sane cases. Now check whether nonexistent headers - # can be detected and how. - cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ -#include -_ACEOF -if ac_fn_c_try_cpp "$LINENO" -then : - # Broken: success on invalid input. -continue -else $as_nop - # Passes both tests. -ac_preproc_ok=: -break -fi -rm -f conftest.err conftest.i conftest.$ac_ext +#ifdef HAVE_SYS_SOCKET_H +#include +#endif +#ifdef HAVE_NETINET_IN_H +#include +#endif -done -# Because of `break', _AC_PREPROC_IFELSE's cleaning code was skipped. -rm -f conftest.i conftest.err conftest.$ac_ext -if $ac_preproc_ok + +" +if test "x$ac_cv_header_stdbool_h" = xyes then : + printf "%s\n" "#define HAVE_STDBOOL_H 1" >>confdefs.h -else $as_nop - { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 -printf "%s\n" "$as_me: error: in \`$ac_pwd':" >&2;} -as_fn_error $? "C preprocessor \"$CPP\" fails sanity check -See \`config.log' for more details" "$LINENO" 5; } fi +ac_fn_c_check_header_compile "$LINENO" "time.h" "ac_cv_header_time_h" " +#ifdef HAVE_SYS_TYPES_H +#include +#endif +#ifdef HAVE_SYS_TIME_H +#include +#endif +#ifdef HAVE_ARPA_NAMESER_H +#include +#endif -ac_ext=c -ac_cpp='$CPP $CPPFLAGS' -ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5' -ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5' -ac_compiler_gnu=$ac_cv_c_compiler_gnu +#ifdef HAVE_SYS_SOCKET_H +#include +#endif +#ifdef HAVE_NETINET_IN_H +#include +#endif -ac_fn_c_check_header_preproc "$LINENO" "windows.h" "ac_cv_header_windows_h" -if test "x$ac_cv_header_windows_h" = xyes +" +if test "x$ac_cv_header_time_h" = xyes then : - printf "%s\n" "#define HAVE_WINDOWS_H 1" >>confdefs.h + printf "%s\n" "#define HAVE_TIME_H 1" >>confdefs.h fi -ac_fn_c_check_header_preproc "$LINENO" "winsock2.h" "ac_cv_header_winsock2_h" -if test "x$ac_cv_header_winsock2_h" = xyes -then : - printf "%s\n" "#define HAVE_WINSOCK2_H 1" >>confdefs.h +ac_fn_c_check_header_compile "$LINENO" "poll.h" "ac_cv_header_poll_h" " +#ifdef HAVE_SYS_TYPES_H +#include +#endif +#ifdef HAVE_SYS_TIME_H +#include +#endif +#ifdef HAVE_ARPA_NAMESER_H +#include +#endif -fi -ac_fn_c_check_header_preproc "$LINENO" "ws2tcpip.h" "ac_cv_header_ws2tcpip_h" -if test "x$ac_cv_header_ws2tcpip_h" = xyes +#ifdef HAVE_SYS_SOCKET_H +#include +#endif +#ifdef HAVE_NETINET_IN_H +#include +#endif + + +" +if test "x$ac_cv_header_poll_h" = xyes then : - printf "%s\n" "#define HAVE_WS2TCPIP_H 1" >>confdefs.h + printf "%s\n" "#define HAVE_POLL_H 1" >>confdefs.h fi -ac_fn_c_check_header_preproc "$LINENO" "iphlpapi.h" "ac_cv_header_iphlpapi_h" -if test "x$ac_cv_header_iphlpapi_h" = xyes +ac_fn_c_check_header_compile "$LINENO" "limits.h" "ac_cv_header_limits_h" " +#ifdef HAVE_SYS_TYPES_H +#include +#endif +#ifdef HAVE_SYS_TIME_H +#include +#endif +#ifdef HAVE_ARPA_NAMESER_H +#include +#endif + +#ifdef HAVE_SYS_SOCKET_H +#include +#endif +#ifdef HAVE_NETINET_IN_H +#include +#endif + + +" +if test "x$ac_cv_header_limits_h" = xyes then : - printf "%s\n" "#define HAVE_IPHLPAPI_H 1" >>confdefs.h + printf "%s\n" "#define HAVE_LIMITS_H 1" >>confdefs.h fi -ac_fn_c_check_header_preproc "$LINENO" "netioapi.h" "ac_cv_header_netioapi_h" -if test "x$ac_cv_header_netioapi_h" = xyes +ac_fn_c_check_header_compile "$LINENO" "arpa/nameser.h" "ac_cv_header_arpa_nameser_h" " +#ifdef HAVE_SYS_TYPES_H +#include +#endif +#ifdef HAVE_SYS_TIME_H +#include +#endif +#ifdef HAVE_ARPA_NAMESER_H +#include +#endif + +#ifdef HAVE_SYS_SOCKET_H +#include +#endif +#ifdef HAVE_NETINET_IN_H +#include +#endif + + +" +if test "x$ac_cv_header_arpa_nameser_h" = xyes then : - printf "%s\n" "#define HAVE_NETIOAPI_H 1" >>confdefs.h + printf "%s\n" "#define HAVE_ARPA_NAMESER_H 1" >>confdefs.h fi -ac_fn_c_check_header_preproc "$LINENO" "ws2ipdef.h" "ac_cv_header_ws2ipdef_h" -if test "x$ac_cv_header_ws2ipdef_h" = xyes +ac_fn_c_check_header_compile "$LINENO" "arpa/nameser_compat.h" "ac_cv_header_arpa_nameser_compat_h" " +#ifdef HAVE_SYS_TYPES_H +#include +#endif +#ifdef HAVE_SYS_TIME_H +#include +#endif +#ifdef HAVE_ARPA_NAMESER_H +#include +#endif + +#ifdef HAVE_SYS_SOCKET_H +#include +#endif +#ifdef HAVE_NETINET_IN_H +#include +#endif + + +" +if test "x$ac_cv_header_arpa_nameser_compat_h" = xyes then : - printf "%s\n" "#define HAVE_WS2IPDEF_H 1" >>confdefs.h + printf "%s\n" "#define HAVE_ARPA_NAMESER_COMPAT_H 1" >>confdefs.h fi -ac_fn_c_check_header_preproc "$LINENO" "winternl.h" "ac_cv_header_winternl_h" -if test "x$ac_cv_header_winternl_h" = xyes +ac_fn_c_check_header_compile "$LINENO" "arpa/inet.h" "ac_cv_header_arpa_inet_h" " +#ifdef HAVE_SYS_TYPES_H +#include +#endif +#ifdef HAVE_SYS_TIME_H +#include +#endif +#ifdef HAVE_ARPA_NAMESER_H +#include +#endif + +#ifdef HAVE_SYS_SOCKET_H +#include +#endif +#ifdef HAVE_NETINET_IN_H +#include +#endif + + +" +if test "x$ac_cv_header_arpa_inet_h" = xyes then : - printf "%s\n" "#define HAVE_WINTERNL_H 1" >>confdefs.h + printf "%s\n" "#define HAVE_ARPA_INET_H 1" >>confdefs.h fi -ac_fn_c_check_header_preproc "$LINENO" "ntdef.h" "ac_cv_header_ntdef_h" -if test "x$ac_cv_header_ntdef_h" = xyes -then : - printf "%s\n" "#define HAVE_NTDEF_H 1" >>confdefs.h +ac_fn_c_check_header_compile "$LINENO" "sys/system_properties.h" "ac_cv_header_sys_system_properties_h" " +#ifdef HAVE_SYS_TYPES_H +#include +#endif +#ifdef HAVE_SYS_TIME_H +#include +#endif +#ifdef HAVE_ARPA_NAMESER_H +#include +#endif -fi -ac_fn_c_check_header_preproc "$LINENO" "ntstatus.h" "ac_cv_header_ntstatus_h" -if test "x$ac_cv_header_ntstatus_h" = xyes -then : - printf "%s\n" "#define HAVE_NTSTATUS_H 1" >>confdefs.h +#ifdef HAVE_SYS_SOCKET_H +#include +#endif +#ifdef HAVE_NETINET_IN_H +#include +#endif -fi -ac_fn_c_check_header_preproc "$LINENO" "mswsock.h" "ac_cv_header_mswsock_h" -if test "x$ac_cv_header_mswsock_h" = xyes + +" +if test "x$ac_cv_header_sys_system_properties_h" = xyes then : - printf "%s\n" "#define HAVE_MSWSOCK_H 1" >>confdefs.h + printf "%s\n" "#define HAVE_SYS_SYSTEM_PROPERTIES_H 1" >>confdefs.h fi - if test "$ac_cv_header_winsock2_h" = "yes"; then - LIBS="$LIBS -lws2_32 -liphlpapi" - fi -fi +cares_all_includes=" +#include +#include +#ifdef HAVE_AVAILABILITYMACROS_H +# include +#endif +#ifdef HAVE_SYS_UIO_H +# include +#endif +#ifdef HAVE_NETINET_IN_H +# include +#endif +#ifdef HAVE_TCP_H +# include +#endif +#ifdef HAVE_SYS_FILIO_H +# include +#endif +#ifdef HAVE_SYS_IOCTL_H +# include +#endif +#ifdef HAVE_UNISTD_H +# include +#endif +#ifdef HAVE_STRING_H +# include +#endif +#ifdef HAVE_STRINGS_H +# include +#endif +#ifdef HAVE_TIME_H +# include +#endif +#ifdef HAVE_SYS_TIME_H +# include +#endif +#ifdef HAVE_SYS_TYPES_H +# include +#endif +#ifdef HAVE_SYS_STAT_H +# include +#endif +#ifdef HAVE_SYS_RANDOM_H +# include +#endif +#ifdef HAVE_SYS_EVENT_H +# include +#endif +#ifdef HAVE_SYS_EPOLL_H +# include +#endif +#ifdef HAVE_SYS_SOCKET_H +# include +#endif +#ifdef HAVE_SYS_PARAM_H +# include +#endif +#ifdef HAVE_FCNTL_H +# include +#endif +#ifdef HAVE_POLL_H +# include +#endif +#ifdef HAVE_NET_IF_H +# include +#endif +#ifdef HAVE_IFADDRS_H +# include +#endif +#ifdef HAVE_NETINET_IN_H +# include +#endif +#ifdef HAVE_NETINET_TCP_H +# include +#endif +#ifdef HAVE_NETDB_H +# include +#endif +#ifdef HAVE_ARPA_INET_H +# include +#endif +#ifdef HAVE_RESOLV_H +# include +#endif +#ifdef HAVE_SYS_SYSTEM_PROPERTIES_H +# include +#endif +#ifdef HAVE_IPHLPAPI_H +# include +#endif +#ifdef HAVE_NETIOAPI_H +# include +#endif +#ifdef HAVE_WINSOCK2_H +# include +#endif +#ifdef HAVE_WS2IPDEF_H +# include +#endif +#ifdef HAVE_WS2TCPIP_H +# include +#endif +#ifdef HAVE_WINDOWS_H +# include +#endif +" -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for library containing getservbyport" >&5 -printf %s "checking for library containing getservbyport... " >&6; } -if test ${ac_cv_search_getservbyport+y} +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for $CC options needed to detect all undeclared functions" >&5 +printf %s "checking for $CC options needed to detect all undeclared functions... " >&6; } +if test ${ac_cv_c_undeclared_builtin_options+y} then : printf %s "(cached) " >&6 else $as_nop - ac_func_search_save_LIBS=$LIBS -cat confdefs.h - <<_ACEOF >conftest.$ac_ext + ac_save_CFLAGS=$CFLAGS + ac_cv_c_undeclared_builtin_options='cannot detect' + for ac_arg in '' -fno-builtin; do + CFLAGS="$ac_save_CFLAGS $ac_arg" + # This test program should *not* compile successfully. + cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ -/* Override any GCC internal prototype to avoid an error. - Use char because int might match the return type of a GCC - builtin and then its argument prototype would still apply. */ -char getservbyport (); int main (void) { -return getservbyport (); +(void) strchr; ; return 0; } _ACEOF -for ac_lib in '' nsl socket resolv -do - if test -z "$ac_lib"; then - ac_res="none required" - else - ac_res=-l$ac_lib - LIBS="-l$ac_lib $ac_func_search_save_LIBS" - fi - if ac_fn_c_try_link "$LINENO" +if ac_fn_c_try_compile "$LINENO" then : - ac_cv_search_getservbyport=$ac_res -fi -rm -f core conftest.err conftest.$ac_objext conftest.beam \ - conftest$ac_exeext - if test ${ac_cv_search_getservbyport+y} + +else $as_nop + # This test program should compile successfully. + # No library function is consistently available on + # freestanding implementations, so test against a dummy + # declaration. Include always-available headers on the + # off chance that they somehow elicit warnings. + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ +#include +#include +#include +#include +extern void ac_decl (int, char *); + +int +main (void) +{ +(void) ac_decl (0, (char *) 0); + (void) ac_decl; + + ; + return 0; +} +_ACEOF +if ac_fn_c_try_compile "$LINENO" then : - break -fi -done -if test ${ac_cv_search_getservbyport+y} + if test x"$ac_arg" = x then : - + ac_cv_c_undeclared_builtin_options='none needed' else $as_nop - ac_cv_search_getservbyport=no + ac_cv_c_undeclared_builtin_options=$ac_arg fi -rm conftest.$ac_ext -LIBS=$ac_func_search_save_LIBS + break fi -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_search_getservbyport" >&5 -printf "%s\n" "$ac_cv_search_getservbyport" >&6; } -ac_res=$ac_cv_search_getservbyport -if test "$ac_res" != no -then : - test "$ac_res" = "none required" || LIBS="$ac_res $LIBS" - +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext fi +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext + done + CFLAGS=$ac_save_CFLAGS +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_c_undeclared_builtin_options" >&5 +printf "%s\n" "$ac_cv_c_undeclared_builtin_options" >&6; } + case $ac_cv_c_undeclared_builtin_options in #( + 'cannot detect') : + { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 +printf "%s\n" "$as_me: error: in \`$ac_pwd':" >&2;} +as_fn_error $? "cannot make $CC report undeclared builtins +See \`config.log' for more details" "$LINENO" 5; } ;; #( + 'none needed') : + ac_c_undeclared_builtin_options='' ;; #( + *) : + ac_c_undeclared_builtin_options=$ac_cv_c_undeclared_builtin_options ;; +esac -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking if libxnet is required" >&5 -printf %s "checking if libxnet is required... " >&6; } -need_xnet=no -case $host_os in - hpux*) - XNET_LIBS="" +ac_fn_check_decl "$LINENO" "HAVE_ARPA_NAMESER_H" "ac_cv_have_decl_HAVE_ARPA_NAMESER_H" "$ac_includes_default" "$ac_c_undeclared_builtin_options" "CFLAGS" +if test "x$ac_cv_have_decl_HAVE_ARPA_NAMESER_H" = xyes +then : +cat >>confdefs.h <<_EOF +#define CARES_HAVE_ARPA_NAMESER_H 1 +_EOF +fi +ac_fn_check_decl "$LINENO" "HAVE_ARPA_NAMESER_COMPAT_H" "ac_cv_have_decl_HAVE_ARPA_NAMESER_COMPAT_H" "$ac_includes_default" "$ac_c_undeclared_builtin_options" "CFLAGS" +if test "x$ac_cv_have_decl_HAVE_ARPA_NAMESER_COMPAT_H" = xyes +then : +cat >>confdefs.h <<_EOF +#define CARES_HAVE_ARPA_NAMESER_COMPAT_H 1 +_EOF -for flag in -lxnet; do - as_CACHEVAR=`printf "%s\n" "ax_cv_check_ldflags__$flag" | $as_tr_sh` -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether the linker accepts $flag" >&5 -printf %s "checking whether the linker accepts $flag... " >&6; } -if eval test \${$as_CACHEVAR+y} +fi +ac_fn_c_check_type "$LINENO" "long long" "ac_cv_type_long_long" "$ac_includes_default" +if test "x$ac_cv_type_long_long" = xyes then : - printf %s "(cached) " >&6 -else $as_nop - ax_check_save_flags=$LDFLAGS - LDFLAGS="$LDFLAGS $flag" - cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ +printf "%s\n" "#define HAVE_LONGLONG 1" >>confdefs.h -int -main (void) -{ +fi - ; - return 0; -} -_ACEOF -if ac_fn_c_try_link "$LINENO" +ac_fn_c_check_type "$LINENO" "ssize_t" "ac_cv_type_ssize_t" "$ac_includes_default" +if test "x$ac_cv_type_ssize_t" = xyes then : - eval "$as_CACHEVAR=yes" + CARES_TYPEOF_ARES_SSIZE_T=ssize_t else $as_nop - eval "$as_CACHEVAR=no" -fi -rm -f core conftest.err conftest.$ac_objext conftest.beam \ - conftest$ac_exeext conftest.$ac_ext - LDFLAGS=$ax_check_save_flags + CARES_TYPEOF_ARES_SSIZE_T=int fi -eval ac_res=\$$as_CACHEVAR - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_res" >&5 -printf "%s\n" "$ac_res" >&6; } -if eval test \"x\$"$as_CACHEVAR"\" = x"yes" -then : -if test ${XNET_LIBS+y} + +printf "%s\n" "#define CARES_TYPEOF_ARES_SSIZE_T ${CARES_TYPEOF_ARES_SSIZE_T}" >>confdefs.h + + +ac_fn_c_check_type "$LINENO" "socklen_t" "ac_cv_type_socklen_t" "$cares_all_includes + +" +if test "x$ac_cv_type_socklen_t" = xyes then : - case " $XNET_LIBS " in #( - *" $flag "*) : - { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: : XNET_LIBS already contains \$flag"; } >&5 - (: XNET_LIBS already contains $flag) 2>&5 - ac_status=$? - printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 - test $ac_status = 0; } ;; #( - *) : - as_fn_append XNET_LIBS " $flag" - { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: : XNET_LIBS=\"\$XNET_LIBS\""; } >&5 - (: XNET_LIBS="$XNET_LIBS") 2>&5 - ac_status=$? - printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 - test $ac_status = 0; } - ;; -esac +printf "%s\n" "#define HAVE_SOCKLEN_T /**/" >>confdefs.h -else $as_nop - XNET_LIBS=$flag - { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: : XNET_LIBS=\"\$XNET_LIBS\""; } >&5 - (: XNET_LIBS="$XNET_LIBS") 2>&5 - ac_status=$? - printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 - test $ac_status = 0; } +cat >>confdefs.h <<_EOF +#define CARES_TYPEOF_ARES_SOCKLEN_T socklen_t +_EOF -fi else $as_nop - : -fi -done +cat >>confdefs.h <<_EOF +#define CARES_TYPEOF_ARES_SOCKLEN_T int +_EOF - if test "x$XNET_LIBS" != "x" ; then - LIBS="$LIBS $XNET_LIBS" - need_xnet=yes - fi - ;; -esac -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $need_xnet" >&5 -printf "%s\n" "$need_xnet" >&6; } +fi -if test "x$host_vendor" = "xibm" -a "x$host_os" = "xopenedition" + +ac_fn_c_check_type "$LINENO" "SOCKET" "ac_cv_type_SOCKET" "$cares_all_includes +" +if test "x$ac_cv_type_SOCKET" = xyes then : - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for library containing res_init" >&5 -printf %s "checking for library containing res_init... " >&6; } -if test ${ac_cv_search_res_init+y} +fi + + + +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for library containing clock_gettime" >&5 +printf %s "checking for library containing clock_gettime... " >&6; } +if test ${ac_cv_search_clock_gettime+y} then : printf %s "(cached) " >&6 else $as_nop @@ -20986,16 +23333,16 @@ cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* Override any GCC internal prototype to avoid an error. Use char because int might match the return type of a GCC builtin and then its argument prototype would still apply. */ -char res_init (); +char clock_gettime (); int main (void) { -return res_init (); +return clock_gettime (); ; return 0; } _ACEOF -for ac_lib in '' resolv +for ac_lib in '' rt posix4 do if test -z "$ac_lib"; then ac_res="none required" @@ -21005,3418 +23352,3045 @@ do fi if ac_fn_c_try_link "$LINENO" then : - ac_cv_search_res_init=$ac_res + ac_cv_search_clock_gettime=$ac_res fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext - if test ${ac_cv_search_res_init+y} + if test ${ac_cv_search_clock_gettime+y} then : break fi done -if test ${ac_cv_search_res_init+y} +if test ${ac_cv_search_clock_gettime+y} then : else $as_nop - ac_cv_search_res_init=no + ac_cv_search_clock_gettime=no fi rm conftest.$ac_ext LIBS=$ac_func_search_save_LIBS fi -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_search_res_init" >&5 -printf "%s\n" "$ac_cv_search_res_init" >&6; } -ac_res=$ac_cv_search_res_init +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_search_clock_gettime" >&5 +printf "%s\n" "$ac_cv_search_clock_gettime" >&6; } +ac_res=$ac_cv_search_clock_gettime if test "$ac_res" != no then : test "$ac_res" = "none required" || LIBS="$ac_res $LIBS" +fi -printf "%s\n" "#define CARES_USE_LIBRESOLV 1" >>confdefs.h -else $as_nop +ac_fn_check_decl "$LINENO" "strnlen" "ac_cv_have_decl_strnlen" "$cares_all_includes +" "$ac_c_undeclared_builtin_options" "CFLAGS" +if test "x$ac_cv_have_decl_strnlen" = xyes +then : - as_fn_error $? "Unable to find libresolv which is required for z/OS" "$LINENO" 5 +printf "%s\n" "#define HAVE_STRNLEN 1" >>confdefs.h fi +ac_fn_check_decl "$LINENO" "memmem" "ac_cv_have_decl_memmem" "$cares_all_includes +" "$ac_c_undeclared_builtin_options" "CFLAGS" +if test "x$ac_cv_have_decl_memmem" = xyes +then : +printf "%s\n" "#define HAVE_MEMMEM 1" >>confdefs.h fi - - -if test "x$host_vendor" = "xapple" +ac_fn_check_decl "$LINENO" "recv" "ac_cv_have_decl_recv" "$cares_all_includes +" "$ac_c_undeclared_builtin_options" "CFLAGS" +if test "x$ac_cv_have_decl_recv" = xyes then : - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for iOS minimum version 10 or later" >&5 -printf %s "checking for iOS minimum version 10 or later... " >&6; } - cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ - - -#include -#include -#include - -int -main (void) -{ - -#if TARGET_OS_IPHONE == 0 || __IPHONE_OS_VERSION_MIN_REQUIRED < 100000 -#error Not iOS 10 or later -#endif -return 0; - - ; - return 0; -} +printf "%s\n" "#define HAVE_RECV 1" >>confdefs.h -_ACEOF -if ac_fn_c_try_compile "$LINENO" +fi +ac_fn_check_decl "$LINENO" "recvfrom" "ac_cv_have_decl_recvfrom" "$cares_all_includes +" "$ac_c_undeclared_builtin_options" "CFLAGS" +if test "x$ac_cv_have_decl_recvfrom" = xyes then : - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: yes" >&5 -printf "%s\n" "yes" >&6; } - ac_cv_ios_10="yes" - -else $as_nop - - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 -printf "%s\n" "no" >&6; } - -fi -rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext +printf "%s\n" "#define HAVE_RECVFROM 1" >>confdefs.h fi - -if test "x$host_vendor" = "xapple" +ac_fn_check_decl "$LINENO" "send" "ac_cv_have_decl_send" "$cares_all_includes +" "$ac_c_undeclared_builtin_options" "CFLAGS" +if test "x$ac_cv_have_decl_send" = xyes then : - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for macOS minimum version 10.12 or later" >&5 -printf %s "checking for macOS minimum version 10.12 or later... " >&6; } - cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ - - -#include -#include -#include - -int -main (void) -{ - -#ifndef MAC_OS_X_VERSION_10_12 -# define MAC_OS_X_VERSION_10_12 101200 -#endif -#if MAC_OS_X_VERSION_MIN_REQUIRED < MAC_OS_X_VERSION_10_12 -#error Not macOS 10.12 or later -#endif -return 0; - - ; - return 0; -} +printf "%s\n" "#define HAVE_SEND 1" >>confdefs.h -_ACEOF -if ac_fn_c_try_compile "$LINENO" +fi +ac_fn_check_decl "$LINENO" "sendto" "ac_cv_have_decl_sendto" "$cares_all_includes +" "$ac_c_undeclared_builtin_options" "CFLAGS" +if test "x$ac_cv_have_decl_sendto" = xyes then : - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: yes" >&5 -printf "%s\n" "yes" >&6; } - ac_cv_macos_10_12="yes" - -else $as_nop - - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 -printf "%s\n" "no" >&6; } +printf "%s\n" "#define HAVE_SENDTO 1" >>confdefs.h fi -rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext +ac_fn_check_decl "$LINENO" "getnameinfo" "ac_cv_have_decl_getnameinfo" "$cares_all_includes +" "$ac_c_undeclared_builtin_options" "CFLAGS" +if test "x$ac_cv_have_decl_getnameinfo" = xyes +then : -fi +printf "%s\n" "#define HAVE_GETNAMEINFO 1" >>confdefs.h -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether to use libgcc" >&5 -printf %s "checking whether to use libgcc... " >&6; } -# Check whether --enable-libgcc was given. -if test ${enable_libgcc+y} +fi +ac_fn_check_decl "$LINENO" "gethostname" "ac_cv_have_decl_gethostname" "$cares_all_includes +" "$ac_c_undeclared_builtin_options" "CFLAGS" +if test "x$ac_cv_have_decl_gethostname" = xyes then : - enableval=$enable_libgcc; case "$enableval" in - yes) - LIBS="$LIBS -lgcc" - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: yes" >&5 -printf "%s\n" "yes" >&6; } - ;; - *) - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 -printf "%s\n" "no" >&6; } - ;; - esac -else $as_nop - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 -printf "%s\n" "no" >&6; } + +printf "%s\n" "#define HAVE_GETHOSTNAME 1" >>confdefs.h fi +ac_fn_check_decl "$LINENO" "connect" "ac_cv_have_decl_connect" "$cares_all_includes +" "$ac_c_undeclared_builtin_options" "CFLAGS" +if test "x$ac_cv_have_decl_connect" = xyes +then : +printf "%s\n" "#define HAVE_CONNECT 1" >>confdefs.h -ac_fn_c_check_header_compile "$LINENO" "malloc.h" "ac_cv_header_malloc_h" " -#ifdef HAVE_SYS_TYPES_H -#include -#endif -#ifdef HAVE_SYS_TIME_H -#include -#endif -#ifdef HAVE_ARPA_NAMESER_H -#include -#endif - -#ifdef HAVE_SYS_SOCKET_H -#include -#endif -#ifdef HAVE_NETINET_IN_H -#include -#endif +fi +ac_fn_check_decl "$LINENO" "connectx" "ac_cv_have_decl_connectx" "$cares_all_includes +" "$ac_c_undeclared_builtin_options" "CFLAGS" +if test "x$ac_cv_have_decl_connectx" = xyes +then : +printf "%s\n" "#define HAVE_CONNECTX 1" >>confdefs.h -" -if test "x$ac_cv_header_malloc_h" = xyes +fi +ac_fn_check_decl "$LINENO" "closesocket" "ac_cv_have_decl_closesocket" "$cares_all_includes +" "$ac_c_undeclared_builtin_options" "CFLAGS" +if test "x$ac_cv_have_decl_closesocket" = xyes then : - printf "%s\n" "#define HAVE_MALLOC_H 1" >>confdefs.h + +printf "%s\n" "#define HAVE_CLOSESOCKET 1" >>confdefs.h fi -ac_fn_c_check_header_compile "$LINENO" "memory.h" "ac_cv_header_memory_h" " -#ifdef HAVE_SYS_TYPES_H -#include -#endif -#ifdef HAVE_SYS_TIME_H -#include -#endif -#ifdef HAVE_ARPA_NAMESER_H -#include -#endif +ac_fn_check_decl "$LINENO" "CloseSocket" "ac_cv_have_decl_CloseSocket" "$cares_all_includes +" "$ac_c_undeclared_builtin_options" "CFLAGS" +if test "x$ac_cv_have_decl_CloseSocket" = xyes +then : -#ifdef HAVE_SYS_SOCKET_H -#include -#endif -#ifdef HAVE_NETINET_IN_H -#include -#endif +printf "%s\n" "#define HAVE_CLOSESOCKET_CAMEL 1" >>confdefs.h +fi +ac_fn_check_decl "$LINENO" "fcntl" "ac_cv_have_decl_fcntl" "$cares_all_includes +" "$ac_c_undeclared_builtin_options" "CFLAGS" +if test "x$ac_cv_have_decl_fcntl" = xyes +then : -" -if test "x$ac_cv_header_memory_h" = xyes +printf "%s\n" "#define HAVE_FCNTL 1" >>confdefs.h + +fi +ac_fn_check_decl "$LINENO" "getenv" "ac_cv_have_decl_getenv" "$cares_all_includes +" "$ac_c_undeclared_builtin_options" "CFLAGS" +if test "x$ac_cv_have_decl_getenv" = xyes then : - printf "%s\n" "#define HAVE_MEMORY_H 1" >>confdefs.h + +printf "%s\n" "#define HAVE_GETENV 1" >>confdefs.h fi -ac_fn_c_check_header_compile "$LINENO" "AvailabilityMacros.h" "ac_cv_header_AvailabilityMacros_h" " -#ifdef HAVE_SYS_TYPES_H -#include -#endif -#ifdef HAVE_SYS_TIME_H -#include -#endif -#ifdef HAVE_ARPA_NAMESER_H -#include -#endif +ac_fn_check_decl "$LINENO" "gethostname" "ac_cv_have_decl_gethostname" "$cares_all_includes +" "$ac_c_undeclared_builtin_options" "CFLAGS" +if test "x$ac_cv_have_decl_gethostname" = xyes +then : -#ifdef HAVE_SYS_SOCKET_H -#include -#endif -#ifdef HAVE_NETINET_IN_H -#include -#endif +printf "%s\n" "#define HAVE_GETHOSTNAME 1" >>confdefs.h + +fi +ac_fn_check_decl "$LINENO" "getrandom" "ac_cv_have_decl_getrandom" "$cares_all_includes +" "$ac_c_undeclared_builtin_options" "CFLAGS" +if test "x$ac_cv_have_decl_getrandom" = xyes +then : +printf "%s\n" "#define HAVE_GETRANDOM 1" >>confdefs.h -" -if test "x$ac_cv_header_AvailabilityMacros_h" = xyes +fi +ac_fn_check_decl "$LINENO" "getservbyport_r" "ac_cv_have_decl_getservbyport_r" "$cares_all_includes +" "$ac_c_undeclared_builtin_options" "CFLAGS" +if test "x$ac_cv_have_decl_getservbyport_r" = xyes then : - printf "%s\n" "#define HAVE_AVAILABILITYMACROS_H 1" >>confdefs.h + +printf "%s\n" "#define HAVE_GETSERVBYPORT_R 1" >>confdefs.h fi -ac_fn_c_check_header_compile "$LINENO" "sys/types.h" "ac_cv_header_sys_types_h" " -#ifdef HAVE_SYS_TYPES_H -#include -#endif -#ifdef HAVE_SYS_TIME_H -#include -#endif -#ifdef HAVE_ARPA_NAMESER_H -#include -#endif +ac_fn_check_decl "$LINENO" "inet_net_pton" "ac_cv_have_decl_inet_net_pton" "$cares_all_includes +" "$ac_c_undeclared_builtin_options" "CFLAGS" +if test "x$ac_cv_have_decl_inet_net_pton" = xyes +then : -#ifdef HAVE_SYS_SOCKET_H -#include -#endif -#ifdef HAVE_NETINET_IN_H -#include -#endif +printf "%s\n" "#define HAVE_INET_NET_PTON 1" >>confdefs.h + +fi +ac_fn_check_decl "$LINENO" "inet_ntop" "ac_cv_have_decl_inet_ntop" "$cares_all_includes +" "$ac_c_undeclared_builtin_options" "CFLAGS" +if test "x$ac_cv_have_decl_inet_ntop" = xyes +then : +printf "%s\n" "#define HAVE_INET_NTOP 1" >>confdefs.h -" -if test "x$ac_cv_header_sys_types_h" = xyes +fi +ac_fn_check_decl "$LINENO" "inet_pton" "ac_cv_have_decl_inet_pton" "$cares_all_includes +" "$ac_c_undeclared_builtin_options" "CFLAGS" +if test "x$ac_cv_have_decl_inet_pton" = xyes then : - printf "%s\n" "#define HAVE_SYS_TYPES_H 1" >>confdefs.h + +printf "%s\n" "#define HAVE_INET_PTON 1" >>confdefs.h fi -ac_fn_c_check_header_compile "$LINENO" "sys/time.h" "ac_cv_header_sys_time_h" " -#ifdef HAVE_SYS_TYPES_H -#include -#endif -#ifdef HAVE_SYS_TIME_H -#include -#endif -#ifdef HAVE_ARPA_NAMESER_H -#include -#endif +ac_fn_check_decl "$LINENO" "ioctl" "ac_cv_have_decl_ioctl" "$cares_all_includes +" "$ac_c_undeclared_builtin_options" "CFLAGS" +if test "x$ac_cv_have_decl_ioctl" = xyes +then : -#ifdef HAVE_SYS_SOCKET_H -#include -#endif -#ifdef HAVE_NETINET_IN_H -#include -#endif +printf "%s\n" "#define HAVE_IOCTL 1" >>confdefs.h +fi +ac_fn_check_decl "$LINENO" "ioctlsocket" "ac_cv_have_decl_ioctlsocket" "$cares_all_includes +" "$ac_c_undeclared_builtin_options" "CFLAGS" +if test "x$ac_cv_have_decl_ioctlsocket" = xyes +then : -" -if test "x$ac_cv_header_sys_time_h" = xyes +printf "%s\n" "#define HAVE_IOCTLSOCKET 1" >>confdefs.h + +fi +ac_fn_check_decl "$LINENO" "IoctlSocket" "ac_cv_have_decl_IoctlSocket" "$cares_all_includes +" "$ac_c_undeclared_builtin_options" "CFLAGS" +if test "x$ac_cv_have_decl_IoctlSocket" = xyes then : - printf "%s\n" "#define HAVE_SYS_TIME_H 1" >>confdefs.h + +printf "%s\n" "#define HAVE_IOCTLSOCKET_CAMEL 1" >>confdefs.h fi -ac_fn_c_check_header_compile "$LINENO" "sys/select.h" "ac_cv_header_sys_select_h" " -#ifdef HAVE_SYS_TYPES_H -#include -#endif -#ifdef HAVE_SYS_TIME_H -#include -#endif -#ifdef HAVE_ARPA_NAMESER_H -#include -#endif +ac_fn_check_decl "$LINENO" "setsockopt" "ac_cv_have_decl_setsockopt" "$cares_all_includes +" "$ac_c_undeclared_builtin_options" "CFLAGS" +if test "x$ac_cv_have_decl_setsockopt" = xyes +then : -#ifdef HAVE_SYS_SOCKET_H -#include -#endif -#ifdef HAVE_NETINET_IN_H -#include -#endif +printf "%s\n" "#define HAVE_SETSOCKOPT 1" >>confdefs.h + +fi +ac_fn_check_decl "$LINENO" "socket" "ac_cv_have_decl_socket" "$cares_all_includes +" "$ac_c_undeclared_builtin_options" "CFLAGS" +if test "x$ac_cv_have_decl_socket" = xyes +then : +printf "%s\n" "#define HAVE_SOCKET 1" >>confdefs.h -" -if test "x$ac_cv_header_sys_select_h" = xyes +fi +ac_fn_check_decl "$LINENO" "strcasecmp" "ac_cv_have_decl_strcasecmp" "$cares_all_includes +" "$ac_c_undeclared_builtin_options" "CFLAGS" +if test "x$ac_cv_have_decl_strcasecmp" = xyes then : - printf "%s\n" "#define HAVE_SYS_SELECT_H 1" >>confdefs.h + +printf "%s\n" "#define HAVE_STRCASECMP 1" >>confdefs.h fi -ac_fn_c_check_header_compile "$LINENO" "sys/socket.h" "ac_cv_header_sys_socket_h" " -#ifdef HAVE_SYS_TYPES_H -#include -#endif -#ifdef HAVE_SYS_TIME_H -#include -#endif -#ifdef HAVE_ARPA_NAMESER_H -#include -#endif +ac_fn_check_decl "$LINENO" "strdup" "ac_cv_have_decl_strdup" "$cares_all_includes +" "$ac_c_undeclared_builtin_options" "CFLAGS" +if test "x$ac_cv_have_decl_strdup" = xyes +then : -#ifdef HAVE_SYS_SOCKET_H -#include -#endif -#ifdef HAVE_NETINET_IN_H -#include -#endif +printf "%s\n" "#define HAVE_STRDUP 1" >>confdefs.h + +fi +ac_fn_check_decl "$LINENO" "stricmp" "ac_cv_have_decl_stricmp" "$cares_all_includes +" "$ac_c_undeclared_builtin_options" "CFLAGS" +if test "x$ac_cv_have_decl_stricmp" = xyes +then : +printf "%s\n" "#define HAVE_STRICMP 1" >>confdefs.h -" -if test "x$ac_cv_header_sys_socket_h" = xyes +fi +ac_fn_check_decl "$LINENO" "strncasecmp" "ac_cv_have_decl_strncasecmp" "$cares_all_includes +" "$ac_c_undeclared_builtin_options" "CFLAGS" +if test "x$ac_cv_have_decl_strncasecmp" = xyes then : - printf "%s\n" "#define HAVE_SYS_SOCKET_H 1" >>confdefs.h + +printf "%s\n" "#define HAVE_STRNCASECMP 1" >>confdefs.h fi -ac_fn_c_check_header_compile "$LINENO" "sys/filio.h" "ac_cv_header_sys_filio_h" " -#ifdef HAVE_SYS_TYPES_H -#include -#endif -#ifdef HAVE_SYS_TIME_H -#include -#endif -#ifdef HAVE_ARPA_NAMESER_H -#include -#endif +ac_fn_check_decl "$LINENO" "strncmpi" "ac_cv_have_decl_strncmpi" "$cares_all_includes +" "$ac_c_undeclared_builtin_options" "CFLAGS" +if test "x$ac_cv_have_decl_strncmpi" = xyes +then : -#ifdef HAVE_SYS_SOCKET_H -#include -#endif -#ifdef HAVE_NETINET_IN_H -#include -#endif +printf "%s\n" "#define HAVE_STRNCMPI 1" >>confdefs.h +fi +ac_fn_check_decl "$LINENO" "strnicmp" "ac_cv_have_decl_strnicmp" "$cares_all_includes +" "$ac_c_undeclared_builtin_options" "CFLAGS" +if test "x$ac_cv_have_decl_strnicmp" = xyes +then : -" -if test "x$ac_cv_header_sys_filio_h" = xyes +printf "%s\n" "#define HAVE_STRNICMP 1" >>confdefs.h + +fi +ac_fn_check_decl "$LINENO" "writev" "ac_cv_have_decl_writev" "$cares_all_includes +" "$ac_c_undeclared_builtin_options" "CFLAGS" +if test "x$ac_cv_have_decl_writev" = xyes then : - printf "%s\n" "#define HAVE_SYS_FILIO_H 1" >>confdefs.h + +printf "%s\n" "#define HAVE_WRITEV 1" >>confdefs.h fi -ac_fn_c_check_header_compile "$LINENO" "sys/ioctl.h" "ac_cv_header_sys_ioctl_h" " -#ifdef HAVE_SYS_TYPES_H -#include -#endif -#ifdef HAVE_SYS_TIME_H -#include -#endif -#ifdef HAVE_ARPA_NAMESER_H -#include -#endif +ac_fn_check_decl "$LINENO" "arc4random_buf" "ac_cv_have_decl_arc4random_buf" "$cares_all_includes +" "$ac_c_undeclared_builtin_options" "CFLAGS" +if test "x$ac_cv_have_decl_arc4random_buf" = xyes +then : -#ifdef HAVE_SYS_SOCKET_H -#include -#endif -#ifdef HAVE_NETINET_IN_H -#include -#endif +printf "%s\n" "#define HAVE_ARC4RANDOM_BUF 1" >>confdefs.h + +fi +ac_fn_check_decl "$LINENO" "stat" "ac_cv_have_decl_stat" "$cares_all_includes +" "$ac_c_undeclared_builtin_options" "CFLAGS" +if test "x$ac_cv_have_decl_stat" = xyes +then : +printf "%s\n" "#define HAVE_STAT 1" >>confdefs.h -" -if test "x$ac_cv_header_sys_ioctl_h" = xyes +fi +ac_fn_check_decl "$LINENO" "gettimeofday" "ac_cv_have_decl_gettimeofday" "$cares_all_includes +" "$ac_c_undeclared_builtin_options" "CFLAGS" +if test "x$ac_cv_have_decl_gettimeofday" = xyes then : - printf "%s\n" "#define HAVE_SYS_IOCTL_H 1" >>confdefs.h + +printf "%s\n" "#define HAVE_GETTIMEOFDAY 1" >>confdefs.h fi -ac_fn_c_check_header_compile "$LINENO" "sys/param.h" "ac_cv_header_sys_param_h" " -#ifdef HAVE_SYS_TYPES_H -#include -#endif -#ifdef HAVE_SYS_TIME_H -#include -#endif -#ifdef HAVE_ARPA_NAMESER_H -#include -#endif +ac_fn_check_decl "$LINENO" "clock_gettime" "ac_cv_have_decl_clock_gettime" "$cares_all_includes +" "$ac_c_undeclared_builtin_options" "CFLAGS" +if test "x$ac_cv_have_decl_clock_gettime" = xyes +then : -#ifdef HAVE_SYS_SOCKET_H -#include -#endif -#ifdef HAVE_NETINET_IN_H -#include -#endif +printf "%s\n" "#define HAVE_CLOCK_GETTIME 1" >>confdefs.h +fi +ac_fn_check_decl "$LINENO" "if_indextoname" "ac_cv_have_decl_if_indextoname" "$cares_all_includes +" "$ac_c_undeclared_builtin_options" "CFLAGS" +if test "x$ac_cv_have_decl_if_indextoname" = xyes +then : -" -if test "x$ac_cv_header_sys_param_h" = xyes +printf "%s\n" "#define HAVE_IF_INDEXTONAME 1" >>confdefs.h + +fi +ac_fn_check_decl "$LINENO" "if_nametoindex" "ac_cv_have_decl_if_nametoindex" "$cares_all_includes +" "$ac_c_undeclared_builtin_options" "CFLAGS" +if test "x$ac_cv_have_decl_if_nametoindex" = xyes then : - printf "%s\n" "#define HAVE_SYS_PARAM_H 1" >>confdefs.h + +printf "%s\n" "#define HAVE_IF_NAMETOINDEX 1" >>confdefs.h fi -ac_fn_c_check_header_compile "$LINENO" "sys/uio.h" "ac_cv_header_sys_uio_h" " -#ifdef HAVE_SYS_TYPES_H -#include -#endif -#ifdef HAVE_SYS_TIME_H -#include -#endif -#ifdef HAVE_ARPA_NAMESER_H -#include -#endif +ac_fn_check_decl "$LINENO" "getifaddrs" "ac_cv_have_decl_getifaddrs" "$cares_all_includes +" "$ac_c_undeclared_builtin_options" "CFLAGS" +if test "x$ac_cv_have_decl_getifaddrs" = xyes +then : -#ifdef HAVE_SYS_SOCKET_H -#include -#endif -#ifdef HAVE_NETINET_IN_H -#include -#endif +printf "%s\n" "#define HAVE_GETIFADDRS 1" >>confdefs.h + +fi +ac_fn_check_decl "$LINENO" "poll" "ac_cv_have_decl_poll" "$cares_all_includes +" "$ac_c_undeclared_builtin_options" "CFLAGS" +if test "x$ac_cv_have_decl_poll" = xyes +then : +printf "%s\n" "#define HAVE_POLL 1" >>confdefs.h -" -if test "x$ac_cv_header_sys_uio_h" = xyes +fi +ac_fn_check_decl "$LINENO" "pipe" "ac_cv_have_decl_pipe" "$cares_all_includes +" "$ac_c_undeclared_builtin_options" "CFLAGS" +if test "x$ac_cv_have_decl_pipe" = xyes then : - printf "%s\n" "#define HAVE_SYS_UIO_H 1" >>confdefs.h + +printf "%s\n" "#define HAVE_PIPE 1" >>confdefs.h fi -ac_fn_c_check_header_compile "$LINENO" "sys/random.h" "ac_cv_header_sys_random_h" " -#ifdef HAVE_SYS_TYPES_H -#include -#endif -#ifdef HAVE_SYS_TIME_H -#include -#endif -#ifdef HAVE_ARPA_NAMESER_H -#include -#endif +ac_fn_check_decl "$LINENO" "pipe2" "ac_cv_have_decl_pipe2" "$cares_all_includes +" "$ac_c_undeclared_builtin_options" "CFLAGS" +if test "x$ac_cv_have_decl_pipe2" = xyes +then : -#ifdef HAVE_SYS_SOCKET_H -#include -#endif -#ifdef HAVE_NETINET_IN_H -#include -#endif +printf "%s\n" "#define HAVE_PIPE2 1" >>confdefs.h + +fi +ac_fn_check_decl "$LINENO" "kqueue" "ac_cv_have_decl_kqueue" "$cares_all_includes +" "$ac_c_undeclared_builtin_options" "CFLAGS" +if test "x$ac_cv_have_decl_kqueue" = xyes +then : +printf "%s\n" "#define HAVE_KQUEUE 1" >>confdefs.h -" -if test "x$ac_cv_header_sys_random_h" = xyes +fi +ac_fn_check_decl "$LINENO" "epoll_create1" "ac_cv_have_decl_epoll_create1" "$cares_all_includes +" "$ac_c_undeclared_builtin_options" "CFLAGS" +if test "x$ac_cv_have_decl_epoll_create1" = xyes then : - printf "%s\n" "#define HAVE_SYS_RANDOM_H 1" >>confdefs.h + +printf "%s\n" "#define HAVE_EPOLL 1" >>confdefs.h fi -ac_fn_c_check_header_compile "$LINENO" "sys/event.h" "ac_cv_header_sys_event_h" " -#ifdef HAVE_SYS_TYPES_H -#include -#endif -#ifdef HAVE_SYS_TIME_H -#include -#endif -#ifdef HAVE_ARPA_NAMESER_H -#include -#endif +ac_fn_check_decl "$LINENO" "GetBestRoute2" "ac_cv_have_decl_GetBestRoute2" "$cares_all_includes +" "$ac_c_undeclared_builtin_options" "CFLAGS" +if test "x$ac_cv_have_decl_GetBestRoute2" = xyes +then : -#ifdef HAVE_SYS_SOCKET_H -#include -#endif -#ifdef HAVE_NETINET_IN_H -#include -#endif +printf "%s\n" "#define HAVE_GETBESTROUTE2 1" >>confdefs.h + +fi +ac_fn_check_decl "$LINENO" "ConvertInterfaceIndexToLuid" "ac_cv_have_decl_ConvertInterfaceIndexToLuid" "$cares_all_includes +" "$ac_c_undeclared_builtin_options" "CFLAGS" +if test "x$ac_cv_have_decl_ConvertInterfaceIndexToLuid" = xyes +then : +printf "%s\n" "#define HAVE_CONVERTINTERFACEINDEXTOLUID 1" >>confdefs.h -" -if test "x$ac_cv_header_sys_event_h" = xyes +fi +ac_fn_check_decl "$LINENO" "ConvertInterfaceLuidToNameA" "ac_cv_have_decl_ConvertInterfaceLuidToNameA" "$cares_all_includes +" "$ac_c_undeclared_builtin_options" "CFLAGS" +if test "x$ac_cv_have_decl_ConvertInterfaceLuidToNameA" = xyes then : - printf "%s\n" "#define HAVE_SYS_EVENT_H 1" >>confdefs.h + +printf "%s\n" "#define HAVE_CONVERTINTERFACELUIDTONAMEA 1" >>confdefs.h fi -ac_fn_c_check_header_compile "$LINENO" "sys/epoll.h" "ac_cv_header_sys_epoll_h" " -#ifdef HAVE_SYS_TYPES_H -#include -#endif -#ifdef HAVE_SYS_TIME_H -#include -#endif -#ifdef HAVE_ARPA_NAMESER_H -#include -#endif +ac_fn_check_decl "$LINENO" "NotifyIpInterfaceChange" "ac_cv_have_decl_NotifyIpInterfaceChange" "$cares_all_includes +" "$ac_c_undeclared_builtin_options" "CFLAGS" +if test "x$ac_cv_have_decl_NotifyIpInterfaceChange" = xyes +then : -#ifdef HAVE_SYS_SOCKET_H -#include -#endif -#ifdef HAVE_NETINET_IN_H -#include -#endif +printf "%s\n" "#define HAVE_NOTIFYIPINTERFACECHANGE 1" >>confdefs.h + +fi +ac_fn_check_decl "$LINENO" "RegisterWaitForSingleObject" "ac_cv_have_decl_RegisterWaitForSingleObject" "$cares_all_includes +" "$ac_c_undeclared_builtin_options" "CFLAGS" +if test "x$ac_cv_have_decl_RegisterWaitForSingleObject" = xyes +then : +printf "%s\n" "#define HAVE_REGISTERWAITFORSINGLEOBJECT 1" >>confdefs.h -" -if test "x$ac_cv_header_sys_epoll_h" = xyes +fi +ac_fn_check_decl "$LINENO" "__system_property_get" "ac_cv_have_decl___system_property_get" "$cares_all_includes +" "$ac_c_undeclared_builtin_options" "CFLAGS" +if test "x$ac_cv_have_decl___system_property_get" = xyes then : - printf "%s\n" "#define HAVE_SYS_EPOLL_H 1" >>confdefs.h + +printf "%s\n" "#define HAVE___SYSTEM_PROPERTY_GET 1" >>confdefs.h fi -ac_fn_c_check_header_compile "$LINENO" "assert.h" "ac_cv_header_assert_h" " -#ifdef HAVE_SYS_TYPES_H -#include -#endif -#ifdef HAVE_SYS_TIME_H -#include -#endif -#ifdef HAVE_ARPA_NAMESER_H -#include -#endif -#ifdef HAVE_SYS_SOCKET_H -#include -#endif -#ifdef HAVE_NETINET_IN_H -#include -#endif -" -if test "x$ac_cv_header_assert_h" = xyes -then : - printf "%s\n" "#define HAVE_ASSERT_H 1" >>confdefs.h +if test "x$ac_cv_type_ssize_t" = "xyes" -a "x$ac_cv_type_socklen_t" = "xyes" -a "x$ac_cv_native_windows" != "xyes" ; then + recvfrom_type_retv="ssize_t" + recvfrom_type_arg3="size_t" +else + recvfrom_type_retv="int" + recvfrom_type_arg3="int" +fi +if test "x$ac_cv_type_SOCKET" = "xyes" ; then + recvfrom_type_arg1="SOCKET" +else + recvfrom_type_arg1="int" fi -ac_fn_c_check_header_compile "$LINENO" "iphlpapi.h" "ac_cv_header_iphlpapi_h" " -#ifdef HAVE_SYS_TYPES_H -#include -#endif -#ifdef HAVE_SYS_TIME_H -#include -#endif -#ifdef HAVE_ARPA_NAMESER_H -#include -#endif -#ifdef HAVE_SYS_SOCKET_H -#include -#endif -#ifdef HAVE_NETINET_IN_H -#include -#endif +if test "x$ac_cv_type_socklen_t" = "xyes" ; then + recvfrom_type_arg6="socklen_t *" + getnameinfo_type_arg2="socklen_t" + getnameinfo_type_arg46="socklen_t" +else + recvfrom_type_arg6="int *" + getnameinfo_type_arg2="int" + getnameinfo_type_arg46="int" +fi +if test "x$ac_cv_native_windows" = "xyes" ; then + recv_type_arg2="char *" +else + recv_type_arg2="void *" +fi -" -if test "x$ac_cv_header_iphlpapi_h" = xyes -then : - printf "%s\n" "#define HAVE_IPHLPAPI_H 1" >>confdefs.h +recv_type_retv=${recvfrom_type_retv} +send_type_retv=${recvfrom_type_retv} +recv_type_arg1=${recvfrom_type_arg1} +recvfrom_type_arg2=${recv_type_arg2} +send_type_arg1=${recvfrom_type_arg1} +recv_type_arg3=${recvfrom_type_arg3} +send_type_arg3=${recvfrom_type_arg3} +gethostname_type_arg2=${recvfrom_type_arg3} -fi -ac_fn_c_check_header_compile "$LINENO" "netioapi.h" "ac_cv_header_netioapi_h" " -#ifdef HAVE_SYS_TYPES_H -#include -#endif -#ifdef HAVE_SYS_TIME_H -#include -#endif -#ifdef HAVE_ARPA_NAMESER_H -#include -#endif +recvfrom_qual_arg5= +recvfrom_type_arg4=int +recvfrom_type_arg5="struct sockaddr *" +recv_type_arg4=int +getnameinfo_type_arg1="struct sockaddr *" +getnameinfo_type_arg7=int +send_type_arg2="const void *" +send_type_arg4=int -#ifdef HAVE_SYS_SOCKET_H -#include -#endif -#ifdef HAVE_NETINET_IN_H -#include -#endif +printf "%s\n" "#define RECVFROM_TYPE_RETV ${recvfrom_type_retv} " >>confdefs.h -" -if test "x$ac_cv_header_netioapi_h" = xyes -then : - printf "%s\n" "#define HAVE_NETIOAPI_H 1" >>confdefs.h -fi -ac_fn_c_check_header_compile "$LINENO" "netdb.h" "ac_cv_header_netdb_h" " -#ifdef HAVE_SYS_TYPES_H -#include -#endif -#ifdef HAVE_SYS_TIME_H -#include -#endif -#ifdef HAVE_ARPA_NAMESER_H -#include -#endif +printf "%s\n" "#define RECVFROM_TYPE_ARG1 ${recvfrom_type_arg1} " >>confdefs.h -#ifdef HAVE_SYS_SOCKET_H -#include -#endif -#ifdef HAVE_NETINET_IN_H -#include -#endif +printf "%s\n" "#define RECVFROM_TYPE_ARG2 ${recvfrom_type_arg2} " >>confdefs.h -" -if test "x$ac_cv_header_netdb_h" = xyes -then : - printf "%s\n" "#define HAVE_NETDB_H 1" >>confdefs.h -fi -ac_fn_c_check_header_compile "$LINENO" "netinet/in.h" "ac_cv_header_netinet_in_h" " -#ifdef HAVE_SYS_TYPES_H -#include -#endif -#ifdef HAVE_SYS_TIME_H -#include -#endif -#ifdef HAVE_ARPA_NAMESER_H -#include -#endif +printf "%s\n" "#define RECVFROM_TYPE_ARG3 ${recvfrom_type_arg3} " >>confdefs.h -#ifdef HAVE_SYS_SOCKET_H -#include -#endif -#ifdef HAVE_NETINET_IN_H -#include -#endif +printf "%s\n" "#define RECVFROM_TYPE_ARG4 ${recvfrom_type_arg4} " >>confdefs.h -" -if test "x$ac_cv_header_netinet_in_h" = xyes -then : - printf "%s\n" "#define HAVE_NETINET_IN_H 1" >>confdefs.h -fi -ac_fn_c_check_header_compile "$LINENO" "netinet6/in6.h" "ac_cv_header_netinet6_in6_h" " -#ifdef HAVE_SYS_TYPES_H -#include -#endif -#ifdef HAVE_SYS_TIME_H -#include -#endif -#ifdef HAVE_ARPA_NAMESER_H -#include -#endif +printf "%s\n" "#define RECVFROM_TYPE_ARG5 ${recvfrom_type_arg5} " >>confdefs.h -#ifdef HAVE_SYS_SOCKET_H -#include -#endif -#ifdef HAVE_NETINET_IN_H -#include -#endif +printf "%s\n" "#define RECVFROM_QUAL_ARG5 ${recvfrom_qual_arg5}" >>confdefs.h -" -if test "x$ac_cv_header_netinet6_in6_h" = xyes -then : - printf "%s\n" "#define HAVE_NETINET6_IN6_H 1" >>confdefs.h -fi -ac_fn_c_check_header_compile "$LINENO" "netinet/tcp.h" "ac_cv_header_netinet_tcp_h" " -#ifdef HAVE_SYS_TYPES_H -#include -#endif -#ifdef HAVE_SYS_TIME_H -#include -#endif -#ifdef HAVE_ARPA_NAMESER_H -#include -#endif -#ifdef HAVE_SYS_SOCKET_H -#include -#endif -#ifdef HAVE_NETINET_IN_H -#include -#endif +printf "%s\n" "#define RECV_TYPE_RETV ${recv_type_retv} " >>confdefs.h -" -if test "x$ac_cv_header_netinet_tcp_h" = xyes -then : - printf "%s\n" "#define HAVE_NETINET_TCP_H 1" >>confdefs.h +printf "%s\n" "#define RECV_TYPE_ARG1 ${recv_type_arg1} " >>confdefs.h -fi -ac_fn_c_check_header_compile "$LINENO" "net/if.h" "ac_cv_header_net_if_h" " -#ifdef HAVE_SYS_TYPES_H -#include -#endif -#ifdef HAVE_SYS_TIME_H -#include -#endif -#ifdef HAVE_ARPA_NAMESER_H -#include -#endif -#ifdef HAVE_SYS_SOCKET_H -#include -#endif -#ifdef HAVE_NETINET_IN_H -#include -#endif +printf "%s\n" "#define RECV_TYPE_ARG2 ${recv_type_arg2} " >>confdefs.h -" -if test "x$ac_cv_header_net_if_h" = xyes -then : - printf "%s\n" "#define HAVE_NET_IF_H 1" >>confdefs.h +printf "%s\n" "#define RECV_TYPE_ARG3 ${recv_type_arg3} " >>confdefs.h -fi -ac_fn_c_check_header_compile "$LINENO" "ifaddrs.h" "ac_cv_header_ifaddrs_h" " -#ifdef HAVE_SYS_TYPES_H -#include -#endif -#ifdef HAVE_SYS_TIME_H -#include -#endif -#ifdef HAVE_ARPA_NAMESER_H -#include -#endif -#ifdef HAVE_SYS_SOCKET_H -#include -#endif -#ifdef HAVE_NETINET_IN_H -#include -#endif +printf "%s\n" "#define RECV_TYPE_ARG4 ${recv_type_arg4} " >>confdefs.h -" -if test "x$ac_cv_header_ifaddrs_h" = xyes -then : - printf "%s\n" "#define HAVE_IFADDRS_H 1" >>confdefs.h -fi -ac_fn_c_check_header_compile "$LINENO" "fcntl.h" "ac_cv_header_fcntl_h" " -#ifdef HAVE_SYS_TYPES_H -#include -#endif -#ifdef HAVE_SYS_TIME_H -#include -#endif -#ifdef HAVE_ARPA_NAMESER_H -#include -#endif +printf "%s\n" "#define SEND_TYPE_RETV ${send_type_retv} " >>confdefs.h + -#ifdef HAVE_SYS_SOCKET_H -#include -#endif -#ifdef HAVE_NETINET_IN_H -#include -#endif +printf "%s\n" "#define SEND_TYPE_ARG1 ${send_type_arg1} " >>confdefs.h -" -if test "x$ac_cv_header_fcntl_h" = xyes -then : - printf "%s\n" "#define HAVE_FCNTL_H 1" >>confdefs.h +printf "%s\n" "#define SEND_TYPE_ARG2 ${send_type_arg2} " >>confdefs.h -fi -ac_fn_c_check_header_compile "$LINENO" "errno.h" "ac_cv_header_errno_h" " -#ifdef HAVE_SYS_TYPES_H -#include -#endif -#ifdef HAVE_SYS_TIME_H -#include -#endif -#ifdef HAVE_ARPA_NAMESER_H -#include -#endif -#ifdef HAVE_SYS_SOCKET_H -#include -#endif -#ifdef HAVE_NETINET_IN_H -#include -#endif +printf "%s\n" "#define SEND_TYPE_ARG3 ${send_type_arg3} " >>confdefs.h -" -if test "x$ac_cv_header_errno_h" = xyes -then : - printf "%s\n" "#define HAVE_ERRNO_H 1" >>confdefs.h +printf "%s\n" "#define SEND_TYPE_ARG4 ${send_type_arg4} " >>confdefs.h -fi -ac_fn_c_check_header_compile "$LINENO" "socket.h" "ac_cv_header_socket_h" " -#ifdef HAVE_SYS_TYPES_H -#include -#endif -#ifdef HAVE_SYS_TIME_H -#include -#endif -#ifdef HAVE_ARPA_NAMESER_H -#include -#endif -#ifdef HAVE_SYS_SOCKET_H -#include -#endif -#ifdef HAVE_NETINET_IN_H -#include -#endif +printf "%s\n" "#define GETNAMEINFO_TYPE_ARG1 ${getnameinfo_type_arg1} " >>confdefs.h -" -if test "x$ac_cv_header_socket_h" = xyes -then : - printf "%s\n" "#define HAVE_SOCKET_H 1" >>confdefs.h -fi -ac_fn_c_check_header_compile "$LINENO" "strings.h" "ac_cv_header_strings_h" " -#ifdef HAVE_SYS_TYPES_H -#include -#endif -#ifdef HAVE_SYS_TIME_H -#include -#endif -#ifdef HAVE_ARPA_NAMESER_H -#include -#endif +printf "%s\n" "#define GETNAMEINFO_TYPE_ARG2 ${getnameinfo_type_arg2} " >>confdefs.h -#ifdef HAVE_SYS_SOCKET_H -#include -#endif -#ifdef HAVE_NETINET_IN_H -#include -#endif +printf "%s\n" "#define GETNAMEINFO_TYPE_ARG7 ${getnameinfo_type_arg7} " >>confdefs.h -" -if test "x$ac_cv_header_strings_h" = xyes -then : - printf "%s\n" "#define HAVE_STRINGS_H 1" >>confdefs.h -fi -ac_fn_c_check_header_compile "$LINENO" "stdbool.h" "ac_cv_header_stdbool_h" " -#ifdef HAVE_SYS_TYPES_H -#include -#endif -#ifdef HAVE_SYS_TIME_H -#include -#endif -#ifdef HAVE_ARPA_NAMESER_H -#include -#endif +printf "%s\n" "#define GETNAMEINFO_TYPE_ARG46 ${getnameinfo_type_arg46} " >>confdefs.h -#ifdef HAVE_SYS_SOCKET_H -#include -#endif -#ifdef HAVE_NETINET_IN_H -#include -#endif -" -if test "x$ac_cv_header_stdbool_h" = xyes -then : - printf "%s\n" "#define HAVE_STDBOOL_H 1" >>confdefs.h +printf "%s\n" "#define GETHOSTNAME_TYPE_ARG2 ${gethostname_type_arg2} " >>confdefs.h -fi -ac_fn_c_check_header_compile "$LINENO" "time.h" "ac_cv_header_time_h" " -#ifdef HAVE_SYS_TYPES_H -#include -#endif -#ifdef HAVE_SYS_TIME_H -#include -#endif -#ifdef HAVE_ARPA_NAMESER_H -#include -#endif -#ifdef HAVE_SYS_SOCKET_H -#include -#endif -#ifdef HAVE_NETINET_IN_H -#include -#endif +if test "$ac_cv_have_decl_getservbyport_r" = "yes" ; then + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking number of arguments for getservbyport_r()" >&5 +printf %s "checking number of arguments for getservbyport_r()... " >&6; } + getservbyport_r_args=6 + case $host_os in + solaris*) + getservbyport_r_args=5 + ;; + aix*|openbsd*) + getservbyport_r_args=4 + ;; + esac + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $getservbyport_r_args" >&5 +printf "%s\n" "$getservbyport_r_args" >&6; } -" -if test "x$ac_cv_header_time_h" = xyes -then : - printf "%s\n" "#define HAVE_TIME_H 1" >>confdefs.h +printf "%s\n" "#define GETSERVBYPORT_R_ARGS $getservbyport_r_args " >>confdefs.h fi -ac_fn_c_check_header_compile "$LINENO" "poll.h" "ac_cv_header_poll_h" " -#ifdef HAVE_SYS_TYPES_H -#include -#endif -#ifdef HAVE_SYS_TIME_H -#include -#endif -#ifdef HAVE_ARPA_NAMESER_H -#include -#endif -#ifdef HAVE_SYS_SOCKET_H -#include -#endif -#ifdef HAVE_NETINET_IN_H -#include -#endif +if test "$ac_cv_have_decl_getservbyname_r" = "yes" ; then + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking number of arguments for getservbyname_r()" >&5 +printf %s "checking number of arguments for getservbyname_r()... " >&6; } + getservbyname_r_args=6 + case $host_os in + solaris*) + getservbyname_r_args=5 + ;; + aix*|openbsd*) + getservbyname_r_args=4 + ;; + esac + +printf "%s\n" "#define GETSERVBYNAME_R_ARGS $getservbyname_r_args " >>confdefs.h + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $getservbyname_r_args" >&5 +printf "%s\n" "$getservbyname_r_args" >&6; } +fi -" -if test "x$ac_cv_header_poll_h" = xyes +ac_fn_c_check_type "$LINENO" "size_t" "ac_cv_type_size_t" "$ac_includes_default" +if test "x$ac_cv_type_size_t" = xyes then : - printf "%s\n" "#define HAVE_POLL_H 1" >>confdefs.h -fi -ac_fn_c_check_header_compile "$LINENO" "limits.h" "ac_cv_header_limits_h" " -#ifdef HAVE_SYS_TYPES_H -#include -#endif -#ifdef HAVE_SYS_TIME_H -#include -#endif -#ifdef HAVE_ARPA_NAMESER_H -#include -#endif +else $as_nop -#ifdef HAVE_SYS_SOCKET_H -#include -#endif -#ifdef HAVE_NETINET_IN_H -#include -#endif +printf "%s\n" "#define size_t unsigned int" >>confdefs.h +fi -" -if test "x$ac_cv_header_limits_h" = xyes +ac_fn_check_decl "$LINENO" "AF_INET6" "ac_cv_have_decl_AF_INET6" "$cares_all_includes +" "$ac_c_undeclared_builtin_options" "CFLAGS" +if test "x$ac_cv_have_decl_AF_INET6" = xyes then : - printf "%s\n" "#define HAVE_LIMITS_H 1" >>confdefs.h -fi -ac_fn_c_check_header_compile "$LINENO" "arpa/nameser.h" "ac_cv_header_arpa_nameser_h" " -#ifdef HAVE_SYS_TYPES_H -#include -#endif -#ifdef HAVE_SYS_TIME_H -#include -#endif -#ifdef HAVE_ARPA_NAMESER_H -#include -#endif +printf "%s\n" "#define HAVE_AF_INET6 1" >>confdefs.h -#ifdef HAVE_SYS_SOCKET_H -#include -#endif -#ifdef HAVE_NETINET_IN_H -#include -#endif +fi +ac_fn_check_decl "$LINENO" "PF_INET6" "ac_cv_have_decl_PF_INET6" "$cares_all_includes +" "$ac_c_undeclared_builtin_options" "CFLAGS" +if test "x$ac_cv_have_decl_PF_INET6" = xyes +then : +printf "%s\n" "#define HAVE_PF_INET6 1" >>confdefs.h +fi +ac_fn_c_check_type "$LINENO" "struct in6_addr" "ac_cv_type_struct_in6_addr" "$cares_all_includes " -if test "x$ac_cv_header_arpa_nameser_h" = xyes +if test "x$ac_cv_type_struct_in6_addr" = xyes then : - printf "%s\n" "#define HAVE_ARPA_NAMESER_H 1" >>confdefs.h + +printf "%s\n" "#define HAVE_STRUCT_IN6_ADDR 1" >>confdefs.h + fi -ac_fn_c_check_header_compile "$LINENO" "arpa/nameser_compat.h" "ac_cv_header_arpa_nameser_compat_h" " -#ifdef HAVE_SYS_TYPES_H -#include -#endif -#ifdef HAVE_SYS_TIME_H -#include -#endif -#ifdef HAVE_ARPA_NAMESER_H -#include -#endif -#ifdef HAVE_SYS_SOCKET_H -#include -#endif -#ifdef HAVE_NETINET_IN_H -#include -#endif +ac_fn_c_check_type "$LINENO" "struct sockaddr_in6" "ac_cv_type_struct_sockaddr_in6" "$cares_all_includes +" +if test "x$ac_cv_type_struct_sockaddr_in6" = xyes +then : + +printf "%s\n" "#define HAVE_STRUCT_SOCKADDR_IN6 1" >>confdefs.h + +fi +ac_fn_c_check_type "$LINENO" "struct sockaddr_storage" "ac_cv_type_struct_sockaddr_storage" "$cares_all_includes " -if test "x$ac_cv_header_arpa_nameser_compat_h" = xyes +if test "x$ac_cv_type_struct_sockaddr_storage" = xyes then : - printf "%s\n" "#define HAVE_ARPA_NAMESER_COMPAT_H 1" >>confdefs.h -fi -ac_fn_c_check_header_compile "$LINENO" "arpa/inet.h" "ac_cv_header_arpa_inet_h" " -#ifdef HAVE_SYS_TYPES_H -#include -#endif -#ifdef HAVE_SYS_TIME_H -#include -#endif -#ifdef HAVE_ARPA_NAMESER_H -#include -#endif +printf "%s\n" "#define HAVE_STRUCT_SOCKADDR_STORAGE 1" >>confdefs.h -#ifdef HAVE_SYS_SOCKET_H -#include -#endif -#ifdef HAVE_NETINET_IN_H -#include -#endif +fi +ac_fn_c_check_type "$LINENO" "struct addrinfo" "ac_cv_type_struct_addrinfo" "$cares_all_includes " -if test "x$ac_cv_header_arpa_inet_h" = xyes +if test "x$ac_cv_type_struct_addrinfo" = xyes then : - printf "%s\n" "#define HAVE_ARPA_INET_H 1" >>confdefs.h -fi -ac_fn_c_check_header_compile "$LINENO" "sys/system_properties.h" "ac_cv_header_sys_system_properties_h" " -#ifdef HAVE_SYS_TYPES_H -#include -#endif -#ifdef HAVE_SYS_TIME_H -#include -#endif -#ifdef HAVE_ARPA_NAMESER_H -#include -#endif +printf "%s\n" "#define HAVE_STRUCT_ADDRINFO 1" >>confdefs.h -#ifdef HAVE_SYS_SOCKET_H -#include -#endif -#ifdef HAVE_NETINET_IN_H -#include -#endif +fi +ac_fn_c_check_type "$LINENO" "struct timeval" "ac_cv_type_struct_timeval" "$cares_all_includes " -if test "x$ac_cv_header_sys_system_properties_h" = xyes +if test "x$ac_cv_type_struct_timeval" = xyes then : - printf "%s\n" "#define HAVE_SYS_SYSTEM_PROPERTIES_H 1" >>confdefs.h -fi +printf "%s\n" "#define HAVE_STRUCT_TIMEVAL 1" >>confdefs.h +fi -cares_all_includes=" -#include -#include -#ifdef HAVE_AVAILABILITYMACROS_H -# include -#endif -#ifdef HAVE_SYS_UIO_H -# include -#endif -#ifdef HAVE_NETINET_IN_H -# include -#endif -#ifdef HAVE_TCP_H -# include -#endif -#ifdef HAVE_SYS_FILIO_H -# include -#endif -#ifdef HAVE_SYS_IOCTL_H -# include -#endif -#ifdef HAVE_UNISTD_H -# include -#endif -#ifdef HAVE_STRING_H -# include -#endif -#ifdef HAVE_STRINGS_H -# include -#endif -#ifdef HAVE_TIME_H -# include -#endif -#ifdef HAVE_SYS_TIME_H -# include -#endif -#ifdef HAVE_SYS_TYPES_H -# include -#endif -#ifdef HAVE_SYS_STAT_H -# include -#endif -#ifdef HAVE_SYS_RANDOM_H -# include -#endif -#ifdef HAVE_SYS_EVENT_H -# include -#endif -#ifdef HAVE_SYS_EPOLL_H -# include -#endif -#ifdef HAVE_SYS_SOCKET_H -# include -#endif -#ifdef HAVE_SYS_PARAM_H -# include -#endif -#ifdef HAVE_FCNTL_H -# include -#endif -#ifdef HAVE_POLL_H -# include -#endif -#ifdef HAVE_NET_IF_H -# include -#endif -#ifdef HAVE_IFADDRS_H -# include -#endif -#ifdef HAVE_NETINET_IN_H -# include -#endif -#ifdef HAVE_NETINET_TCP_H -# include -#endif -#ifdef HAVE_NETDB_H -# include -#endif -#ifdef HAVE_ARPA_INET_H -# include -#endif -#ifdef HAVE_RESOLV_H -# include -#endif -#ifdef HAVE_SYS_SYSTEM_PROPERTIES_H -# include -#endif -#ifdef HAVE_IPHLPAPI_H -# include -#endif -#ifdef HAVE_NETIOAPI_H -# include -#endif -#ifdef HAVE_WINSOCK2_H -# include -#endif -#ifdef HAVE_WS2IPDEF_H -# include -#endif -#ifdef HAVE_WS2TCPIP_H -# include -#endif -#ifdef HAVE_WINDOWS_H -# include -#endif +ac_fn_c_check_member "$LINENO" "struct sockaddr_in6" "sin6_scope_id" "ac_cv_member_struct_sockaddr_in6_sin6_scope_id" "$cares_all_includes " +if test "x$ac_cv_member_struct_sockaddr_in6_sin6_scope_id" = xyes +then : -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for $CC options needed to detect all undeclared functions" >&5 -printf %s "checking for $CC options needed to detect all undeclared functions... " >&6; } -if test ${ac_cv_c_undeclared_builtin_options+y} +printf "%s\n" "#define HAVE_STRUCT_SOCKADDR_IN6_SIN6_SCOPE_ID 1" >>confdefs.h + + +fi + +ac_fn_c_check_member "$LINENO" "struct addrinfo" "ai_flags" "ac_cv_member_struct_addrinfo_ai_flags" "$cares_all_includes +" +if test "x$ac_cv_member_struct_addrinfo_ai_flags" = xyes then : - printf %s "(cached) " >&6 -else $as_nop - ac_save_CFLAGS=$CFLAGS - ac_cv_c_undeclared_builtin_options='cannot detect' - for ac_arg in '' -fno-builtin; do - CFLAGS="$ac_save_CFLAGS $ac_arg" - # This test program should *not* compile successfully. - cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ -int -main (void) -{ -(void) strchr; - ; - return 0; -} -_ACEOF -if ac_fn_c_try_compile "$LINENO" +printf "%s\n" "#define HAVE_STRUCT_ADDRINFO_AI_FLAGS 1" >>confdefs.h + + +fi + +ac_fn_check_decl "$LINENO" "FIONBIO" "ac_cv_have_decl_FIONBIO" "$cares_all_includes +" "$ac_c_undeclared_builtin_options" "CFLAGS" +if test "x$ac_cv_have_decl_FIONBIO" = xyes then : -else $as_nop - # This test program should compile successfully. - # No library function is consistently available on - # freestanding implementations, so test against a dummy - # declaration. Include always-available headers on the - # off chance that they somehow elicit warnings. - cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ -#include -#include -#include -#include -extern void ac_decl (int, char *); +fi +ac_fn_check_decl "$LINENO" "O_NONBLOCK" "ac_cv_have_decl_O_NONBLOCK" "$cares_all_includes +" "$ac_c_undeclared_builtin_options" "CFLAGS" +if test "x$ac_cv_have_decl_O_NONBLOCK" = xyes +then : -int -main (void) -{ -(void) ac_decl (0, (char *) 0); - (void) ac_decl; +fi +ac_fn_check_decl "$LINENO" "SO_NONBLOCK" "ac_cv_have_decl_SO_NONBLOCK" "$cares_all_includes +" "$ac_c_undeclared_builtin_options" "CFLAGS" +if test "x$ac_cv_have_decl_SO_NONBLOCK" = xyes +then : - ; - return 0; -} -_ACEOF -if ac_fn_c_try_compile "$LINENO" +fi +ac_fn_check_decl "$LINENO" "MSG_NOSIGNAL" "ac_cv_have_decl_MSG_NOSIGNAL" "$cares_all_includes +" "$ac_c_undeclared_builtin_options" "CFLAGS" +if test "x$ac_cv_have_decl_MSG_NOSIGNAL" = xyes then : - if test x"$ac_arg" = x + +fi +ac_fn_check_decl "$LINENO" "CLOCK_MONOTONIC" "ac_cv_have_decl_CLOCK_MONOTONIC" "$cares_all_includes +" "$ac_c_undeclared_builtin_options" "CFLAGS" +if test "x$ac_cv_have_decl_CLOCK_MONOTONIC" = xyes then : - ac_cv_c_undeclared_builtin_options='none needed' -else $as_nop - ac_cv_c_undeclared_builtin_options=$ac_arg + +fi + +if test "$ac_cv_have_decl_CLOCK_MONOTONIC" = "yes" -a "$ac_cv_have_decl_clock_gettime" = "yes" ; then + +printf "%s\n" "#define HAVE_CLOCK_GETTIME_MONOTONIC 1 " >>confdefs.h + +fi + +if test "$ac_cv_have_decl_FIONBIO" = "yes" -a "$ac_cv_have_decl_ioctl" = "yes" ; then + +printf "%s\n" "#define HAVE_IOCTL_FIONBIO 1 " >>confdefs.h + fi - break +if test "$ac_cv_have_decl_FIONBIO" = "yes" -a "$ac_cv_have_decl_ioctlsocket" = "yes" ; then + +printf "%s\n" "#define HAVE_IOCTLSOCKET_FIONBIO 1 " >>confdefs.h + fi -rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext +if test "$ac_cv_have_decl_SO_NONBLOCK" = "yes" -a "$ac_cv_have_decl_setsockopt" = "yes" ; then + +printf "%s\n" "#define HAVE_SETSOCKOPT_SO_NONBLOCK 1 " >>confdefs.h + fi -rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext - done - CFLAGS=$ac_save_CFLAGS +if test "$ac_cv_have_decl_O_NONBLOCK" = "yes" -a "$ac_cv_have_decl_fcntl" = "yes" ; then + +printf "%s\n" "#define HAVE_FCNTL_O_NONBLOCK 1 " >>confdefs.h fi -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_c_undeclared_builtin_options" >&5 -printf "%s\n" "$ac_cv_c_undeclared_builtin_options" >&6; } - case $ac_cv_c_undeclared_builtin_options in #( - 'cannot detect') : - { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 -printf "%s\n" "$as_me: error: in \`$ac_pwd':" >&2;} -as_fn_error $? "cannot make $CC report undeclared builtins -See \`config.log' for more details" "$LINENO" 5; } ;; #( - 'none needed') : - ac_c_undeclared_builtin_options='' ;; #( - *) : - ac_c_undeclared_builtin_options=$ac_cv_c_undeclared_builtin_options ;; -esac -ac_fn_check_decl "$LINENO" "HAVE_ARPA_NAMESER_H" "ac_cv_have_decl_HAVE_ARPA_NAMESER_H" "$ac_includes_default" "$ac_c_undeclared_builtin_options" "CFLAGS" -if test "x$ac_cv_have_decl_HAVE_ARPA_NAMESER_H" = xyes -then : +if test "x$ac_cv_header_sys_types_h" = "xyes" ; then cat >>confdefs.h <<_EOF -#define CARES_HAVE_ARPA_NAMESER_H 1 +#define CARES_HAVE_SYS_TYPES_H 1 _EOF fi -ac_fn_check_decl "$LINENO" "HAVE_ARPA_NAMESER_COMPAT_H" "ac_cv_have_decl_HAVE_ARPA_NAMESER_COMPAT_H" "$ac_includes_default" "$ac_c_undeclared_builtin_options" "CFLAGS" -if test "x$ac_cv_have_decl_HAVE_ARPA_NAMESER_COMPAT_H" = xyes -then : +if test "x$ac_cv_header_sys_socket_h" = "xyes" ; then cat >>confdefs.h <<_EOF -#define CARES_HAVE_ARPA_NAMESER_COMPAT_H 1 +#define CARES_HAVE_SYS_SOCKET_H 1 _EOF fi -ac_fn_c_check_type "$LINENO" "long long" "ac_cv_type_long_long" "$ac_includes_default" -if test "x$ac_cv_type_long_long" = xyes -then : - -printf "%s\n" "#define HAVE_LONGLONG 1" >>confdefs.h +if test "x$ac_cv_header_sys_select_h" = "xyes" ; then -fi +cat >>confdefs.h <<_EOF +#define CARES_HAVE_SYS_SELECT_H 1 +_EOF -ac_fn_c_check_type "$LINENO" "ssize_t" "ac_cv_type_ssize_t" "$ac_includes_default" -if test "x$ac_cv_type_ssize_t" = xyes -then : - CARES_TYPEOF_ARES_SSIZE_T=ssize_t -else $as_nop - CARES_TYPEOF_ARES_SSIZE_T=int fi +if test "x$ac_cv_header_ws2tcpip_h" = "xyes" ; then +cat >>confdefs.h <<_EOF +#define CARES_HAVE_WS2TCPIP_H 1 +_EOF -printf "%s\n" "#define CARES_TYPEOF_ARES_SSIZE_T ${CARES_TYPEOF_ARES_SSIZE_T}" >>confdefs.h - - -ac_fn_c_check_type "$LINENO" "socklen_t" "ac_cv_type_socklen_t" "$cares_all_includes +fi +if test "x$ac_cv_header_winsock2_h" = "xyes" ; then -" -if test "x$ac_cv_type_socklen_t" = xyes -then : +cat >>confdefs.h <<_EOF +#define CARES_HAVE_WINSOCK2_H 1 +_EOF +fi +if test "x$ac_cv_header_windows_h" = "xyes" ; then -printf "%s\n" "#define HAVE_SOCKLEN_T /**/" >>confdefs.h +cat >>confdefs.h <<_EOF +#define CARES_HAVE_WINDOWS_H 1 +_EOF +fi +if test "x$ac_cv_header_arpa_nameser_h" = "xyes" ; then cat >>confdefs.h <<_EOF -#define CARES_TYPEOF_ARES_SOCKLEN_T socklen_t +#define CARES_HAVE_ARPA_NAMESER_H 1 _EOF - -else $as_nop +fi +if test "x$ac_cv_header_arpa_nameser_compa_h" = "xyes" ; then cat >>confdefs.h <<_EOF -#define CARES_TYPEOF_ARES_SOCKLEN_T int +#define CARES_HAVE_ARPA_NAMESER_COMPA_H 1 _EOF fi -ac_fn_c_check_type "$LINENO" "SOCKET" "ac_cv_type_SOCKET" "$cares_all_includes -" -if test "x$ac_cv_type_SOCKET" = xyes -then : -fi +if test "${CARES_THREADS}" = "yes" -a "x${ac_cv_native_windows}" != "xyes" ; then -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for library containing clock_gettime" >&5 -printf %s "checking for library containing clock_gettime... " >&6; } -if test ${ac_cv_search_clock_gettime+y} + +ac_ext=c +ac_cpp='$CPP $CPPFLAGS' +ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5' +ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5' +ac_compiler_gnu=$ac_cv_c_compiler_gnu + +ax_pthread_ok=no + +# We used to check for pthread.h first, but this fails if pthread.h +# requires special compiler flags (e.g. on Tru64 or Sequent). +# It gets checked for in the link test anyway. + +# First of all, check if the user has set any of the PTHREAD_LIBS, +# etcetera environment variables, and if threads linking works using +# them: +if test "x$PTHREAD_CFLAGS$PTHREAD_LIBS" != "x"; then + ax_pthread_save_CC="$CC" + ax_pthread_save_CFLAGS="$CFLAGS" + ax_pthread_save_LIBS="$LIBS" + if test "x$PTHREAD_CC" != "x" then : - printf %s "(cached) " >&6 -else $as_nop - ac_func_search_save_LIBS=$LIBS -cat confdefs.h - <<_ACEOF >conftest.$ac_ext + CC="$PTHREAD_CC" +fi + if test "x$PTHREAD_CXX" != "x" +then : + CXX="$PTHREAD_CXX" +fi + CFLAGS="$CFLAGS $PTHREAD_CFLAGS" + LIBS="$PTHREAD_LIBS $LIBS" + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for pthread_join using $CC $PTHREAD_CFLAGS $PTHREAD_LIBS" >&5 +printf %s "checking for pthread_join using $CC $PTHREAD_CFLAGS $PTHREAD_LIBS... " >&6; } + cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ /* Override any GCC internal prototype to avoid an error. Use char because int might match the return type of a GCC builtin and then its argument prototype would still apply. */ -char clock_gettime (); +char pthread_join (); int main (void) { -return clock_gettime (); +return pthread_join (); ; return 0; } _ACEOF -for ac_lib in '' rt posix4 -do - if test -z "$ac_lib"; then - ac_res="none required" - else - ac_res=-l$ac_lib - LIBS="-l$ac_lib $ac_func_search_save_LIBS" - fi - if ac_fn_c_try_link "$LINENO" +if ac_fn_c_try_link "$LINENO" then : - ac_cv_search_clock_gettime=$ac_res + ax_pthread_ok=yes fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ - conftest$ac_exeext - if test ${ac_cv_search_clock_gettime+y} -then : - break + conftest$ac_exeext conftest.$ac_ext + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ax_pthread_ok" >&5 +printf "%s\n" "$ax_pthread_ok" >&6; } + if test "x$ax_pthread_ok" = "xno"; then + PTHREAD_LIBS="" + PTHREAD_CFLAGS="" + fi + CC="$ax_pthread_save_CC" + CFLAGS="$ax_pthread_save_CFLAGS" + LIBS="$ax_pthread_save_LIBS" fi -done -if test ${ac_cv_search_clock_gettime+y} -then : -else $as_nop - ac_cv_search_clock_gettime=no -fi -rm conftest.$ac_ext -LIBS=$ac_func_search_save_LIBS -fi -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_search_clock_gettime" >&5 -printf "%s\n" "$ac_cv_search_clock_gettime" >&6; } -ac_res=$ac_cv_search_clock_gettime -if test "$ac_res" != no -then : - test "$ac_res" = "none required" || LIBS="$ac_res $LIBS" +# We must check for the threads library under a number of different +# names; the ordering is very important because some systems +# (e.g. DEC) have both -lpthread and -lpthreads, where one of the +# libraries is broken (non-POSIX). + +# Create a list of thread flags to try. Items with a "," contain both +# C compiler flags (before ",") and linker flags (after ","). Other items +# starting with a "-" are C compiler flags, and remaining items are +# library names, except for "none" which indicates that we try without +# any flags at all, and "pthread-config" which is a program returning +# the flags for the Pth emulation library. + +ax_pthread_flags="pthreads none -Kthread -pthread -pthreads -mthreads pthread --thread-safe -mt pthread-config" + +# The ordering *is* (sometimes) important. Some notes on the +# individual items follow: + +# pthreads: AIX (must check this before -lpthread) +# none: in case threads are in libc; should be tried before -Kthread and +# other compiler flags to prevent continual compiler warnings +# -Kthread: Sequent (threads in libc, but -Kthread needed for pthread.h) +# -pthread: Linux/gcc (kernel threads), BSD/gcc (userland threads), Tru64 +# (Note: HP C rejects this with "bad form for `-t' option") +# -pthreads: Solaris/gcc (Note: HP C also rejects) +# -mt: Sun Workshop C (may only link SunOS threads [-lthread], but it +# doesn't hurt to check since this sometimes defines pthreads and +# -D_REENTRANT too), HP C (must be checked before -lpthread, which +# is present but should not be used directly; and before -mthreads, +# because the compiler interprets this as "-mt" + "-hreads") +# -mthreads: Mingw32/gcc, Lynx/gcc +# pthread: Linux, etcetera +# --thread-safe: KAI C++ +# pthread-config: use pthread-config program (for GNU Pth library) + +case $host_os in + + freebsd*|midnightbsd*) + + # -kthread: FreeBSD kernel threads (preferred to -pthread since SMP-able) + # lthread: LinuxThreads port on FreeBSD (also preferred to -pthread) + + ax_pthread_flags="-kthread lthread $ax_pthread_flags" + ;; + + hpux*) + + # From the cc(1) man page: "[-mt] Sets various -D flags to enable + # multi-threading and also sets -lpthread." + + ax_pthread_flags="-mt -pthread pthread $ax_pthread_flags" + ;; + + openedition*) + + # IBM z/OS requires a feature-test macro to be defined in order to + # enable POSIX threads at all, so give the user a hint if this is + # not set. (We don't define these ourselves, as they can affect + # other portions of the system API in unpredictable ways.) + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +# if !defined(_OPEN_THREADS) && !defined(_UNIX03_THREADS) + AX_PTHREAD_ZOS_MISSING +# endif + +_ACEOF +if (eval "$ac_cpp conftest.$ac_ext") 2>&5 | + $EGREP "AX_PTHREAD_ZOS_MISSING" >/dev/null 2>&1 +then : + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: WARNING: IBM z/OS requires -D_OPEN_THREADS or -D_UNIX03_THREADS to enable pthreads support." >&5 +printf "%s\n" "$as_me: WARNING: IBM z/OS requires -D_OPEN_THREADS or -D_UNIX03_THREADS to enable pthreads support." >&2;} fi +rm -rf conftest* + + ;; + solaris*) + # On Solaris (at least, for some versions), libc contains stubbed + # (non-functional) versions of the pthreads routines, so link-based + # tests will erroneously succeed. (N.B.: The stubs are missing + # pthread_cleanup_push, or rather a function called by this macro, + # so we could check for that, but who knows whether they'll stub + # that too in a future libc.) So we'll check first for the + # standard Solaris way of linking pthreads (-mt -lpthread). -ac_fn_check_decl "$LINENO" "strnlen" "ac_cv_have_decl_strnlen" "$cares_all_includes -" "$ac_c_undeclared_builtin_options" "CFLAGS" -if test "x$ac_cv_have_decl_strnlen" = xyes -then : + ax_pthread_flags="-mt,-lpthread pthread $ax_pthread_flags" + ;; +esac -printf "%s\n" "#define HAVE_STRNLEN 1" >>confdefs.h +# Are we compiling with Clang? -fi -ac_fn_check_decl "$LINENO" "memmem" "ac_cv_have_decl_memmem" "$cares_all_includes -" "$ac_c_undeclared_builtin_options" "CFLAGS" -if test "x$ac_cv_have_decl_memmem" = xyes +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether $CC is Clang" >&5 +printf %s "checking whether $CC is Clang... " >&6; } +if test ${ax_cv_PTHREAD_CLANG+y} then : + printf %s "(cached) " >&6 +else $as_nop + ax_cv_PTHREAD_CLANG=no + # Note that Autoconf sets GCC=yes for Clang as well as GCC + if test "x$GCC" = "xyes"; then + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ +/* Note: Clang 2.7 lacks __clang_[a-z]+__ */ +# if defined(__clang__) && defined(__llvm__) + AX_PTHREAD_CC_IS_CLANG +# endif -printf "%s\n" "#define HAVE_MEMMEM 1" >>confdefs.h - -fi -ac_fn_check_decl "$LINENO" "recv" "ac_cv_have_decl_recv" "$cares_all_includes -" "$ac_c_undeclared_builtin_options" "CFLAGS" -if test "x$ac_cv_have_decl_recv" = xyes +_ACEOF +if (eval "$ac_cpp conftest.$ac_ext") 2>&5 | + $EGREP "AX_PTHREAD_CC_IS_CLANG" >/dev/null 2>&1 then : - -printf "%s\n" "#define HAVE_RECV 1" >>confdefs.h - + ax_cv_PTHREAD_CLANG=yes fi -ac_fn_check_decl "$LINENO" "recvfrom" "ac_cv_have_decl_recvfrom" "$cares_all_includes -" "$ac_c_undeclared_builtin_options" "CFLAGS" -if test "x$ac_cv_have_decl_recvfrom" = xyes -then : +rm -rf conftest* -printf "%s\n" "#define HAVE_RECVFROM 1" >>confdefs.h + fi fi -ac_fn_check_decl "$LINENO" "send" "ac_cv_have_decl_send" "$cares_all_includes -" "$ac_c_undeclared_builtin_options" "CFLAGS" -if test "x$ac_cv_have_decl_send" = xyes -then : +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ax_cv_PTHREAD_CLANG" >&5 +printf "%s\n" "$ax_cv_PTHREAD_CLANG" >&6; } +ax_pthread_clang="$ax_cv_PTHREAD_CLANG" -printf "%s\n" "#define HAVE_SEND 1" >>confdefs.h -fi -ac_fn_check_decl "$LINENO" "sendto" "ac_cv_have_decl_sendto" "$cares_all_includes -" "$ac_c_undeclared_builtin_options" "CFLAGS" -if test "x$ac_cv_have_decl_sendto" = xyes -then : +# GCC generally uses -pthread, or -pthreads on some platforms (e.g. SPARC) -printf "%s\n" "#define HAVE_SENDTO 1" >>confdefs.h +# Note that for GCC and Clang -pthread generally implies -lpthread, +# except when -nostdlib is passed. +# This is problematic using libtool to build C++ shared libraries with pthread: +# [1] https://gcc.gnu.org/bugzilla/show_bug.cgi?id=25460 +# [2] https://bugzilla.redhat.com/show_bug.cgi?id=661333 +# [3] https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=468555 +# To solve this, first try -pthread together with -lpthread for GCC -fi -ac_fn_check_decl "$LINENO" "getnameinfo" "ac_cv_have_decl_getnameinfo" "$cares_all_includes -" "$ac_c_undeclared_builtin_options" "CFLAGS" -if test "x$ac_cv_have_decl_getnameinfo" = xyes +if test "x$GCC" = "xyes" then : - -printf "%s\n" "#define HAVE_GETNAMEINFO 1" >>confdefs.h - + ax_pthread_flags="-pthread,-lpthread -pthread -pthreads $ax_pthread_flags" fi -ac_fn_check_decl "$LINENO" "gethostname" "ac_cv_have_decl_gethostname" "$cares_all_includes -" "$ac_c_undeclared_builtin_options" "CFLAGS" -if test "x$ac_cv_have_decl_gethostname" = xyes -then : -printf "%s\n" "#define HAVE_GETHOSTNAME 1" >>confdefs.h +# Clang takes -pthread (never supported any other flag), but we'll try with -lpthread first -fi -ac_fn_check_decl "$LINENO" "connect" "ac_cv_have_decl_connect" "$cares_all_includes -" "$ac_c_undeclared_builtin_options" "CFLAGS" -if test "x$ac_cv_have_decl_connect" = xyes +if test "x$ax_pthread_clang" = "xyes" then : + ax_pthread_flags="-pthread,-lpthread -pthread" +fi -printf "%s\n" "#define HAVE_CONNECT 1" >>confdefs.h -fi -ac_fn_check_decl "$LINENO" "connectx" "ac_cv_have_decl_connectx" "$cares_all_includes -" "$ac_c_undeclared_builtin_options" "CFLAGS" -if test "x$ac_cv_have_decl_connectx" = xyes -then : +# The presence of a feature test macro requesting re-entrant function +# definitions is, on some systems, a strong hint that pthreads support is +# correctly enabled -printf "%s\n" "#define HAVE_CONNECTX 1" >>confdefs.h +case $host_os in + darwin* | hpux* | linux* | osf* | solaris*) + ax_pthread_check_macro="_REENTRANT" + ;; -fi -ac_fn_check_decl "$LINENO" "closesocket" "ac_cv_have_decl_closesocket" "$cares_all_includes -" "$ac_c_undeclared_builtin_options" "CFLAGS" -if test "x$ac_cv_have_decl_closesocket" = xyes + aix*) + ax_pthread_check_macro="_THREAD_SAFE" + ;; + + *) + ax_pthread_check_macro="--" + ;; +esac +if test "x$ax_pthread_check_macro" = "x--" then : + ax_pthread_check_cond=0 +else $as_nop + ax_pthread_check_cond="!defined($ax_pthread_check_macro)" +fi -printf "%s\n" "#define HAVE_CLOSESOCKET 1" >>confdefs.h -fi -ac_fn_check_decl "$LINENO" "CloseSocket" "ac_cv_have_decl_CloseSocket" "$cares_all_includes -" "$ac_c_undeclared_builtin_options" "CFLAGS" -if test "x$ac_cv_have_decl_CloseSocket" = xyes -then : +if test "x$ax_pthread_ok" = "xno"; then +for ax_pthread_try_flag in $ax_pthread_flags; do -printf "%s\n" "#define HAVE_CLOSESOCKET_CAMEL 1" >>confdefs.h + case $ax_pthread_try_flag in + none) + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether pthreads work without any flags" >&5 +printf %s "checking whether pthreads work without any flags... " >&6; } + ;; -fi -ac_fn_check_decl "$LINENO" "fcntl" "ac_cv_have_decl_fcntl" "$cares_all_includes -" "$ac_c_undeclared_builtin_options" "CFLAGS" -if test "x$ac_cv_have_decl_fcntl" = xyes -then : + *,*) + PTHREAD_CFLAGS=`echo $ax_pthread_try_flag | sed "s/^\(.*\),\(.*\)$/\1/"` + PTHREAD_LIBS=`echo $ax_pthread_try_flag | sed "s/^\(.*\),\(.*\)$/\2/"` + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether pthreads work with \"$PTHREAD_CFLAGS\" and \"$PTHREAD_LIBS\"" >&5 +printf %s "checking whether pthreads work with \"$PTHREAD_CFLAGS\" and \"$PTHREAD_LIBS\"... " >&6; } + ;; -printf "%s\n" "#define HAVE_FCNTL 1" >>confdefs.h + -*) + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether pthreads work with $ax_pthread_try_flag" >&5 +printf %s "checking whether pthreads work with $ax_pthread_try_flag... " >&6; } + PTHREAD_CFLAGS="$ax_pthread_try_flag" + ;; -fi -ac_fn_check_decl "$LINENO" "getenv" "ac_cv_have_decl_getenv" "$cares_all_includes -" "$ac_c_undeclared_builtin_options" "CFLAGS" -if test "x$ac_cv_have_decl_getenv" = xyes + pthread-config) + # Extract the first word of "pthread-config", so it can be a program name with args. +set dummy pthread-config; ac_word=$2 +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 +printf %s "checking for $ac_word... " >&6; } +if test ${ac_cv_prog_ax_pthread_config+y} then : + printf %s "(cached) " >&6 +else $as_nop + if test -n "$ax_pthread_config"; then + ac_cv_prog_ax_pthread_config="$ax_pthread_config" # Let the user override the test. +else +as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +for as_dir in $PATH +do + IFS=$as_save_IFS + case $as_dir in #((( + '') as_dir=./ ;; + */) ;; + *) as_dir=$as_dir/ ;; + esac + for ac_exec_ext in '' $ac_executable_extensions; do + if as_fn_executable_p "$as_dir$ac_word$ac_exec_ext"; then + ac_cv_prog_ax_pthread_config="yes" + printf "%s\n" "$as_me:${as_lineno-$LINENO}: found $as_dir$ac_word$ac_exec_ext" >&5 + break 2 + fi +done + done +IFS=$as_save_IFS -printf "%s\n" "#define HAVE_GETENV 1" >>confdefs.h - + test -z "$ac_cv_prog_ax_pthread_config" && ac_cv_prog_ax_pthread_config="no" +fi +fi +ax_pthread_config=$ac_cv_prog_ax_pthread_config +if test -n "$ax_pthread_config"; then + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ax_pthread_config" >&5 +printf "%s\n" "$ax_pthread_config" >&6; } +else + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 +printf "%s\n" "no" >&6; } fi -ac_fn_check_decl "$LINENO" "gethostname" "ac_cv_have_decl_gethostname" "$cares_all_includes -" "$ac_c_undeclared_builtin_options" "CFLAGS" -if test "x$ac_cv_have_decl_gethostname" = xyes -then : -printf "%s\n" "#define HAVE_GETHOSTNAME 1" >>confdefs.h -fi -ac_fn_check_decl "$LINENO" "getrandom" "ac_cv_have_decl_getrandom" "$cares_all_includes -" "$ac_c_undeclared_builtin_options" "CFLAGS" -if test "x$ac_cv_have_decl_getrandom" = xyes + if test "x$ax_pthread_config" = "xno" then : + continue +fi + PTHREAD_CFLAGS="`pthread-config --cflags`" + PTHREAD_LIBS="`pthread-config --ldflags` `pthread-config --libs`" + ;; -printf "%s\n" "#define HAVE_GETRANDOM 1" >>confdefs.h + *) + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for the pthreads library -l$ax_pthread_try_flag" >&5 +printf %s "checking for the pthreads library -l$ax_pthread_try_flag... " >&6; } + PTHREAD_LIBS="-l$ax_pthread_try_flag" + ;; + esac -fi -ac_fn_check_decl "$LINENO" "getservbyport_r" "ac_cv_have_decl_getservbyport_r" "$cares_all_includes -" "$ac_c_undeclared_builtin_options" "CFLAGS" -if test "x$ac_cv_have_decl_getservbyport_r" = xyes -then : + ax_pthread_save_CFLAGS="$CFLAGS" + ax_pthread_save_LIBS="$LIBS" + CFLAGS="$CFLAGS $PTHREAD_CFLAGS" + LIBS="$PTHREAD_LIBS $LIBS" -printf "%s\n" "#define HAVE_GETSERVBYPORT_R 1" >>confdefs.h + # Check for various functions. We must include pthread.h, + # since some functions may be macros. (On the Sequent, we + # need a special flag -Kthread to make this header compile.) + # We check for pthread_join because it is in -lpthread on IRIX + # while pthread_create is in libc. We check for pthread_attr_init + # due to DEC craziness with -lpthreads. We check for + # pthread_cleanup_push because it is one of the few pthread + # functions on Solaris that doesn't have a non-functional libc stub. + # We try pthread_create on general principles. -fi -ac_fn_check_decl "$LINENO" "inet_net_pton" "ac_cv_have_decl_inet_net_pton" "$cares_all_includes -" "$ac_c_undeclared_builtin_options" "CFLAGS" -if test "x$ac_cv_have_decl_inet_net_pton" = xyes + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ +#include +# if $ax_pthread_check_cond +# error "$ax_pthread_check_macro must be defined" +# endif + static void *some_global = NULL; + static void routine(void *a) + { + /* To avoid any unused-parameter or + unused-but-set-parameter warning. */ + some_global = a; + } + static void *start_routine(void *a) { return a; } +int +main (void) +{ +pthread_t th; pthread_attr_t attr; + pthread_create(&th, 0, start_routine, 0); + pthread_join(th, 0); + pthread_attr_init(&attr); + pthread_cleanup_push(routine, 0); + pthread_cleanup_pop(0) /* ; */ + ; + return 0; +} +_ACEOF +if ac_fn_c_try_link "$LINENO" then : - -printf "%s\n" "#define HAVE_INET_NET_PTON 1" >>confdefs.h - + ax_pthread_ok=yes fi -ac_fn_check_decl "$LINENO" "inet_ntop" "ac_cv_have_decl_inet_ntop" "$cares_all_includes -" "$ac_c_undeclared_builtin_options" "CFLAGS" -if test "x$ac_cv_have_decl_inet_ntop" = xyes -then : +rm -f core conftest.err conftest.$ac_objext conftest.beam \ + conftest$ac_exeext conftest.$ac_ext -printf "%s\n" "#define HAVE_INET_NTOP 1" >>confdefs.h + CFLAGS="$ax_pthread_save_CFLAGS" + LIBS="$ax_pthread_save_LIBS" -fi -ac_fn_check_decl "$LINENO" "inet_pton" "ac_cv_have_decl_inet_pton" "$cares_all_includes -" "$ac_c_undeclared_builtin_options" "CFLAGS" -if test "x$ac_cv_have_decl_inet_pton" = xyes + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ax_pthread_ok" >&5 +printf "%s\n" "$ax_pthread_ok" >&6; } + if test "x$ax_pthread_ok" = "xyes" then : - -printf "%s\n" "#define HAVE_INET_PTON 1" >>confdefs.h - + break fi -ac_fn_check_decl "$LINENO" "ioctl" "ac_cv_have_decl_ioctl" "$cares_all_includes -" "$ac_c_undeclared_builtin_options" "CFLAGS" -if test "x$ac_cv_have_decl_ioctl" = xyes -then : - -printf "%s\n" "#define HAVE_IOCTL 1" >>confdefs.h + PTHREAD_LIBS="" + PTHREAD_CFLAGS="" +done fi -ac_fn_check_decl "$LINENO" "ioctlsocket" "ac_cv_have_decl_ioctlsocket" "$cares_all_includes -" "$ac_c_undeclared_builtin_options" "CFLAGS" -if test "x$ac_cv_have_decl_ioctlsocket" = xyes -then : -printf "%s\n" "#define HAVE_IOCTLSOCKET 1" >>confdefs.h -fi -ac_fn_check_decl "$LINENO" "IoctlSocket" "ac_cv_have_decl_IoctlSocket" "$cares_all_includes -" "$ac_c_undeclared_builtin_options" "CFLAGS" -if test "x$ac_cv_have_decl_IoctlSocket" = xyes -then : +# Clang needs special handling, because older versions handle the -pthread +# option in a rather... idiosyncratic way -printf "%s\n" "#define HAVE_IOCTLSOCKET_CAMEL 1" >>confdefs.h +if test "x$ax_pthread_clang" = "xyes"; then -fi -ac_fn_check_decl "$LINENO" "setsockopt" "ac_cv_have_decl_setsockopt" "$cares_all_includes -" "$ac_c_undeclared_builtin_options" "CFLAGS" -if test "x$ac_cv_have_decl_setsockopt" = xyes -then : + # Clang takes -pthread; it has never supported any other flag -printf "%s\n" "#define HAVE_SETSOCKOPT 1" >>confdefs.h + # (Note 1: This will need to be revisited if a system that Clang + # supports has POSIX threads in a separate library. This tends not + # to be the way of modern systems, but it's conceivable.) -fi -ac_fn_check_decl "$LINENO" "socket" "ac_cv_have_decl_socket" "$cares_all_includes -" "$ac_c_undeclared_builtin_options" "CFLAGS" -if test "x$ac_cv_have_decl_socket" = xyes -then : + # (Note 2: On some systems, notably Darwin, -pthread is not needed + # to get POSIX threads support; the API is always present and + # active. We could reasonably leave PTHREAD_CFLAGS empty. But + # -pthread does define _REENTRANT, and while the Darwin headers + # ignore this macro, third-party headers might not.) -printf "%s\n" "#define HAVE_SOCKET 1" >>confdefs.h + # However, older versions of Clang make a point of warning the user + # that, in an invocation where only linking and no compilation is + # taking place, the -pthread option has no effect ("argument unused + # during compilation"). They expect -pthread to be passed in only + # when source code is being compiled. + # + # Problem is, this is at odds with the way Automake and most other + # C build frameworks function, which is that the same flags used in + # compilation (CFLAGS) are also used in linking. Many systems + # supported by AX_PTHREAD require exactly this for POSIX threads + # support, and in fact it is often not straightforward to specify a + # flag that is used only in the compilation phase and not in + # linking. Such a scenario is extremely rare in practice. + # + # Even though use of the -pthread flag in linking would only print + # a warning, this can be a nuisance for well-run software projects + # that build with -Werror. So if the active version of Clang has + # this misfeature, we search for an option to squash it. -fi -ac_fn_check_decl "$LINENO" "strcasecmp" "ac_cv_have_decl_strcasecmp" "$cares_all_includes -" "$ac_c_undeclared_builtin_options" "CFLAGS" -if test "x$ac_cv_have_decl_strcasecmp" = xyes + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether Clang needs flag to prevent \"argument unused\" warning when linking with -pthread" >&5 +printf %s "checking whether Clang needs flag to prevent \"argument unused\" warning when linking with -pthread... " >&6; } +if test ${ax_cv_PTHREAD_CLANG_NO_WARN_FLAG+y} then : - -printf "%s\n" "#define HAVE_STRCASECMP 1" >>confdefs.h - -fi -ac_fn_check_decl "$LINENO" "strdup" "ac_cv_have_decl_strdup" "$cares_all_includes -" "$ac_c_undeclared_builtin_options" "CFLAGS" -if test "x$ac_cv_have_decl_strdup" = xyes + printf %s "(cached) " >&6 +else $as_nop + ax_cv_PTHREAD_CLANG_NO_WARN_FLAG=unknown + # Create an alternate version of $ac_link that compiles and + # links in two steps (.c -> .o, .o -> exe) instead of one + # (.c -> exe), because the warning occurs only in the second + # step + ax_pthread_save_ac_link="$ac_link" + ax_pthread_sed='s/conftest\.\$ac_ext/conftest.$ac_objext/g' + ax_pthread_link_step=`printf "%s\n" "$ac_link" | sed "$ax_pthread_sed"` + ax_pthread_2step_ac_link="($ac_compile) && (echo ==== >&5) && ($ax_pthread_link_step)" + ax_pthread_save_CFLAGS="$CFLAGS" + for ax_pthread_try in '' -Qunused-arguments -Wno-unused-command-line-argument unknown; do + if test "x$ax_pthread_try" = "xunknown" then : - -printf "%s\n" "#define HAVE_STRDUP 1" >>confdefs.h - + break fi -ac_fn_check_decl "$LINENO" "stricmp" "ac_cv_have_decl_stricmp" "$cares_all_includes -" "$ac_c_undeclared_builtin_options" "CFLAGS" -if test "x$ac_cv_have_decl_stricmp" = xyes + CFLAGS="-Werror -Wunknown-warning-option $ax_pthread_try -pthread $ax_pthread_save_CFLAGS" + ac_link="$ax_pthread_save_ac_link" + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ +int main(void){return 0;} +_ACEOF +if ac_fn_c_try_link "$LINENO" then : - -printf "%s\n" "#define HAVE_STRICMP 1" >>confdefs.h - -fi -ac_fn_check_decl "$LINENO" "strncasecmp" "ac_cv_have_decl_strncasecmp" "$cares_all_includes -" "$ac_c_undeclared_builtin_options" "CFLAGS" -if test "x$ac_cv_have_decl_strncasecmp" = xyes + ac_link="$ax_pthread_2step_ac_link" + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ +int main(void){return 0;} +_ACEOF +if ac_fn_c_try_link "$LINENO" then : - -printf "%s\n" "#define HAVE_STRNCASECMP 1" >>confdefs.h - + break fi -ac_fn_check_decl "$LINENO" "strncmpi" "ac_cv_have_decl_strncmpi" "$cares_all_includes -" "$ac_c_undeclared_builtin_options" "CFLAGS" -if test "x$ac_cv_have_decl_strncmpi" = xyes -then : - -printf "%s\n" "#define HAVE_STRNCMPI 1" >>confdefs.h +rm -f core conftest.err conftest.$ac_objext conftest.beam \ + conftest$ac_exeext conftest.$ac_ext fi -ac_fn_check_decl "$LINENO" "strnicmp" "ac_cv_have_decl_strnicmp" "$cares_all_includes -" "$ac_c_undeclared_builtin_options" "CFLAGS" -if test "x$ac_cv_have_decl_strnicmp" = xyes +rm -f core conftest.err conftest.$ac_objext conftest.beam \ + conftest$ac_exeext conftest.$ac_ext + done + ac_link="$ax_pthread_save_ac_link" + CFLAGS="$ax_pthread_save_CFLAGS" + if test "x$ax_pthread_try" = "x" then : - -printf "%s\n" "#define HAVE_STRNICMP 1" >>confdefs.h - + ax_pthread_try=no fi -ac_fn_check_decl "$LINENO" "writev" "ac_cv_have_decl_writev" "$cares_all_includes -" "$ac_c_undeclared_builtin_options" "CFLAGS" -if test "x$ac_cv_have_decl_writev" = xyes -then : - -printf "%s\n" "#define HAVE_WRITEV 1" >>confdefs.h + ax_cv_PTHREAD_CLANG_NO_WARN_FLAG="$ax_pthread_try" fi -ac_fn_check_decl "$LINENO" "arc4random_buf" "ac_cv_have_decl_arc4random_buf" "$cares_all_includes -" "$ac_c_undeclared_builtin_options" "CFLAGS" -if test "x$ac_cv_have_decl_arc4random_buf" = xyes -then : +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ax_cv_PTHREAD_CLANG_NO_WARN_FLAG" >&5 +printf "%s\n" "$ax_cv_PTHREAD_CLANG_NO_WARN_FLAG" >&6; } -printf "%s\n" "#define HAVE_ARC4RANDOM_BUF 1" >>confdefs.h + case "$ax_cv_PTHREAD_CLANG_NO_WARN_FLAG" in + no | unknown) ;; + *) PTHREAD_CFLAGS="$ax_cv_PTHREAD_CLANG_NO_WARN_FLAG $PTHREAD_CFLAGS" ;; + esac -fi -ac_fn_check_decl "$LINENO" "stat" "ac_cv_have_decl_stat" "$cares_all_includes -" "$ac_c_undeclared_builtin_options" "CFLAGS" -if test "x$ac_cv_have_decl_stat" = xyes -then : +fi # $ax_pthread_clang = yes -printf "%s\n" "#define HAVE_STAT 1" >>confdefs.h -fi -ac_fn_check_decl "$LINENO" "gettimeofday" "ac_cv_have_decl_gettimeofday" "$cares_all_includes -" "$ac_c_undeclared_builtin_options" "CFLAGS" -if test "x$ac_cv_have_decl_gettimeofday" = xyes -then : -printf "%s\n" "#define HAVE_GETTIMEOFDAY 1" >>confdefs.h +# Various other checks: +if test "x$ax_pthread_ok" = "xyes"; then + ax_pthread_save_CFLAGS="$CFLAGS" + ax_pthread_save_LIBS="$LIBS" + CFLAGS="$CFLAGS $PTHREAD_CFLAGS" + LIBS="$PTHREAD_LIBS $LIBS" -fi -ac_fn_check_decl "$LINENO" "clock_gettime" "ac_cv_have_decl_clock_gettime" "$cares_all_includes -" "$ac_c_undeclared_builtin_options" "CFLAGS" -if test "x$ac_cv_have_decl_clock_gettime" = xyes + # Detect AIX lossage: JOINABLE attribute is called UNDETACHED. + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for joinable pthread attribute" >&5 +printf %s "checking for joinable pthread attribute... " >&6; } +if test ${ax_cv_PTHREAD_JOINABLE_ATTR+y} then : - -printf "%s\n" "#define HAVE_CLOCK_GETTIME 1" >>confdefs.h - -fi -ac_fn_check_decl "$LINENO" "if_indextoname" "ac_cv_have_decl_if_indextoname" "$cares_all_includes -" "$ac_c_undeclared_builtin_options" "CFLAGS" -if test "x$ac_cv_have_decl_if_indextoname" = xyes + printf %s "(cached) " >&6 +else $as_nop + ax_cv_PTHREAD_JOINABLE_ATTR=unknown + for ax_pthread_attr in PTHREAD_CREATE_JOINABLE PTHREAD_CREATE_UNDETACHED; do + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ +#include +int +main (void) +{ +int attr = $ax_pthread_attr; return attr /* ; */ + ; + return 0; +} +_ACEOF +if ac_fn_c_try_link "$LINENO" then : - -printf "%s\n" "#define HAVE_IF_INDEXTONAME 1" >>confdefs.h - + ax_cv_PTHREAD_JOINABLE_ATTR=$ax_pthread_attr; break fi -ac_fn_check_decl "$LINENO" "if_nametoindex" "ac_cv_have_decl_if_nametoindex" "$cares_all_includes -" "$ac_c_undeclared_builtin_options" "CFLAGS" -if test "x$ac_cv_have_decl_if_nametoindex" = xyes -then : - -printf "%s\n" "#define HAVE_IF_NAMETOINDEX 1" >>confdefs.h +rm -f core conftest.err conftest.$ac_objext conftest.beam \ + conftest$ac_exeext conftest.$ac_ext + done fi -ac_fn_check_decl "$LINENO" "getifaddrs" "ac_cv_have_decl_getifaddrs" "$cares_all_includes -" "$ac_c_undeclared_builtin_options" "CFLAGS" -if test "x$ac_cv_have_decl_getifaddrs" = xyes +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ax_cv_PTHREAD_JOINABLE_ATTR" >&5 +printf "%s\n" "$ax_cv_PTHREAD_JOINABLE_ATTR" >&6; } + if test "x$ax_cv_PTHREAD_JOINABLE_ATTR" != "xunknown" && \ + test "x$ax_cv_PTHREAD_JOINABLE_ATTR" != "xPTHREAD_CREATE_JOINABLE" && \ + test "x$ax_pthread_joinable_attr_defined" != "xyes" then : -printf "%s\n" "#define HAVE_GETIFADDRS 1" >>confdefs.h - -fi -ac_fn_check_decl "$LINENO" "poll" "ac_cv_have_decl_poll" "$cares_all_includes -" "$ac_c_undeclared_builtin_options" "CFLAGS" -if test "x$ac_cv_have_decl_poll" = xyes -then : +printf "%s\n" "#define PTHREAD_CREATE_JOINABLE $ax_cv_PTHREAD_JOINABLE_ATTR" >>confdefs.h -printf "%s\n" "#define HAVE_POLL 1" >>confdefs.h + ax_pthread_joinable_attr_defined=yes fi -ac_fn_check_decl "$LINENO" "pipe" "ac_cv_have_decl_pipe" "$cares_all_includes -" "$ac_c_undeclared_builtin_options" "CFLAGS" -if test "x$ac_cv_have_decl_pipe" = xyes -then : -printf "%s\n" "#define HAVE_PIPE 1" >>confdefs.h - -fi -ac_fn_check_decl "$LINENO" "pipe2" "ac_cv_have_decl_pipe2" "$cares_all_includes -" "$ac_c_undeclared_builtin_options" "CFLAGS" -if test "x$ac_cv_have_decl_pipe2" = xyes + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether more special flags are required for pthreads" >&5 +printf %s "checking whether more special flags are required for pthreads... " >&6; } +if test ${ax_cv_PTHREAD_SPECIAL_FLAGS+y} then : - -printf "%s\n" "#define HAVE_PIPE2 1" >>confdefs.h + printf %s "(cached) " >&6 +else $as_nop + ax_cv_PTHREAD_SPECIAL_FLAGS=no + case $host_os in + solaris*) + ax_cv_PTHREAD_SPECIAL_FLAGS="-D_POSIX_PTHREAD_SEMANTICS" + ;; + esac fi -ac_fn_check_decl "$LINENO" "kqueue" "ac_cv_have_decl_kqueue" "$cares_all_includes -" "$ac_c_undeclared_builtin_options" "CFLAGS" -if test "x$ac_cv_have_decl_kqueue" = xyes +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ax_cv_PTHREAD_SPECIAL_FLAGS" >&5 +printf "%s\n" "$ax_cv_PTHREAD_SPECIAL_FLAGS" >&6; } + if test "x$ax_cv_PTHREAD_SPECIAL_FLAGS" != "xno" && \ + test "x$ax_pthread_special_flags_added" != "xyes" then : - -printf "%s\n" "#define HAVE_KQUEUE 1" >>confdefs.h - + PTHREAD_CFLAGS="$ax_cv_PTHREAD_SPECIAL_FLAGS $PTHREAD_CFLAGS" + ax_pthread_special_flags_added=yes fi -ac_fn_check_decl "$LINENO" "epoll_create1" "ac_cv_have_decl_epoll_create1" "$cares_all_includes -" "$ac_c_undeclared_builtin_options" "CFLAGS" -if test "x$ac_cv_have_decl_epoll_create1" = xyes -then : - -printf "%s\n" "#define HAVE_EPOLL 1" >>confdefs.h -fi -ac_fn_check_decl "$LINENO" "ConvertInterfaceIndexToLuid" "ac_cv_have_decl_ConvertInterfaceIndexToLuid" "$cares_all_includes -" "$ac_c_undeclared_builtin_options" "CFLAGS" -if test "x$ac_cv_have_decl_ConvertInterfaceIndexToLuid" = xyes + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for PTHREAD_PRIO_INHERIT" >&5 +printf %s "checking for PTHREAD_PRIO_INHERIT... " >&6; } +if test ${ax_cv_PTHREAD_PRIO_INHERIT+y} then : - -printf "%s\n" "#define HAVE_CONVERTINTERFACEINDEXTOLUID 1" >>confdefs.h - -fi -ac_fn_check_decl "$LINENO" "ConvertInterfaceLuidToNameA" "ac_cv_have_decl_ConvertInterfaceLuidToNameA" "$cares_all_includes -" "$ac_c_undeclared_builtin_options" "CFLAGS" -if test "x$ac_cv_have_decl_ConvertInterfaceLuidToNameA" = xyes + printf %s "(cached) " >&6 +else $as_nop + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ +#include +int +main (void) +{ +int i = PTHREAD_PRIO_INHERIT; + return i; + ; + return 0; +} +_ACEOF +if ac_fn_c_try_link "$LINENO" then : - -printf "%s\n" "#define HAVE_CONVERTINTERFACELUIDTONAMEA 1" >>confdefs.h - + ax_cv_PTHREAD_PRIO_INHERIT=yes +else $as_nop + ax_cv_PTHREAD_PRIO_INHERIT=no fi -ac_fn_check_decl "$LINENO" "NotifyIpInterfaceChange" "ac_cv_have_decl_NotifyIpInterfaceChange" "$cares_all_includes -" "$ac_c_undeclared_builtin_options" "CFLAGS" -if test "x$ac_cv_have_decl_NotifyIpInterfaceChange" = xyes -then : - -printf "%s\n" "#define HAVE_NOTIFYIPINTERFACECHANGE 1" >>confdefs.h +rm -f core conftest.err conftest.$ac_objext conftest.beam \ + conftest$ac_exeext conftest.$ac_ext fi -ac_fn_check_decl "$LINENO" "RegisterWaitForSingleObject" "ac_cv_have_decl_RegisterWaitForSingleObject" "$cares_all_includes -" "$ac_c_undeclared_builtin_options" "CFLAGS" -if test "x$ac_cv_have_decl_RegisterWaitForSingleObject" = xyes +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ax_cv_PTHREAD_PRIO_INHERIT" >&5 +printf "%s\n" "$ax_cv_PTHREAD_PRIO_INHERIT" >&6; } + if test "x$ax_cv_PTHREAD_PRIO_INHERIT" = "xyes" && \ + test "x$ax_pthread_prio_inherit_defined" != "xyes" then : -printf "%s\n" "#define HAVE_REGISTERWAITFORSINGLEOBJECT 1" >>confdefs.h - -fi -ac_fn_check_decl "$LINENO" "__system_property_get" "ac_cv_have_decl___system_property_get" "$cares_all_includes -" "$ac_c_undeclared_builtin_options" "CFLAGS" -if test "x$ac_cv_have_decl___system_property_get" = xyes -then : +printf "%s\n" "#define HAVE_PTHREAD_PRIO_INHERIT 1" >>confdefs.h -printf "%s\n" "#define HAVE___SYSTEM_PROPERTY_GET 1" >>confdefs.h + ax_pthread_prio_inherit_defined=yes fi + CFLAGS="$ax_pthread_save_CFLAGS" + LIBS="$ax_pthread_save_LIBS" + # More AIX lossage: compile with *_r variant + if test "x$GCC" != "xyes"; then + case $host_os in + aix*) + case "x/$CC" in #( + x*/c89|x*/c89_128|x*/c99|x*/c99_128|x*/cc|x*/cc128|x*/xlc|x*/xlc_v6|x*/xlc128|x*/xlc128_v6) : + #handle absolute path differently from PATH based program lookup + case "x$CC" in #( + x/*) : -if test "x$ac_cv_type_ssize_t" = "xyes" -a "x$ac_cv_type_socklen_t" = "xyes" -a "x$ac_cv_native_windows" != "xyes" ; then - recvfrom_type_retv="ssize_t" - recvfrom_type_arg3="size_t" -else - recvfrom_type_retv="int" - recvfrom_type_arg3="int" + if as_fn_executable_p ${CC}_r +then : + PTHREAD_CC="${CC}_r" +fi + if test "x${CXX}" != "x" +then : + if as_fn_executable_p ${CXX}_r +then : + PTHREAD_CXX="${CXX}_r" fi - -if test "x$ac_cv_type_SOCKET" = "xyes" ; then - recvfrom_type_arg1="SOCKET" -else - recvfrom_type_arg1="int" fi + ;; #( + *) : -if test "x$ac_cv_type_socklen_t" = "xyes" ; then - recvfrom_type_arg6="socklen_t *" - getnameinfo_type_arg2="socklen_t" - getnameinfo_type_arg46="socklen_t" + for ac_prog in ${CC}_r +do + # Extract the first word of "$ac_prog", so it can be a program name with args. +set dummy $ac_prog; ac_word=$2 +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 +printf %s "checking for $ac_word... " >&6; } +if test ${ac_cv_prog_PTHREAD_CC+y} +then : + printf %s "(cached) " >&6 +else $as_nop + if test -n "$PTHREAD_CC"; then + ac_cv_prog_PTHREAD_CC="$PTHREAD_CC" # Let the user override the test. else - recvfrom_type_arg6="int *" - getnameinfo_type_arg2="int" - getnameinfo_type_arg46="int" -fi +as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +for as_dir in $PATH +do + IFS=$as_save_IFS + case $as_dir in #((( + '') as_dir=./ ;; + */) ;; + *) as_dir=$as_dir/ ;; + esac + for ac_exec_ext in '' $ac_executable_extensions; do + if as_fn_executable_p "$as_dir$ac_word$ac_exec_ext"; then + ac_cv_prog_PTHREAD_CC="$ac_prog" + printf "%s\n" "$as_me:${as_lineno-$LINENO}: found $as_dir$ac_word$ac_exec_ext" >&5 + break 2 + fi +done + done +IFS=$as_save_IFS -if test "x$ac_cv_native_windows" = "xyes" ; then - recv_type_arg2="char *" +fi +fi +PTHREAD_CC=$ac_cv_prog_PTHREAD_CC +if test -n "$PTHREAD_CC"; then + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $PTHREAD_CC" >&5 +printf "%s\n" "$PTHREAD_CC" >&6; } else - recv_type_arg2="void *" + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 +printf "%s\n" "no" >&6; } fi -recv_type_retv=${recvfrom_type_retv} -send_type_retv=${recvfrom_type_retv} -recv_type_arg1=${recvfrom_type_arg1} -recvfrom_type_arg2=${recv_type_arg2} -send_type_arg1=${recvfrom_type_arg1} -recv_type_arg3=${recvfrom_type_arg3} -send_type_arg3=${recvfrom_type_arg3} -gethostname_type_arg2=${recvfrom_type_arg3} - -recvfrom_qual_arg5= -recvfrom_type_arg4=int -recvfrom_type_arg5="struct sockaddr *" -recv_type_arg4=int -getnameinfo_type_arg1="struct sockaddr *" -getnameinfo_type_arg7=int -send_type_arg2="const void *" -send_type_arg4=int + test -n "$PTHREAD_CC" && break +done +test -n "$PTHREAD_CC" || PTHREAD_CC="$CC" -printf "%s\n" "#define RECVFROM_TYPE_RETV ${recvfrom_type_retv} " >>confdefs.h - - -printf "%s\n" "#define RECVFROM_TYPE_ARG1 ${recvfrom_type_arg1} " >>confdefs.h - - -printf "%s\n" "#define RECVFROM_TYPE_ARG2 ${recvfrom_type_arg2} " >>confdefs.h - + if test "x${CXX}" != "x" +then : + for ac_prog in ${CXX}_r +do + # Extract the first word of "$ac_prog", so it can be a program name with args. +set dummy $ac_prog; ac_word=$2 +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 +printf %s "checking for $ac_word... " >&6; } +if test ${ac_cv_prog_PTHREAD_CXX+y} +then : + printf %s "(cached) " >&6 +else $as_nop + if test -n "$PTHREAD_CXX"; then + ac_cv_prog_PTHREAD_CXX="$PTHREAD_CXX" # Let the user override the test. +else +as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +for as_dir in $PATH +do + IFS=$as_save_IFS + case $as_dir in #((( + '') as_dir=./ ;; + */) ;; + *) as_dir=$as_dir/ ;; + esac + for ac_exec_ext in '' $ac_executable_extensions; do + if as_fn_executable_p "$as_dir$ac_word$ac_exec_ext"; then + ac_cv_prog_PTHREAD_CXX="$ac_prog" + printf "%s\n" "$as_me:${as_lineno-$LINENO}: found $as_dir$ac_word$ac_exec_ext" >&5 + break 2 + fi +done + done +IFS=$as_save_IFS -printf "%s\n" "#define RECVFROM_TYPE_ARG3 ${recvfrom_type_arg3} " >>confdefs.h +fi +fi +PTHREAD_CXX=$ac_cv_prog_PTHREAD_CXX +if test -n "$PTHREAD_CXX"; then + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $PTHREAD_CXX" >&5 +printf "%s\n" "$PTHREAD_CXX" >&6; } +else + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 +printf "%s\n" "no" >&6; } +fi -printf "%s\n" "#define RECVFROM_TYPE_ARG4 ${recvfrom_type_arg4} " >>confdefs.h + test -n "$PTHREAD_CXX" && break +done +test -n "$PTHREAD_CXX" || PTHREAD_CXX="$CXX" +fi -printf "%s\n" "#define RECVFROM_TYPE_ARG5 ${recvfrom_type_arg5} " >>confdefs.h + ;; +esac + ;; #( + *) : + ;; +esac + ;; + esac + fi +fi +test -n "$PTHREAD_CC" || PTHREAD_CC="$CC" +test -n "$PTHREAD_CXX" || PTHREAD_CXX="$CXX" -printf "%s\n" "#define RECVFROM_QUAL_ARG5 ${recvfrom_qual_arg5}" >>confdefs.h -printf "%s\n" "#define RECV_TYPE_RETV ${recv_type_retv} " >>confdefs.h -printf "%s\n" "#define RECV_TYPE_ARG1 ${recv_type_arg1} " >>confdefs.h +# Finally, execute ACTION-IF-FOUND/ACTION-IF-NOT-FOUND: +if test "x$ax_pthread_ok" = "xyes"; then + : +else + ax_pthread_ok=no -printf "%s\n" "#define RECV_TYPE_ARG2 ${recv_type_arg2} " >>confdefs.h + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: WARNING: threads requested but not supported" >&5 +printf "%s\n" "$as_me: WARNING: threads requested but not supported" >&2;} + CARES_THREADS=no +fi +ac_ext=c +ac_cpp='$CPP $CPPFLAGS' +ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5' +ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5' +ac_compiler_gnu=$ac_cv_c_compiler_gnu -printf "%s\n" "#define RECV_TYPE_ARG3 ${recv_type_arg3} " >>confdefs.h -printf "%s\n" "#define RECV_TYPE_ARG4 ${recv_type_arg4} " >>confdefs.h + if test "${CARES_THREADS}" = "yes" ; then + ac_fn_c_check_header_compile "$LINENO" "pthread.h" "ac_cv_header_pthread_h" "$ac_includes_default" +if test "x$ac_cv_header_pthread_h" = xyes +then : + printf "%s\n" "#define HAVE_PTHREAD_H 1" >>confdefs.h +fi +ac_fn_c_check_header_compile "$LINENO" "pthread_np.h" "ac_cv_header_pthread_np_h" "$ac_includes_default" +if test "x$ac_cv_header_pthread_np_h" = xyes +then : + printf "%s\n" "#define HAVE_PTHREAD_NP_H 1" >>confdefs.h +fi -printf "%s\n" "#define SEND_TYPE_RETV ${send_type_retv} " >>confdefs.h + LIBS="$PTHREAD_LIBS $LIBS" + AM_CFLAGS="$AM_CFLAGS $PTHREAD_CFLAGS" + CC="$PTHREAD_CC" + CXX="$PTHREAD_CXX" + fi +fi +if test "${CARES_THREADS}" = "yes" ; then -printf "%s\n" "#define SEND_TYPE_ARG1 ${send_type_arg1} " >>confdefs.h +printf "%s\n" "#define CARES_THREADS 1 " >>confdefs.h +fi -printf "%s\n" "#define SEND_TYPE_ARG2 ${send_type_arg2} " >>confdefs.h +CARES_PRIVATE_LIBS="$LIBS" -printf "%s\n" "#define SEND_TYPE_ARG3 ${send_type_arg3} " >>confdefs.h +BUILD_SUBDIRS="include src docs" -printf "%s\n" "#define SEND_TYPE_ARG4 ${send_type_arg4} " >>confdefs.h +if test "x$build_tests" != "xno" -a "x$cross_compiling" = "xyes" ; then + if test "x$build_tests" = "xmaybe" ; then + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: WARNING: cannot build tests when cross compiling" >&5 +printf "%s\n" "$as_me: WARNING: cannot build tests when cross compiling" >&2;} + build_tests=no + else + as_fn_error $? "*** Tests not supported when cross compiling" "$LINENO" 5 + fi +fi +# Check whether --enable-tests-crossbuild was given. +if test ${enable_tests_crossbuild+y} +then : + enableval=$enable_tests_crossbuild; build_tests="$enableval" -printf "%s\n" "#define GETNAMEINFO_TYPE_ARG1 ${getnameinfo_type_arg1} " >>confdefs.h +fi -printf "%s\n" "#define GETNAMEINFO_TYPE_ARG2 ${getnameinfo_type_arg2} " >>confdefs.h +if test "x$build_tests" != "xno" ; then -printf "%s\n" "#define GETNAMEINFO_TYPE_ARG7 ${getnameinfo_type_arg7} " >>confdefs.h -printf "%s\n" "#define GETNAMEINFO_TYPE_ARG46 ${getnameinfo_type_arg46} " >>confdefs.h -printf "%s\n" "#define GETHOSTNAME_TYPE_ARG2 ${gethostname_type_arg2} " >>confdefs.h +if test "x$ac_cv_env_PKG_CONFIG_set" != "xset"; then + if test -n "$ac_tool_prefix"; then + # Extract the first word of "${ac_tool_prefix}pkg-config", so it can be a program name with args. +set dummy ${ac_tool_prefix}pkg-config; ac_word=$2 +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 +printf %s "checking for $ac_word... " >&6; } +if test ${ac_cv_path_PKG_CONFIG+y} +then : + printf %s "(cached) " >&6 +else $as_nop + case $PKG_CONFIG in + [\\/]* | ?:[\\/]*) + ac_cv_path_PKG_CONFIG="$PKG_CONFIG" # Let the user override the test with a path. + ;; + *) + as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +for as_dir in $PATH +do + IFS=$as_save_IFS + case $as_dir in #((( + '') as_dir=./ ;; + */) ;; + *) as_dir=$as_dir/ ;; + esac + for ac_exec_ext in '' $ac_executable_extensions; do + if as_fn_executable_p "$as_dir$ac_word$ac_exec_ext"; then + ac_cv_path_PKG_CONFIG="$as_dir$ac_word$ac_exec_ext" + printf "%s\n" "$as_me:${as_lineno-$LINENO}: found $as_dir$ac_word$ac_exec_ext" >&5 + break 2 + fi +done + done +IFS=$as_save_IFS + ;; +esac +fi +PKG_CONFIG=$ac_cv_path_PKG_CONFIG +if test -n "$PKG_CONFIG"; then + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $PKG_CONFIG" >&5 +printf "%s\n" "$PKG_CONFIG" >&6; } +else + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 +printf "%s\n" "no" >&6; } +fi -if test "$ac_cv_have_decl_getservbyport_r" = "yes" ; then - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking number of arguments for getservbyport_r()" >&5 -printf %s "checking number of arguments for getservbyport_r()... " >&6; } - getservbyport_r_args=6 - case $host_os in - solaris*) - getservbyport_r_args=5 - ;; - aix*|openbsd*) - getservbyport_r_args=4 - ;; +fi +if test -z "$ac_cv_path_PKG_CONFIG"; then + ac_pt_PKG_CONFIG=$PKG_CONFIG + # Extract the first word of "pkg-config", so it can be a program name with args. +set dummy pkg-config; ac_word=$2 +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 +printf %s "checking for $ac_word... " >&6; } +if test ${ac_cv_path_ac_pt_PKG_CONFIG+y} +then : + printf %s "(cached) " >&6 +else $as_nop + case $ac_pt_PKG_CONFIG in + [\\/]* | ?:[\\/]*) + ac_cv_path_ac_pt_PKG_CONFIG="$ac_pt_PKG_CONFIG" # Let the user override the test with a path. + ;; + *) + as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +for as_dir in $PATH +do + IFS=$as_save_IFS + case $as_dir in #((( + '') as_dir=./ ;; + */) ;; + *) as_dir=$as_dir/ ;; esac - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $getservbyport_r_args" >&5 -printf "%s\n" "$getservbyport_r_args" >&6; } + for ac_exec_ext in '' $ac_executable_extensions; do + if as_fn_executable_p "$as_dir$ac_word$ac_exec_ext"; then + ac_cv_path_ac_pt_PKG_CONFIG="$as_dir$ac_word$ac_exec_ext" + printf "%s\n" "$as_me:${as_lineno-$LINENO}: found $as_dir$ac_word$ac_exec_ext" >&5 + break 2 + fi +done + done +IFS=$as_save_IFS -printf "%s\n" "#define GETSERVBYPORT_R_ARGS $getservbyport_r_args " >>confdefs.h + ;; +esac +fi +ac_pt_PKG_CONFIG=$ac_cv_path_ac_pt_PKG_CONFIG +if test -n "$ac_pt_PKG_CONFIG"; then + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_pt_PKG_CONFIG" >&5 +printf "%s\n" "$ac_pt_PKG_CONFIG" >&6; } +else + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 +printf "%s\n" "no" >&6; } +fi + if test "x$ac_pt_PKG_CONFIG" = x; then + PKG_CONFIG="" + else + case $cross_compiling:$ac_tool_warned in +yes:) +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: WARNING: using cross tools not prefixed with host triplet" >&5 +printf "%s\n" "$as_me: WARNING: using cross tools not prefixed with host triplet" >&2;} +ac_tool_warned=yes ;; +esac + PKG_CONFIG=$ac_pt_PKG_CONFIG + fi +else + PKG_CONFIG="$ac_cv_path_PKG_CONFIG" fi -if test "$ac_cv_have_decl_getservbyname_r" = "yes" ; then - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking number of arguments for getservbyname_r()" >&5 -printf %s "checking number of arguments for getservbyname_r()... " >&6; } - getservbyname_r_args=6 - case $host_os in - solaris*) - getservbyname_r_args=5 - ;; - aix*|openbsd*) - getservbyname_r_args=4 - ;; - esac +fi +if test -n "$PKG_CONFIG"; then + _pkg_min_version=0.9.0 + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking pkg-config is at least version $_pkg_min_version" >&5 +printf %s "checking pkg-config is at least version $_pkg_min_version... " >&6; } + if $PKG_CONFIG --atleast-pkgconfig-version $_pkg_min_version; then + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +printf "%s\n" "yes" >&6; } + else + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 +printf "%s\n" "no" >&6; } + PKG_CONFIG="" + fi +fi -printf "%s\n" "#define GETSERVBYNAME_R_ARGS $getservbyname_r_args " >>confdefs.h +pkg_failed=no +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for gmock" >&5 +printf %s "checking for gmock... " >&6; } - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $getservbyname_r_args" >&5 -printf "%s\n" "$getservbyname_r_args" >&6; } +if test -n "$GMOCK_CFLAGS"; then + pkg_cv_GMOCK_CFLAGS="$GMOCK_CFLAGS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"gmock\""; } >&5 + ($PKG_CONFIG --exists --print-errors "gmock") 2>&5 + ac_status=$? + printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_GMOCK_CFLAGS=`$PKG_CONFIG --cflags "gmock" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi +if test -n "$GMOCK_LIBS"; then + pkg_cv_GMOCK_LIBS="$GMOCK_LIBS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"gmock\""; } >&5 + ($PKG_CONFIG --exists --print-errors "gmock") 2>&5 + ac_status=$? + printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_GMOCK_LIBS=`$PKG_CONFIG --libs "gmock" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried fi -ac_fn_c_check_type "$LINENO" "size_t" "ac_cv_type_size_t" "$ac_includes_default" -if test "x$ac_cv_type_size_t" = xyes -then : -else $as_nop -printf "%s\n" "#define size_t unsigned int" >>confdefs.h +if test $pkg_failed = yes; then + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 +printf "%s\n" "no" >&6; } +if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then + _pkg_short_errors_supported=yes +else + _pkg_short_errors_supported=no fi + if test $_pkg_short_errors_supported = yes; then + GMOCK_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "gmock" 2>&1` + else + GMOCK_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "gmock" 2>&1` + fi + # Put the nasty error message in config.log where it belongs + echo "$GMOCK_PKG_ERRORS" >&5 -ac_fn_check_decl "$LINENO" "AF_INET6" "ac_cv_have_decl_AF_INET6" "$cares_all_includes -" "$ac_c_undeclared_builtin_options" "CFLAGS" -if test "x$ac_cv_have_decl_AF_INET6" = xyes -then : - -printf "%s\n" "#define HAVE_AF_INET6 1" >>confdefs.h - + have_gmock=no +elif test $pkg_failed = untried; then + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 +printf "%s\n" "no" >&6; } + have_gmock=no +else + GMOCK_CFLAGS=$pkg_cv_GMOCK_CFLAGS + GMOCK_LIBS=$pkg_cv_GMOCK_LIBS + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +printf "%s\n" "yes" >&6; } + have_gmock=yes fi -ac_fn_check_decl "$LINENO" "PF_INET6" "ac_cv_have_decl_PF_INET6" "$cares_all_includes -" "$ac_c_undeclared_builtin_options" "CFLAGS" -if test "x$ac_cv_have_decl_PF_INET6" = xyes -then : + if test "x$have_gmock" = "xno" ; then + if test "x$build_tests" = "xmaybe" ; then + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: WARNING: gmock could not be found, not building tests" >&5 +printf "%s\n" "$as_me: WARNING: gmock could not be found, not building tests" >&2;} + build_tests=no + else + as_fn_error $? "tests require gmock" "$LINENO" 5 + fi + else -printf "%s\n" "#define HAVE_PF_INET6 1" >>confdefs.h +pkg_failed=no +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for gmock >= 1.12.0" >&5 +printf %s "checking for gmock >= 1.12.0... " >&6; } +if test -n "$GMOCK112_CFLAGS"; then + pkg_cv_GMOCK112_CFLAGS="$GMOCK112_CFLAGS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"gmock >= 1.12.0\""; } >&5 + ($PKG_CONFIG --exists --print-errors "gmock >= 1.12.0") 2>&5 + ac_status=$? + printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_GMOCK112_CFLAGS=`$PKG_CONFIG --cflags "gmock >= 1.12.0" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes fi -ac_fn_c_check_type "$LINENO" "struct in6_addr" "ac_cv_type_struct_in6_addr" "$cares_all_includes -" -if test "x$ac_cv_type_struct_in6_addr" = xyes -then : - -printf "%s\n" "#define HAVE_STRUCT_IN6_ADDR 1" >>confdefs.h - - + else + pkg_failed=untried +fi +if test -n "$GMOCK112_LIBS"; then + pkg_cv_GMOCK112_LIBS="$GMOCK112_LIBS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"gmock >= 1.12.0\""; } >&5 + ($PKG_CONFIG --exists --print-errors "gmock >= 1.12.0") 2>&5 + ac_status=$? + printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_GMOCK112_LIBS=`$PKG_CONFIG --libs "gmock >= 1.12.0" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried fi -ac_fn_c_check_type "$LINENO" "struct sockaddr_in6" "ac_cv_type_struct_sockaddr_in6" "$cares_all_includes -" -if test "x$ac_cv_type_struct_sockaddr_in6" = xyes -then : -printf "%s\n" "#define HAVE_STRUCT_SOCKADDR_IN6 1" >>confdefs.h +if test $pkg_failed = yes; then + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 +printf "%s\n" "no" >&6; } +if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then + _pkg_short_errors_supported=yes +else + _pkg_short_errors_supported=no fi + if test $_pkg_short_errors_supported = yes; then + GMOCK112_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "gmock >= 1.12.0" 2>&1` + else + GMOCK112_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "gmock >= 1.12.0" 2>&1` + fi + # Put the nasty error message in config.log where it belongs + echo "$GMOCK112_PKG_ERRORS" >&5 -ac_fn_c_check_type "$LINENO" "struct sockaddr_storage" "ac_cv_type_struct_sockaddr_storage" "$cares_all_includes -" -if test "x$ac_cv_type_struct_sockaddr_storage" = xyes + have_gmock_v112=no +elif test $pkg_failed = untried; then + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 +printf "%s\n" "no" >&6; } + have_gmock_v112=no +else + GMOCK112_CFLAGS=$pkg_cv_GMOCK112_CFLAGS + GMOCK112_LIBS=$pkg_cv_GMOCK112_LIBS + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +printf "%s\n" "yes" >&6; } + have_gmock_v112=yes +fi + if test "x$have_gmock_v112" = "xyes" ; then + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether user namespaces are supported" >&5 +printf %s "checking whether user namespaces are supported... " >&6; } +if test ${ares_cv_user_namespace+y} then : + printf %s "(cached) " >&6 +else $as_nop -printf "%s\n" "#define HAVE_STRUCT_SOCKADDR_STORAGE 1" >>confdefs.h - - -fi + ac_ext=c +ac_cpp='$CPP $CPPFLAGS' +ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5' +ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5' +ac_compiler_gnu=$ac_cv_c_compiler_gnu -ac_fn_c_check_type "$LINENO" "struct addrinfo" "ac_cv_type_struct_addrinfo" "$cares_all_includes -" -if test "x$ac_cv_type_struct_addrinfo" = xyes + if test "$cross_compiling" = yes then : + ares_cv_user_namespace=no +else $as_nop + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ -printf "%s\n" "#define HAVE_STRUCT_ADDRINFO 1" >>confdefs.h +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +int userfn(void *d) { + usleep(100000); /* synchronize by sleep */ + return (getuid() != 0); +} +char userst[1024*1024]; +int main() { + char buffer[1024]; + int rc, status, fd; + pid_t child = clone(userfn, userst + 1024*1024, CLONE_NEWUSER|SIGCHLD, 0); + if (child < 0) return 1; -fi + snprintf(buffer, sizeof(buffer), "/proc/%d/uid_map", child); + fd = open(buffer, O_CREAT|O_WRONLY|O_TRUNC, 0755); + snprintf(buffer, sizeof(buffer), "0 %d 1\n", getuid()); + write(fd, buffer, strlen(buffer)); + close(fd); -ac_fn_c_check_type "$LINENO" "struct timeval" "ac_cv_type_struct_timeval" "$cares_all_includes -" -if test "x$ac_cv_type_struct_timeval" = xyes + rc = waitpid(child, &status, 0); + if (rc <= 0) return 1; + if (!WIFEXITED(status)) return 1; + return WEXITSTATUS(status); +} + +_ACEOF +if ac_fn_c_try_run "$LINENO" then : + ares_cv_user_namespace=yes +else $as_nop + ares_cv_user_namespace=no +fi +rm -f core *.core core.conftest.* gmon.out bb.out conftest$ac_exeext \ + conftest.$ac_objext conftest.beam conftest.$ac_ext +fi -printf "%s\n" "#define HAVE_STRUCT_TIMEVAL 1" >>confdefs.h + ac_ext=c +ac_cpp='$CPP $CPPFLAGS' +ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5' +ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5' +ac_compiler_gnu=$ac_cv_c_compiler_gnu fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ares_cv_user_namespace" >&5 +printf "%s\n" "$ares_cv_user_namespace" >&6; } + if test "$ares_cv_user_namespace" = yes; then -ac_fn_c_check_member "$LINENO" "struct sockaddr_in6" "sin6_scope_id" "ac_cv_member_struct_sockaddr_in6_sin6_scope_id" "$cares_all_includes -" -if test "x$ac_cv_member_struct_sockaddr_in6_sin6_scope_id" = xyes -then : +printf "%s\n" "#define HAVE_USER_NAMESPACE 1" >>confdefs.h -printf "%s\n" "#define HAVE_STRUCT_SOCKADDR_IN6_SIN6_SCOPE_ID 1" >>confdefs.h + fi + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether UTS namespaces are supported" >&5 +printf %s "checking whether UTS namespaces are supported... " >&6; } +if test ${ares_cv_uts_namespace+y} +then : + printf %s "(cached) " >&6 +else $as_nop -fi + ac_ext=c +ac_cpp='$CPP $CPPFLAGS' +ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5' +ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5' +ac_compiler_gnu=$ac_cv_c_compiler_gnu -ac_fn_c_check_member "$LINENO" "struct addrinfo" "ai_flags" "ac_cv_member_struct_addrinfo_ai_flags" "$cares_all_includes -" -if test "x$ac_cv_member_struct_addrinfo_ai_flags" = xyes + if test "$cross_compiling" = yes then : + ares_cv_uts_namespace=no +else $as_nop + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ -printf "%s\n" "#define HAVE_STRUCT_ADDRINFO_AI_FLAGS 1" >>confdefs.h +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +int utsfn(void *d) { + char buffer[1024]; + const char *name = "autoconftest"; + int rc = sethostname(name, strlen(name)); + if (rc != 0) return 1; + gethostname(buffer, 1024); + return (strcmp(buffer, name) != 0); +} -fi +char st2[1024*1024]; +int fn(void *d) { + pid_t child; + int rc, status; + usleep(100000); /* synchronize by sleep */ + if (getuid() != 0) return 1; + child = clone(utsfn, st2 + 1024*1024, CLONE_NEWUTS|SIGCHLD, 0); + if (child < 0) return 1; + rc = waitpid(child, &status, 0); + if (rc <= 0) return 1; + if (!WIFEXITED(status)) return 1; + return WEXITSTATUS(status); +} +char st[1024*1024]; +int main() { + char buffer[1024]; + int rc, status, fd; + pid_t child = clone(fn, st + 1024*1024, CLONE_NEWUSER|SIGCHLD, 0); + if (child < 0) return 1; -ac_fn_check_decl "$LINENO" "FIONBIO" "ac_cv_have_decl_FIONBIO" "$cares_all_includes -" "$ac_c_undeclared_builtin_options" "CFLAGS" -if test "x$ac_cv_have_decl_FIONBIO" = xyes -then : + snprintf(buffer, sizeof(buffer), "/proc/%d/uid_map", child); + fd = open(buffer, O_CREAT|O_WRONLY|O_TRUNC, 0755); + snprintf(buffer, sizeof(buffer), "0 %d 1\n", getuid()); + write(fd, buffer, strlen(buffer)); + close(fd); -fi -ac_fn_check_decl "$LINENO" "O_NONBLOCK" "ac_cv_have_decl_O_NONBLOCK" "$cares_all_includes -" "$ac_c_undeclared_builtin_options" "CFLAGS" -if test "x$ac_cv_have_decl_O_NONBLOCK" = xyes -then : + rc = waitpid(child, &status, 0); + if (rc <= 0) return 1; + if (!WIFEXITED(status)) return 1; + return WEXITSTATUS(status); +} -fi -ac_fn_check_decl "$LINENO" "SO_NONBLOCK" "ac_cv_have_decl_SO_NONBLOCK" "$cares_all_includes -" "$ac_c_undeclared_builtin_options" "CFLAGS" -if test "x$ac_cv_have_decl_SO_NONBLOCK" = xyes -then : -fi -ac_fn_check_decl "$LINENO" "MSG_NOSIGNAL" "ac_cv_have_decl_MSG_NOSIGNAL" "$cares_all_includes -" "$ac_c_undeclared_builtin_options" "CFLAGS" -if test "x$ac_cv_have_decl_MSG_NOSIGNAL" = xyes +_ACEOF +if ac_fn_c_try_run "$LINENO" then : - + ares_cv_uts_namespace=yes +else $as_nop + ares_cv_uts_namespace=no fi -ac_fn_check_decl "$LINENO" "CLOCK_MONOTONIC" "ac_cv_have_decl_CLOCK_MONOTONIC" "$cares_all_includes -" "$ac_c_undeclared_builtin_options" "CFLAGS" -if test "x$ac_cv_have_decl_CLOCK_MONOTONIC" = xyes -then : - +rm -f core *.core core.conftest.* gmon.out bb.out conftest$ac_exeext \ + conftest.$ac_objext conftest.beam conftest.$ac_ext fi -if test "$ac_cv_have_decl_CLOCK_MONOTONIC" = "yes" -a "$ac_cv_have_decl_clock_gettime" = "yes" ; then + ac_ext=c +ac_cpp='$CPP $CPPFLAGS' +ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5' +ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5' +ac_compiler_gnu=$ac_cv_c_compiler_gnu -printf "%s\n" "#define HAVE_CLOCK_GETTIME_MONOTONIC 1 " >>confdefs.h fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ares_cv_uts_namespace" >&5 +printf "%s\n" "$ares_cv_uts_namespace" >&6; } + if test "$ares_cv_uts_namespace" = yes; then -if test "$ac_cv_have_decl_FIONBIO" = "yes" -a "$ac_cv_have_decl_ioctl" = "yes" ; then +printf "%s\n" "#define HAVE_UTS_NAMESPACE 1" >>confdefs.h -printf "%s\n" "#define HAVE_IOCTL_FIONBIO 1 " >>confdefs.h + fi -fi -if test "$ac_cv_have_decl_FIONBIO" = "yes" -a "$ac_cv_have_decl_ioctlsocket" = "yes" ; then + fi -printf "%s\n" "#define HAVE_IOCTLSOCKET_FIONBIO 1 " >>confdefs.h +pkg_failed=no +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for gmock >= 1.17.0" >&5 +printf %s "checking for gmock >= 1.17.0... " >&6; } +if test -n "$GMOCK117_CFLAGS"; then + pkg_cv_GMOCK117_CFLAGS="$GMOCK117_CFLAGS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"gmock >= 1.17.0\""; } >&5 + ($PKG_CONFIG --exists --print-errors "gmock >= 1.17.0") 2>&5 + ac_status=$? + printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_GMOCK117_CFLAGS=`$PKG_CONFIG --cflags "gmock >= 1.17.0" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes fi -if test "$ac_cv_have_decl_SO_NONBLOCK" = "yes" -a "$ac_cv_have_decl_setsockopt" = "yes" ; then - -printf "%s\n" "#define HAVE_SETSOCKOPT_SO_NONBLOCK 1 " >>confdefs.h - + else + pkg_failed=untried +fi +if test -n "$GMOCK117_LIBS"; then + pkg_cv_GMOCK117_LIBS="$GMOCK117_LIBS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"gmock >= 1.17.0\""; } >&5 + ($PKG_CONFIG --exists --print-errors "gmock >= 1.17.0") 2>&5 + ac_status=$? + printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_GMOCK117_LIBS=`$PKG_CONFIG --libs "gmock >= 1.17.0" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried fi -if test "$ac_cv_have_decl_O_NONBLOCK" = "yes" -a "$ac_cv_have_decl_fcntl" = "yes" ; then -printf "%s\n" "#define HAVE_FCNTL_O_NONBLOCK 1 " >>confdefs.h -fi -if test "x$ac_cv_header_sys_types_h" = "xyes" ; then +if test $pkg_failed = yes; then + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 +printf "%s\n" "no" >&6; } -cat >>confdefs.h <<_EOF -#define CARES_HAVE_SYS_TYPES_H 1 -_EOF +if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then + _pkg_short_errors_supported=yes +else + _pkg_short_errors_supported=no +fi + if test $_pkg_short_errors_supported = yes; then + GMOCK117_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "gmock >= 1.17.0" 2>&1` + else + GMOCK117_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "gmock >= 1.17.0" 2>&1` + fi + # Put the nasty error message in config.log where it belongs + echo "$GMOCK117_PKG_ERRORS" >&5 + have_gmock_v117=no +elif test $pkg_failed = untried; then + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 +printf "%s\n" "no" >&6; } + have_gmock_v117=no +else + GMOCK117_CFLAGS=$pkg_cv_GMOCK117_CFLAGS + GMOCK117_LIBS=$pkg_cv_GMOCK117_LIBS + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +printf "%s\n" "yes" >&6; } + have_gmock_v117=yes fi -if test "x$ac_cv_header_sys_socket_h" = "xyes" ; then + if test "x$have_gmock_v117" = "xyes" ; then + ax_cxx_compile_alternatives="17 1z" ax_cxx_compile_cxx17_required=true + ac_ext=cpp +ac_cpp='$CXXCPP $CPPFLAGS' +ac_compile='$CXX -c $CXXFLAGS $CPPFLAGS conftest.$ac_ext >&5' +ac_link='$CXX -o conftest$ac_exeext $CXXFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5' +ac_compiler_gnu=$ac_cv_cxx_compiler_gnu + ac_success=no -cat >>confdefs.h <<_EOF -#define CARES_HAVE_SYS_SOCKET_H 1 -_EOF -fi -if test "x$ac_cv_header_sys_select_h" = "xyes" ; then -cat >>confdefs.h <<_EOF -#define CARES_HAVE_SYS_SELECT_H 1 -_EOF -fi -if test "x$ac_cv_header_ws2tcpip_h" = "xyes" ; then -cat >>confdefs.h <<_EOF -#define CARES_HAVE_WS2TCPIP_H 1 -_EOF + if test x$ac_success = xno; then + for alternative in ${ax_cxx_compile_alternatives}; do + for switch in -std=c++${alternative} +std=c++${alternative} "-h std=c++${alternative}" MSVC; do + if test x"$switch" = xMSVC; then + switch=-std:c++${alternative} + cachevar=`printf "%s\n" "ax_cv_cxx_compile_cxx17_${switch}_MSVC" | $as_tr_sh` + else + cachevar=`printf "%s\n" "ax_cv_cxx_compile_cxx17_$switch" | $as_tr_sh` + fi + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether $CXX supports C++17 features with $switch" >&5 +printf %s "checking whether $CXX supports C++17 features with $switch... " >&6; } +if eval test \${$cachevar+y} +then : + printf %s "(cached) " >&6 +else $as_nop + ac_save_CXX="$CXX" + CXX="$CXX $switch" + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ -fi -if test "x$ac_cv_header_winsock2_h" = "xyes" ; then -cat >>confdefs.h <<_EOF -#define CARES_HAVE_WINSOCK2_H 1 -_EOF +// If the compiler admits that it is not ready for C++11, why torture it? +// Hopefully, this will speed up the test. -fi -if test "x$ac_cv_header_windows_h" = "xyes" ; then +#ifndef __cplusplus -cat >>confdefs.h <<_EOF -#define CARES_HAVE_WINDOWS_H 1 -_EOF +#error "This is not a C++ compiler" -fi -if test "x$ac_cv_header_arpa_nameser_h" = "xyes" ; then +// MSVC always sets __cplusplus to 199711L in older versions; newer versions +// only set it correctly if /Zc:__cplusplus is specified as well as a +// /std:c++NN switch: +// +// https://devblogs.microsoft.com/cppblog/msvc-now-correctly-reports-__cplusplus/ +// +// The value __cplusplus ought to have is available in _MSVC_LANG since +// Visual Studio 2015 Update 3: +// +// https://learn.microsoft.com/en-us/cpp/preprocessor/predefined-macros +// +// This was also the first MSVC version to support C++14 so we can't use the +// value of either __cplusplus or _MSVC_LANG to quickly rule out MSVC having +// C++11 or C++14 support, but we can check _MSVC_LANG for C++17 and later. +#elif __cplusplus < 201103L && !defined _MSC_VER -cat >>confdefs.h <<_EOF -#define CARES_HAVE_ARPA_NAMESER_H 1 -_EOF +#error "This is not a C++11 compiler" -fi -if test "x$ac_cv_header_arpa_nameser_compa_h" = "xyes" ; then +#else -cat >>confdefs.h <<_EOF -#define CARES_HAVE_ARPA_NAMESER_COMPA_H 1 -_EOF +namespace cxx11 +{ -fi + namespace test_static_assert + { + + template + struct check + { + static_assert(sizeof(int) <= sizeof(T), "not big enough"); + }; + } + namespace test_final_override + { -if test "${CARES_THREADS}" = "yes" -a "x${ac_cv_native_windows}" != "xyes" ; then + struct Base + { + virtual ~Base() {} + virtual void f() {} + }; + struct Derived : public Base + { + virtual ~Derived() override {} + virtual void f() override {} + }; + } + namespace test_double_right_angle_brackets + { -ac_ext=c -ac_cpp='$CPP $CPPFLAGS' -ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5' -ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5' -ac_compiler_gnu=$ac_cv_c_compiler_gnu + template < typename T > + struct check {}; -ax_pthread_ok=no + typedef check single_type; + typedef check> double_type; + typedef check>> triple_type; + typedef check>>> quadruple_type; -# We used to check for pthread.h first, but this fails if pthread.h -# requires special compiler flags (e.g. on Tru64 or Sequent). -# It gets checked for in the link test anyway. + } -# First of all, check if the user has set any of the PTHREAD_LIBS, -# etcetera environment variables, and if threads linking works using -# them: -if test "x$PTHREAD_CFLAGS$PTHREAD_LIBS" != "x"; then - ax_pthread_save_CC="$CC" - ax_pthread_save_CFLAGS="$CFLAGS" - ax_pthread_save_LIBS="$LIBS" - if test "x$PTHREAD_CC" != "x" -then : - CC="$PTHREAD_CC" -fi - if test "x$PTHREAD_CXX" != "x" -then : - CXX="$PTHREAD_CXX" -fi - CFLAGS="$CFLAGS $PTHREAD_CFLAGS" - LIBS="$PTHREAD_LIBS $LIBS" - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for pthread_join using $CC $PTHREAD_CFLAGS $PTHREAD_LIBS" >&5 -printf %s "checking for pthread_join using $CC $PTHREAD_CFLAGS $PTHREAD_LIBS... " >&6; } - cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ + namespace test_decltype + { -/* Override any GCC internal prototype to avoid an error. - Use char because int might match the return type of a GCC - builtin and then its argument prototype would still apply. */ -char pthread_join (); -int -main (void) -{ -return pthread_join (); - ; - return 0; -} -_ACEOF -if ac_fn_c_try_link "$LINENO" -then : - ax_pthread_ok=yes -fi -rm -f core conftest.err conftest.$ac_objext conftest.beam \ - conftest$ac_exeext conftest.$ac_ext - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ax_pthread_ok" >&5 -printf "%s\n" "$ax_pthread_ok" >&6; } - if test "x$ax_pthread_ok" = "xno"; then - PTHREAD_LIBS="" - PTHREAD_CFLAGS="" - fi - CC="$ax_pthread_save_CC" - CFLAGS="$ax_pthread_save_CFLAGS" - LIBS="$ax_pthread_save_LIBS" -fi + int + f() + { + int a = 1; + decltype(a) b = 2; + return a + b; + } -# We must check for the threads library under a number of different -# names; the ordering is very important because some systems -# (e.g. DEC) have both -lpthread and -lpthreads, where one of the -# libraries is broken (non-POSIX). + } -# Create a list of thread flags to try. Items with a "," contain both -# C compiler flags (before ",") and linker flags (after ","). Other items -# starting with a "-" are C compiler flags, and remaining items are -# library names, except for "none" which indicates that we try without -# any flags at all, and "pthread-config" which is a program returning -# the flags for the Pth emulation library. + namespace test_type_deduction + { -ax_pthread_flags="pthreads none -Kthread -pthread -pthreads -mthreads pthread --thread-safe -mt pthread-config" + template < typename T1, typename T2 > + struct is_same + { + static const bool value = false; + }; -# The ordering *is* (sometimes) important. Some notes on the -# individual items follow: + template < typename T > + struct is_same + { + static const bool value = true; + }; -# pthreads: AIX (must check this before -lpthread) -# none: in case threads are in libc; should be tried before -Kthread and -# other compiler flags to prevent continual compiler warnings -# -Kthread: Sequent (threads in libc, but -Kthread needed for pthread.h) -# -pthread: Linux/gcc (kernel threads), BSD/gcc (userland threads), Tru64 -# (Note: HP C rejects this with "bad form for `-t' option") -# -pthreads: Solaris/gcc (Note: HP C also rejects) -# -mt: Sun Workshop C (may only link SunOS threads [-lthread], but it -# doesn't hurt to check since this sometimes defines pthreads and -# -D_REENTRANT too), HP C (must be checked before -lpthread, which -# is present but should not be used directly; and before -mthreads, -# because the compiler interprets this as "-mt" + "-hreads") -# -mthreads: Mingw32/gcc, Lynx/gcc -# pthread: Linux, etcetera -# --thread-safe: KAI C++ -# pthread-config: use pthread-config program (for GNU Pth library) + template < typename T1, typename T2 > + auto + add(T1 a1, T2 a2) -> decltype(a1 + a2) + { + return a1 + a2; + } -case $host_os in + int + test(const int c, volatile int v) + { + static_assert(is_same::value == true, ""); + static_assert(is_same::value == false, ""); + static_assert(is_same::value == false, ""); + auto ac = c; + auto av = v; + auto sumi = ac + av + 'x'; + auto sumf = ac + av + 1.0; + static_assert(is_same::value == true, ""); + static_assert(is_same::value == true, ""); + static_assert(is_same::value == true, ""); + static_assert(is_same::value == false, ""); + static_assert(is_same::value == true, ""); + return (sumf > 0.0) ? sumi : add(c, v); + } - freebsd*) + } - # -kthread: FreeBSD kernel threads (preferred to -pthread since SMP-able) - # lthread: LinuxThreads port on FreeBSD (also preferred to -pthread) + namespace test_noexcept + { - ax_pthread_flags="-kthread lthread $ax_pthread_flags" - ;; + int f() { return 0; } + int g() noexcept { return 0; } - hpux*) + static_assert(noexcept(f()) == false, ""); + static_assert(noexcept(g()) == true, ""); - # From the cc(1) man page: "[-mt] Sets various -D flags to enable - # multi-threading and also sets -lpthread." + } - ax_pthread_flags="-mt -pthread pthread $ax_pthread_flags" - ;; + namespace test_constexpr + { - openedition*) + template < typename CharT > + unsigned long constexpr + strlen_c_r(const CharT *const s, const unsigned long acc) noexcept + { + return *s ? strlen_c_r(s + 1, acc + 1) : acc; + } - # IBM z/OS requires a feature-test macro to be defined in order to - # enable POSIX threads at all, so give the user a hint if this is - # not set. (We don't define these ourselves, as they can affect - # other portions of the system API in unpredictable ways.) + template < typename CharT > + unsigned long constexpr + strlen_c(const CharT *const s) noexcept + { + return strlen_c_r(s, 0UL); + } - cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ + static_assert(strlen_c("") == 0UL, ""); + static_assert(strlen_c("1") == 1UL, ""); + static_assert(strlen_c("example") == 7UL, ""); + static_assert(strlen_c("another\0example") == 7UL, ""); -# if !defined(_OPEN_THREADS) && !defined(_UNIX03_THREADS) - AX_PTHREAD_ZOS_MISSING -# endif + } -_ACEOF -if (eval "$ac_cpp conftest.$ac_ext") 2>&5 | - $EGREP "AX_PTHREAD_ZOS_MISSING" >/dev/null 2>&1 -then : - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: WARNING: IBM z/OS requires -D_OPEN_THREADS or -D_UNIX03_THREADS to enable pthreads support." >&5 -printf "%s\n" "$as_me: WARNING: IBM z/OS requires -D_OPEN_THREADS or -D_UNIX03_THREADS to enable pthreads support." >&2;} -fi -rm -rf conftest* + namespace test_rvalue_references + { - ;; + template < int N > + struct answer + { + static constexpr int value = N; + }; - solaris*) + answer<1> f(int&) { return answer<1>(); } + answer<2> f(const int&) { return answer<2>(); } + answer<3> f(int&&) { return answer<3>(); } - # On Solaris (at least, for some versions), libc contains stubbed - # (non-functional) versions of the pthreads routines, so link-based - # tests will erroneously succeed. (N.B.: The stubs are missing - # pthread_cleanup_push, or rather a function called by this macro, - # so we could check for that, but who knows whether they'll stub - # that too in a future libc.) So we'll check first for the - # standard Solaris way of linking pthreads (-mt -lpthread). + void + test() + { + int i = 0; + const int c = 0; + static_assert(decltype(f(i))::value == 1, ""); + static_assert(decltype(f(c))::value == 2, ""); + static_assert(decltype(f(0))::value == 3, ""); + } - ax_pthread_flags="-mt,-lpthread pthread $ax_pthread_flags" - ;; -esac + } -# Are we compiling with Clang? + namespace test_uniform_initialization + { -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether $CC is Clang" >&5 -printf %s "checking whether $CC is Clang... " >&6; } -if test ${ax_cv_PTHREAD_CLANG+y} -then : - printf %s "(cached) " >&6 -else $as_nop - ax_cv_PTHREAD_CLANG=no - # Note that Autoconf sets GCC=yes for Clang as well as GCC - if test "x$GCC" = "xyes"; then - cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ -/* Note: Clang 2.7 lacks __clang_[a-z]+__ */ -# if defined(__clang__) && defined(__llvm__) - AX_PTHREAD_CC_IS_CLANG -# endif + struct test + { + static const int zero {}; + static const int one {1}; + }; -_ACEOF -if (eval "$ac_cpp conftest.$ac_ext") 2>&5 | - $EGREP "AX_PTHREAD_CC_IS_CLANG" >/dev/null 2>&1 -then : - ax_cv_PTHREAD_CLANG=yes -fi -rm -rf conftest* + static_assert(test::zero == 0, ""); + static_assert(test::one == 1, ""); - fi + } -fi -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ax_cv_PTHREAD_CLANG" >&5 -printf "%s\n" "$ax_cv_PTHREAD_CLANG" >&6; } -ax_pthread_clang="$ax_cv_PTHREAD_CLANG" + namespace test_lambdas + { + void + test1() + { + auto lambda1 = [](){}; + auto lambda2 = lambda1; + lambda1(); + lambda2(); + } -# GCC generally uses -pthread, or -pthreads on some platforms (e.g. SPARC) + int + test2() + { + auto a = [](int i, int j){ return i + j; }(1, 2); + auto b = []() -> int { return '0'; }(); + auto c = [=](){ return a + b; }(); + auto d = [&](){ return c; }(); + auto e = [a, &b](int x) mutable { + const auto identity = [](int y){ return y; }; + for (auto i = 0; i < a; ++i) + a += b--; + return x + identity(a + b); + }(0); + return a + b + c + d + e; + } -# Note that for GCC and Clang -pthread generally implies -lpthread, -# except when -nostdlib is passed. -# This is problematic using libtool to build C++ shared libraries with pthread: -# [1] https://gcc.gnu.org/bugzilla/show_bug.cgi?id=25460 -# [2] https://bugzilla.redhat.com/show_bug.cgi?id=661333 -# [3] https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=468555 -# To solve this, first try -pthread together with -lpthread for GCC + int + test3() + { + const auto nullary = [](){ return 0; }; + const auto unary = [](int x){ return x; }; + using nullary_t = decltype(nullary); + using unary_t = decltype(unary); + const auto higher1st = [](nullary_t f){ return f(); }; + const auto higher2nd = [unary](nullary_t f1){ + return [unary, f1](unary_t f2){ return f2(unary(f1())); }; + }; + return higher1st(nullary) + higher2nd(nullary)(unary); + } -if test "x$GCC" = "xyes" -then : - ax_pthread_flags="-pthread,-lpthread -pthread -pthreads $ax_pthread_flags" -fi + } -# Clang takes -pthread (never supported any other flag), but we'll try with -lpthread first + namespace test_variadic_templates + { -if test "x$ax_pthread_clang" = "xyes" -then : - ax_pthread_flags="-pthread,-lpthread -pthread" -fi + template + struct sum; + template + struct sum + { + static constexpr auto value = N0 + sum::value; + }; -# The presence of a feature test macro requesting re-entrant function -# definitions is, on some systems, a strong hint that pthreads support is -# correctly enabled + template <> + struct sum<> + { + static constexpr auto value = 0; + }; -case $host_os in - darwin* | hpux* | linux* | osf* | solaris*) - ax_pthread_check_macro="_REENTRANT" - ;; + static_assert(sum<>::value == 0, ""); + static_assert(sum<1>::value == 1, ""); + static_assert(sum<23>::value == 23, ""); + static_assert(sum<1, 2>::value == 3, ""); + static_assert(sum<5, 5, 11>::value == 21, ""); + static_assert(sum<2, 3, 5, 7, 11, 13>::value == 41, ""); - aix*) - ax_pthread_check_macro="_THREAD_SAFE" - ;; + } - *) - ax_pthread_check_macro="--" - ;; -esac -if test "x$ax_pthread_check_macro" = "x--" -then : - ax_pthread_check_cond=0 -else $as_nop - ax_pthread_check_cond="!defined($ax_pthread_check_macro)" -fi + // http://stackoverflow.com/questions/13728184/template-aliases-and-sfinae + // Clang 3.1 fails with headers of libstd++ 4.8.3 when using std::function + // because of this. + namespace test_template_alias_sfinae + { + struct foo {}; -if test "x$ax_pthread_ok" = "xno"; then -for ax_pthread_try_flag in $ax_pthread_flags; do + template + using member = typename T::member_type; - case $ax_pthread_try_flag in - none) - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether pthreads work without any flags" >&5 -printf %s "checking whether pthreads work without any flags... " >&6; } - ;; + template + void func(...) {} - *,*) - PTHREAD_CFLAGS=`echo $ax_pthread_try_flag | sed "s/^\(.*\),\(.*\)$/\1/"` - PTHREAD_LIBS=`echo $ax_pthread_try_flag | sed "s/^\(.*\),\(.*\)$/\2/"` - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether pthreads work with \"$PTHREAD_CFLAGS\" and \"$PTHREAD_LIBS\"" >&5 -printf %s "checking whether pthreads work with \"$PTHREAD_CFLAGS\" and \"$PTHREAD_LIBS\"... " >&6; } - ;; + template + void func(member*) {} - -*) - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether pthreads work with $ax_pthread_try_flag" >&5 -printf %s "checking whether pthreads work with $ax_pthread_try_flag... " >&6; } - PTHREAD_CFLAGS="$ax_pthread_try_flag" - ;; + void test(); - pthread-config) - # Extract the first word of "pthread-config", so it can be a program name with args. -set dummy pthread-config; ac_word=$2 -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 -printf %s "checking for $ac_word... " >&6; } -if test ${ac_cv_prog_ax_pthread_config+y} -then : - printf %s "(cached) " >&6 -else $as_nop - if test -n "$ax_pthread_config"; then - ac_cv_prog_ax_pthread_config="$ax_pthread_config" # Let the user override the test. -else -as_save_IFS=$IFS; IFS=$PATH_SEPARATOR -for as_dir in $PATH -do - IFS=$as_save_IFS - case $as_dir in #((( - '') as_dir=./ ;; - */) ;; - *) as_dir=$as_dir/ ;; - esac - for ac_exec_ext in '' $ac_executable_extensions; do - if as_fn_executable_p "$as_dir$ac_word$ac_exec_ext"; then - ac_cv_prog_ax_pthread_config="yes" - printf "%s\n" "$as_me:${as_lineno-$LINENO}: found $as_dir$ac_word$ac_exec_ext" >&5 - break 2 - fi -done - done -IFS=$as_save_IFS + void test() { func(0); } - test -z "$ac_cv_prog_ax_pthread_config" && ac_cv_prog_ax_pthread_config="no" -fi -fi -ax_pthread_config=$ac_cv_prog_ax_pthread_config -if test -n "$ax_pthread_config"; then - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ax_pthread_config" >&5 -printf "%s\n" "$ax_pthread_config" >&6; } -else - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 -printf "%s\n" "no" >&6; } -fi + } +} // namespace cxx11 - if test "x$ax_pthread_config" = "xno" -then : - continue -fi - PTHREAD_CFLAGS="`pthread-config --cflags`" - PTHREAD_LIBS="`pthread-config --ldflags` `pthread-config --libs`" - ;; +#endif // __cplusplus >= 201103L - *) - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for the pthreads library -l$ax_pthread_try_flag" >&5 -printf %s "checking for the pthreads library -l$ax_pthread_try_flag... " >&6; } - PTHREAD_LIBS="-l$ax_pthread_try_flag" - ;; - esac - ax_pthread_save_CFLAGS="$CFLAGS" - ax_pthread_save_LIBS="$LIBS" - CFLAGS="$CFLAGS $PTHREAD_CFLAGS" - LIBS="$PTHREAD_LIBS $LIBS" - # Check for various functions. We must include pthread.h, - # since some functions may be macros. (On the Sequent, we - # need a special flag -Kthread to make this header compile.) - # We check for pthread_join because it is in -lpthread on IRIX - # while pthread_create is in libc. We check for pthread_attr_init - # due to DEC craziness with -lpthreads. We check for - # pthread_cleanup_push because it is one of the few pthread - # functions on Solaris that doesn't have a non-functional libc stub. - # We try pthread_create on general principles. - cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ -#include -# if $ax_pthread_check_cond -# error "$ax_pthread_check_macro must be defined" -# endif - static void *some_global = NULL; - static void routine(void *a) - { - /* To avoid any unused-parameter or - unused-but-set-parameter warning. */ - some_global = a; - } - static void *start_routine(void *a) { return a; } -int -main (void) -{ -pthread_t th; pthread_attr_t attr; - pthread_create(&th, 0, start_routine, 0); - pthread_join(th, 0); - pthread_attr_init(&attr); - pthread_cleanup_push(routine, 0); - pthread_cleanup_pop(0) /* ; */ - ; - return 0; -} -_ACEOF -if ac_fn_c_try_link "$LINENO" -then : - ax_pthread_ok=yes -fi -rm -f core conftest.err conftest.$ac_objext conftest.beam \ - conftest$ac_exeext conftest.$ac_ext +// If the compiler admits that it is not ready for C++14, why torture it? +// Hopefully, this will speed up the test. - CFLAGS="$ax_pthread_save_CFLAGS" - LIBS="$ax_pthread_save_LIBS" +#ifndef __cplusplus - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ax_pthread_ok" >&5 -printf "%s\n" "$ax_pthread_ok" >&6; } - if test "x$ax_pthread_ok" = "xyes" -then : - break -fi +#error "This is not a C++ compiler" - PTHREAD_LIBS="" - PTHREAD_CFLAGS="" -done -fi +#elif __cplusplus < 201402L && !defined _MSC_VER +#error "This is not a C++14 compiler" -# Clang needs special handling, because older versions handle the -pthread -# option in a rather... idiosyncratic way +#else -if test "x$ax_pthread_clang" = "xyes"; then +namespace cxx14 +{ - # Clang takes -pthread; it has never supported any other flag + namespace test_polymorphic_lambdas + { - # (Note 1: This will need to be revisited if a system that Clang - # supports has POSIX threads in a separate library. This tends not - # to be the way of modern systems, but it's conceivable.) + int + test() + { + const auto lambda = [](auto&&... args){ + const auto istiny = [](auto x){ + return (sizeof(x) == 1UL) ? 1 : 0; + }; + const int aretiny[] = { istiny(args)... }; + return aretiny[0]; + }; + return lambda(1, 1L, 1.0f, '1'); + } - # (Note 2: On some systems, notably Darwin, -pthread is not needed - # to get POSIX threads support; the API is always present and - # active. We could reasonably leave PTHREAD_CFLAGS empty. But - # -pthread does define _REENTRANT, and while the Darwin headers - # ignore this macro, third-party headers might not.) + } - # However, older versions of Clang make a point of warning the user - # that, in an invocation where only linking and no compilation is - # taking place, the -pthread option has no effect ("argument unused - # during compilation"). They expect -pthread to be passed in only - # when source code is being compiled. - # - # Problem is, this is at odds with the way Automake and most other - # C build frameworks function, which is that the same flags used in - # compilation (CFLAGS) are also used in linking. Many systems - # supported by AX_PTHREAD require exactly this for POSIX threads - # support, and in fact it is often not straightforward to specify a - # flag that is used only in the compilation phase and not in - # linking. Such a scenario is extremely rare in practice. - # - # Even though use of the -pthread flag in linking would only print - # a warning, this can be a nuisance for well-run software projects - # that build with -Werror. So if the active version of Clang has - # this misfeature, we search for an option to squash it. + namespace test_binary_literals + { - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether Clang needs flag to prevent \"argument unused\" warning when linking with -pthread" >&5 -printf %s "checking whether Clang needs flag to prevent \"argument unused\" warning when linking with -pthread... " >&6; } -if test ${ax_cv_PTHREAD_CLANG_NO_WARN_FLAG+y} -then : - printf %s "(cached) " >&6 -else $as_nop - ax_cv_PTHREAD_CLANG_NO_WARN_FLAG=unknown - # Create an alternate version of $ac_link that compiles and - # links in two steps (.c -> .o, .o -> exe) instead of one - # (.c -> exe), because the warning occurs only in the second - # step - ax_pthread_save_ac_link="$ac_link" - ax_pthread_sed='s/conftest\.\$ac_ext/conftest.$ac_objext/g' - ax_pthread_link_step=`printf "%s\n" "$ac_link" | sed "$ax_pthread_sed"` - ax_pthread_2step_ac_link="($ac_compile) && (echo ==== >&5) && ($ax_pthread_link_step)" - ax_pthread_save_CFLAGS="$CFLAGS" - for ax_pthread_try in '' -Qunused-arguments -Wno-unused-command-line-argument unknown; do - if test "x$ax_pthread_try" = "xunknown" -then : - break -fi - CFLAGS="-Werror -Wunknown-warning-option $ax_pthread_try -pthread $ax_pthread_save_CFLAGS" - ac_link="$ax_pthread_save_ac_link" - cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ -int main(void){return 0;} -_ACEOF -if ac_fn_c_try_link "$LINENO" -then : - ac_link="$ax_pthread_2step_ac_link" - cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ -int main(void){return 0;} -_ACEOF -if ac_fn_c_try_link "$LINENO" -then : - break -fi -rm -f core conftest.err conftest.$ac_objext conftest.beam \ - conftest$ac_exeext conftest.$ac_ext + constexpr auto ivii = 0b0000000000101010; + static_assert(ivii == 42, "wrong value"); -fi -rm -f core conftest.err conftest.$ac_objext conftest.beam \ - conftest$ac_exeext conftest.$ac_ext - done - ac_link="$ax_pthread_save_ac_link" - CFLAGS="$ax_pthread_save_CFLAGS" - if test "x$ax_pthread_try" = "x" -then : - ax_pthread_try=no -fi - ax_cv_PTHREAD_CLANG_NO_WARN_FLAG="$ax_pthread_try" + } -fi -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ax_cv_PTHREAD_CLANG_NO_WARN_FLAG" >&5 -printf "%s\n" "$ax_cv_PTHREAD_CLANG_NO_WARN_FLAG" >&6; } + namespace test_generalized_constexpr + { - case "$ax_cv_PTHREAD_CLANG_NO_WARN_FLAG" in - no | unknown) ;; - *) PTHREAD_CFLAGS="$ax_cv_PTHREAD_CLANG_NO_WARN_FLAG $PTHREAD_CFLAGS" ;; - esac + template < typename CharT > + constexpr unsigned long + strlen_c(const CharT *const s) noexcept + { + auto length = 0UL; + for (auto p = s; *p; ++p) + ++length; + return length; + } -fi # $ax_pthread_clang = yes + static_assert(strlen_c("") == 0UL, ""); + static_assert(strlen_c("x") == 1UL, ""); + static_assert(strlen_c("test") == 4UL, ""); + static_assert(strlen_c("another\0test") == 7UL, ""); + } + namespace test_lambda_init_capture + { -# Various other checks: -if test "x$ax_pthread_ok" = "xyes"; then - ax_pthread_save_CFLAGS="$CFLAGS" - ax_pthread_save_LIBS="$LIBS" - CFLAGS="$CFLAGS $PTHREAD_CFLAGS" - LIBS="$PTHREAD_LIBS $LIBS" + int + test() + { + auto x = 0; + const auto lambda1 = [a = x](int b){ return a + b; }; + const auto lambda2 = [a = lambda1(x)](){ return a; }; + return lambda2(); + } - # Detect AIX lossage: JOINABLE attribute is called UNDETACHED. - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for joinable pthread attribute" >&5 -printf %s "checking for joinable pthread attribute... " >&6; } -if test ${ax_cv_PTHREAD_JOINABLE_ATTR+y} -then : - printf %s "(cached) " >&6 -else $as_nop - ax_cv_PTHREAD_JOINABLE_ATTR=unknown - for ax_pthread_attr in PTHREAD_CREATE_JOINABLE PTHREAD_CREATE_UNDETACHED; do - cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ -#include -int -main (void) -{ -int attr = $ax_pthread_attr; return attr /* ; */ - ; - return 0; -} -_ACEOF -if ac_fn_c_try_link "$LINENO" -then : - ax_cv_PTHREAD_JOINABLE_ATTR=$ax_pthread_attr; break -fi -rm -f core conftest.err conftest.$ac_objext conftest.beam \ - conftest$ac_exeext conftest.$ac_ext - done + } -fi -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ax_cv_PTHREAD_JOINABLE_ATTR" >&5 -printf "%s\n" "$ax_cv_PTHREAD_JOINABLE_ATTR" >&6; } - if test "x$ax_cv_PTHREAD_JOINABLE_ATTR" != "xunknown" && \ - test "x$ax_cv_PTHREAD_JOINABLE_ATTR" != "xPTHREAD_CREATE_JOINABLE" && \ - test "x$ax_pthread_joinable_attr_defined" != "xyes" -then : + namespace test_digit_separators + { -printf "%s\n" "#define PTHREAD_CREATE_JOINABLE $ax_cv_PTHREAD_JOINABLE_ATTR" >>confdefs.h + constexpr auto ten_million = 100'000'000; + static_assert(ten_million == 100000000, ""); - ax_pthread_joinable_attr_defined=yes + } -fi + namespace test_return_type_deduction + { - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether more special flags are required for pthreads" >&5 -printf %s "checking whether more special flags are required for pthreads... " >&6; } -if test ${ax_cv_PTHREAD_SPECIAL_FLAGS+y} -then : - printf %s "(cached) " >&6 -else $as_nop - ax_cv_PTHREAD_SPECIAL_FLAGS=no - case $host_os in - solaris*) - ax_cv_PTHREAD_SPECIAL_FLAGS="-D_POSIX_PTHREAD_SEMANTICS" - ;; - esac + auto f(int& x) { return x; } + decltype(auto) g(int& x) { return x; } -fi -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ax_cv_PTHREAD_SPECIAL_FLAGS" >&5 -printf "%s\n" "$ax_cv_PTHREAD_SPECIAL_FLAGS" >&6; } - if test "x$ax_cv_PTHREAD_SPECIAL_FLAGS" != "xno" && \ - test "x$ax_pthread_special_flags_added" != "xyes" -then : - PTHREAD_CFLAGS="$ax_cv_PTHREAD_SPECIAL_FLAGS $PTHREAD_CFLAGS" - ax_pthread_special_flags_added=yes -fi + template < typename T1, typename T2 > + struct is_same + { + static constexpr auto value = false; + }; - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for PTHREAD_PRIO_INHERIT" >&5 -printf %s "checking for PTHREAD_PRIO_INHERIT... " >&6; } -if test ${ax_cv_PTHREAD_PRIO_INHERIT+y} -then : - printf %s "(cached) " >&6 -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ -#include -int -main (void) -{ -int i = PTHREAD_PRIO_INHERIT; - return i; - ; - return 0; -} -_ACEOF -if ac_fn_c_try_link "$LINENO" -then : - ax_cv_PTHREAD_PRIO_INHERIT=yes -else $as_nop - ax_cv_PTHREAD_PRIO_INHERIT=no -fi -rm -f core conftest.err conftest.$ac_objext conftest.beam \ - conftest$ac_exeext conftest.$ac_ext + template < typename T > + struct is_same + { + static constexpr auto value = true; + }; -fi -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ax_cv_PTHREAD_PRIO_INHERIT" >&5 -printf "%s\n" "$ax_cv_PTHREAD_PRIO_INHERIT" >&6; } - if test "x$ax_cv_PTHREAD_PRIO_INHERIT" = "xyes" && \ - test "x$ax_pthread_prio_inherit_defined" != "xyes" -then : + int + test() + { + auto x = 0; + static_assert(is_same::value, ""); + static_assert(is_same::value, ""); + return x; + } -printf "%s\n" "#define HAVE_PTHREAD_PRIO_INHERIT 1" >>confdefs.h + } - ax_pthread_prio_inherit_defined=yes +} // namespace cxx14 -fi +#endif // __cplusplus >= 201402L - CFLAGS="$ax_pthread_save_CFLAGS" - LIBS="$ax_pthread_save_LIBS" - # More AIX lossage: compile with *_r variant - if test "x$GCC" != "xyes"; then - case $host_os in - aix*) - case "x/$CC" in #( - x*/c89|x*/c89_128|x*/c99|x*/c99_128|x*/cc|x*/cc128|x*/xlc|x*/xlc_v6|x*/xlc128|x*/xlc128_v6) : - #handle absolute path differently from PATH based program lookup - case "x$CC" in #( - x/*) : - if as_fn_executable_p ${CC}_r -then : - PTHREAD_CC="${CC}_r" -fi - if test "x${CXX}" != "x" -then : - if as_fn_executable_p ${CXX}_r -then : - PTHREAD_CXX="${CXX}_r" -fi -fi - ;; #( - *) : - for ac_prog in ${CC}_r -do - # Extract the first word of "$ac_prog", so it can be a program name with args. -set dummy $ac_prog; ac_word=$2 -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 -printf %s "checking for $ac_word... " >&6; } -if test ${ac_cv_prog_PTHREAD_CC+y} -then : - printf %s "(cached) " >&6 -else $as_nop - if test -n "$PTHREAD_CC"; then - ac_cv_prog_PTHREAD_CC="$PTHREAD_CC" # Let the user override the test. -else -as_save_IFS=$IFS; IFS=$PATH_SEPARATOR -for as_dir in $PATH -do - IFS=$as_save_IFS - case $as_dir in #((( - '') as_dir=./ ;; - */) ;; - *) as_dir=$as_dir/ ;; - esac - for ac_exec_ext in '' $ac_executable_extensions; do - if as_fn_executable_p "$as_dir$ac_word$ac_exec_ext"; then - ac_cv_prog_PTHREAD_CC="$ac_prog" - printf "%s\n" "$as_me:${as_lineno-$LINENO}: found $as_dir$ac_word$ac_exec_ext" >&5 - break 2 - fi -done - done -IFS=$as_save_IFS +// If the compiler admits that it is not ready for C++17, why torture it? +// Hopefully, this will speed up the test. -fi -fi -PTHREAD_CC=$ac_cv_prog_PTHREAD_CC -if test -n "$PTHREAD_CC"; then - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $PTHREAD_CC" >&5 -printf "%s\n" "$PTHREAD_CC" >&6; } -else - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 -printf "%s\n" "no" >&6; } -fi +#ifndef __cplusplus +#error "This is not a C++ compiler" - test -n "$PTHREAD_CC" && break -done -test -n "$PTHREAD_CC" || PTHREAD_CC="$CC" +#elif (defined _MSVC_LANG ? _MSVC_LANG : __cplusplus) < 201703L - if test "x${CXX}" != "x" -then : - for ac_prog in ${CXX}_r -do - # Extract the first word of "$ac_prog", so it can be a program name with args. -set dummy $ac_prog; ac_word=$2 -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 -printf %s "checking for $ac_word... " >&6; } -if test ${ac_cv_prog_PTHREAD_CXX+y} -then : - printf %s "(cached) " >&6 -else $as_nop - if test -n "$PTHREAD_CXX"; then - ac_cv_prog_PTHREAD_CXX="$PTHREAD_CXX" # Let the user override the test. -else -as_save_IFS=$IFS; IFS=$PATH_SEPARATOR -for as_dir in $PATH -do - IFS=$as_save_IFS - case $as_dir in #((( - '') as_dir=./ ;; - */) ;; - *) as_dir=$as_dir/ ;; - esac - for ac_exec_ext in '' $ac_executable_extensions; do - if as_fn_executable_p "$as_dir$ac_word$ac_exec_ext"; then - ac_cv_prog_PTHREAD_CXX="$ac_prog" - printf "%s\n" "$as_me:${as_lineno-$LINENO}: found $as_dir$ac_word$ac_exec_ext" >&5 - break 2 - fi -done - done -IFS=$as_save_IFS +#error "This is not a C++17 compiler" -fi -fi -PTHREAD_CXX=$ac_cv_prog_PTHREAD_CXX -if test -n "$PTHREAD_CXX"; then - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $PTHREAD_CXX" >&5 -printf "%s\n" "$PTHREAD_CXX" >&6; } -else - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 -printf "%s\n" "no" >&6; } -fi +#else +#include +#include +#include - test -n "$PTHREAD_CXX" && break -done -test -n "$PTHREAD_CXX" || PTHREAD_CXX="$CXX" +namespace cxx17 +{ -fi + namespace test_constexpr_lambdas + { - ;; -esac - ;; #( - *) : - ;; -esac - ;; - esac - fi -fi + constexpr int foo = [](){return 42;}(); -test -n "$PTHREAD_CC" || PTHREAD_CC="$CC" -test -n "$PTHREAD_CXX" || PTHREAD_CXX="$CXX" + } + namespace test::nested_namespace::definitions + { + } + namespace test_fold_expression + { + template + int multiply(Args... args) + { + return (args * ... * 1); + } + template + bool all(Args... args) + { + return (args && ...); + } -# Finally, execute ACTION-IF-FOUND/ACTION-IF-NOT-FOUND: -if test "x$ax_pthread_ok" = "xyes"; then + } - : -else - ax_pthread_ok=no + namespace test_extended_static_assert + { - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: WARNING: threads requested but not supported" >&5 -printf "%s\n" "$as_me: WARNING: threads requested but not supported" >&2;} - CARES_THREADS=no + static_assert (true); -fi -ac_ext=c -ac_cpp='$CPP $CPPFLAGS' -ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5' -ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5' -ac_compiler_gnu=$ac_cv_c_compiler_gnu + } + namespace test_auto_brace_init_list + { + auto foo = {5}; + auto bar {5}; - if test "${CARES_THREADS}" = "yes" ; then - ac_fn_c_check_header_compile "$LINENO" "pthread.h" "ac_cv_header_pthread_h" "$ac_includes_default" -if test "x$ac_cv_header_pthread_h" = xyes -then : - printf "%s\n" "#define HAVE_PTHREAD_H 1" >>confdefs.h + static_assert(std::is_same, decltype(foo)>::value); + static_assert(std::is_same::value); + } -fi -ac_fn_c_check_header_compile "$LINENO" "pthread_np.h" "ac_cv_header_pthread_np_h" "$ac_includes_default" -if test "x$ac_cv_header_pthread_np_h" = xyes -then : - printf "%s\n" "#define HAVE_PTHREAD_NP_H 1" >>confdefs.h + namespace test_typename_in_template_template_parameter + { -fi + template typename X> struct D; - LIBS="$PTHREAD_LIBS $LIBS" - AM_CFLAGS="$AM_CFLAGS $PTHREAD_CFLAGS" - CC="$PTHREAD_CC" - CXX="$PTHREAD_CXX" - fi -fi + } -if test "${CARES_THREADS}" = "yes" ; then + namespace test_fallthrough_nodiscard_maybe_unused_attributes + { -printf "%s\n" "#define CARES_THREADS 1 " >>confdefs.h + int f1() + { + return 42; + } -fi + [[nodiscard]] int f2() + { + [[maybe_unused]] auto unused = f1(); + + switch (f1()) + { + case 17: + f1(); + [[fallthrough]]; + case 42: + f1(); + } + return f1(); + } -CARES_PRIVATE_LIBS="$LIBS" + } + + namespace test_extended_aggregate_initialization + { + struct base1 + { + int b1, b2 = 42; + }; -BUILD_SUBDIRS="include src docs" + struct base2 + { + base2() { + b3 = 42; + } + int b3; + }; + struct derived : base1, base2 + { + int d; + }; + derived d1 {{1, 2}, {}, 4}; // full initialization + derived d2 {{}, {}, 4}; // value-initialized bases -if test "x$build_tests" != "xno" -a "x$HAVE_CXX14" = "0" ; then - if test "x$build_tests" = "xmaybe" ; then - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: WARNING: cannot build tests without a CXX14 compiler" >&5 -printf "%s\n" "$as_me: WARNING: cannot build tests without a CXX14 compiler" >&2;} - build_tests=no - else - as_fn_error $? "*** Building tests requires a CXX14 compiler" "$LINENO" 5 - fi -fi -if test "x$build_tests" != "xno" -a "x$cross_compiling" = "xyes" ; then - if test "x$build_tests" = "xmaybe" ; then - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: WARNING: cannot build tests when cross compiling" >&5 -printf "%s\n" "$as_me: WARNING: cannot build tests when cross compiling" >&2;} - build_tests=no - else - as_fn_error $? "*** Tests not supported when cross compiling" "$LINENO" 5 - fi -fi + } -# Check whether --enable-tests-crossbuild was given. -if test ${enable_tests_crossbuild+y} -then : - enableval=$enable_tests_crossbuild; build_tests="$enableval" + namespace test_general_range_based_for_loop + { -fi + struct iter + { + int i; + int& operator* () + { + return i; + } -if test "x$build_tests" != "xno" ; then + const int& operator* () const + { + return i; + } + iter& operator++() + { + ++i; + return *this; + } + }; + struct sentinel + { + int i; + }; + bool operator== (const iter& i, const sentinel& s) + { + return i.i == s.i; + } + bool operator!= (const iter& i, const sentinel& s) + { + return !(i == s); + } + struct range + { + iter begin() const + { + return {0}; + } + sentinel end() const + { + return {5}; + } + }; -if test "x$ac_cv_env_PKG_CONFIG_set" != "xset"; then - if test -n "$ac_tool_prefix"; then - # Extract the first word of "${ac_tool_prefix}pkg-config", so it can be a program name with args. -set dummy ${ac_tool_prefix}pkg-config; ac_word=$2 -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 -printf %s "checking for $ac_word... " >&6; } -if test ${ac_cv_path_PKG_CONFIG+y} -then : - printf %s "(cached) " >&6 -else $as_nop - case $PKG_CONFIG in - [\\/]* | ?:[\\/]*) - ac_cv_path_PKG_CONFIG="$PKG_CONFIG" # Let the user override the test with a path. - ;; - *) - as_save_IFS=$IFS; IFS=$PATH_SEPARATOR -for as_dir in $PATH -do - IFS=$as_save_IFS - case $as_dir in #((( - '') as_dir=./ ;; - */) ;; - *) as_dir=$as_dir/ ;; - esac - for ac_exec_ext in '' $ac_executable_extensions; do - if as_fn_executable_p "$as_dir$ac_word$ac_exec_ext"; then - ac_cv_path_PKG_CONFIG="$as_dir$ac_word$ac_exec_ext" - printf "%s\n" "$as_me:${as_lineno-$LINENO}: found $as_dir$ac_word$ac_exec_ext" >&5 - break 2 - fi -done - done -IFS=$as_save_IFS + void f() + { + range r {}; - ;; -esac -fi -PKG_CONFIG=$ac_cv_path_PKG_CONFIG -if test -n "$PKG_CONFIG"; then - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $PKG_CONFIG" >&5 -printf "%s\n" "$PKG_CONFIG" >&6; } -else - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 -printf "%s\n" "no" >&6; } -fi + for (auto i : r) + { + [[maybe_unused]] auto v = i; + } + } + } -fi -if test -z "$ac_cv_path_PKG_CONFIG"; then - ac_pt_PKG_CONFIG=$PKG_CONFIG - # Extract the first word of "pkg-config", so it can be a program name with args. -set dummy pkg-config; ac_word=$2 -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 -printf %s "checking for $ac_word... " >&6; } -if test ${ac_cv_path_ac_pt_PKG_CONFIG+y} -then : - printf %s "(cached) " >&6 -else $as_nop - case $ac_pt_PKG_CONFIG in - [\\/]* | ?:[\\/]*) - ac_cv_path_ac_pt_PKG_CONFIG="$ac_pt_PKG_CONFIG" # Let the user override the test with a path. - ;; - *) - as_save_IFS=$IFS; IFS=$PATH_SEPARATOR -for as_dir in $PATH -do - IFS=$as_save_IFS - case $as_dir in #((( - '') as_dir=./ ;; - */) ;; - *) as_dir=$as_dir/ ;; - esac - for ac_exec_ext in '' $ac_executable_extensions; do - if as_fn_executable_p "$as_dir$ac_word$ac_exec_ext"; then - ac_cv_path_ac_pt_PKG_CONFIG="$as_dir$ac_word$ac_exec_ext" - printf "%s\n" "$as_me:${as_lineno-$LINENO}: found $as_dir$ac_word$ac_exec_ext" >&5 - break 2 - fi -done - done -IFS=$as_save_IFS + namespace test_lambda_capture_asterisk_this_by_value + { - ;; -esac -fi -ac_pt_PKG_CONFIG=$ac_cv_path_ac_pt_PKG_CONFIG -if test -n "$ac_pt_PKG_CONFIG"; then - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_pt_PKG_CONFIG" >&5 -printf "%s\n" "$ac_pt_PKG_CONFIG" >&6; } -else - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 -printf "%s\n" "no" >&6; } -fi + struct t + { + int i; + int foo() + { + return [*this]() + { + return i; + }(); + } + }; - if test "x$ac_pt_PKG_CONFIG" = x; then - PKG_CONFIG="" - else - case $cross_compiling:$ac_tool_warned in -yes:) -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: WARNING: using cross tools not prefixed with host triplet" >&5 -printf "%s\n" "$as_me: WARNING: using cross tools not prefixed with host triplet" >&2;} -ac_tool_warned=yes ;; -esac - PKG_CONFIG=$ac_pt_PKG_CONFIG - fi -else - PKG_CONFIG="$ac_cv_path_PKG_CONFIG" -fi + } -fi -if test -n "$PKG_CONFIG"; then - _pkg_min_version=0.9.0 - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking pkg-config is at least version $_pkg_min_version" >&5 -printf %s "checking pkg-config is at least version $_pkg_min_version... " >&6; } - if $PKG_CONFIG --atleast-pkgconfig-version $_pkg_min_version; then - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: yes" >&5 -printf "%s\n" "yes" >&6; } - else - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 -printf "%s\n" "no" >&6; } - PKG_CONFIG="" - fi -fi + namespace test_enum_class_construction + { -pkg_failed=no -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for gmock" >&5 -printf %s "checking for gmock... " >&6; } + enum class byte : unsigned char + {}; -if test -n "$GMOCK_CFLAGS"; then - pkg_cv_GMOCK_CFLAGS="$GMOCK_CFLAGS" - elif test -n "$PKG_CONFIG"; then - if test -n "$PKG_CONFIG" && \ - { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"gmock\""; } >&5 - ($PKG_CONFIG --exists --print-errors "gmock") 2>&5 - ac_status=$? - printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 - test $ac_status = 0; }; then - pkg_cv_GMOCK_CFLAGS=`$PKG_CONFIG --cflags "gmock" 2>/dev/null` - test "x$?" != "x0" && pkg_failed=yes -else - pkg_failed=yes -fi - else - pkg_failed=untried -fi -if test -n "$GMOCK_LIBS"; then - pkg_cv_GMOCK_LIBS="$GMOCK_LIBS" - elif test -n "$PKG_CONFIG"; then - if test -n "$PKG_CONFIG" && \ - { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"gmock\""; } >&5 - ($PKG_CONFIG --exists --print-errors "gmock") 2>&5 - ac_status=$? - printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 - test $ac_status = 0; }; then - pkg_cv_GMOCK_LIBS=`$PKG_CONFIG --libs "gmock" 2>/dev/null` - test "x$?" != "x0" && pkg_failed=yes -else - pkg_failed=yes -fi - else - pkg_failed=untried -fi + byte foo {42}; + } + namespace test_constexpr_if + { -if test $pkg_failed = yes; then - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 -printf "%s\n" "no" >&6; } + template + int f () + { + if constexpr(cond) + { + return 13; + } + else + { + return 42; + } + } -if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then - _pkg_short_errors_supported=yes -else - _pkg_short_errors_supported=no -fi - if test $_pkg_short_errors_supported = yes; then - GMOCK_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "gmock" 2>&1` - else - GMOCK_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "gmock" 2>&1` - fi - # Put the nasty error message in config.log where it belongs - echo "$GMOCK_PKG_ERRORS" >&5 + } - have_gmock=no -elif test $pkg_failed = untried; then - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 -printf "%s\n" "no" >&6; } - have_gmock=no -else - GMOCK_CFLAGS=$pkg_cv_GMOCK_CFLAGS - GMOCK_LIBS=$pkg_cv_GMOCK_LIBS - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: yes" >&5 -printf "%s\n" "yes" >&6; } - have_gmock=yes -fi - if test "x$have_gmock" = "xno" ; then - if test "x$build_tests" = "xmaybe" ; then - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: WARNING: gmock could not be found, not building tests" >&5 -printf "%s\n" "$as_me: WARNING: gmock could not be found, not building tests" >&2;} - build_tests=no - else - as_fn_error $? "tests require gmock" "$LINENO" 5 - fi - else + namespace test_selection_statement_with_initializer + { -pkg_failed=no -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for gmock >= 1.12.0" >&5 -printf %s "checking for gmock >= 1.12.0... " >&6; } + int f() + { + return 13; + } -if test -n "$GMOCK112_CFLAGS"; then - pkg_cv_GMOCK112_CFLAGS="$GMOCK112_CFLAGS" - elif test -n "$PKG_CONFIG"; then - if test -n "$PKG_CONFIG" && \ - { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"gmock >= 1.12.0\""; } >&5 - ($PKG_CONFIG --exists --print-errors "gmock >= 1.12.0") 2>&5 - ac_status=$? - printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 - test $ac_status = 0; }; then - pkg_cv_GMOCK112_CFLAGS=`$PKG_CONFIG --cflags "gmock >= 1.12.0" 2>/dev/null` - test "x$?" != "x0" && pkg_failed=yes -else - pkg_failed=yes -fi - else - pkg_failed=untried -fi -if test -n "$GMOCK112_LIBS"; then - pkg_cv_GMOCK112_LIBS="$GMOCK112_LIBS" - elif test -n "$PKG_CONFIG"; then - if test -n "$PKG_CONFIG" && \ - { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"gmock >= 1.12.0\""; } >&5 - ($PKG_CONFIG --exists --print-errors "gmock >= 1.12.0") 2>&5 - ac_status=$? - printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 - test $ac_status = 0; }; then - pkg_cv_GMOCK112_LIBS=`$PKG_CONFIG --libs "gmock >= 1.12.0" 2>/dev/null` - test "x$?" != "x0" && pkg_failed=yes -else - pkg_failed=yes -fi - else - pkg_failed=untried -fi + int f2() + { + if (auto i = f(); i > 0) + { + return 3; + } + switch (auto i = f(); i + 4) + { + case 17: + return 2; + default: + return 1; + } + } -if test $pkg_failed = yes; then - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 -printf "%s\n" "no" >&6; } + } -if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then - _pkg_short_errors_supported=yes -else - _pkg_short_errors_supported=no -fi - if test $_pkg_short_errors_supported = yes; then - GMOCK112_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "gmock >= 1.12.0" 2>&1` - else - GMOCK112_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "gmock >= 1.12.0" 2>&1` - fi - # Put the nasty error message in config.log where it belongs - echo "$GMOCK112_PKG_ERRORS" >&5 + namespace test_template_argument_deduction_for_class_templates + { - have_gmock_v112=no -elif test $pkg_failed = untried; then - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 -printf "%s\n" "no" >&6; } - have_gmock_v112=no -else - GMOCK112_CFLAGS=$pkg_cv_GMOCK112_CFLAGS - GMOCK112_LIBS=$pkg_cv_GMOCK112_LIBS - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: yes" >&5 -printf "%s\n" "yes" >&6; } - have_gmock_v112=yes -fi - if test "x$have_gmock_v112" = "xyes" ; then - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether user namespaces are supported" >&5 -printf %s "checking whether user namespaces are supported... " >&6; } -if test ${ares_cv_user_namespace+y} -then : - printf %s "(cached) " >&6 -else $as_nop + template + struct pair + { + pair (T1 p1, T2 p2) + : m1 {p1}, + m2 {p2} + {} - ac_ext=c -ac_cpp='$CPP $CPPFLAGS' -ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5' -ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5' -ac_compiler_gnu=$ac_cv_c_compiler_gnu + T1 m1; + T2 m2; + }; - if test "$cross_compiling" = yes -then : - ares_cv_user_namespace=no -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ + void f() + { + [[maybe_unused]] auto p = pair{13, 42u}; + } -#define _GNU_SOURCE -#include -#include -#include -#include -#include -#include -#include + } -int userfn(void *d) { - usleep(100000); /* synchronize by sleep */ - return (getuid() != 0); -} -char userst[1024*1024]; -int main() { - char buffer[1024]; - int rc, status, fd; - pid_t child = clone(userfn, userst + 1024*1024, CLONE_NEWUSER|SIGCHLD, 0); - if (child < 0) return 1; + namespace test_non_type_auto_template_parameters + { - snprintf(buffer, sizeof(buffer), "/proc/%d/uid_map", child); - fd = open(buffer, O_CREAT|O_WRONLY|O_TRUNC, 0755); - snprintf(buffer, sizeof(buffer), "0 %d 1\n", getuid()); - write(fd, buffer, strlen(buffer)); - close(fd); + template + struct B + {}; - rc = waitpid(child, &status, 0); - if (rc <= 0) return 1; - if (!WIFEXITED(status)) return 1; - return WEXITSTATUS(status); -} + B<5> b1; + B<'a'> b2; -_ACEOF -if ac_fn_c_try_run "$LINENO" -then : - ares_cv_user_namespace=yes -else $as_nop - ares_cv_user_namespace=no -fi -rm -f core *.core core.conftest.* gmon.out bb.out conftest$ac_exeext \ - conftest.$ac_objext conftest.beam conftest.$ac_ext -fi + } - ac_ext=c -ac_cpp='$CPP $CPPFLAGS' -ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5' -ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5' -ac_compiler_gnu=$ac_cv_c_compiler_gnu + namespace test_structured_bindings + { + int arr[2] = { 1, 2 }; + std::pair pr = { 1, 2 }; -fi -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ares_cv_user_namespace" >&5 -printf "%s\n" "$ares_cv_user_namespace" >&6; } - if test "$ares_cv_user_namespace" = yes; then + auto f1() -> int(&)[2] + { + return arr; + } -printf "%s\n" "#define HAVE_USER_NAMESPACE 1" >>confdefs.h + auto f2() -> std::pair& + { + return pr; + } - fi + struct S + { + int x1 : 2; + volatile double y1; + }; - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether UTS namespaces are supported" >&5 -printf %s "checking whether UTS namespaces are supported... " >&6; } -if test ${ares_cv_uts_namespace+y} -then : - printf %s "(cached) " >&6 -else $as_nop + S f3() + { + return {}; + } - ac_ext=c -ac_cpp='$CPP $CPPFLAGS' -ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5' -ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5' -ac_compiler_gnu=$ac_cv_c_compiler_gnu + auto [ x1, y1 ] = f1(); + auto& [ xr1, yr1 ] = f1(); + auto [ x2, y2 ] = f2(); + auto& [ xr2, yr2 ] = f2(); + const auto [ x3, y3 ] = f3(); - if test "$cross_compiling" = yes -then : - ares_cv_uts_namespace=no -else $as_nop - cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ + } -#define _GNU_SOURCE -#include -#include -#include -#include -#include -#include -#include -#include + namespace test_exception_spec_type_system + { -int utsfn(void *d) { - char buffer[1024]; - const char *name = "autoconftest"; - int rc = sethostname(name, strlen(name)); - if (rc != 0) return 1; - gethostname(buffer, 1024); - return (strcmp(buffer, name) != 0); -} + struct Good {}; + struct Bad {}; -char st2[1024*1024]; -int fn(void *d) { - pid_t child; - int rc, status; - usleep(100000); /* synchronize by sleep */ - if (getuid() != 0) return 1; - child = clone(utsfn, st2 + 1024*1024, CLONE_NEWUTS|SIGCHLD, 0); - if (child < 0) return 1; - rc = waitpid(child, &status, 0); - if (rc <= 0) return 1; - if (!WIFEXITED(status)) return 1; - return WEXITSTATUS(status); -} -char st[1024*1024]; -int main() { - char buffer[1024]; - int rc, status, fd; - pid_t child = clone(fn, st + 1024*1024, CLONE_NEWUSER|SIGCHLD, 0); - if (child < 0) return 1; + void g1() noexcept; + void g2(); - snprintf(buffer, sizeof(buffer), "/proc/%d/uid_map", child); - fd = open(buffer, O_CREAT|O_WRONLY|O_TRUNC, 0755); - snprintf(buffer, sizeof(buffer), "0 %d 1\n", getuid()); - write(fd, buffer, strlen(buffer)); - close(fd); + template + Bad + f(T*, T*); + + template + Good + f(T1*, T2*); + + static_assert (std::is_same_v); + + } + + namespace test_inline_variables + { + + template void f(T) + {} + + template inline T g(T) + { + return T{}; + } + + template<> inline void f<>(int) + {} + + template<> int g<>(int) + { + return 5; + } + + } + +} // namespace cxx17 + +#endif // (defined _MSVC_LANG ? _MSVC_LANG : __cplusplus) < 201703L - rc = waitpid(child, &status, 0); - if (rc <= 0) return 1; - if (!WIFEXITED(status)) return 1; - return WEXITSTATUS(status); -} _ACEOF -if ac_fn_c_try_run "$LINENO" +if ac_fn_cxx_try_compile "$LINENO" then : - ares_cv_uts_namespace=yes + eval $cachevar=yes else $as_nop - ares_cv_uts_namespace=no + eval $cachevar=no fi -rm -f core *.core core.conftest.* gmon.out bb.out conftest$ac_exeext \ - conftest.$ac_objext conftest.beam conftest.$ac_ext +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext + CXX="$ac_save_CXX" fi - - ac_ext=c +eval ac_res=\$$cachevar + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_res" >&5 +printf "%s\n" "$ac_res" >&6; } + if eval test x\$$cachevar = xyes; then + CXX="$CXX $switch" + if test -n "$CXXCPP" ; then + CXXCPP="$CXXCPP $switch" + fi + ac_success=yes + break + fi + done + if test x$ac_success = xyes; then + break + fi + done + fi + ac_ext=c ac_cpp='$CPP $CPPFLAGS' ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5' ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5' ac_compiler_gnu=$ac_cv_c_compiler_gnu + if test x$ax_cxx_compile_cxx17_required = xtrue; then + if test x$ac_success = xno; then + as_fn_error $? "*** A compiler with support for C++17 language features is required." "$LINENO" 5 + fi + fi + if test x$ac_success = xno; then + HAVE_CXX17=0 + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: No compiler with C++17 support was found" >&5 +printf "%s\n" "$as_me: No compiler with C++17 support was found" >&6;} + else + HAVE_CXX17=1 -fi -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ares_cv_uts_namespace" >&5 -printf "%s\n" "$ares_cv_uts_namespace" >&6; } - if test "$ares_cv_uts_namespace" = yes; then - -printf "%s\n" "#define HAVE_UTS_NAMESPACE 1" >>confdefs.h - - fi +printf "%s\n" "#define HAVE_CXX17 1" >>confdefs.h - fi fi -fi -if test "x$build_tests" != "xno" ; then - build_tests=yes - ax_cxx_compile_alternatives="14 1y" ax_cxx_compile_cxx14_required=true + + else + ax_cxx_compile_alternatives="14 1y" ax_cxx_compile_cxx14_required=true ac_ext=cpp ac_cpp='$CXXCPP $CPPFLAGS' ac_compile='$CXX -c $CXXFLAGS $CPPFLAGS conftest.$ac_ext >&5' @@ -24919,6 +26893,12 @@ printf "%s\n" "#define HAVE_CXX14 1" >>confdefs.h fi + fi + fi +fi +if test "x$build_tests" != "xno" ; then + build_tests=yes + if test "$ac_cv_native_windows" != "yes" ; then @@ -25023,7 +27003,7 @@ ax_pthread_flags="pthreads none -Kthread -pthread -pthreads -mthreads pthread -- case $host_os in - freebsd*) + freebsd*|midnightbsd*) # -kthread: FreeBSD kernel threads (preferred to -pthread since SMP-able) # lthread: LinuxThreads port on FreeBSD (also preferred to -pthread) @@ -26258,7 +28238,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 # report actual input values of CONFIG_FILES etc. instead of their # values after options handling. ac_log=" -This file was extended by c-ares $as_me 1.34.4, which was +This file was extended by c-ares $as_me 1.34.6, which was generated by GNU Autoconf 2.71. Invocation command line was CONFIG_FILES = $CONFIG_FILES @@ -26326,7 +28306,7 @@ ac_cs_config_escaped=`printf "%s\n" "$ac_cs_config" | sed "s/^ //; s/'/'\\\\\\\\ cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 ac_cs_config='$ac_cs_config_escaped' ac_cs_version="\\ -c-ares config.status 1.34.4 +c-ares config.status 1.34.6 configured by $0, generated by GNU Autoconf 2.71, with options \\"\$ac_cs_config\\" @@ -26492,12 +28472,14 @@ lt_cv_to_host_file_cmd='`$ECHO "$lt_cv_to_host_file_cmd" | $SED "$delay_single_q lt_cv_to_tool_file_cmd='`$ECHO "$lt_cv_to_tool_file_cmd" | $SED "$delay_single_quote_subst"`' reload_flag='`$ECHO "$reload_flag" | $SED "$delay_single_quote_subst"`' reload_cmds='`$ECHO "$reload_cmds" | $SED "$delay_single_quote_subst"`' +FILECMD='`$ECHO "$FILECMD" | $SED "$delay_single_quote_subst"`' deplibs_check_method='`$ECHO "$deplibs_check_method" | $SED "$delay_single_quote_subst"`' file_magic_cmd='`$ECHO "$file_magic_cmd" | $SED "$delay_single_quote_subst"`' file_magic_glob='`$ECHO "$file_magic_glob" | $SED "$delay_single_quote_subst"`' want_nocaseglob='`$ECHO "$want_nocaseglob" | $SED "$delay_single_quote_subst"`' sharedlib_from_linklib_cmd='`$ECHO "$sharedlib_from_linklib_cmd" | $SED "$delay_single_quote_subst"`' AR='`$ECHO "$AR" | $SED "$delay_single_quote_subst"`' +lt_ar_flags='`$ECHO "$lt_ar_flags" | $SED "$delay_single_quote_subst"`' AR_FLAGS='`$ECHO "$AR_FLAGS" | $SED "$delay_single_quote_subst"`' archiver_list_spec='`$ECHO "$archiver_list_spec" | $SED "$delay_single_quote_subst"`' STRIP='`$ECHO "$STRIP" | $SED "$delay_single_quote_subst"`' @@ -26675,13 +28657,13 @@ LN_S \ lt_SP2NL \ lt_NL2SP \ reload_flag \ +FILECMD \ deplibs_check_method \ file_magic_cmd \ file_magic_glob \ want_nocaseglob \ sharedlib_from_linklib_cmd \ AR \ -AR_FLAGS \ archiver_list_spec \ STRIP \ RANLIB \ @@ -27684,6 +29666,9 @@ to_host_file_cmd=$lt_cv_to_host_file_cmd # convert \$build files to toolchain format. to_tool_file_cmd=$lt_cv_to_tool_file_cmd +# A file(cmd) program that detects file types. +FILECMD=$lt_FILECMD + # Method to check whether dependent libraries are shared objects. deplibs_check_method=$lt_deplibs_check_method @@ -27702,8 +29687,11 @@ sharedlib_from_linklib_cmd=$lt_sharedlib_from_linklib_cmd # The archiver. AR=$lt_AR +# Flags to create an archive (by configure). +lt_ar_flags=$lt_ar_flags + # Flags to create an archive. -AR_FLAGS=$lt_AR_FLAGS +AR_FLAGS=\${ARFLAGS-"\$lt_ar_flags"} # How to feed a file listing to the archiver. archiver_list_spec=$lt_archiver_list_spec @@ -28093,7 +30081,7 @@ ltmain=$ac_aux_dir/ltmain.sh # if finds mixed CR/LF and LF-only lines. Since sed operates in # text mode, it properly converts lines to CR/LF. This bash problem # is reportedly fixed, but why not run on old versions too? - sed '$q' "$ltmain" >> "$cfgfile" \ + $SED '$q' "$ltmain" >> "$cfgfile" \ || (rm -f "$cfgfile"; exit 1) mv -f "$cfgfile" "$ofile" || diff --git a/source/lib/c-ares-1.34.4/configure.ac b/source/lib/c-ares-1.34.6/configure.ac similarity index 98% rename from source/lib/c-ares-1.34.4/configure.ac rename to source/lib/c-ares-1.34.6/configure.ac index 9dacf1fb..744a99be 100644 --- a/source/lib/c-ares-1.34.4/configure.ac +++ b/source/lib/c-ares-1.34.6/configure.ac @@ -2,10 +2,10 @@ dnl Copyright (C) The c-ares project and its contributors dnl SPDX-License-Identifier: MIT AC_PREREQ([2.69]) -AC_INIT([c-ares], [1.34.4], +AC_INIT([c-ares], [1.34.6], [c-ares mailing list: http://lists.haxx.se/listinfo/c-ares]) -CARES_VERSION_INFO="21:3:19" +CARES_VERSION_INFO="21:5:19" dnl This flag accepts an argument of the form current[:revision[:age]]. So, dnl passing -version-info 3:12:1 sets current to 3, revision to 12, and age to dnl 1. @@ -40,7 +40,8 @@ AC_CONFIG_HEADERS([src/lib/ares_config.h include/ares_build.h]) AC_CONFIG_AUX_DIR(config) AC_CONFIG_MACRO_DIR([m4]) AC_USE_SYSTEM_EXTENSIONS -AX_CXX_COMPILE_STDCXX_14([noext],[optional]) +AX_CXX_COMPILE_STDCXX(14,[noext],[optional]) +AX_CXX_COMPILE_STDCXX(17,[noext],[optional]) AM_INIT_AUTOMAKE([foreign subdir-objects 1.9.6]) LT_INIT([win32-dll,pic,disable-fast-install,aix-soname=svr4]) AC_LANG([C]) @@ -594,6 +595,7 @@ AC_CHECK_DECL(pipe, [AC_DEFINE([HAVE_PIPE], 1, [Define t AC_CHECK_DECL(pipe2, [AC_DEFINE([HAVE_PIPE2], 1, [Define to 1 if you have `pipe2`] )], [], $cares_all_includes) AC_CHECK_DECL(kqueue, [AC_DEFINE([HAVE_KQUEUE], 1, [Define to 1 if you have `kqueue`] )], [], $cares_all_includes) AC_CHECK_DECL(epoll_create1, [AC_DEFINE([HAVE_EPOLL], 1, [Define to 1 if you have `epoll_{create1,ctl,wait}`])], [], $cares_all_includes) +AC_CHECK_DECL(GetBestRoute2, [AC_DEFINE([HAVE_GETBESTROUTE2], 1, [Define to 1 if you have `GetBestRoute2`] )], [], $cares_all_includes) AC_CHECK_DECL(ConvertInterfaceIndexToLuid, [AC_DEFINE([HAVE_CONVERTINTERFACEINDEXTOLUID], 1, [Define to 1 if you have `ConvertInterfaceIndexToLuid`])], [], $cares_all_includes) AC_CHECK_DECL(ConvertInterfaceLuidToNameA, [AC_DEFINE([HAVE_CONVERTINTERFACELUIDTONAMEA], 1, [Define to 1 if you have `ConvertInterfaceLuidToNameA`])], [], $cares_all_includes) AC_CHECK_DECL(NotifyIpInterfaceChange, [AC_DEFINE([HAVE_NOTIFYIPINTERFACECHANGE], 1, [Define to 1 if you have `NotifyIpInterfaceChange`] )], [], $cares_all_includes) @@ -805,14 +807,6 @@ BUILD_SUBDIRS="include src docs" dnl ******** TESTS ******* -if test "x$build_tests" != "xno" -a "x$HAVE_CXX14" = "0" ; then - if test "x$build_tests" = "xmaybe" ; then - AC_MSG_WARN([cannot build tests without a CXX14 compiler]) - build_tests=no - else - AC_MSG_ERROR([*** Building tests requires a CXX14 compiler]) - fi -fi if test "x$build_tests" != "xno" -a "x$cross_compiling" = "xyes" ; then if test "x$build_tests" = "xmaybe" ; then AC_MSG_WARN([cannot build tests when cross compiling]) @@ -843,12 +837,19 @@ if test "x$build_tests" != "xno" ; then ARES_CHECK_USER_NAMESPACE ARES_CHECK_UTS_NAMESPACE fi + PKG_CHECK_MODULES([GMOCK117], [gmock >= 1.17.0], [ have_gmock_v117=yes ], [ have_gmock_v117=no ]) + if test "x$have_gmock_v117" = "xyes" ; then + dnl GMock v1.17.0 requires C++17 or higher + AX_CXX_COMPILE_STDCXX(17,[noext],[mandatory]) + else + dnl older version needed v1.14.0 + AX_CXX_COMPILE_STDCXX(14,[noext],[mandatory]) + fi fi fi if test "x$build_tests" != "xno" ; then build_tests=yes - AX_CXX_COMPILE_STDCXX_14([noext],[mandatory]) if test "$ac_cv_native_windows" != "yes" ; then AX_PTHREAD([ CARES_TEST_PTHREADS="yes" ], [ AC_MSG_ERROR([threading required for tests]) ]) fi diff --git a/source/lib/c-ares-1.34.4/docs/CMakeLists.txt b/source/lib/c-ares-1.34.6/docs/CMakeLists.txt similarity index 100% rename from source/lib/c-ares-1.34.4/docs/CMakeLists.txt rename to source/lib/c-ares-1.34.6/docs/CMakeLists.txt diff --git a/source/lib/c-ares-1.34.4/docs/Makefile.am b/source/lib/c-ares-1.34.6/docs/Makefile.am similarity index 100% rename from source/lib/c-ares-1.34.4/docs/Makefile.am rename to source/lib/c-ares-1.34.6/docs/Makefile.am diff --git a/source/lib/c-ares-1.34.4/docs/Makefile.in b/source/lib/c-ares-1.34.6/docs/Makefile.in similarity index 99% rename from source/lib/c-ares-1.34.4/docs/Makefile.in rename to source/lib/c-ares-1.34.6/docs/Makefile.in index 0d1873c9..501364eb 100644 --- a/source/lib/c-ares-1.34.4/docs/Makefile.in +++ b/source/lib/c-ares-1.34.6/docs/Makefile.in @@ -107,7 +107,6 @@ am__aclocal_m4_deps = $(top_srcdir)/m4/ares_check_user_namespace.m4 \ $(top_srcdir)/m4/ax_code_coverage.m4 \ $(top_srcdir)/m4/ax_compiler_vendor.m4 \ $(top_srcdir)/m4/ax_cxx_compile_stdcxx.m4 \ - $(top_srcdir)/m4/ax_cxx_compile_stdcxx_14.m4 \ $(top_srcdir)/m4/ax_file_escapes.m4 \ $(top_srcdir)/m4/ax_pthread.m4 \ $(top_srcdir)/m4/ax_require_defined.m4 \ @@ -223,14 +222,18 @@ EGREP = @EGREP@ ETAGS = @ETAGS@ EXEEXT = @EXEEXT@ FGREP = @FGREP@ +FILECMD = @FILECMD@ GCOV = @GCOV@ GENHTML = @GENHTML@ GMOCK112_CFLAGS = @GMOCK112_CFLAGS@ GMOCK112_LIBS = @GMOCK112_LIBS@ +GMOCK117_CFLAGS = @GMOCK117_CFLAGS@ +GMOCK117_LIBS = @GMOCK117_LIBS@ GMOCK_CFLAGS = @GMOCK_CFLAGS@ GMOCK_LIBS = @GMOCK_LIBS@ GREP = @GREP@ HAVE_CXX14 = @HAVE_CXX14@ +HAVE_CXX17 = @HAVE_CXX17@ INSTALL = @INSTALL@ INSTALL_DATA = @INSTALL_DATA@ INSTALL_PROGRAM = @INSTALL_PROGRAM@ diff --git a/source/lib/c-ares-1.34.4/docs/Makefile.inc b/source/lib/c-ares-1.34.6/docs/Makefile.inc similarity index 100% rename from source/lib/c-ares-1.34.4/docs/Makefile.inc rename to source/lib/c-ares-1.34.6/docs/Makefile.inc diff --git a/source/lib/c-ares-1.34.4/docs/adig.1 b/source/lib/c-ares-1.34.6/docs/adig.1 similarity index 100% rename from source/lib/c-ares-1.34.4/docs/adig.1 rename to source/lib/c-ares-1.34.6/docs/adig.1 diff --git a/source/lib/c-ares-1.34.4/docs/ahost.1 b/source/lib/c-ares-1.34.6/docs/ahost.1 similarity index 100% rename from source/lib/c-ares-1.34.4/docs/ahost.1 rename to source/lib/c-ares-1.34.6/docs/ahost.1 diff --git a/source/lib/c-ares-1.34.4/docs/ares_cancel.3 b/source/lib/c-ares-1.34.6/docs/ares_cancel.3 similarity index 100% rename from source/lib/c-ares-1.34.4/docs/ares_cancel.3 rename to source/lib/c-ares-1.34.6/docs/ares_cancel.3 diff --git a/source/lib/c-ares-1.34.4/docs/ares_create_query.3 b/source/lib/c-ares-1.34.6/docs/ares_create_query.3 similarity index 100% rename from source/lib/c-ares-1.34.4/docs/ares_create_query.3 rename to source/lib/c-ares-1.34.6/docs/ares_create_query.3 diff --git a/source/lib/c-ares-1.34.4/docs/ares_destroy.3 b/source/lib/c-ares-1.34.6/docs/ares_destroy.3 similarity index 100% rename from source/lib/c-ares-1.34.4/docs/ares_destroy.3 rename to source/lib/c-ares-1.34.6/docs/ares_destroy.3 diff --git a/source/lib/c-ares-1.34.4/docs/ares_destroy_options.3 b/source/lib/c-ares-1.34.6/docs/ares_destroy_options.3 similarity index 100% rename from source/lib/c-ares-1.34.4/docs/ares_destroy_options.3 rename to source/lib/c-ares-1.34.6/docs/ares_destroy_options.3 diff --git a/source/lib/c-ares-1.34.4/docs/ares_dns_class_fromstr.3 b/source/lib/c-ares-1.34.6/docs/ares_dns_class_fromstr.3 similarity index 100% rename from source/lib/c-ares-1.34.4/docs/ares_dns_class_fromstr.3 rename to source/lib/c-ares-1.34.6/docs/ares_dns_class_fromstr.3 diff --git a/source/lib/c-ares-1.34.4/docs/ares_dns_class_t.3 b/source/lib/c-ares-1.34.6/docs/ares_dns_class_t.3 similarity index 100% rename from source/lib/c-ares-1.34.4/docs/ares_dns_class_t.3 rename to source/lib/c-ares-1.34.6/docs/ares_dns_class_t.3 diff --git a/source/lib/c-ares-1.34.4/docs/ares_dns_class_tostr.3 b/source/lib/c-ares-1.34.6/docs/ares_dns_class_tostr.3 similarity index 100% rename from source/lib/c-ares-1.34.4/docs/ares_dns_class_tostr.3 rename to source/lib/c-ares-1.34.6/docs/ares_dns_class_tostr.3 diff --git a/source/lib/c-ares-1.34.4/docs/ares_dns_datatype_t.3 b/source/lib/c-ares-1.34.6/docs/ares_dns_datatype_t.3 similarity index 100% rename from source/lib/c-ares-1.34.4/docs/ares_dns_datatype_t.3 rename to source/lib/c-ares-1.34.6/docs/ares_dns_datatype_t.3 diff --git a/source/lib/c-ares-1.34.4/docs/ares_dns_flags_t.3 b/source/lib/c-ares-1.34.6/docs/ares_dns_flags_t.3 similarity index 100% rename from source/lib/c-ares-1.34.4/docs/ares_dns_flags_t.3 rename to source/lib/c-ares-1.34.6/docs/ares_dns_flags_t.3 diff --git a/source/lib/c-ares-1.34.4/docs/ares_dns_mapping.3 b/source/lib/c-ares-1.34.6/docs/ares_dns_mapping.3 similarity index 100% rename from source/lib/c-ares-1.34.4/docs/ares_dns_mapping.3 rename to source/lib/c-ares-1.34.6/docs/ares_dns_mapping.3 diff --git a/source/lib/c-ares-1.34.4/docs/ares_dns_opcode_t.3 b/source/lib/c-ares-1.34.6/docs/ares_dns_opcode_t.3 similarity index 100% rename from source/lib/c-ares-1.34.4/docs/ares_dns_opcode_t.3 rename to source/lib/c-ares-1.34.6/docs/ares_dns_opcode_t.3 diff --git a/source/lib/c-ares-1.34.4/docs/ares_dns_opcode_tostr.3 b/source/lib/c-ares-1.34.6/docs/ares_dns_opcode_tostr.3 similarity index 100% rename from source/lib/c-ares-1.34.4/docs/ares_dns_opcode_tostr.3 rename to source/lib/c-ares-1.34.6/docs/ares_dns_opcode_tostr.3 diff --git a/source/lib/c-ares-1.34.4/docs/ares_dns_opt_datatype_t.3 b/source/lib/c-ares-1.34.6/docs/ares_dns_opt_datatype_t.3 similarity index 100% rename from source/lib/c-ares-1.34.4/docs/ares_dns_opt_datatype_t.3 rename to source/lib/c-ares-1.34.6/docs/ares_dns_opt_datatype_t.3 diff --git a/source/lib/c-ares-1.34.4/docs/ares_dns_opt_get_datatype.3 b/source/lib/c-ares-1.34.6/docs/ares_dns_opt_get_datatype.3 similarity index 100% rename from source/lib/c-ares-1.34.4/docs/ares_dns_opt_get_datatype.3 rename to source/lib/c-ares-1.34.6/docs/ares_dns_opt_get_datatype.3 diff --git a/source/lib/c-ares-1.34.4/docs/ares_dns_opt_get_name.3 b/source/lib/c-ares-1.34.6/docs/ares_dns_opt_get_name.3 similarity index 100% rename from source/lib/c-ares-1.34.4/docs/ares_dns_opt_get_name.3 rename to source/lib/c-ares-1.34.6/docs/ares_dns_opt_get_name.3 diff --git a/source/lib/c-ares-1.34.4/docs/ares_dns_parse.3 b/source/lib/c-ares-1.34.6/docs/ares_dns_parse.3 similarity index 100% rename from source/lib/c-ares-1.34.4/docs/ares_dns_parse.3 rename to source/lib/c-ares-1.34.6/docs/ares_dns_parse.3 diff --git a/source/lib/c-ares-1.34.4/docs/ares_dns_rcode_t.3 b/source/lib/c-ares-1.34.6/docs/ares_dns_rcode_t.3 similarity index 100% rename from source/lib/c-ares-1.34.4/docs/ares_dns_rcode_t.3 rename to source/lib/c-ares-1.34.6/docs/ares_dns_rcode_t.3 diff --git a/source/lib/c-ares-1.34.4/docs/ares_dns_rcode_tostr.3 b/source/lib/c-ares-1.34.6/docs/ares_dns_rcode_tostr.3 similarity index 100% rename from source/lib/c-ares-1.34.4/docs/ares_dns_rcode_tostr.3 rename to source/lib/c-ares-1.34.6/docs/ares_dns_rcode_tostr.3 diff --git a/source/lib/c-ares-1.34.4/docs/ares_dns_rec_type_fromstr.3 b/source/lib/c-ares-1.34.6/docs/ares_dns_rec_type_fromstr.3 similarity index 100% rename from source/lib/c-ares-1.34.4/docs/ares_dns_rec_type_fromstr.3 rename to source/lib/c-ares-1.34.6/docs/ares_dns_rec_type_fromstr.3 diff --git a/source/lib/c-ares-1.34.4/docs/ares_dns_rec_type_t.3 b/source/lib/c-ares-1.34.6/docs/ares_dns_rec_type_t.3 similarity index 100% rename from source/lib/c-ares-1.34.4/docs/ares_dns_rec_type_t.3 rename to source/lib/c-ares-1.34.6/docs/ares_dns_rec_type_t.3 diff --git a/source/lib/c-ares-1.34.4/docs/ares_dns_rec_type_tostr.3 b/source/lib/c-ares-1.34.6/docs/ares_dns_rec_type_tostr.3 similarity index 100% rename from source/lib/c-ares-1.34.4/docs/ares_dns_rec_type_tostr.3 rename to source/lib/c-ares-1.34.6/docs/ares_dns_rec_type_tostr.3 diff --git a/source/lib/c-ares-1.34.4/docs/ares_dns_record.3 b/source/lib/c-ares-1.34.6/docs/ares_dns_record.3 similarity index 100% rename from source/lib/c-ares-1.34.4/docs/ares_dns_record.3 rename to source/lib/c-ares-1.34.6/docs/ares_dns_record.3 diff --git a/source/lib/c-ares-1.34.4/docs/ares_dns_record_create.3 b/source/lib/c-ares-1.34.6/docs/ares_dns_record_create.3 similarity index 100% rename from source/lib/c-ares-1.34.4/docs/ares_dns_record_create.3 rename to source/lib/c-ares-1.34.6/docs/ares_dns_record_create.3 diff --git a/source/lib/c-ares-1.34.4/docs/ares_dns_record_destroy.3 b/source/lib/c-ares-1.34.6/docs/ares_dns_record_destroy.3 similarity index 100% rename from source/lib/c-ares-1.34.4/docs/ares_dns_record_destroy.3 rename to source/lib/c-ares-1.34.6/docs/ares_dns_record_destroy.3 diff --git a/source/lib/c-ares-1.34.4/docs/ares_dns_record_duplicate.3 b/source/lib/c-ares-1.34.6/docs/ares_dns_record_duplicate.3 similarity index 100% rename from source/lib/c-ares-1.34.4/docs/ares_dns_record_duplicate.3 rename to source/lib/c-ares-1.34.6/docs/ares_dns_record_duplicate.3 diff --git a/source/lib/c-ares-1.34.4/docs/ares_dns_record_get_flags.3 b/source/lib/c-ares-1.34.6/docs/ares_dns_record_get_flags.3 similarity index 100% rename from source/lib/c-ares-1.34.4/docs/ares_dns_record_get_flags.3 rename to source/lib/c-ares-1.34.6/docs/ares_dns_record_get_flags.3 diff --git a/source/lib/c-ares-1.34.4/docs/ares_dns_record_get_id.3 b/source/lib/c-ares-1.34.6/docs/ares_dns_record_get_id.3 similarity index 100% rename from source/lib/c-ares-1.34.4/docs/ares_dns_record_get_id.3 rename to source/lib/c-ares-1.34.6/docs/ares_dns_record_get_id.3 diff --git a/source/lib/c-ares-1.34.4/docs/ares_dns_record_get_opcode.3 b/source/lib/c-ares-1.34.6/docs/ares_dns_record_get_opcode.3 similarity index 100% rename from source/lib/c-ares-1.34.4/docs/ares_dns_record_get_opcode.3 rename to source/lib/c-ares-1.34.6/docs/ares_dns_record_get_opcode.3 diff --git a/source/lib/c-ares-1.34.4/docs/ares_dns_record_get_rcode.3 b/source/lib/c-ares-1.34.6/docs/ares_dns_record_get_rcode.3 similarity index 100% rename from source/lib/c-ares-1.34.4/docs/ares_dns_record_get_rcode.3 rename to source/lib/c-ares-1.34.6/docs/ares_dns_record_get_rcode.3 diff --git a/source/lib/c-ares-1.34.4/docs/ares_dns_record_query_add.3 b/source/lib/c-ares-1.34.6/docs/ares_dns_record_query_add.3 similarity index 100% rename from source/lib/c-ares-1.34.4/docs/ares_dns_record_query_add.3 rename to source/lib/c-ares-1.34.6/docs/ares_dns_record_query_add.3 diff --git a/source/lib/c-ares-1.34.4/docs/ares_dns_record_query_cnt.3 b/source/lib/c-ares-1.34.6/docs/ares_dns_record_query_cnt.3 similarity index 100% rename from source/lib/c-ares-1.34.4/docs/ares_dns_record_query_cnt.3 rename to source/lib/c-ares-1.34.6/docs/ares_dns_record_query_cnt.3 diff --git a/source/lib/c-ares-1.34.4/docs/ares_dns_record_query_get.3 b/source/lib/c-ares-1.34.6/docs/ares_dns_record_query_get.3 similarity index 100% rename from source/lib/c-ares-1.34.4/docs/ares_dns_record_query_get.3 rename to source/lib/c-ares-1.34.6/docs/ares_dns_record_query_get.3 diff --git a/source/lib/c-ares-1.34.4/docs/ares_dns_record_query_set_name.3 b/source/lib/c-ares-1.34.6/docs/ares_dns_record_query_set_name.3 similarity index 100% rename from source/lib/c-ares-1.34.4/docs/ares_dns_record_query_set_name.3 rename to source/lib/c-ares-1.34.6/docs/ares_dns_record_query_set_name.3 diff --git a/source/lib/c-ares-1.34.4/docs/ares_dns_record_query_set_type.3 b/source/lib/c-ares-1.34.6/docs/ares_dns_record_query_set_type.3 similarity index 100% rename from source/lib/c-ares-1.34.4/docs/ares_dns_record_query_set_type.3 rename to source/lib/c-ares-1.34.6/docs/ares_dns_record_query_set_type.3 diff --git a/source/lib/c-ares-1.34.4/docs/ares_dns_record_rr_add.3 b/source/lib/c-ares-1.34.6/docs/ares_dns_record_rr_add.3 similarity index 100% rename from source/lib/c-ares-1.34.4/docs/ares_dns_record_rr_add.3 rename to source/lib/c-ares-1.34.6/docs/ares_dns_record_rr_add.3 diff --git a/source/lib/c-ares-1.34.4/docs/ares_dns_record_rr_cnt.3 b/source/lib/c-ares-1.34.6/docs/ares_dns_record_rr_cnt.3 similarity index 100% rename from source/lib/c-ares-1.34.4/docs/ares_dns_record_rr_cnt.3 rename to source/lib/c-ares-1.34.6/docs/ares_dns_record_rr_cnt.3 diff --git a/source/lib/c-ares-1.34.4/docs/ares_dns_record_rr_del.3 b/source/lib/c-ares-1.34.6/docs/ares_dns_record_rr_del.3 similarity index 100% rename from source/lib/c-ares-1.34.4/docs/ares_dns_record_rr_del.3 rename to source/lib/c-ares-1.34.6/docs/ares_dns_record_rr_del.3 diff --git a/source/lib/c-ares-1.34.4/docs/ares_dns_record_rr_get.3 b/source/lib/c-ares-1.34.6/docs/ares_dns_record_rr_get.3 similarity index 100% rename from source/lib/c-ares-1.34.4/docs/ares_dns_record_rr_get.3 rename to source/lib/c-ares-1.34.6/docs/ares_dns_record_rr_get.3 diff --git a/source/lib/c-ares-1.34.4/docs/ares_dns_record_rr_get_const.3 b/source/lib/c-ares-1.34.6/docs/ares_dns_record_rr_get_const.3 similarity index 100% rename from source/lib/c-ares-1.34.4/docs/ares_dns_record_rr_get_const.3 rename to source/lib/c-ares-1.34.6/docs/ares_dns_record_rr_get_const.3 diff --git a/source/lib/c-ares-1.34.4/docs/ares_dns_record_set_id.3 b/source/lib/c-ares-1.34.6/docs/ares_dns_record_set_id.3 similarity index 100% rename from source/lib/c-ares-1.34.4/docs/ares_dns_record_set_id.3 rename to source/lib/c-ares-1.34.6/docs/ares_dns_record_set_id.3 diff --git a/source/lib/c-ares-1.34.4/docs/ares_dns_rr.3 b/source/lib/c-ares-1.34.6/docs/ares_dns_rr.3 similarity index 100% rename from source/lib/c-ares-1.34.4/docs/ares_dns_rr.3 rename to source/lib/c-ares-1.34.6/docs/ares_dns_rr.3 diff --git a/source/lib/c-ares-1.34.4/docs/ares_dns_rr_add_abin.3 b/source/lib/c-ares-1.34.6/docs/ares_dns_rr_add_abin.3 similarity index 100% rename from source/lib/c-ares-1.34.4/docs/ares_dns_rr_add_abin.3 rename to source/lib/c-ares-1.34.6/docs/ares_dns_rr_add_abin.3 diff --git a/source/lib/c-ares-1.34.4/docs/ares_dns_rr_del_abin.3 b/source/lib/c-ares-1.34.6/docs/ares_dns_rr_del_abin.3 similarity index 100% rename from source/lib/c-ares-1.34.4/docs/ares_dns_rr_del_abin.3 rename to source/lib/c-ares-1.34.6/docs/ares_dns_rr_del_abin.3 diff --git a/source/lib/c-ares-1.34.4/docs/ares_dns_rr_del_opt_byid.3 b/source/lib/c-ares-1.34.6/docs/ares_dns_rr_del_opt_byid.3 similarity index 100% rename from source/lib/c-ares-1.34.4/docs/ares_dns_rr_del_opt_byid.3 rename to source/lib/c-ares-1.34.6/docs/ares_dns_rr_del_opt_byid.3 diff --git a/source/lib/c-ares-1.34.4/docs/ares_dns_rr_get_abin.3 b/source/lib/c-ares-1.34.6/docs/ares_dns_rr_get_abin.3 similarity index 100% rename from source/lib/c-ares-1.34.4/docs/ares_dns_rr_get_abin.3 rename to source/lib/c-ares-1.34.6/docs/ares_dns_rr_get_abin.3 diff --git a/source/lib/c-ares-1.34.4/docs/ares_dns_rr_get_abin_cnt.3 b/source/lib/c-ares-1.34.6/docs/ares_dns_rr_get_abin_cnt.3 similarity index 100% rename from source/lib/c-ares-1.34.4/docs/ares_dns_rr_get_abin_cnt.3 rename to source/lib/c-ares-1.34.6/docs/ares_dns_rr_get_abin_cnt.3 diff --git a/source/lib/c-ares-1.34.4/docs/ares_dns_rr_get_addr.3 b/source/lib/c-ares-1.34.6/docs/ares_dns_rr_get_addr.3 similarity index 100% rename from source/lib/c-ares-1.34.4/docs/ares_dns_rr_get_addr.3 rename to source/lib/c-ares-1.34.6/docs/ares_dns_rr_get_addr.3 diff --git a/source/lib/c-ares-1.34.4/docs/ares_dns_rr_get_addr6.3 b/source/lib/c-ares-1.34.6/docs/ares_dns_rr_get_addr6.3 similarity index 100% rename from source/lib/c-ares-1.34.4/docs/ares_dns_rr_get_addr6.3 rename to source/lib/c-ares-1.34.6/docs/ares_dns_rr_get_addr6.3 diff --git a/source/lib/c-ares-1.34.4/docs/ares_dns_rr_get_bin.3 b/source/lib/c-ares-1.34.6/docs/ares_dns_rr_get_bin.3 similarity index 100% rename from source/lib/c-ares-1.34.4/docs/ares_dns_rr_get_bin.3 rename to source/lib/c-ares-1.34.6/docs/ares_dns_rr_get_bin.3 diff --git a/source/lib/c-ares-1.34.4/docs/ares_dns_rr_get_class.3 b/source/lib/c-ares-1.34.6/docs/ares_dns_rr_get_class.3 similarity index 100% rename from source/lib/c-ares-1.34.4/docs/ares_dns_rr_get_class.3 rename to source/lib/c-ares-1.34.6/docs/ares_dns_rr_get_class.3 diff --git a/source/lib/c-ares-1.34.4/docs/ares_dns_rr_get_keys.3 b/source/lib/c-ares-1.34.6/docs/ares_dns_rr_get_keys.3 similarity index 100% rename from source/lib/c-ares-1.34.4/docs/ares_dns_rr_get_keys.3 rename to source/lib/c-ares-1.34.6/docs/ares_dns_rr_get_keys.3 diff --git a/source/lib/c-ares-1.34.4/docs/ares_dns_rr_get_name.3 b/source/lib/c-ares-1.34.6/docs/ares_dns_rr_get_name.3 similarity index 100% rename from source/lib/c-ares-1.34.4/docs/ares_dns_rr_get_name.3 rename to source/lib/c-ares-1.34.6/docs/ares_dns_rr_get_name.3 diff --git a/source/lib/c-ares-1.34.4/docs/ares_dns_rr_get_opt.3 b/source/lib/c-ares-1.34.6/docs/ares_dns_rr_get_opt.3 similarity index 100% rename from source/lib/c-ares-1.34.4/docs/ares_dns_rr_get_opt.3 rename to source/lib/c-ares-1.34.6/docs/ares_dns_rr_get_opt.3 diff --git a/source/lib/c-ares-1.34.4/docs/ares_dns_rr_get_opt_byid.3 b/source/lib/c-ares-1.34.6/docs/ares_dns_rr_get_opt_byid.3 similarity index 100% rename from source/lib/c-ares-1.34.4/docs/ares_dns_rr_get_opt_byid.3 rename to source/lib/c-ares-1.34.6/docs/ares_dns_rr_get_opt_byid.3 diff --git a/source/lib/c-ares-1.34.4/docs/ares_dns_rr_get_opt_cnt.3 b/source/lib/c-ares-1.34.6/docs/ares_dns_rr_get_opt_cnt.3 similarity index 100% rename from source/lib/c-ares-1.34.4/docs/ares_dns_rr_get_opt_cnt.3 rename to source/lib/c-ares-1.34.6/docs/ares_dns_rr_get_opt_cnt.3 diff --git a/source/lib/c-ares-1.34.4/docs/ares_dns_rr_get_str.3 b/source/lib/c-ares-1.34.6/docs/ares_dns_rr_get_str.3 similarity index 100% rename from source/lib/c-ares-1.34.4/docs/ares_dns_rr_get_str.3 rename to source/lib/c-ares-1.34.6/docs/ares_dns_rr_get_str.3 diff --git a/source/lib/c-ares-1.34.4/docs/ares_dns_rr_get_ttl.3 b/source/lib/c-ares-1.34.6/docs/ares_dns_rr_get_ttl.3 similarity index 100% rename from source/lib/c-ares-1.34.4/docs/ares_dns_rr_get_ttl.3 rename to source/lib/c-ares-1.34.6/docs/ares_dns_rr_get_ttl.3 diff --git a/source/lib/c-ares-1.34.4/docs/ares_dns_rr_get_type.3 b/source/lib/c-ares-1.34.6/docs/ares_dns_rr_get_type.3 similarity index 100% rename from source/lib/c-ares-1.34.4/docs/ares_dns_rr_get_type.3 rename to source/lib/c-ares-1.34.6/docs/ares_dns_rr_get_type.3 diff --git a/source/lib/c-ares-1.34.4/docs/ares_dns_rr_get_u16.3 b/source/lib/c-ares-1.34.6/docs/ares_dns_rr_get_u16.3 similarity index 100% rename from source/lib/c-ares-1.34.4/docs/ares_dns_rr_get_u16.3 rename to source/lib/c-ares-1.34.6/docs/ares_dns_rr_get_u16.3 diff --git a/source/lib/c-ares-1.34.4/docs/ares_dns_rr_get_u32.3 b/source/lib/c-ares-1.34.6/docs/ares_dns_rr_get_u32.3 similarity index 100% rename from source/lib/c-ares-1.34.4/docs/ares_dns_rr_get_u32.3 rename to source/lib/c-ares-1.34.6/docs/ares_dns_rr_get_u32.3 diff --git a/source/lib/c-ares-1.34.4/docs/ares_dns_rr_get_u8.3 b/source/lib/c-ares-1.34.6/docs/ares_dns_rr_get_u8.3 similarity index 100% rename from source/lib/c-ares-1.34.4/docs/ares_dns_rr_get_u8.3 rename to source/lib/c-ares-1.34.6/docs/ares_dns_rr_get_u8.3 diff --git a/source/lib/c-ares-1.34.4/docs/ares_dns_rr_key_datatype.3 b/source/lib/c-ares-1.34.6/docs/ares_dns_rr_key_datatype.3 similarity index 100% rename from source/lib/c-ares-1.34.4/docs/ares_dns_rr_key_datatype.3 rename to source/lib/c-ares-1.34.6/docs/ares_dns_rr_key_datatype.3 diff --git a/source/lib/c-ares-1.34.4/docs/ares_dns_rr_key_t.3 b/source/lib/c-ares-1.34.6/docs/ares_dns_rr_key_t.3 similarity index 100% rename from source/lib/c-ares-1.34.4/docs/ares_dns_rr_key_t.3 rename to source/lib/c-ares-1.34.6/docs/ares_dns_rr_key_t.3 diff --git a/source/lib/c-ares-1.34.4/docs/ares_dns_rr_key_to_rec_type.3 b/source/lib/c-ares-1.34.6/docs/ares_dns_rr_key_to_rec_type.3 similarity index 100% rename from source/lib/c-ares-1.34.4/docs/ares_dns_rr_key_to_rec_type.3 rename to source/lib/c-ares-1.34.6/docs/ares_dns_rr_key_to_rec_type.3 diff --git a/source/lib/c-ares-1.34.4/docs/ares_dns_rr_key_tostr.3 b/source/lib/c-ares-1.34.6/docs/ares_dns_rr_key_tostr.3 similarity index 100% rename from source/lib/c-ares-1.34.4/docs/ares_dns_rr_key_tostr.3 rename to source/lib/c-ares-1.34.6/docs/ares_dns_rr_key_tostr.3 diff --git a/source/lib/c-ares-1.34.4/docs/ares_dns_rr_set_addr.3 b/source/lib/c-ares-1.34.6/docs/ares_dns_rr_set_addr.3 similarity index 100% rename from source/lib/c-ares-1.34.4/docs/ares_dns_rr_set_addr.3 rename to source/lib/c-ares-1.34.6/docs/ares_dns_rr_set_addr.3 diff --git a/source/lib/c-ares-1.34.4/docs/ares_dns_rr_set_addr6.3 b/source/lib/c-ares-1.34.6/docs/ares_dns_rr_set_addr6.3 similarity index 100% rename from source/lib/c-ares-1.34.4/docs/ares_dns_rr_set_addr6.3 rename to source/lib/c-ares-1.34.6/docs/ares_dns_rr_set_addr6.3 diff --git a/source/lib/c-ares-1.34.4/docs/ares_dns_rr_set_bin.3 b/source/lib/c-ares-1.34.6/docs/ares_dns_rr_set_bin.3 similarity index 100% rename from source/lib/c-ares-1.34.4/docs/ares_dns_rr_set_bin.3 rename to source/lib/c-ares-1.34.6/docs/ares_dns_rr_set_bin.3 diff --git a/source/lib/c-ares-1.34.4/docs/ares_dns_rr_set_opt.3 b/source/lib/c-ares-1.34.6/docs/ares_dns_rr_set_opt.3 similarity index 100% rename from source/lib/c-ares-1.34.4/docs/ares_dns_rr_set_opt.3 rename to source/lib/c-ares-1.34.6/docs/ares_dns_rr_set_opt.3 diff --git a/source/lib/c-ares-1.34.4/docs/ares_dns_rr_set_str.3 b/source/lib/c-ares-1.34.6/docs/ares_dns_rr_set_str.3 similarity index 100% rename from source/lib/c-ares-1.34.4/docs/ares_dns_rr_set_str.3 rename to source/lib/c-ares-1.34.6/docs/ares_dns_rr_set_str.3 diff --git a/source/lib/c-ares-1.34.4/docs/ares_dns_rr_set_u16.3 b/source/lib/c-ares-1.34.6/docs/ares_dns_rr_set_u16.3 similarity index 100% rename from source/lib/c-ares-1.34.4/docs/ares_dns_rr_set_u16.3 rename to source/lib/c-ares-1.34.6/docs/ares_dns_rr_set_u16.3 diff --git a/source/lib/c-ares-1.34.4/docs/ares_dns_rr_set_u32.3 b/source/lib/c-ares-1.34.6/docs/ares_dns_rr_set_u32.3 similarity index 100% rename from source/lib/c-ares-1.34.4/docs/ares_dns_rr_set_u32.3 rename to source/lib/c-ares-1.34.6/docs/ares_dns_rr_set_u32.3 diff --git a/source/lib/c-ares-1.34.4/docs/ares_dns_rr_set_u8.3 b/source/lib/c-ares-1.34.6/docs/ares_dns_rr_set_u8.3 similarity index 100% rename from source/lib/c-ares-1.34.4/docs/ares_dns_rr_set_u8.3 rename to source/lib/c-ares-1.34.6/docs/ares_dns_rr_set_u8.3 diff --git a/source/lib/c-ares-1.34.4/docs/ares_dns_section_t.3 b/source/lib/c-ares-1.34.6/docs/ares_dns_section_t.3 similarity index 100% rename from source/lib/c-ares-1.34.4/docs/ares_dns_section_t.3 rename to source/lib/c-ares-1.34.6/docs/ares_dns_section_t.3 diff --git a/source/lib/c-ares-1.34.4/docs/ares_dns_section_tostr.3 b/source/lib/c-ares-1.34.6/docs/ares_dns_section_tostr.3 similarity index 100% rename from source/lib/c-ares-1.34.4/docs/ares_dns_section_tostr.3 rename to source/lib/c-ares-1.34.6/docs/ares_dns_section_tostr.3 diff --git a/source/lib/c-ares-1.34.4/docs/ares_dns_write.3 b/source/lib/c-ares-1.34.6/docs/ares_dns_write.3 similarity index 100% rename from source/lib/c-ares-1.34.4/docs/ares_dns_write.3 rename to source/lib/c-ares-1.34.6/docs/ares_dns_write.3 diff --git a/source/lib/c-ares-1.34.4/docs/ares_dup.3 b/source/lib/c-ares-1.34.6/docs/ares_dup.3 similarity index 100% rename from source/lib/c-ares-1.34.4/docs/ares_dup.3 rename to source/lib/c-ares-1.34.6/docs/ares_dup.3 diff --git a/source/lib/c-ares-1.34.4/docs/ares_expand_name.3 b/source/lib/c-ares-1.34.6/docs/ares_expand_name.3 similarity index 100% rename from source/lib/c-ares-1.34.4/docs/ares_expand_name.3 rename to source/lib/c-ares-1.34.6/docs/ares_expand_name.3 diff --git a/source/lib/c-ares-1.34.4/docs/ares_expand_string.3 b/source/lib/c-ares-1.34.6/docs/ares_expand_string.3 similarity index 100% rename from source/lib/c-ares-1.34.4/docs/ares_expand_string.3 rename to source/lib/c-ares-1.34.6/docs/ares_expand_string.3 diff --git a/source/lib/c-ares-1.34.4/docs/ares_fds.3 b/source/lib/c-ares-1.34.6/docs/ares_fds.3 similarity index 100% rename from source/lib/c-ares-1.34.4/docs/ares_fds.3 rename to source/lib/c-ares-1.34.6/docs/ares_fds.3 diff --git a/source/lib/c-ares-1.34.4/docs/ares_free_data.3 b/source/lib/c-ares-1.34.6/docs/ares_free_data.3 similarity index 100% rename from source/lib/c-ares-1.34.4/docs/ares_free_data.3 rename to source/lib/c-ares-1.34.6/docs/ares_free_data.3 diff --git a/source/lib/c-ares-1.34.4/docs/ares_free_hostent.3 b/source/lib/c-ares-1.34.6/docs/ares_free_hostent.3 similarity index 100% rename from source/lib/c-ares-1.34.4/docs/ares_free_hostent.3 rename to source/lib/c-ares-1.34.6/docs/ares_free_hostent.3 diff --git a/source/lib/c-ares-1.34.4/docs/ares_free_string.3 b/source/lib/c-ares-1.34.6/docs/ares_free_string.3 similarity index 100% rename from source/lib/c-ares-1.34.4/docs/ares_free_string.3 rename to source/lib/c-ares-1.34.6/docs/ares_free_string.3 diff --git a/source/lib/c-ares-1.34.4/docs/ares_freeaddrinfo.3 b/source/lib/c-ares-1.34.6/docs/ares_freeaddrinfo.3 similarity index 100% rename from source/lib/c-ares-1.34.4/docs/ares_freeaddrinfo.3 rename to source/lib/c-ares-1.34.6/docs/ares_freeaddrinfo.3 diff --git a/source/lib/c-ares-1.34.4/docs/ares_get_servers.3 b/source/lib/c-ares-1.34.6/docs/ares_get_servers.3 similarity index 100% rename from source/lib/c-ares-1.34.4/docs/ares_get_servers.3 rename to source/lib/c-ares-1.34.6/docs/ares_get_servers.3 diff --git a/source/lib/c-ares-1.34.4/docs/ares_get_servers_csv.3 b/source/lib/c-ares-1.34.6/docs/ares_get_servers_csv.3 similarity index 100% rename from source/lib/c-ares-1.34.4/docs/ares_get_servers_csv.3 rename to source/lib/c-ares-1.34.6/docs/ares_get_servers_csv.3 diff --git a/source/lib/c-ares-1.34.4/docs/ares_get_servers_ports.3 b/source/lib/c-ares-1.34.6/docs/ares_get_servers_ports.3 similarity index 100% rename from source/lib/c-ares-1.34.4/docs/ares_get_servers_ports.3 rename to source/lib/c-ares-1.34.6/docs/ares_get_servers_ports.3 diff --git a/source/lib/c-ares-1.34.4/docs/ares_getaddrinfo.3 b/source/lib/c-ares-1.34.6/docs/ares_getaddrinfo.3 similarity index 100% rename from source/lib/c-ares-1.34.4/docs/ares_getaddrinfo.3 rename to source/lib/c-ares-1.34.6/docs/ares_getaddrinfo.3 diff --git a/source/lib/c-ares-1.34.4/docs/ares_gethostbyaddr.3 b/source/lib/c-ares-1.34.6/docs/ares_gethostbyaddr.3 similarity index 100% rename from source/lib/c-ares-1.34.4/docs/ares_gethostbyaddr.3 rename to source/lib/c-ares-1.34.6/docs/ares_gethostbyaddr.3 diff --git a/source/lib/c-ares-1.34.4/docs/ares_gethostbyname.3 b/source/lib/c-ares-1.34.6/docs/ares_gethostbyname.3 similarity index 100% rename from source/lib/c-ares-1.34.4/docs/ares_gethostbyname.3 rename to source/lib/c-ares-1.34.6/docs/ares_gethostbyname.3 diff --git a/source/lib/c-ares-1.34.4/docs/ares_gethostbyname_file.3 b/source/lib/c-ares-1.34.6/docs/ares_gethostbyname_file.3 similarity index 100% rename from source/lib/c-ares-1.34.4/docs/ares_gethostbyname_file.3 rename to source/lib/c-ares-1.34.6/docs/ares_gethostbyname_file.3 diff --git a/source/lib/c-ares-1.34.4/docs/ares_getnameinfo.3 b/source/lib/c-ares-1.34.6/docs/ares_getnameinfo.3 similarity index 100% rename from source/lib/c-ares-1.34.4/docs/ares_getnameinfo.3 rename to source/lib/c-ares-1.34.6/docs/ares_getnameinfo.3 diff --git a/source/lib/c-ares-1.34.4/docs/ares_getsock.3 b/source/lib/c-ares-1.34.6/docs/ares_getsock.3 similarity index 100% rename from source/lib/c-ares-1.34.4/docs/ares_getsock.3 rename to source/lib/c-ares-1.34.6/docs/ares_getsock.3 diff --git a/source/lib/c-ares-1.34.4/docs/ares_inet_ntop.3 b/source/lib/c-ares-1.34.6/docs/ares_inet_ntop.3 similarity index 100% rename from source/lib/c-ares-1.34.4/docs/ares_inet_ntop.3 rename to source/lib/c-ares-1.34.6/docs/ares_inet_ntop.3 diff --git a/source/lib/c-ares-1.34.4/docs/ares_inet_pton.3 b/source/lib/c-ares-1.34.6/docs/ares_inet_pton.3 similarity index 100% rename from source/lib/c-ares-1.34.4/docs/ares_inet_pton.3 rename to source/lib/c-ares-1.34.6/docs/ares_inet_pton.3 diff --git a/source/lib/c-ares-1.34.4/docs/ares_init.3 b/source/lib/c-ares-1.34.6/docs/ares_init.3 similarity index 100% rename from source/lib/c-ares-1.34.4/docs/ares_init.3 rename to source/lib/c-ares-1.34.6/docs/ares_init.3 diff --git a/source/lib/c-ares-1.34.4/docs/ares_init_options.3 b/source/lib/c-ares-1.34.6/docs/ares_init_options.3 similarity index 100% rename from source/lib/c-ares-1.34.4/docs/ares_init_options.3 rename to source/lib/c-ares-1.34.6/docs/ares_init_options.3 diff --git a/source/lib/c-ares-1.34.4/docs/ares_library_cleanup.3 b/source/lib/c-ares-1.34.6/docs/ares_library_cleanup.3 similarity index 100% rename from source/lib/c-ares-1.34.4/docs/ares_library_cleanup.3 rename to source/lib/c-ares-1.34.6/docs/ares_library_cleanup.3 diff --git a/source/lib/c-ares-1.34.4/docs/ares_library_init.3 b/source/lib/c-ares-1.34.6/docs/ares_library_init.3 similarity index 100% rename from source/lib/c-ares-1.34.4/docs/ares_library_init.3 rename to source/lib/c-ares-1.34.6/docs/ares_library_init.3 diff --git a/source/lib/c-ares-1.34.4/docs/ares_library_init_android.3 b/source/lib/c-ares-1.34.6/docs/ares_library_init_android.3 similarity index 100% rename from source/lib/c-ares-1.34.4/docs/ares_library_init_android.3 rename to source/lib/c-ares-1.34.6/docs/ares_library_init_android.3 diff --git a/source/lib/c-ares-1.34.4/docs/ares_library_initialized.3 b/source/lib/c-ares-1.34.6/docs/ares_library_initialized.3 similarity index 100% rename from source/lib/c-ares-1.34.4/docs/ares_library_initialized.3 rename to source/lib/c-ares-1.34.6/docs/ares_library_initialized.3 diff --git a/source/lib/c-ares-1.34.4/docs/ares_mkquery.3 b/source/lib/c-ares-1.34.6/docs/ares_mkquery.3 similarity index 100% rename from source/lib/c-ares-1.34.4/docs/ares_mkquery.3 rename to source/lib/c-ares-1.34.6/docs/ares_mkquery.3 diff --git a/source/lib/c-ares-1.34.4/docs/ares_opt_param_t.3 b/source/lib/c-ares-1.34.6/docs/ares_opt_param_t.3 similarity index 100% rename from source/lib/c-ares-1.34.4/docs/ares_opt_param_t.3 rename to source/lib/c-ares-1.34.6/docs/ares_opt_param_t.3 diff --git a/source/lib/c-ares-1.34.4/docs/ares_parse_a_reply.3 b/source/lib/c-ares-1.34.6/docs/ares_parse_a_reply.3 similarity index 100% rename from source/lib/c-ares-1.34.4/docs/ares_parse_a_reply.3 rename to source/lib/c-ares-1.34.6/docs/ares_parse_a_reply.3 diff --git a/source/lib/c-ares-1.34.4/docs/ares_parse_aaaa_reply.3 b/source/lib/c-ares-1.34.6/docs/ares_parse_aaaa_reply.3 similarity index 100% rename from source/lib/c-ares-1.34.4/docs/ares_parse_aaaa_reply.3 rename to source/lib/c-ares-1.34.6/docs/ares_parse_aaaa_reply.3 diff --git a/source/lib/c-ares-1.34.4/docs/ares_parse_caa_reply.3 b/source/lib/c-ares-1.34.6/docs/ares_parse_caa_reply.3 similarity index 100% rename from source/lib/c-ares-1.34.4/docs/ares_parse_caa_reply.3 rename to source/lib/c-ares-1.34.6/docs/ares_parse_caa_reply.3 diff --git a/source/lib/c-ares-1.34.4/docs/ares_parse_mx_reply.3 b/source/lib/c-ares-1.34.6/docs/ares_parse_mx_reply.3 similarity index 100% rename from source/lib/c-ares-1.34.4/docs/ares_parse_mx_reply.3 rename to source/lib/c-ares-1.34.6/docs/ares_parse_mx_reply.3 diff --git a/source/lib/c-ares-1.34.4/docs/ares_parse_naptr_reply.3 b/source/lib/c-ares-1.34.6/docs/ares_parse_naptr_reply.3 similarity index 100% rename from source/lib/c-ares-1.34.4/docs/ares_parse_naptr_reply.3 rename to source/lib/c-ares-1.34.6/docs/ares_parse_naptr_reply.3 diff --git a/source/lib/c-ares-1.34.4/docs/ares_parse_ns_reply.3 b/source/lib/c-ares-1.34.6/docs/ares_parse_ns_reply.3 similarity index 100% rename from source/lib/c-ares-1.34.4/docs/ares_parse_ns_reply.3 rename to source/lib/c-ares-1.34.6/docs/ares_parse_ns_reply.3 diff --git a/source/lib/c-ares-1.34.4/docs/ares_parse_ptr_reply.3 b/source/lib/c-ares-1.34.6/docs/ares_parse_ptr_reply.3 similarity index 100% rename from source/lib/c-ares-1.34.4/docs/ares_parse_ptr_reply.3 rename to source/lib/c-ares-1.34.6/docs/ares_parse_ptr_reply.3 diff --git a/source/lib/c-ares-1.34.4/docs/ares_parse_soa_reply.3 b/source/lib/c-ares-1.34.6/docs/ares_parse_soa_reply.3 similarity index 100% rename from source/lib/c-ares-1.34.4/docs/ares_parse_soa_reply.3 rename to source/lib/c-ares-1.34.6/docs/ares_parse_soa_reply.3 diff --git a/source/lib/c-ares-1.34.4/docs/ares_parse_srv_reply.3 b/source/lib/c-ares-1.34.6/docs/ares_parse_srv_reply.3 similarity index 100% rename from source/lib/c-ares-1.34.4/docs/ares_parse_srv_reply.3 rename to source/lib/c-ares-1.34.6/docs/ares_parse_srv_reply.3 diff --git a/source/lib/c-ares-1.34.4/docs/ares_parse_txt_reply.3 b/source/lib/c-ares-1.34.6/docs/ares_parse_txt_reply.3 similarity index 100% rename from source/lib/c-ares-1.34.4/docs/ares_parse_txt_reply.3 rename to source/lib/c-ares-1.34.6/docs/ares_parse_txt_reply.3 diff --git a/source/lib/c-ares-1.34.4/docs/ares_parse_uri_reply.3 b/source/lib/c-ares-1.34.6/docs/ares_parse_uri_reply.3 similarity index 100% rename from source/lib/c-ares-1.34.4/docs/ares_parse_uri_reply.3 rename to source/lib/c-ares-1.34.6/docs/ares_parse_uri_reply.3 diff --git a/source/lib/c-ares-1.34.4/docs/ares_process.3 b/source/lib/c-ares-1.34.6/docs/ares_process.3 similarity index 100% rename from source/lib/c-ares-1.34.4/docs/ares_process.3 rename to source/lib/c-ares-1.34.6/docs/ares_process.3 diff --git a/source/lib/c-ares-1.34.4/docs/ares_process_fd.3 b/source/lib/c-ares-1.34.6/docs/ares_process_fd.3 similarity index 100% rename from source/lib/c-ares-1.34.4/docs/ares_process_fd.3 rename to source/lib/c-ares-1.34.6/docs/ares_process_fd.3 diff --git a/source/lib/c-ares-1.34.4/docs/ares_process_fds.3 b/source/lib/c-ares-1.34.6/docs/ares_process_fds.3 similarity index 100% rename from source/lib/c-ares-1.34.4/docs/ares_process_fds.3 rename to source/lib/c-ares-1.34.6/docs/ares_process_fds.3 diff --git a/source/lib/c-ares-1.34.4/docs/ares_process_pending_write.3 b/source/lib/c-ares-1.34.6/docs/ares_process_pending_write.3 similarity index 100% rename from source/lib/c-ares-1.34.4/docs/ares_process_pending_write.3 rename to source/lib/c-ares-1.34.6/docs/ares_process_pending_write.3 diff --git a/source/lib/c-ares-1.34.4/docs/ares_query.3 b/source/lib/c-ares-1.34.6/docs/ares_query.3 similarity index 100% rename from source/lib/c-ares-1.34.4/docs/ares_query.3 rename to source/lib/c-ares-1.34.6/docs/ares_query.3 diff --git a/source/lib/c-ares-1.34.4/docs/ares_query_dnsrec.3 b/source/lib/c-ares-1.34.6/docs/ares_query_dnsrec.3 similarity index 100% rename from source/lib/c-ares-1.34.4/docs/ares_query_dnsrec.3 rename to source/lib/c-ares-1.34.6/docs/ares_query_dnsrec.3 diff --git a/source/lib/c-ares-1.34.4/docs/ares_queue.3 b/source/lib/c-ares-1.34.6/docs/ares_queue.3 similarity index 100% rename from source/lib/c-ares-1.34.4/docs/ares_queue.3 rename to source/lib/c-ares-1.34.6/docs/ares_queue.3 diff --git a/source/lib/c-ares-1.34.4/docs/ares_queue_active_queries.3 b/source/lib/c-ares-1.34.6/docs/ares_queue_active_queries.3 similarity index 100% rename from source/lib/c-ares-1.34.4/docs/ares_queue_active_queries.3 rename to source/lib/c-ares-1.34.6/docs/ares_queue_active_queries.3 diff --git a/source/lib/c-ares-1.34.4/docs/ares_queue_wait_empty.3 b/source/lib/c-ares-1.34.6/docs/ares_queue_wait_empty.3 similarity index 100% rename from source/lib/c-ares-1.34.4/docs/ares_queue_wait_empty.3 rename to source/lib/c-ares-1.34.6/docs/ares_queue_wait_empty.3 diff --git a/source/lib/c-ares-1.34.4/docs/ares_reinit.3 b/source/lib/c-ares-1.34.6/docs/ares_reinit.3 similarity index 100% rename from source/lib/c-ares-1.34.4/docs/ares_reinit.3 rename to source/lib/c-ares-1.34.6/docs/ares_reinit.3 diff --git a/source/lib/c-ares-1.34.4/docs/ares_save_options.3 b/source/lib/c-ares-1.34.6/docs/ares_save_options.3 similarity index 100% rename from source/lib/c-ares-1.34.4/docs/ares_save_options.3 rename to source/lib/c-ares-1.34.6/docs/ares_save_options.3 diff --git a/source/lib/c-ares-1.34.4/docs/ares_search.3 b/source/lib/c-ares-1.34.6/docs/ares_search.3 similarity index 100% rename from source/lib/c-ares-1.34.4/docs/ares_search.3 rename to source/lib/c-ares-1.34.6/docs/ares_search.3 diff --git a/source/lib/c-ares-1.34.4/docs/ares_search_dnsrec.3 b/source/lib/c-ares-1.34.6/docs/ares_search_dnsrec.3 similarity index 100% rename from source/lib/c-ares-1.34.4/docs/ares_search_dnsrec.3 rename to source/lib/c-ares-1.34.6/docs/ares_search_dnsrec.3 diff --git a/source/lib/c-ares-1.34.4/docs/ares_send.3 b/source/lib/c-ares-1.34.6/docs/ares_send.3 similarity index 100% rename from source/lib/c-ares-1.34.4/docs/ares_send.3 rename to source/lib/c-ares-1.34.6/docs/ares_send.3 diff --git a/source/lib/c-ares-1.34.4/docs/ares_send_dnsrec.3 b/source/lib/c-ares-1.34.6/docs/ares_send_dnsrec.3 similarity index 100% rename from source/lib/c-ares-1.34.4/docs/ares_send_dnsrec.3 rename to source/lib/c-ares-1.34.6/docs/ares_send_dnsrec.3 diff --git a/source/lib/c-ares-1.34.4/docs/ares_set_local_dev.3 b/source/lib/c-ares-1.34.6/docs/ares_set_local_dev.3 similarity index 100% rename from source/lib/c-ares-1.34.4/docs/ares_set_local_dev.3 rename to source/lib/c-ares-1.34.6/docs/ares_set_local_dev.3 diff --git a/source/lib/c-ares-1.34.4/docs/ares_set_local_ip4.3 b/source/lib/c-ares-1.34.6/docs/ares_set_local_ip4.3 similarity index 100% rename from source/lib/c-ares-1.34.4/docs/ares_set_local_ip4.3 rename to source/lib/c-ares-1.34.6/docs/ares_set_local_ip4.3 diff --git a/source/lib/c-ares-1.34.4/docs/ares_set_local_ip6.3 b/source/lib/c-ares-1.34.6/docs/ares_set_local_ip6.3 similarity index 100% rename from source/lib/c-ares-1.34.4/docs/ares_set_local_ip6.3 rename to source/lib/c-ares-1.34.6/docs/ares_set_local_ip6.3 diff --git a/source/lib/c-ares-1.34.4/docs/ares_set_pending_write_cb.3 b/source/lib/c-ares-1.34.6/docs/ares_set_pending_write_cb.3 similarity index 100% rename from source/lib/c-ares-1.34.4/docs/ares_set_pending_write_cb.3 rename to source/lib/c-ares-1.34.6/docs/ares_set_pending_write_cb.3 diff --git a/source/lib/c-ares-1.34.4/docs/ares_set_server_state_callback.3 b/source/lib/c-ares-1.34.6/docs/ares_set_server_state_callback.3 similarity index 100% rename from source/lib/c-ares-1.34.4/docs/ares_set_server_state_callback.3 rename to source/lib/c-ares-1.34.6/docs/ares_set_server_state_callback.3 diff --git a/source/lib/c-ares-1.34.4/docs/ares_set_servers.3 b/source/lib/c-ares-1.34.6/docs/ares_set_servers.3 similarity index 100% rename from source/lib/c-ares-1.34.4/docs/ares_set_servers.3 rename to source/lib/c-ares-1.34.6/docs/ares_set_servers.3 diff --git a/source/lib/c-ares-1.34.4/docs/ares_set_servers_csv.3 b/source/lib/c-ares-1.34.6/docs/ares_set_servers_csv.3 similarity index 100% rename from source/lib/c-ares-1.34.4/docs/ares_set_servers_csv.3 rename to source/lib/c-ares-1.34.6/docs/ares_set_servers_csv.3 diff --git a/source/lib/c-ares-1.34.4/docs/ares_set_servers_ports.3 b/source/lib/c-ares-1.34.6/docs/ares_set_servers_ports.3 similarity index 100% rename from source/lib/c-ares-1.34.4/docs/ares_set_servers_ports.3 rename to source/lib/c-ares-1.34.6/docs/ares_set_servers_ports.3 diff --git a/source/lib/c-ares-1.34.4/docs/ares_set_servers_ports_csv.3 b/source/lib/c-ares-1.34.6/docs/ares_set_servers_ports_csv.3 similarity index 100% rename from source/lib/c-ares-1.34.4/docs/ares_set_servers_ports_csv.3 rename to source/lib/c-ares-1.34.6/docs/ares_set_servers_ports_csv.3 diff --git a/source/lib/c-ares-1.34.4/docs/ares_set_socket_callback.3 b/source/lib/c-ares-1.34.6/docs/ares_set_socket_callback.3 similarity index 100% rename from source/lib/c-ares-1.34.4/docs/ares_set_socket_callback.3 rename to source/lib/c-ares-1.34.6/docs/ares_set_socket_callback.3 diff --git a/source/lib/c-ares-1.34.4/docs/ares_set_socket_configure_callback.3 b/source/lib/c-ares-1.34.6/docs/ares_set_socket_configure_callback.3 similarity index 100% rename from source/lib/c-ares-1.34.4/docs/ares_set_socket_configure_callback.3 rename to source/lib/c-ares-1.34.6/docs/ares_set_socket_configure_callback.3 diff --git a/source/lib/c-ares-1.34.4/docs/ares_set_socket_functions.3 b/source/lib/c-ares-1.34.6/docs/ares_set_socket_functions.3 similarity index 100% rename from source/lib/c-ares-1.34.4/docs/ares_set_socket_functions.3 rename to source/lib/c-ares-1.34.6/docs/ares_set_socket_functions.3 diff --git a/source/lib/c-ares-1.34.4/docs/ares_set_socket_functions_ex.3 b/source/lib/c-ares-1.34.6/docs/ares_set_socket_functions_ex.3 similarity index 100% rename from source/lib/c-ares-1.34.4/docs/ares_set_socket_functions_ex.3 rename to source/lib/c-ares-1.34.6/docs/ares_set_socket_functions_ex.3 diff --git a/source/lib/c-ares-1.34.4/docs/ares_set_sortlist.3 b/source/lib/c-ares-1.34.6/docs/ares_set_sortlist.3 similarity index 100% rename from source/lib/c-ares-1.34.4/docs/ares_set_sortlist.3 rename to source/lib/c-ares-1.34.6/docs/ares_set_sortlist.3 diff --git a/source/lib/c-ares-1.34.4/docs/ares_strerror.3 b/source/lib/c-ares-1.34.6/docs/ares_strerror.3 similarity index 100% rename from source/lib/c-ares-1.34.4/docs/ares_strerror.3 rename to source/lib/c-ares-1.34.6/docs/ares_strerror.3 diff --git a/source/lib/c-ares-1.34.4/docs/ares_svcb_param_t.3 b/source/lib/c-ares-1.34.6/docs/ares_svcb_param_t.3 similarity index 100% rename from source/lib/c-ares-1.34.4/docs/ares_svcb_param_t.3 rename to source/lib/c-ares-1.34.6/docs/ares_svcb_param_t.3 diff --git a/source/lib/c-ares-1.34.4/docs/ares_threadsafety.3 b/source/lib/c-ares-1.34.6/docs/ares_threadsafety.3 similarity index 100% rename from source/lib/c-ares-1.34.4/docs/ares_threadsafety.3 rename to source/lib/c-ares-1.34.6/docs/ares_threadsafety.3 diff --git a/source/lib/c-ares-1.34.4/docs/ares_timeout.3 b/source/lib/c-ares-1.34.6/docs/ares_timeout.3 similarity index 100% rename from source/lib/c-ares-1.34.4/docs/ares_timeout.3 rename to source/lib/c-ares-1.34.6/docs/ares_timeout.3 diff --git a/source/lib/c-ares-1.34.4/docs/ares_tlsa_match_t.3 b/source/lib/c-ares-1.34.6/docs/ares_tlsa_match_t.3 similarity index 100% rename from source/lib/c-ares-1.34.4/docs/ares_tlsa_match_t.3 rename to source/lib/c-ares-1.34.6/docs/ares_tlsa_match_t.3 diff --git a/source/lib/c-ares-1.34.4/docs/ares_tlsa_selector_t.3 b/source/lib/c-ares-1.34.6/docs/ares_tlsa_selector_t.3 similarity index 100% rename from source/lib/c-ares-1.34.4/docs/ares_tlsa_selector_t.3 rename to source/lib/c-ares-1.34.6/docs/ares_tlsa_selector_t.3 diff --git a/source/lib/c-ares-1.34.4/docs/ares_tlsa_usage_t.3 b/source/lib/c-ares-1.34.6/docs/ares_tlsa_usage_t.3 similarity index 100% rename from source/lib/c-ares-1.34.4/docs/ares_tlsa_usage_t.3 rename to source/lib/c-ares-1.34.6/docs/ares_tlsa_usage_t.3 diff --git a/source/lib/c-ares-1.34.4/docs/ares_version.3 b/source/lib/c-ares-1.34.6/docs/ares_version.3 similarity index 100% rename from source/lib/c-ares-1.34.4/docs/ares_version.3 rename to source/lib/c-ares-1.34.6/docs/ares_version.3 diff --git a/source/lib/c-ares-1.34.4/include/CMakeLists.txt b/source/lib/c-ares-1.34.6/include/CMakeLists.txt similarity index 100% rename from source/lib/c-ares-1.34.4/include/CMakeLists.txt rename to source/lib/c-ares-1.34.6/include/CMakeLists.txt diff --git a/source/lib/c-ares-1.34.4/include/Makefile.am b/source/lib/c-ares-1.34.6/include/Makefile.am similarity index 100% rename from source/lib/c-ares-1.34.4/include/Makefile.am rename to source/lib/c-ares-1.34.6/include/Makefile.am diff --git a/source/lib/c-ares-1.34.4/include/Makefile.in b/source/lib/c-ares-1.34.6/include/Makefile.in similarity index 99% rename from source/lib/c-ares-1.34.4/include/Makefile.in rename to source/lib/c-ares-1.34.6/include/Makefile.in index 7dc40eb0..3588b27a 100644 --- a/source/lib/c-ares-1.34.4/include/Makefile.in +++ b/source/lib/c-ares-1.34.6/include/Makefile.in @@ -105,7 +105,6 @@ am__aclocal_m4_deps = $(top_srcdir)/m4/ares_check_user_namespace.m4 \ $(top_srcdir)/m4/ax_code_coverage.m4 \ $(top_srcdir)/m4/ax_compiler_vendor.m4 \ $(top_srcdir)/m4/ax_cxx_compile_stdcxx.m4 \ - $(top_srcdir)/m4/ax_cxx_compile_stdcxx_14.m4 \ $(top_srcdir)/m4/ax_file_escapes.m4 \ $(top_srcdir)/m4/ax_pthread.m4 \ $(top_srcdir)/m4/ax_require_defined.m4 \ @@ -234,14 +233,18 @@ EGREP = @EGREP@ ETAGS = @ETAGS@ EXEEXT = @EXEEXT@ FGREP = @FGREP@ +FILECMD = @FILECMD@ GCOV = @GCOV@ GENHTML = @GENHTML@ GMOCK112_CFLAGS = @GMOCK112_CFLAGS@ GMOCK112_LIBS = @GMOCK112_LIBS@ +GMOCK117_CFLAGS = @GMOCK117_CFLAGS@ +GMOCK117_LIBS = @GMOCK117_LIBS@ GMOCK_CFLAGS = @GMOCK_CFLAGS@ GMOCK_LIBS = @GMOCK_LIBS@ GREP = @GREP@ HAVE_CXX14 = @HAVE_CXX14@ +HAVE_CXX17 = @HAVE_CXX17@ INSTALL = @INSTALL@ INSTALL_DATA = @INSTALL_DATA@ INSTALL_PROGRAM = @INSTALL_PROGRAM@ diff --git a/source/lib/c-ares-1.34.4/include/ares.h b/source/lib/c-ares-1.34.6/include/ares.h similarity index 100% rename from source/lib/c-ares-1.34.4/include/ares.h rename to source/lib/c-ares-1.34.6/include/ares.h diff --git a/source/lib/c-ares-1.34.4/include/ares_build.h b/source/lib/c-ares-1.34.6/include/ares_build.h similarity index 100% rename from source/lib/c-ares-1.34.4/include/ares_build.h rename to source/lib/c-ares-1.34.6/include/ares_build.h diff --git a/source/lib/c-ares-1.34.4/include/ares_build.h.cmake b/source/lib/c-ares-1.34.6/include/ares_build.h.cmake similarity index 100% rename from source/lib/c-ares-1.34.4/include/ares_build.h.cmake rename to source/lib/c-ares-1.34.6/include/ares_build.h.cmake diff --git a/source/lib/c-ares-1.34.4/include/ares_build.h.in b/source/lib/c-ares-1.34.6/include/ares_build.h.in similarity index 100% rename from source/lib/c-ares-1.34.4/include/ares_build.h.in rename to source/lib/c-ares-1.34.6/include/ares_build.h.in diff --git a/source/lib/c-ares-1.34.4/include/ares_dns.h b/source/lib/c-ares-1.34.6/include/ares_dns.h similarity index 100% rename from source/lib/c-ares-1.34.4/include/ares_dns.h rename to source/lib/c-ares-1.34.6/include/ares_dns.h diff --git a/source/lib/c-ares-1.34.4/include/ares_dns_record.h b/source/lib/c-ares-1.34.6/include/ares_dns_record.h similarity index 99% rename from source/lib/c-ares-1.34.4/include/ares_dns_record.h rename to source/lib/c-ares-1.34.6/include/ares_dns_record.h index 2896eab2..cec9f47f 100644 --- a/source/lib/c-ares-1.34.4/include/ares_dns_record.h +++ b/source/lib/c-ares-1.34.6/include/ares_dns_record.h @@ -1104,7 +1104,7 @@ CARES_EXTERN ares_status_t ares_dns_write(const ares_dns_record_t *dnsrec, * (such as the ttl decrement capability). * * \param[in] dnsrec Pointer to initialized and filled DNS record object. - * \return duplicted DNS record object, or NULL on out of memory. + * \return duplicated DNS record object, or NULL on out of memory. */ CARES_EXTERN ares_dns_record_t * ares_dns_record_duplicate(const ares_dns_record_t *dnsrec); diff --git a/source/lib/c-ares-1.34.4/include/ares_nameser.h b/source/lib/c-ares-1.34.6/include/ares_nameser.h similarity index 100% rename from source/lib/c-ares-1.34.4/include/ares_nameser.h rename to source/lib/c-ares-1.34.6/include/ares_nameser.h diff --git a/source/lib/c-ares-1.34.4/include/ares_version.h b/source/lib/c-ares-1.34.6/include/ares_version.h similarity index 93% rename from source/lib/c-ares-1.34.4/include/ares_version.h rename to source/lib/c-ares-1.34.6/include/ares_version.h index 782046bd..006029c1 100644 --- a/source/lib/c-ares-1.34.4/include/ares_version.h +++ b/source/lib/c-ares-1.34.6/include/ares_version.h @@ -28,12 +28,12 @@ #define ARES__VERSION_H /* This is the global package copyright */ -#define ARES_COPYRIGHT "2004 - 2024 Daniel Stenberg, ." +#define ARES_COPYRIGHT "2004 - 2025 Daniel Stenberg, ." #define ARES_VERSION_MAJOR 1 #define ARES_VERSION_MINOR 34 -#define ARES_VERSION_PATCH 4 -#define ARES_VERSION_STR "1.34.4" +#define ARES_VERSION_PATCH 6 +#define ARES_VERSION_STR "1.34.6" /* NOTE: We cannot make the version string a C preprocessor stringify operation * due to assumptions made by integrators that aren't properly using diff --git a/source/lib/c-ares-1.34.4/libcares.pc.cmake b/source/lib/c-ares-1.34.6/libcares.pc.cmake similarity index 100% rename from source/lib/c-ares-1.34.4/libcares.pc.cmake rename to source/lib/c-ares-1.34.6/libcares.pc.cmake diff --git a/source/lib/c-ares-1.34.4/libcares.pc.in b/source/lib/c-ares-1.34.6/libcares.pc.in similarity index 100% rename from source/lib/c-ares-1.34.4/libcares.pc.in rename to source/lib/c-ares-1.34.6/libcares.pc.in diff --git a/source/lib/c-ares-1.34.4/m4/ares_check_user_namespace.m4 b/source/lib/c-ares-1.34.6/m4/ares_check_user_namespace.m4 similarity index 100% rename from source/lib/c-ares-1.34.4/m4/ares_check_user_namespace.m4 rename to source/lib/c-ares-1.34.6/m4/ares_check_user_namespace.m4 diff --git a/source/lib/c-ares-1.34.4/m4/ares_check_uts_namespace.m4 b/source/lib/c-ares-1.34.6/m4/ares_check_uts_namespace.m4 similarity index 100% rename from source/lib/c-ares-1.34.4/m4/ares_check_uts_namespace.m4 rename to source/lib/c-ares-1.34.6/m4/ares_check_uts_namespace.m4 diff --git a/source/lib/c-ares-1.34.4/m4/ax_ac_append_to_file.m4 b/source/lib/c-ares-1.34.6/m4/ax_ac_append_to_file.m4 similarity index 100% rename from source/lib/c-ares-1.34.4/m4/ax_ac_append_to_file.m4 rename to source/lib/c-ares-1.34.6/m4/ax_ac_append_to_file.m4 diff --git a/source/lib/c-ares-1.34.4/m4/ax_ac_print_to_file.m4 b/source/lib/c-ares-1.34.6/m4/ax_ac_print_to_file.m4 similarity index 100% rename from source/lib/c-ares-1.34.4/m4/ax_ac_print_to_file.m4 rename to source/lib/c-ares-1.34.6/m4/ax_ac_print_to_file.m4 diff --git a/source/lib/c-ares-1.34.4/m4/ax_add_am_macro_static.m4 b/source/lib/c-ares-1.34.6/m4/ax_add_am_macro_static.m4 similarity index 100% rename from source/lib/c-ares-1.34.4/m4/ax_add_am_macro_static.m4 rename to source/lib/c-ares-1.34.6/m4/ax_add_am_macro_static.m4 diff --git a/source/lib/c-ares-1.34.4/m4/ax_am_macros_static.m4 b/source/lib/c-ares-1.34.6/m4/ax_am_macros_static.m4 similarity index 100% rename from source/lib/c-ares-1.34.4/m4/ax_am_macros_static.m4 rename to source/lib/c-ares-1.34.6/m4/ax_am_macros_static.m4 diff --git a/source/lib/c-ares-1.34.4/m4/ax_append_compile_flags.m4 b/source/lib/c-ares-1.34.6/m4/ax_append_compile_flags.m4 similarity index 100% rename from source/lib/c-ares-1.34.4/m4/ax_append_compile_flags.m4 rename to source/lib/c-ares-1.34.6/m4/ax_append_compile_flags.m4 diff --git a/source/lib/c-ares-1.34.4/m4/ax_append_flag.m4 b/source/lib/c-ares-1.34.6/m4/ax_append_flag.m4 similarity index 100% rename from source/lib/c-ares-1.34.4/m4/ax_append_flag.m4 rename to source/lib/c-ares-1.34.6/m4/ax_append_flag.m4 diff --git a/source/lib/c-ares-1.34.4/m4/ax_append_link_flags.m4 b/source/lib/c-ares-1.34.6/m4/ax_append_link_flags.m4 similarity index 100% rename from source/lib/c-ares-1.34.4/m4/ax_append_link_flags.m4 rename to source/lib/c-ares-1.34.6/m4/ax_append_link_flags.m4 diff --git a/source/lib/c-ares-1.34.4/m4/ax_check_compile_flag.m4 b/source/lib/c-ares-1.34.6/m4/ax_check_compile_flag.m4 similarity index 100% rename from source/lib/c-ares-1.34.4/m4/ax_check_compile_flag.m4 rename to source/lib/c-ares-1.34.6/m4/ax_check_compile_flag.m4 diff --git a/source/lib/c-ares-1.34.4/m4/ax_check_gnu_make.m4 b/source/lib/c-ares-1.34.6/m4/ax_check_gnu_make.m4 similarity index 100% rename from source/lib/c-ares-1.34.4/m4/ax_check_gnu_make.m4 rename to source/lib/c-ares-1.34.6/m4/ax_check_gnu_make.m4 diff --git a/source/lib/c-ares-1.34.4/m4/ax_check_link_flag.m4 b/source/lib/c-ares-1.34.6/m4/ax_check_link_flag.m4 similarity index 100% rename from source/lib/c-ares-1.34.4/m4/ax_check_link_flag.m4 rename to source/lib/c-ares-1.34.6/m4/ax_check_link_flag.m4 diff --git a/source/lib/c-ares-1.34.4/m4/ax_code_coverage.m4 b/source/lib/c-ares-1.34.6/m4/ax_code_coverage.m4 similarity index 100% rename from source/lib/c-ares-1.34.4/m4/ax_code_coverage.m4 rename to source/lib/c-ares-1.34.6/m4/ax_code_coverage.m4 diff --git a/source/lib/c-ares-1.34.4/m4/ax_compiler_vendor.m4 b/source/lib/c-ares-1.34.6/m4/ax_compiler_vendor.m4 similarity index 100% rename from source/lib/c-ares-1.34.4/m4/ax_compiler_vendor.m4 rename to source/lib/c-ares-1.34.6/m4/ax_compiler_vendor.m4 diff --git a/source/lib/c-ares-1.34.4/m4/ax_cxx_compile_stdcxx.m4 b/source/lib/c-ares-1.34.6/m4/ax_cxx_compile_stdcxx.m4 similarity index 100% rename from source/lib/c-ares-1.34.4/m4/ax_cxx_compile_stdcxx.m4 rename to source/lib/c-ares-1.34.6/m4/ax_cxx_compile_stdcxx.m4 diff --git a/source/lib/c-ares-1.34.4/m4/ax_file_escapes.m4 b/source/lib/c-ares-1.34.6/m4/ax_file_escapes.m4 similarity index 100% rename from source/lib/c-ares-1.34.4/m4/ax_file_escapes.m4 rename to source/lib/c-ares-1.34.6/m4/ax_file_escapes.m4 diff --git a/source/lib/c-ares-1.34.4/m4/ax_pthread.m4 b/source/lib/c-ares-1.34.6/m4/ax_pthread.m4 similarity index 99% rename from source/lib/c-ares-1.34.4/m4/ax_pthread.m4 rename to source/lib/c-ares-1.34.6/m4/ax_pthread.m4 index 9f35d139..f8823176 100644 --- a/source/lib/c-ares-1.34.4/m4/ax_pthread.m4 +++ b/source/lib/c-ares-1.34.6/m4/ax_pthread.m4 @@ -160,7 +160,7 @@ ax_pthread_flags="pthreads none -Kthread -pthread -pthreads -mthreads pthread -- case $host_os in - freebsd*) + freebsd*|midnightbsd*) # -kthread: FreeBSD kernel threads (preferred to -pthread since SMP-able) # lthread: LinuxThreads port on FreeBSD (also preferred to -pthread) diff --git a/source/lib/c-ares-1.34.4/m4/ax_require_defined.m4 b/source/lib/c-ares-1.34.6/m4/ax_require_defined.m4 similarity index 100% rename from source/lib/c-ares-1.34.4/m4/ax_require_defined.m4 rename to source/lib/c-ares-1.34.6/m4/ax_require_defined.m4 diff --git a/source/lib/c-ares-1.34.4/m4/libtool.m4 b/source/lib/c-ares-1.34.6/m4/libtool.m4 similarity index 97% rename from source/lib/c-ares-1.34.4/m4/libtool.m4 rename to source/lib/c-ares-1.34.6/m4/libtool.m4 index c4c02946..e7b68334 100755 --- a/source/lib/c-ares-1.34.4/m4/libtool.m4 +++ b/source/lib/c-ares-1.34.6/m4/libtool.m4 @@ -1,6 +1,7 @@ # libtool.m4 - Configure libtool for the host system. -*-Autoconf-*- # -# Copyright (C) 1996-2001, 2003-2015 Free Software Foundation, Inc. +# Copyright (C) 1996-2001, 2003-2019, 2021-2022 Free Software +# Foundation, Inc. # Written by Gordon Matzigkeit, 1996 # # This file is free software; the Free Software Foundation gives @@ -31,7 +32,7 @@ m4_define([_LT_COPYING], [dnl # along with this program. If not, see . ]) -# serial 58 LT_INIT +# serial 59 LT_INIT # LT_PREREQ(VERSION) @@ -181,6 +182,7 @@ m4_require([_LT_FILEUTILS_DEFAULTS])dnl m4_require([_LT_CHECK_SHELL_FEATURES])dnl m4_require([_LT_PATH_CONVERSION_FUNCTIONS])dnl m4_require([_LT_CMD_RELOAD])dnl +m4_require([_LT_DECL_FILECMD])dnl m4_require([_LT_CHECK_MAGIC_METHOD])dnl m4_require([_LT_CHECK_SHAREDLIB_FROM_LINKLIB])dnl m4_require([_LT_CMD_OLD_ARCHIVE])dnl @@ -219,8 +221,8 @@ esac ofile=libtool can_build_shared=yes -# All known linkers require a '.a' archive for static linking (except MSVC, -# which needs '.lib'). +# All known linkers require a '.a' archive for static linking (except MSVC and +# ICC, which need '.lib'). libext=a with_gnu_ld=$lt_cv_prog_gnu_ld @@ -777,7 +779,7 @@ _LT_EOF # if finds mixed CR/LF and LF-only lines. Since sed operates in # text mode, it properly converts lines to CR/LF. This bash problem # is reportedly fixed, but why not run on old versions too? - sed '$q' "$ltmain" >> "$cfgfile" \ + $SED '$q' "$ltmain" >> "$cfgfile" \ || (rm -f "$cfgfile"; exit 1) mv -f "$cfgfile" "$ofile" || @@ -1041,8 +1043,8 @@ int forced_loaded() { return 2;} _LT_EOF echo "$LTCC $LTCFLAGS -c -o conftest.o conftest.c" >&AS_MESSAGE_LOG_FD $LTCC $LTCFLAGS -c -o conftest.o conftest.c 2>&AS_MESSAGE_LOG_FD - echo "$AR cr libconftest.a conftest.o" >&AS_MESSAGE_LOG_FD - $AR cr libconftest.a conftest.o 2>&AS_MESSAGE_LOG_FD + echo "$AR $AR_FLAGS libconftest.a conftest.o" >&AS_MESSAGE_LOG_FD + $AR $AR_FLAGS libconftest.a conftest.o 2>&AS_MESSAGE_LOG_FD echo "$RANLIB libconftest.a" >&AS_MESSAGE_LOG_FD $RANLIB libconftest.a 2>&AS_MESSAGE_LOG_FD cat > conftest.c << _LT_EOF @@ -1066,17 +1068,12 @@ _LT_EOF _lt_dar_allow_undefined='$wl-undefined ${wl}suppress' ;; darwin1.*) _lt_dar_allow_undefined='$wl-flat_namespace $wl-undefined ${wl}suppress' ;; - darwin*) # darwin 5.x on - # if running on 10.5 or later, the deployment target defaults - # to the OS version, if on x86, and 10.4, the deployment - # target defaults to 10.4. Don't you love it? - case ${MACOSX_DEPLOYMENT_TARGET-10.0},$host in - 10.0,*86*-darwin8*|10.0,*-darwin[[912]]*) - _lt_dar_allow_undefined='$wl-undefined ${wl}dynamic_lookup' ;; - 10.[[012]][[,.]]*) - _lt_dar_allow_undefined='$wl-flat_namespace $wl-undefined ${wl}suppress' ;; - 10.*|11.*) - _lt_dar_allow_undefined='$wl-undefined ${wl}dynamic_lookup' ;; + darwin*) + case $MACOSX_DEPLOYMENT_TARGET,$host in + 10.[[012]],*|,*powerpc*-darwin[[5-8]]*) + _lt_dar_allow_undefined='$wl-flat_namespace $wl-undefined ${wl}suppress' ;; + *) + _lt_dar_allow_undefined='$wl-undefined ${wl}dynamic_lookup' ;; esac ;; esac @@ -1125,12 +1122,12 @@ m4_defun([_LT_DARWIN_LINKER_FEATURES], output_verbose_link_cmd=func_echo_all _LT_TAGVAR(archive_cmds, $1)="\$CC -dynamiclib \$allow_undefined_flag -o \$lib \$libobjs \$deplibs \$compiler_flags -install_name \$rpath/\$soname \$verstring $_lt_dar_single_mod$_lt_dsymutil" _LT_TAGVAR(module_cmds, $1)="\$CC \$allow_undefined_flag -o \$lib -bundle \$libobjs \$deplibs \$compiler_flags$_lt_dsymutil" - _LT_TAGVAR(archive_expsym_cmds, $1)="sed 's|^|_|' < \$export_symbols > \$output_objdir/\$libname-symbols.expsym~\$CC -dynamiclib \$allow_undefined_flag -o \$lib \$libobjs \$deplibs \$compiler_flags -install_name \$rpath/\$soname \$verstring $_lt_dar_single_mod$_lt_dar_export_syms$_lt_dsymutil" - _LT_TAGVAR(module_expsym_cmds, $1)="sed -e 's|^|_|' < \$export_symbols > \$output_objdir/\$libname-symbols.expsym~\$CC \$allow_undefined_flag -o \$lib -bundle \$libobjs \$deplibs \$compiler_flags$_lt_dar_export_syms$_lt_dsymutil" + _LT_TAGVAR(archive_expsym_cmds, $1)="$SED 's|^|_|' < \$export_symbols > \$output_objdir/\$libname-symbols.expsym~\$CC -dynamiclib \$allow_undefined_flag -o \$lib \$libobjs \$deplibs \$compiler_flags -install_name \$rpath/\$soname \$verstring $_lt_dar_single_mod$_lt_dar_export_syms$_lt_dsymutil" + _LT_TAGVAR(module_expsym_cmds, $1)="$SED -e 's|^|_|' < \$export_symbols > \$output_objdir/\$libname-symbols.expsym~\$CC \$allow_undefined_flag -o \$lib -bundle \$libobjs \$deplibs \$compiler_flags$_lt_dar_export_syms$_lt_dsymutil" m4_if([$1], [CXX], [ if test yes != "$lt_cv_apple_cc_single_mod"; then _LT_TAGVAR(archive_cmds, $1)="\$CC -r -keep_private_externs -nostdlib -o \$lib-master.o \$libobjs~\$CC -dynamiclib \$allow_undefined_flag -o \$lib \$lib-master.o \$deplibs \$compiler_flags -install_name \$rpath/\$soname \$verstring$_lt_dsymutil" - _LT_TAGVAR(archive_expsym_cmds, $1)="sed 's|^|_|' < \$export_symbols > \$output_objdir/\$libname-symbols.expsym~\$CC -r -keep_private_externs -nostdlib -o \$lib-master.o \$libobjs~\$CC -dynamiclib \$allow_undefined_flag -o \$lib \$lib-master.o \$deplibs \$compiler_flags -install_name \$rpath/\$soname \$verstring$_lt_dar_export_syms$_lt_dsymutil" + _LT_TAGVAR(archive_expsym_cmds, $1)="$SED 's|^|_|' < \$export_symbols > \$output_objdir/\$libname-symbols.expsym~\$CC -r -keep_private_externs -nostdlib -o \$lib-master.o \$libobjs~\$CC -dynamiclib \$allow_undefined_flag -o \$lib \$lib-master.o \$deplibs \$compiler_flags -install_name \$rpath/\$soname \$verstring$_lt_dar_export_syms$_lt_dsymutil" fi ],[]) else @@ -1244,7 +1241,8 @@ _LT_DECL([], [ECHO], [1], [An echo program that protects backslashes]) # _LT_WITH_SYSROOT # ---------------- AC_DEFUN([_LT_WITH_SYSROOT], -[AC_MSG_CHECKING([for sysroot]) +[m4_require([_LT_DECL_SED])dnl +AC_MSG_CHECKING([for sysroot]) AC_ARG_WITH([sysroot], [AS_HELP_STRING([--with-sysroot@<:@=DIR@:>@], [Search for dependent libraries within DIR (or the compiler's sysroot @@ -1261,7 +1259,7 @@ case $with_sysroot in #( fi ;; #( /*) - lt_sysroot=`echo "$with_sysroot" | sed -e "$sed_quote_subst"` + lt_sysroot=`echo "$with_sysroot" | $SED -e "$sed_quote_subst"` ;; #( no|'') ;; #( @@ -1291,7 +1289,7 @@ ia64-*-hpux*) # options accordingly. echo 'int i;' > conftest.$ac_ext if AC_TRY_EVAL(ac_compile); then - case `/usr/bin/file conftest.$ac_objext` in + case `$FILECMD conftest.$ac_objext` in *ELF-32*) HPUX_IA64_MODE=32 ;; @@ -1308,7 +1306,7 @@ ia64-*-hpux*) echo '[#]line '$LINENO' "configure"' > conftest.$ac_ext if AC_TRY_EVAL(ac_compile); then if test yes = "$lt_cv_prog_gnu_ld"; then - case `/usr/bin/file conftest.$ac_objext` in + case `$FILECMD conftest.$ac_objext` in *32-bit*) LD="${LD-ld} -melf32bsmip" ;; @@ -1320,7 +1318,7 @@ ia64-*-hpux*) ;; esac else - case `/usr/bin/file conftest.$ac_objext` in + case `$FILECMD conftest.$ac_objext` in *32-bit*) LD="${LD-ld} -32" ;; @@ -1342,7 +1340,7 @@ mips64*-*linux*) echo '[#]line '$LINENO' "configure"' > conftest.$ac_ext if AC_TRY_EVAL(ac_compile); then emul=elf - case `/usr/bin/file conftest.$ac_objext` in + case `$FILECMD conftest.$ac_objext` in *32-bit*) emul="${emul}32" ;; @@ -1350,7 +1348,7 @@ mips64*-*linux*) emul="${emul}64" ;; esac - case `/usr/bin/file conftest.$ac_objext` in + case `$FILECMD conftest.$ac_objext` in *MSB*) emul="${emul}btsmip" ;; @@ -1358,7 +1356,7 @@ mips64*-*linux*) emul="${emul}ltsmip" ;; esac - case `/usr/bin/file conftest.$ac_objext` in + case `$FILECMD conftest.$ac_objext` in *N32*) emul="${emul}n32" ;; @@ -1378,14 +1376,14 @@ s390*-*linux*|s390*-*tpf*|sparc*-*linux*) # not appear in the list. echo 'int i;' > conftest.$ac_ext if AC_TRY_EVAL(ac_compile); then - case `/usr/bin/file conftest.o` in + case `$FILECMD conftest.o` in *32-bit*) case $host in x86_64-*kfreebsd*-gnu) LD="${LD-ld} -m elf_i386_fbsd" ;; x86_64-*linux*) - case `/usr/bin/file conftest.o` in + case `$FILECMD conftest.o` in *x86-64*) LD="${LD-ld} -m elf32_x86_64" ;; @@ -1453,7 +1451,7 @@ s390*-*linux*|s390*-*tpf*|sparc*-*linux*) # options accordingly. echo 'int i;' > conftest.$ac_ext if AC_TRY_EVAL(ac_compile); then - case `/usr/bin/file conftest.o` in + case `$FILECMD conftest.o` in *64-bit*) case $lt_cv_prog_gnu_ld in yes*) @@ -1492,9 +1490,22 @@ need_locks=$enable_libtool_lock m4_defun([_LT_PROG_AR], [AC_CHECK_TOOLS(AR, [ar], false) : ${AR=ar} -: ${AR_FLAGS=cr} _LT_DECL([], [AR], [1], [The archiver]) -_LT_DECL([], [AR_FLAGS], [1], [Flags to create an archive]) + +# Use ARFLAGS variable as AR's operation code to sync the variable naming with +# Automake. If both AR_FLAGS and ARFLAGS are specified, AR_FLAGS should have +# higher priority because thats what people were doing historically (setting +# ARFLAGS for automake and AR_FLAGS for libtool). FIXME: Make the AR_FLAGS +# variable obsoleted/removed. + +test ${AR_FLAGS+y} || AR_FLAGS=${ARFLAGS-cr} +lt_ar_flags=$AR_FLAGS +_LT_DECL([], [lt_ar_flags], [0], [Flags to create an archive (by configure)]) + +# Make AR_FLAGS overridable by 'make ARFLAGS='. Don't try to run-time override +# by AR_FLAGS because that was never working and AR_FLAGS is about to die. +_LT_DECL([], [AR_FLAGS], [\@S|@{ARFLAGS-"\@S|@lt_ar_flags"}], + [Flags to create an archive]) AC_CACHE_CHECK([for archiver @FILE support], [lt_cv_ar_at_file], [lt_cv_ar_at_file=no @@ -1713,7 +1724,7 @@ AC_CACHE_VAL([lt_cv_sys_max_cmd_len], [dnl lt_cv_sys_max_cmd_len=8192; ;; - bitrig* | darwin* | dragonfly* | freebsd* | netbsd* | openbsd*) + bitrig* | darwin* | dragonfly* | freebsd* | midnightbsd* | netbsd* | openbsd*) # This has been around since 386BSD, at least. Likely further. if test -x /sbin/sysctl; then lt_cv_sys_max_cmd_len=`/sbin/sysctl -n kern.argmax` @@ -1756,7 +1767,7 @@ AC_CACHE_VAL([lt_cv_sys_max_cmd_len], [dnl sysv5* | sco5v6* | sysv4.2uw2*) kargmax=`grep ARG_MAX /etc/conf/cf.d/stune 2>/dev/null` if test -n "$kargmax"; then - lt_cv_sys_max_cmd_len=`echo $kargmax | sed 's/.*[[ ]]//'` + lt_cv_sys_max_cmd_len=`echo $kargmax | $SED 's/.*[[ ]]//'` else lt_cv_sys_max_cmd_len=32768 fi @@ -2206,26 +2217,35 @@ m4_defun([_LT_CMD_STRIPLIB], striplib= old_striplib= AC_MSG_CHECKING([whether stripping libraries is possible]) -if test -n "$STRIP" && $STRIP -V 2>&1 | $GREP "GNU strip" >/dev/null; then - test -z "$old_striplib" && old_striplib="$STRIP --strip-debug" - test -z "$striplib" && striplib="$STRIP --strip-unneeded" - AC_MSG_RESULT([yes]) +if test -z "$STRIP"; then + AC_MSG_RESULT([no]) else -# FIXME - insert some real tests, host_os isn't really good enough - case $host_os in - darwin*) - if test -n "$STRIP"; then + if $STRIP -V 2>&1 | $GREP "GNU strip" >/dev/null; then + old_striplib="$STRIP --strip-debug" + striplib="$STRIP --strip-unneeded" + AC_MSG_RESULT([yes]) + else + case $host_os in + darwin*) + # FIXME - insert some real tests, host_os isn't really good enough striplib="$STRIP -x" old_striplib="$STRIP -S" AC_MSG_RESULT([yes]) - else + ;; + freebsd*) + if $STRIP -V 2>&1 | $GREP "elftoolchain" >/dev/null; then + old_striplib="$STRIP --strip-debug" + striplib="$STRIP --strip-unneeded" + AC_MSG_RESULT([yes]) + else + AC_MSG_RESULT([no]) + fi + ;; + *) AC_MSG_RESULT([no]) - fi - ;; - *) - AC_MSG_RESULT([no]) - ;; - esac + ;; + esac + fi fi _LT_DECL([], [old_striplib], [1], [Commands to strip libraries]) _LT_DECL([], [striplib], [1]) @@ -2548,7 +2568,7 @@ cygwin* | mingw* | pw32* | cegcc*) case $host_os in cygwin*) # Cygwin DLLs use 'cyg' prefix rather than 'lib' - soname_spec='`echo $libname | sed -e 's/^lib/cyg/'``echo $release | $SED -e 's/[[.]]/-/g'`$versuffix$shared_ext' + soname_spec='`echo $libname | $SED -e 's/^lib/cyg/'``echo $release | $SED -e 's/[[.]]/-/g'`$versuffix$shared_ext' m4_if([$1], [],[ sys_lib_search_path_spec="$sys_lib_search_path_spec /usr/lib/w32api"]) ;; @@ -2558,14 +2578,14 @@ m4_if([$1], [],[ ;; pw32*) # pw32 DLLs use 'pw' prefix rather than 'lib' - library_names_spec='`echo $libname | sed -e 's/^lib/pw/'``echo $release | $SED -e 's/[[.]]/-/g'`$versuffix$shared_ext' + library_names_spec='`echo $libname | $SED -e 's/^lib/pw/'``echo $release | $SED -e 's/[[.]]/-/g'`$versuffix$shared_ext' ;; esac dynamic_linker='Win32 ld.exe' ;; - *,cl*) - # Native MSVC + *,cl* | *,icl*) + # Native MSVC or ICC libname_spec='$name' soname_spec='$libname`echo $release | $SED -e 's/[[.]]/-/g'`$versuffix$shared_ext' library_names_spec='$libname.dll.lib' @@ -2584,7 +2604,7 @@ m4_if([$1], [],[ done IFS=$lt_save_ifs # Convert to MSYS style. - sys_lib_search_path_spec=`$ECHO "$sys_lib_search_path_spec" | sed -e 's|\\\\|/|g' -e 's| \\([[a-zA-Z]]\\):| /\\1|g' -e 's|^ ||'` + sys_lib_search_path_spec=`$ECHO "$sys_lib_search_path_spec" | $SED -e 's|\\\\|/|g' -e 's| \\([[a-zA-Z]]\\):| /\\1|g' -e 's|^ ||'` ;; cygwin*) # Convert to unix form, then to dos form, then back to unix form @@ -2621,7 +2641,7 @@ m4_if([$1], [],[ ;; *) - # Assume MSVC wrapper + # Assume MSVC and ICC wrapper library_names_spec='$libname`echo $release | $SED -e 's/[[.]]/-/g'`$versuffix$shared_ext $libname.lib' dynamic_linker='Win32 ld.exe' ;; @@ -2654,7 +2674,7 @@ dgux*) shlibpath_var=LD_LIBRARY_PATH ;; -freebsd* | dragonfly*) +freebsd* | dragonfly* | midnightbsd*) # DragonFly does not have aout. When/if they implement a new # versioning mechanism, adjust this. if test -x /usr/bin/objformat; then @@ -3465,7 +3485,7 @@ beos*) bsdi[[45]]*) lt_cv_deplibs_check_method='file_magic ELF [[0-9]][[0-9]]*-bit [[ML]]SB (shared object|dynamic lib)' - lt_cv_file_magic_cmd='/usr/bin/file -L' + lt_cv_file_magic_cmd='$FILECMD -L' lt_cv_file_magic_test_file=/shlib/libc.so ;; @@ -3499,14 +3519,14 @@ darwin* | rhapsody*) lt_cv_deplibs_check_method=pass_all ;; -freebsd* | dragonfly*) +freebsd* | dragonfly* | midnightbsd*) if echo __ELF__ | $CC -E - | $GREP __ELF__ > /dev/null; then case $host_cpu in i*86 ) # Not sure whether the presence of OpenBSD here was a mistake. # Let's accept both of them until this is cleared up. lt_cv_deplibs_check_method='file_magic (FreeBSD|OpenBSD|DragonFly)/i[[3-9]]86 (compact )?demand paged shared library' - lt_cv_file_magic_cmd=/usr/bin/file + lt_cv_file_magic_cmd=$FILECMD lt_cv_file_magic_test_file=`echo /usr/lib/libc.so.*` ;; esac @@ -3520,7 +3540,7 @@ haiku*) ;; hpux10.20* | hpux11*) - lt_cv_file_magic_cmd=/usr/bin/file + lt_cv_file_magic_cmd=$FILECMD case $host_cpu in ia64*) lt_cv_deplibs_check_method='file_magic (s[[0-9]][[0-9]][[0-9]]|ELF-[[0-9]][[0-9]]) shared object file - IA64' @@ -3567,7 +3587,7 @@ netbsd* | netbsdelf*-gnu) newos6*) lt_cv_deplibs_check_method='file_magic ELF [[0-9]][[0-9]]*-bit [[ML]]SB (executable|dynamic lib)' - lt_cv_file_magic_cmd=/usr/bin/file + lt_cv_file_magic_cmd=$FILECMD lt_cv_file_magic_test_file=/usr/lib/libnls.so ;; @@ -3694,13 +3714,13 @@ else mingw*) lt_bad_file=conftest.nm/nofile ;; *) lt_bad_file=/dev/null ;; esac - case `"$tmp_nm" -B $lt_bad_file 2>&1 | sed '1q'` in + case `"$tmp_nm" -B $lt_bad_file 2>&1 | $SED '1q'` in *$lt_bad_file* | *'Invalid file or object type'*) lt_cv_path_NM="$tmp_nm -B" break 2 ;; *) - case `"$tmp_nm" -p /dev/null 2>&1 | sed '1q'` in + case `"$tmp_nm" -p /dev/null 2>&1 | $SED '1q'` in */dev/null*) lt_cv_path_NM="$tmp_nm -p" break 2 @@ -3726,7 +3746,7 @@ else # Let the user override the test. else AC_CHECK_TOOLS(DUMPBIN, [dumpbin "link -dump"], :) - case `$DUMPBIN -symbols -headers /dev/null 2>&1 | sed '1q'` in + case `$DUMPBIN -symbols -headers /dev/null 2>&1 | $SED '1q'` in *COFF*) DUMPBIN="$DUMPBIN -symbols -headers" ;; @@ -3966,7 +3986,7 @@ esac if test "$lt_cv_nm_interface" = "MS dumpbin"; then # Gets list of data symbols to import. - lt_cv_sys_global_symbol_to_import="sed -n -e 's/^I .* \(.*\)$/\1/p'" + lt_cv_sys_global_symbol_to_import="$SED -n -e 's/^I .* \(.*\)$/\1/p'" # Adjust the below global symbol transforms to fixup imported variables. lt_cdecl_hook=" -e 's/^I .* \(.*\)$/extern __declspec(dllimport) char \1;/p'" lt_c_name_hook=" -e 's/^I .* \(.*\)$/ {\"\1\", (void *) 0},/p'" @@ -3984,20 +4004,20 @@ fi # Transform an extracted symbol line into a proper C declaration. # Some systems (esp. on ia64) link data and code symbols differently, # so use this general approach. -lt_cv_sys_global_symbol_to_cdecl="sed -n"\ +lt_cv_sys_global_symbol_to_cdecl="$SED -n"\ $lt_cdecl_hook\ " -e 's/^T .* \(.*\)$/extern int \1();/p'"\ " -e 's/^$symcode$symcode* .* \(.*\)$/extern char \1;/p'" # Transform an extracted symbol line into symbol name and symbol address -lt_cv_sys_global_symbol_to_c_name_address="sed -n"\ +lt_cv_sys_global_symbol_to_c_name_address="$SED -n"\ $lt_c_name_hook\ " -e 's/^: \(.*\) .*$/ {\"\1\", (void *) 0},/p'"\ " -e 's/^$symcode$symcode* .* \(.*\)$/ {\"\1\", (void *) \&\1},/p'" # Transform an extracted symbol line into symbol name with lib prefix and # symbol address. -lt_cv_sys_global_symbol_to_c_name_address_lib_prefix="sed -n"\ +lt_cv_sys_global_symbol_to_c_name_address_lib_prefix="$SED -n"\ $lt_c_name_lib_hook\ " -e 's/^: \(.*\) .*$/ {\"\1\", (void *) 0},/p'"\ " -e 's/^$symcode$symcode* .* \(lib.*\)$/ {\"\1\", (void *) \&\1},/p'"\ @@ -4021,7 +4041,7 @@ for ac_symprfx in "" "_"; do if test "$lt_cv_nm_interface" = "MS dumpbin"; then # Fake it for dumpbin and say T for any non-static function, # D for any global variable and I for any imported variable. - # Also find C++ and __fastcall symbols from MSVC++, + # Also find C++ and __fastcall symbols from MSVC++ or ICC, # which start with @ or ?. lt_cv_sys_global_symbol_pipe="$AWK ['"\ " {last_section=section; section=\$ 3};"\ @@ -4039,9 +4059,9 @@ for ac_symprfx in "" "_"; do " s[1]~prfx {split(s[1],t,\"@\"); print f,t[1],substr(t[1],length(prfx))}"\ " ' prfx=^$ac_symprfx]" else - lt_cv_sys_global_symbol_pipe="sed -n -e 's/^.*[[ ]]\($symcode$symcode*\)[[ ]][[ ]]*$ac_symprfx$sympat$opt_cr$/$symxfrm/p'" + lt_cv_sys_global_symbol_pipe="$SED -n -e 's/^.*[[ ]]\($symcode$symcode*\)[[ ]][[ ]]*$ac_symprfx$sympat$opt_cr$/$symxfrm/p'" fi - lt_cv_sys_global_symbol_pipe="$lt_cv_sys_global_symbol_pipe | sed '/ __gnu_lto/d'" + lt_cv_sys_global_symbol_pipe="$lt_cv_sys_global_symbol_pipe | $SED '/ __gnu_lto/d'" # Check to see that the pipe works correctly. pipe_works=no @@ -4329,7 +4349,7 @@ m4_if([$1], [CXX], [ ;; esac ;; - freebsd* | dragonfly*) + freebsd* | dragonfly* | midnightbsd*) # FreeBSD uses GNU C++ ;; hpux9* | hpux10* | hpux11*) @@ -4412,7 +4432,7 @@ m4_if([$1], [CXX], [ _LT_TAGVAR(lt_prog_compiler_static, $1)='-qstaticlink' ;; *) - case `$CC -V 2>&1 | sed 5q` in + case `$CC -V 2>&1 | $SED 5q` in *Sun\ C*) # Sun C++ 5.9 _LT_TAGVAR(lt_prog_compiler_pic, $1)='-KPIC' @@ -4754,7 +4774,7 @@ m4_if([$1], [CXX], [ _LT_TAGVAR(lt_prog_compiler_static, $1)='-qstaticlink' ;; *) - case `$CC -V 2>&1 | sed 5q` in + case `$CC -V 2>&1 | $SED 5q` in *Sun\ Ceres\ Fortran* | *Sun*Fortran*\ [[1-7]].* | *Sun*Fortran*\ 8.[[0-3]]*) # Sun Fortran 8.3 passes all unrecognized flags to the linker _LT_TAGVAR(lt_prog_compiler_pic, $1)='-KPIC' @@ -4937,7 +4957,7 @@ m4_if([$1], [CXX], [ if $NM -V 2>&1 | $GREP 'GNU' > /dev/null; then _LT_TAGVAR(export_symbols_cmds, $1)='$NM -Bpg $libobjs $convenience | awk '\''{ if (((\$ 2 == "T") || (\$ 2 == "D") || (\$ 2 == "B") || (\$ 2 == "W")) && ([substr](\$ 3,1,1) != ".")) { if (\$ 2 == "W") { print \$ 3 " weak" } else { print \$ 3 } } }'\'' | sort -u > $export_symbols' else - _LT_TAGVAR(export_symbols_cmds, $1)='`func_echo_all $NM | $SED -e '\''s/B\([[^B]]*\)$/P\1/'\''` -PCpgl $libobjs $convenience | awk '\''{ if (((\$ 2 == "T") || (\$ 2 == "D") || (\$ 2 == "B") || (\$ 2 == "W") || (\$ 2 == "V") || (\$ 2 == "Z")) && ([substr](\$ 1,1,1) != ".")) { if ((\$ 2 == "W") || (\$ 2 == "V") || (\$ 2 == "Z")) { print \$ 1 " weak" } else { print \$ 1 } } }'\'' | sort -u > $export_symbols' + _LT_TAGVAR(export_symbols_cmds, $1)='`func_echo_all $NM | $SED -e '\''s/B\([[^B]]*\)$/P\1/'\''` -PCpgl $libobjs $convenience | awk '\''{ if (((\$ 2 == "T") || (\$ 2 == "D") || (\$ 2 == "B") || (\$ 2 == "L") || (\$ 2 == "W") || (\$ 2 == "V") || (\$ 2 == "Z")) && ([substr](\$ 1,1,1) != ".")) { if ((\$ 2 == "W") || (\$ 2 == "V") || (\$ 2 == "Z")) { print \$ 1 " weak" } else { print \$ 1 } } }'\'' | sort -u > $export_symbols' fi ;; pw32*) @@ -4945,7 +4965,7 @@ m4_if([$1], [CXX], [ ;; cygwin* | mingw* | cegcc*) case $cc_basename in - cl*) + cl* | icl*) _LT_TAGVAR(exclude_expsyms, $1)='_NULL_IMPORT_DESCRIPTOR|_IMPORT_DESCRIPTOR_.*' ;; *) @@ -5005,15 +5025,15 @@ dnl Note also adjust exclude_expsyms for C++ above. case $host_os in cygwin* | mingw* | pw32* | cegcc*) - # FIXME: the MSVC++ port hasn't been tested in a loooong time + # FIXME: the MSVC++ and ICC port hasn't been tested in a loooong time # When not using gcc, we currently assume that we are using - # Microsoft Visual C++. + # Microsoft Visual C++ or Intel C++ Compiler. if test yes != "$GCC"; then with_gnu_ld=no fi ;; interix*) - # we just hope/assume this is gcc and not c89 (= MSVC++) + # we just hope/assume this is gcc and not c89 (= MSVC++ or ICC) with_gnu_ld=yes ;; openbsd* | bitrig*) @@ -5068,7 +5088,7 @@ dnl Note also adjust exclude_expsyms for C++ above. _LT_TAGVAR(whole_archive_flag_spec, $1)= fi supports_anon_versioning=no - case `$LD -v | $SED -e 's/([^)]\+)\s\+//' 2>&1` in + case `$LD -v | $SED -e 's/([[^)]]\+)\s\+//' 2>&1` in *GNU\ gold*) supports_anon_versioning=yes ;; *\ [[01]].* | *\ 2.[[0-9]].* | *\ 2.10.*) ;; # catch versions < 2.11 *\ 2.11.93.0.2\ *) supports_anon_versioning=yes ;; # RH7.3 ... @@ -5180,6 +5200,7 @@ _LT_EOF emximp -o $lib $output_objdir/$libname.def' _LT_TAGVAR(old_archive_From_new_cmds, $1)='emximp -o $output_objdir/${libname}_dll.a $output_objdir/$libname.def' _LT_TAGVAR(enable_shared_with_static_runtimes, $1)=yes + _LT_TAGVAR(file_list_spec, $1)='@' ;; interix[[3-9]]*) @@ -5194,7 +5215,7 @@ _LT_EOF # 256 KiB-aligned image base between 0x50000000 and 0x6FFC0000 at link # time. Moving up from 0x10000000 also allows more sbrk(2) space. _LT_TAGVAR(archive_cmds, $1)='$CC -shared $pic_flag $libobjs $deplibs $compiler_flags $wl-h,$soname $wl--image-base,`expr ${RANDOM-$$} % 4096 / 2 \* 262144 + 1342177280` -o $lib' - _LT_TAGVAR(archive_expsym_cmds, $1)='sed "s|^|_|" $export_symbols >$output_objdir/$soname.expsym~$CC -shared $pic_flag $libobjs $deplibs $compiler_flags $wl-h,$soname $wl--retain-symbols-file,$output_objdir/$soname.expsym $wl--image-base,`expr ${RANDOM-$$} % 4096 / 2 \* 262144 + 1342177280` -o $lib' + _LT_TAGVAR(archive_expsym_cmds, $1)='$SED "s|^|_|" $export_symbols >$output_objdir/$soname.expsym~$CC -shared $pic_flag $libobjs $deplibs $compiler_flags $wl-h,$soname $wl--retain-symbols-file,$output_objdir/$soname.expsym $wl--image-base,`expr ${RANDOM-$$} % 4096 / 2 \* 262144 + 1342177280` -o $lib' ;; gnu* | linux* | tpf* | k*bsd*-gnu | kopensolaris*-gnu) @@ -5237,7 +5258,7 @@ _LT_EOF _LT_TAGVAR(compiler_needs_object, $1)=yes ;; esac - case `$CC -V 2>&1 | sed 5q` in + case `$CC -V 2>&1 | $SED 5q` in *Sun\ C*) # Sun C 5.9 _LT_TAGVAR(whole_archive_flag_spec, $1)='$wl--whole-archive`new_convenience=; for conv in $convenience\"\"; do test -z \"$conv\" || new_convenience=\"$new_convenience,$conv\"; done; func_echo_all \"$new_convenience\"` $wl--no-whole-archive' _LT_TAGVAR(compiler_needs_object, $1)=yes @@ -5249,13 +5270,14 @@ _LT_EOF if test yes = "$supports_anon_versioning"; then _LT_TAGVAR(archive_expsym_cmds, $1)='echo "{ global:" > $output_objdir/$libname.ver~ - cat $export_symbols | sed -e "s/\(.*\)/\1;/" >> $output_objdir/$libname.ver~ + cat $export_symbols | $SED -e "s/\(.*\)/\1;/" >> $output_objdir/$libname.ver~ echo "local: *; };" >> $output_objdir/$libname.ver~ $CC '"$tmp_sharedflag""$tmp_addflag"' $libobjs $deplibs $compiler_flags $wl-soname $wl$soname $wl-version-script $wl$output_objdir/$libname.ver -o $lib' fi case $cc_basename in tcc*) + _LT_TAGVAR(hardcode_libdir_flag_spec, $1)='$wl-rpath $wl$libdir' _LT_TAGVAR(export_dynamic_flag_spec, $1)='-rdynamic' ;; xlf* | bgf* | bgxlf* | mpixlf*) @@ -5265,7 +5287,7 @@ _LT_EOF _LT_TAGVAR(archive_cmds, $1)='$LD -shared $libobjs $deplibs $linker_flags -soname $soname -o $lib' if test yes = "$supports_anon_versioning"; then _LT_TAGVAR(archive_expsym_cmds, $1)='echo "{ global:" > $output_objdir/$libname.ver~ - cat $export_symbols | sed -e "s/\(.*\)/\1;/" >> $output_objdir/$libname.ver~ + cat $export_symbols | $SED -e "s/\(.*\)/\1;/" >> $output_objdir/$libname.ver~ echo "local: *; };" >> $output_objdir/$libname.ver~ $LD -shared $libobjs $deplibs $linker_flags -soname $soname -version-script $output_objdir/$libname.ver -o $lib' fi @@ -5397,7 +5419,7 @@ _LT_EOF if $NM -V 2>&1 | $GREP 'GNU' > /dev/null; then _LT_TAGVAR(export_symbols_cmds, $1)='$NM -Bpg $libobjs $convenience | awk '\''{ if (((\$ 2 == "T") || (\$ 2 == "D") || (\$ 2 == "B") || (\$ 2 == "W")) && ([substr](\$ 3,1,1) != ".")) { if (\$ 2 == "W") { print \$ 3 " weak" } else { print \$ 3 } } }'\'' | sort -u > $export_symbols' else - _LT_TAGVAR(export_symbols_cmds, $1)='`func_echo_all $NM | $SED -e '\''s/B\([[^B]]*\)$/P\1/'\''` -PCpgl $libobjs $convenience | awk '\''{ if (((\$ 2 == "T") || (\$ 2 == "D") || (\$ 2 == "B") || (\$ 2 == "W") || (\$ 2 == "V") || (\$ 2 == "Z")) && ([substr](\$ 1,1,1) != ".")) { if ((\$ 2 == "W") || (\$ 2 == "V") || (\$ 2 == "Z")) { print \$ 1 " weak" } else { print \$ 1 } } }'\'' | sort -u > $export_symbols' + _LT_TAGVAR(export_symbols_cmds, $1)='`func_echo_all $NM | $SED -e '\''s/B\([[^B]]*\)$/P\1/'\''` -PCpgl $libobjs $convenience | awk '\''{ if (((\$ 2 == "T") || (\$ 2 == "D") || (\$ 2 == "B") || (\$ 2 == "L") || (\$ 2 == "W") || (\$ 2 == "V") || (\$ 2 == "Z")) && ([substr](\$ 1,1,1) != ".")) { if ((\$ 2 == "W") || (\$ 2 == "V") || (\$ 2 == "Z")) { print \$ 1 " weak" } else { print \$ 1 } } }'\'' | sort -u > $export_symbols' fi aix_use_runtimelinking=no @@ -5580,12 +5602,12 @@ _LT_EOF cygwin* | mingw* | pw32* | cegcc*) # When not using gcc, we currently assume that we are using - # Microsoft Visual C++. + # Microsoft Visual C++ or Intel C++ Compiler. # hardcode_libdir_flag_spec is actually meaningless, as there is # no search path for DLLs. case $cc_basename in - cl*) - # Native MSVC + cl* | icl*) + # Native MSVC or ICC _LT_TAGVAR(hardcode_libdir_flag_spec, $1)=' ' _LT_TAGVAR(allow_undefined_flag, $1)=unsupported _LT_TAGVAR(always_export_symbols, $1)=yes @@ -5626,7 +5648,7 @@ _LT_EOF fi' ;; *) - # Assume MSVC wrapper + # Assume MSVC and ICC wrapper _LT_TAGVAR(hardcode_libdir_flag_spec, $1)=' ' _LT_TAGVAR(allow_undefined_flag, $1)=unsupported # Tell ltmain to make .lib files, not .a files. @@ -5674,7 +5696,7 @@ _LT_EOF ;; # FreeBSD 3 and greater uses gcc -shared to do shared libraries. - freebsd* | dragonfly*) + freebsd* | dragonfly* | midnightbsd*) _LT_TAGVAR(archive_cmds, $1)='$CC -shared $pic_flag -o $lib $libobjs $deplibs $compiler_flags' _LT_TAGVAR(hardcode_libdir_flag_spec, $1)='-R$libdir' _LT_TAGVAR(hardcode_direct, $1)=yes @@ -5815,6 +5837,7 @@ _LT_EOF # Fabrice Bellard et al's Tiny C Compiler _LT_TAGVAR(ld_shlibs, $1)=yes _LT_TAGVAR(archive_cmds, $1)='$CC -shared $pic_flag -o $lib $libobjs $deplibs $compiler_flags' + _LT_TAGVAR(hardcode_libdir_flag_spec, $1)='$wl-rpath $wl$libdir' ;; esac ;; @@ -5886,6 +5909,7 @@ _LT_EOF emximp -o $lib $output_objdir/$libname.def' _LT_TAGVAR(old_archive_From_new_cmds, $1)='emximp -o $output_objdir/${libname}_dll.a $output_objdir/$libname.def' _LT_TAGVAR(enable_shared_with_static_runtimes, $1)=yes + _LT_TAGVAR(file_list_spec, $1)='@' ;; osf3*) @@ -6656,8 +6680,8 @@ if test yes != "$_lt_caught_CXX_error"; then cygwin* | mingw* | pw32* | cegcc*) case $GXX,$cc_basename in - ,cl* | no,cl*) - # Native MSVC + ,cl* | no,cl* | ,icl* | no,icl*) + # Native MSVC or ICC # hardcode_libdir_flag_spec is actually meaningless, as there is # no search path for DLLs. _LT_TAGVAR(hardcode_libdir_flag_spec, $1)=' ' @@ -6755,6 +6779,7 @@ if test yes != "$_lt_caught_CXX_error"; then emximp -o $lib $output_objdir/$libname.def' _LT_TAGVAR(old_archive_From_new_cmds, $1)='emximp -o $output_objdir/${libname}_dll.a $output_objdir/$libname.def' _LT_TAGVAR(enable_shared_with_static_runtimes, $1)=yes + _LT_TAGVAR(file_list_spec, $1)='@' ;; dgux*) @@ -6785,7 +6810,7 @@ if test yes != "$_lt_caught_CXX_error"; then _LT_TAGVAR(archive_cmds_need_lc, $1)=no ;; - freebsd* | dragonfly*) + freebsd* | dragonfly* | midnightbsd*) # FreeBSD 3 and later use GNU C++ and GNU ld with standard ELF # conventions _LT_TAGVAR(ld_shlibs, $1)=yes @@ -6922,7 +6947,7 @@ if test yes != "$_lt_caught_CXX_error"; then # 256 KiB-aligned image base between 0x50000000 and 0x6FFC0000 at link # time. Moving up from 0x10000000 also allows more sbrk(2) space. _LT_TAGVAR(archive_cmds, $1)='$CC -shared $pic_flag $libobjs $deplibs $compiler_flags $wl-h,$soname $wl--image-base,`expr ${RANDOM-$$} % 4096 / 2 \* 262144 + 1342177280` -o $lib' - _LT_TAGVAR(archive_expsym_cmds, $1)='sed "s|^|_|" $export_symbols >$output_objdir/$soname.expsym~$CC -shared $pic_flag $libobjs $deplibs $compiler_flags $wl-h,$soname $wl--retain-symbols-file,$output_objdir/$soname.expsym $wl--image-base,`expr ${RANDOM-$$} % 4096 / 2 \* 262144 + 1342177280` -o $lib' + _LT_TAGVAR(archive_expsym_cmds, $1)='$SED "s|^|_|" $export_symbols >$output_objdir/$soname.expsym~$CC -shared $pic_flag $libobjs $deplibs $compiler_flags $wl-h,$soname $wl--retain-symbols-file,$output_objdir/$soname.expsym $wl--image-base,`expr ${RANDOM-$$} % 4096 / 2 \* 262144 + 1342177280` -o $lib' ;; irix5* | irix6*) case $cc_basename in @@ -7062,13 +7087,13 @@ if test yes != "$_lt_caught_CXX_error"; then _LT_TAGVAR(archive_cmds, $1)='$CC -qmkshrobj $libobjs $deplibs $compiler_flags $wl-soname $wl$soname -o $lib' if test yes = "$supports_anon_versioning"; then _LT_TAGVAR(archive_expsym_cmds, $1)='echo "{ global:" > $output_objdir/$libname.ver~ - cat $export_symbols | sed -e "s/\(.*\)/\1;/" >> $output_objdir/$libname.ver~ + cat $export_symbols | $SED -e "s/\(.*\)/\1;/" >> $output_objdir/$libname.ver~ echo "local: *; };" >> $output_objdir/$libname.ver~ $CC -qmkshrobj $libobjs $deplibs $compiler_flags $wl-soname $wl$soname $wl-version-script $wl$output_objdir/$libname.ver -o $lib' fi ;; *) - case `$CC -V 2>&1 | sed 5q` in + case `$CC -V 2>&1 | $SED 5q` in *Sun\ C*) # Sun C++ 5.9 _LT_TAGVAR(no_undefined_flag, $1)=' -zdefs' @@ -8214,6 +8239,14 @@ _LT_DECL([], [DLLTOOL], [1], [DLL creation program]) AC_SUBST([DLLTOOL]) ]) +# _LT_DECL_FILECMD +# ---------------- +# Check for a file(cmd) program that can be used to detect file type and magic +m4_defun([_LT_DECL_FILECMD], +[AC_CHECK_TOOL([FILECMD], [file], [:]) +_LT_DECL([], [FILECMD], [1], [A file(cmd) program that detects file types]) +])# _LD_DECL_FILECMD + # _LT_DECL_SED # ------------ # Check for a fully-functional sed program, that truncates diff --git a/source/lib/c-ares-1.34.4/m4/ltoptions.m4 b/source/lib/c-ares-1.34.6/m4/ltoptions.m4 similarity index 99% rename from source/lib/c-ares-1.34.4/m4/ltoptions.m4 rename to source/lib/c-ares-1.34.6/m4/ltoptions.m4 index 94b08297..b0b5e9c2 100755 --- a/source/lib/c-ares-1.34.4/m4/ltoptions.m4 +++ b/source/lib/c-ares-1.34.6/m4/ltoptions.m4 @@ -1,7 +1,7 @@ # Helper functions for option handling. -*- Autoconf -*- # -# Copyright (C) 2004-2005, 2007-2009, 2011-2015 Free Software -# Foundation, Inc. +# Copyright (C) 2004-2005, 2007-2009, 2011-2019, 2021-2022 Free +# Software Foundation, Inc. # Written by Gary V. Vaughan, 2004 # # This file is free software; the Free Software Foundation gives diff --git a/source/lib/c-ares-1.34.4/m4/ltsugar.m4 b/source/lib/c-ares-1.34.6/m4/ltsugar.m4 similarity index 98% rename from source/lib/c-ares-1.34.4/m4/ltsugar.m4 rename to source/lib/c-ares-1.34.6/m4/ltsugar.m4 index 48bc9344..902508bd 100755 --- a/source/lib/c-ares-1.34.4/m4/ltsugar.m4 +++ b/source/lib/c-ares-1.34.6/m4/ltsugar.m4 @@ -1,6 +1,6 @@ # ltsugar.m4 -- libtool m4 base layer. -*-Autoconf-*- # -# Copyright (C) 2004-2005, 2007-2008, 2011-2015 Free Software +# Copyright (C) 2004-2005, 2007-2008, 2011-2019, 2021-2022 Free Software # Foundation, Inc. # Written by Gary V. Vaughan, 2004 # diff --git a/source/lib/c-ares-1.34.4/m4/ltversion.m4 b/source/lib/c-ares-1.34.6/m4/ltversion.m4 similarity index 66% rename from source/lib/c-ares-1.34.4/m4/ltversion.m4 rename to source/lib/c-ares-1.34.6/m4/ltversion.m4 index fa04b52a..b155d0ac 100755 --- a/source/lib/c-ares-1.34.4/m4/ltversion.m4 +++ b/source/lib/c-ares-1.34.6/m4/ltversion.m4 @@ -1,6 +1,7 @@ # ltversion.m4 -- version numbers -*- Autoconf -*- # -# Copyright (C) 2004, 2011-2015 Free Software Foundation, Inc. +# Copyright (C) 2004, 2011-2019, 2021-2022 Free Software Foundation, +# Inc. # Written by Scott James Remnant, 2004 # # This file is free software; the Free Software Foundation gives @@ -9,15 +10,15 @@ # @configure_input@ -# serial 4179 ltversion.m4 +# serial 4245 ltversion.m4 # This file is part of GNU Libtool -m4_define([LT_PACKAGE_VERSION], [2.4.6]) -m4_define([LT_PACKAGE_REVISION], [2.4.6]) +m4_define([LT_PACKAGE_VERSION], [2.4.7]) +m4_define([LT_PACKAGE_REVISION], [2.4.7]) AC_DEFUN([LTVERSION_VERSION], -[macro_version='2.4.6' -macro_revision='2.4.6' +[macro_version='2.4.7' +macro_revision='2.4.7' _LT_DECL(, macro_version, 0, [Which release of libtool.m4 was used?]) _LT_DECL(, macro_revision, 0) ]) diff --git a/source/lib/c-ares-1.34.4/m4/lt~obsolete.m4 b/source/lib/c-ares-1.34.6/m4/lt~obsolete.m4 similarity index 98% rename from source/lib/c-ares-1.34.4/m4/lt~obsolete.m4 rename to source/lib/c-ares-1.34.6/m4/lt~obsolete.m4 index c6b26f88..0f7a8759 100755 --- a/source/lib/c-ares-1.34.4/m4/lt~obsolete.m4 +++ b/source/lib/c-ares-1.34.6/m4/lt~obsolete.m4 @@ -1,7 +1,7 @@ # lt~obsolete.m4 -- aclocal satisfying obsolete definitions. -*-Autoconf-*- # -# Copyright (C) 2004-2005, 2007, 2009, 2011-2015 Free Software -# Foundation, Inc. +# Copyright (C) 2004-2005, 2007, 2009, 2011-2019, 2021-2022 Free +# Software Foundation, Inc. # Written by Scott James Remnant, 2004. # # This file is free software; the Free Software Foundation gives diff --git a/source/lib/c-ares-1.34.4/m4/pkg.m4 b/source/lib/c-ares-1.34.6/m4/pkg.m4 similarity index 100% rename from source/lib/c-ares-1.34.4/m4/pkg.m4 rename to source/lib/c-ares-1.34.6/m4/pkg.m4 diff --git a/source/lib/c-ares-1.34.4/src/CMakeLists.txt b/source/lib/c-ares-1.34.6/src/CMakeLists.txt similarity index 100% rename from source/lib/c-ares-1.34.4/src/CMakeLists.txt rename to source/lib/c-ares-1.34.6/src/CMakeLists.txt diff --git a/source/lib/c-ares-1.34.4/src/Makefile.am b/source/lib/c-ares-1.34.6/src/Makefile.am similarity index 100% rename from source/lib/c-ares-1.34.4/src/Makefile.am rename to source/lib/c-ares-1.34.6/src/Makefile.am diff --git a/source/lib/c-ares-1.34.4/src/Makefile.in b/source/lib/c-ares-1.34.6/src/Makefile.in similarity index 99% rename from source/lib/c-ares-1.34.4/src/Makefile.in rename to source/lib/c-ares-1.34.6/src/Makefile.in index 1f286880..ff4275b6 100644 --- a/source/lib/c-ares-1.34.4/src/Makefile.in +++ b/source/lib/c-ares-1.34.6/src/Makefile.in @@ -104,7 +104,6 @@ am__aclocal_m4_deps = $(top_srcdir)/m4/ares_check_user_namespace.m4 \ $(top_srcdir)/m4/ax_code_coverage.m4 \ $(top_srcdir)/m4/ax_compiler_vendor.m4 \ $(top_srcdir)/m4/ax_cxx_compile_stdcxx.m4 \ - $(top_srcdir)/m4/ax_cxx_compile_stdcxx_14.m4 \ $(top_srcdir)/m4/ax_file_escapes.m4 \ $(top_srcdir)/m4/ax_pthread.m4 \ $(top_srcdir)/m4/ax_require_defined.m4 \ @@ -245,14 +244,18 @@ EGREP = @EGREP@ ETAGS = @ETAGS@ EXEEXT = @EXEEXT@ FGREP = @FGREP@ +FILECMD = @FILECMD@ GCOV = @GCOV@ GENHTML = @GENHTML@ GMOCK112_CFLAGS = @GMOCK112_CFLAGS@ GMOCK112_LIBS = @GMOCK112_LIBS@ +GMOCK117_CFLAGS = @GMOCK117_CFLAGS@ +GMOCK117_LIBS = @GMOCK117_LIBS@ GMOCK_CFLAGS = @GMOCK_CFLAGS@ GMOCK_LIBS = @GMOCK_LIBS@ GREP = @GREP@ HAVE_CXX14 = @HAVE_CXX14@ +HAVE_CXX17 = @HAVE_CXX17@ INSTALL = @INSTALL@ INSTALL_DATA = @INSTALL_DATA@ INSTALL_PROGRAM = @INSTALL_PROGRAM@ diff --git a/source/lib/c-ares-1.34.4/src/lib/CMakeLists.txt b/source/lib/c-ares-1.34.6/src/lib/CMakeLists.txt similarity index 92% rename from source/lib/c-ares-1.34.4/src/lib/CMakeLists.txt rename to source/lib/c-ares-1.34.6/src/lib/CMakeLists.txt index 9d4e1092..bdb2690f 100644 --- a/source/lib/c-ares-1.34.4/src/lib/CMakeLists.txt +++ b/source/lib/c-ares-1.34.6/src/lib/CMakeLists.txt @@ -15,6 +15,14 @@ IF (CARES_SHARED) # Include resource file in windows builds for versioned DLLs IF (WIN32) TARGET_SOURCES (${PROJECT_NAME} PRIVATE cares.rc) + + # Check for use of llvm-rc (implies clang is being used). We need to set the + # compile flags to use the correct codepage for the resource compiler. + # This is needed for the copyright symbol to be encoded correctly. + # The default codepage is 1252 (Windows Latin-1) but llvm-rc defaults to UTF-8. + if (CMAKE_RC_COMPILER MATCHES "llvm-rc") + set_source_files_properties(cares.rc PROPERTIES COMPILE_FLAGS "/C 1252") + endif() ENDIF() # Convert CARES_LIB_VERSIONINFO libtool version format into VERSION and SOVERSION diff --git a/source/lib/c-ares-1.34.4/src/lib/Makefile.am b/source/lib/c-ares-1.34.6/src/lib/Makefile.am similarity index 100% rename from source/lib/c-ares-1.34.4/src/lib/Makefile.am rename to source/lib/c-ares-1.34.6/src/lib/Makefile.am diff --git a/source/lib/c-ares-1.34.4/src/lib/Makefile.in b/source/lib/c-ares-1.34.6/src/lib/Makefile.in similarity index 99% rename from source/lib/c-ares-1.34.4/src/lib/Makefile.in rename to source/lib/c-ares-1.34.6/src/lib/Makefile.in index a45fc10b..e92732ea 100644 --- a/source/lib/c-ares-1.34.4/src/lib/Makefile.in +++ b/source/lib/c-ares-1.34.6/src/lib/Makefile.in @@ -15,7 +15,7 @@ @SET_MAKE@ # aminclude_static.am generated automatically by Autoconf -# from AX_AM_MACROS_STATIC on Sat Dec 14 15:15:44 UTC 2024 +# from AX_AM_MACROS_STATIC on Mon Dec 8 16:21:41 UTC 2025 # Copyright (C) The c-ares project and its contributors # SPDX-License-Identifier: MIT @@ -115,7 +115,6 @@ am__aclocal_m4_deps = $(top_srcdir)/m4/ares_check_user_namespace.m4 \ $(top_srcdir)/m4/ax_code_coverage.m4 \ $(top_srcdir)/m4/ax_compiler_vendor.m4 \ $(top_srcdir)/m4/ax_cxx_compile_stdcxx.m4 \ - $(top_srcdir)/m4/ax_cxx_compile_stdcxx_14.m4 \ $(top_srcdir)/m4/ax_file_escapes.m4 \ $(top_srcdir)/m4/ax_pthread.m4 \ $(top_srcdir)/m4/ax_require_defined.m4 \ @@ -490,14 +489,18 @@ EGREP = @EGREP@ ETAGS = @ETAGS@ EXEEXT = @EXEEXT@ FGREP = @FGREP@ +FILECMD = @FILECMD@ GCOV = @GCOV@ GENHTML = @GENHTML@ GMOCK112_CFLAGS = @GMOCK112_CFLAGS@ GMOCK112_LIBS = @GMOCK112_LIBS@ +GMOCK117_CFLAGS = @GMOCK117_CFLAGS@ +GMOCK117_LIBS = @GMOCK117_LIBS@ GMOCK_CFLAGS = @GMOCK_CFLAGS@ GMOCK_LIBS = @GMOCK_LIBS@ GREP = @GREP@ HAVE_CXX14 = @HAVE_CXX14@ +HAVE_CXX17 = @HAVE_CXX17@ INSTALL = @INSTALL@ INSTALL_DATA = @INSTALL_DATA@ INSTALL_PROGRAM = @INSTALL_PROGRAM@ diff --git a/source/lib/c-ares-1.34.4/src/lib/Makefile.inc b/source/lib/c-ares-1.34.6/src/lib/Makefile.inc similarity index 100% rename from source/lib/c-ares-1.34.4/src/lib/Makefile.inc rename to source/lib/c-ares-1.34.6/src/lib/Makefile.inc diff --git a/source/lib/c-ares-1.34.4/src/lib/ares_addrinfo2hostent.c b/source/lib/c-ares-1.34.6/src/lib/ares_addrinfo2hostent.c similarity index 67% rename from source/lib/c-ares-1.34.4/src/lib/ares_addrinfo2hostent.c rename to source/lib/c-ares-1.34.6/src/lib/ares_addrinfo2hostent.c index 2bbc7911..239ca5bc 100644 --- a/source/lib/c-ares-1.34.4/src/lib/ares_addrinfo2hostent.c +++ b/source/lib/c-ares-1.34.6/src/lib/ares_addrinfo2hostent.c @@ -47,119 +47,154 @@ # include #endif +static size_t hostent_nalias(const struct hostent *host) +{ + size_t i; + for (i=0; host->h_aliases != NULL && host->h_aliases[i] != NULL; i++) + ; + + return i; +} + +static size_t ai_nalias(const struct ares_addrinfo *ai) +{ + const struct ares_addrinfo_cname *cname; + size_t i = 0; + + for (cname = ai->cnames; cname != NULL; cname=cname->next) { + i++; + } + + return i; +} + +static size_t hostent_naddr(const struct hostent *host) +{ + size_t i; + for (i=0; host->h_addr_list != NULL && host->h_addr_list[i] != NULL; i++) + ; + + return i; +} + +static size_t ai_naddr(const struct ares_addrinfo *ai, int af) +{ + const struct ares_addrinfo_node *node; + size_t i = 0; + + for (node = ai->nodes; node != NULL; node=node->ai_next) { + if (af != AF_UNSPEC && af != node->ai_family) + continue; + i++; + } + + return i; +} ares_status_t ares_addrinfo2hostent(const struct ares_addrinfo *ai, int family, struct hostent **host) { struct ares_addrinfo_node *next; - struct ares_addrinfo_cname *next_cname; char **aliases = NULL; - char *addrs = NULL; + char **addrs = NULL; size_t naliases = 0; size_t naddrs = 0; - size_t alias = 0; size_t i; + size_t ealiases = 0; + size_t eaddrs = 0; if (ai == NULL || host == NULL) { return ARES_EBADQUERY; /* LCOV_EXCL_LINE: DefensiveCoding */ } - /* Use the first node of the response as the family, since hostent can only + /* Use either the host set in the passed in hosts to be filled in, or the + * first node of the response as the family, since hostent can only * represent one family. We assume getaddrinfo() returned a sorted list if * the user requested AF_UNSPEC. */ - if (family == AF_UNSPEC && ai->nodes) { - family = ai->nodes->ai_family; + if (family == AF_UNSPEC) { + if (*host != NULL && (*host)->h_addrtype != AF_UNSPEC) { + family = (*host)->h_addrtype; + } else if (ai->nodes != NULL) { + family = ai->nodes->ai_family; + } } if (family != AF_INET && family != AF_INET6) { return ARES_EBADQUERY; /* LCOV_EXCL_LINE: DefensiveCoding */ } - *host = ares_malloc(sizeof(**host)); - if (!(*host)) { - goto enomem; /* LCOV_EXCL_LINE: OutOfMemory */ + if (*host == NULL) { + *host = ares_malloc_zero(sizeof(**host)); + if (!(*host)) { + goto enomem; /* LCOV_EXCL_LINE: OutOfMemory */ + } } - memset(*host, 0, sizeof(**host)); - next = ai->nodes; - while (next) { - if (next->ai_family == family) { - ++naddrs; - } - next = next->ai_next; + (*host)->h_addrtype = (HOSTENT_ADDRTYPE_TYPE)family; + if (family == AF_INET) { + (*host)->h_length = sizeof(struct in_addr); + } else if (family == AF_INET6) { + (*host)->h_length = sizeof(struct ares_in6_addr); } - next_cname = ai->cnames; - while (next_cname) { - if (next_cname->alias) { - ++naliases; + if ((*host)->h_name == NULL) { + if (ai->cnames) { + (*host)->h_name = ares_strdup(ai->cnames->name); + if ((*host)->h_name == NULL && ai->cnames->name) { + goto enomem; /* LCOV_EXCL_LINE: OutOfMemory */ + } + } else { + (*host)->h_name = ares_strdup(ai->name); + if ((*host)->h_name == NULL && ai->name) { + goto enomem; /* LCOV_EXCL_LINE: OutOfMemory */ + } } - next_cname = next_cname->next; } - aliases = ares_malloc((naliases + 1) * sizeof(char *)); + naliases = ai_nalias(ai); + ealiases = hostent_nalias(*host); + aliases = ares_realloc_zero((*host)->h_aliases, + ealiases * sizeof(char *), + (naliases + ealiases + 1) * sizeof(char *)); if (!aliases) { goto enomem; /* LCOV_EXCL_LINE: OutOfMemory */ } (*host)->h_aliases = aliases; - memset(aliases, 0, (naliases + 1) * sizeof(char *)); if (naliases) { - for (next_cname = ai->cnames; next_cname != NULL; - next_cname = next_cname->next) { - if (next_cname->alias == NULL) { + const struct ares_addrinfo_cname *cname; + i = ealiases; + for (cname = ai->cnames; cname != NULL; cname = cname->next) { + if (cname->alias == NULL) { continue; } - aliases[alias] = ares_strdup(next_cname->alias); - if (!aliases[alias]) { + (*host)->h_aliases[i] = ares_strdup(cname->alias); + if ((*host)->h_aliases[i] == NULL) { goto enomem; /* LCOV_EXCL_LINE: OutOfMemory */ } - alias++; + i++; } } - - (*host)->h_addr_list = ares_malloc((naddrs + 1) * sizeof(char *)); - if (!(*host)->h_addr_list) { + naddrs = ai_naddr(ai, family); + eaddrs = hostent_naddr(*host); + addrs = ares_realloc_zero((*host)->h_addr_list, eaddrs * sizeof(char *), + (naddrs + eaddrs + 1) * sizeof(char *)); + if (addrs == NULL) { goto enomem; /* LCOV_EXCL_LINE: OutOfMemory */ } - - memset((*host)->h_addr_list, 0, (naddrs + 1) * sizeof(char *)); - - if (ai->cnames) { - (*host)->h_name = ares_strdup(ai->cnames->name); - if ((*host)->h_name == NULL && ai->cnames->name) { - goto enomem; /* LCOV_EXCL_LINE: OutOfMemory */ - } - } else { - (*host)->h_name = ares_strdup(ai->name); - if ((*host)->h_name == NULL && ai->name) { - goto enomem; /* LCOV_EXCL_LINE: OutOfMemory */ - } - } - - (*host)->h_addrtype = (HOSTENT_ADDRTYPE_TYPE)family; - - if (family == AF_INET) { - (*host)->h_length = sizeof(struct in_addr); - } - - if (family == AF_INET6) { - (*host)->h_length = sizeof(struct ares_in6_addr); - } + (*host)->h_addr_list = addrs; if (naddrs) { - addrs = ares_malloc(naddrs * (size_t)(*host)->h_length); - if (!addrs) { - goto enomem; /* LCOV_EXCL_LINE: OutOfMemory */ - } - - i = 0; + i = eaddrs; for (next = ai->nodes; next != NULL; next = next->ai_next) { if (next->ai_family != family) { continue; } - (*host)->h_addr_list[i] = addrs + (i * (size_t)(*host)->h_length); + (*host)->h_addr_list[i] = ares_malloc_zero((size_t)(*host)->h_length); + if ((*host)->h_addr_list[i] == NULL) { + goto enomem; /* LCOV_EXCL_LINE: OutOfMemory */ + } if (family == AF_INET6) { memcpy((*host)->h_addr_list[i], &(CARES_INADDR_CAST(const struct sockaddr_in6 *, next->ai_addr) @@ -172,15 +207,11 @@ ares_status_t ares_addrinfo2hostent(const struct ares_addrinfo *ai, int family, ->sin_addr), (size_t)(*host)->h_length); } - ++i; - } - - if (i == 0) { - ares_free(addrs); + i++; } } - if (naddrs == 0 && naliases == 0) { + if (naddrs + eaddrs == 0 && naliases + ealiases == 0) { ares_free_hostent(*host); *host = NULL; return ARES_ENODATA; diff --git a/source/lib/c-ares-1.34.4/src/lib/ares_addrinfo_localhost.c b/source/lib/c-ares-1.34.6/src/lib/ares_addrinfo_localhost.c similarity index 83% rename from source/lib/c-ares-1.34.4/src/lib/ares_addrinfo_localhost.c rename to source/lib/c-ares-1.34.6/src/lib/ares_addrinfo_localhost.c index 6f4f2a37..2abb0c48 100644 --- a/source/lib/c-ares-1.34.4/src/lib/ares_addrinfo_localhost.c +++ b/source/lib/c-ares-1.34.6/src/lib/ares_addrinfo_localhost.c @@ -49,6 +49,19 @@ # endif #endif +static ares_bool_t ares_ai_has_family(int aftype, + const struct ares_addrinfo_node *nodes) +{ + const struct ares_addrinfo_node *node; + + for (node = nodes; node != NULL; node = node->ai_next) { + if (node->ai_family == aftype) + return ARES_TRUE; + } + + return ARES_FALSE; +} + ares_status_t ares_append_ai_node(int aftype, unsigned short port, unsigned int ttl, const void *adata, struct ares_addrinfo_node **nodes) @@ -107,7 +120,8 @@ static ares_status_t { ares_status_t status = ARES_SUCCESS; - if (aftype == AF_UNSPEC || aftype == AF_INET6) { + if ((aftype == AF_UNSPEC || aftype == AF_INET6) && + !ares_ai_has_family(AF_INET6, *nodes)) { struct ares_in6_addr addr6; ares_inet_pton(AF_INET6, "::1", &addr6); status = ares_append_ai_node(AF_INET6, port, 0, &addr6, nodes); @@ -116,7 +130,8 @@ static ares_status_t } } - if (aftype == AF_UNSPEC || aftype == AF_INET) { + if ((aftype == AF_UNSPEC || aftype == AF_INET) && + !ares_ai_has_family(AF_INET, *nodes)) { struct in_addr addr4; ares_inet_pton(AF_INET, "127.0.0.1", &addr4); status = ares_append_ai_node(AF_INET, port, 0, &addr4, nodes); @@ -150,11 +165,13 @@ static ares_status_t continue; } - if (table->Table[i].Address.si_family == AF_INET) { + if (table->Table[i].Address.si_family == AF_INET && + !ares_ai_has_family(AF_INET, *nodes)) { status = ares_append_ai_node(table->Table[i].Address.si_family, port, 0, &table->Table[i].Address.Ipv4.sin_addr, nodes); - } else if (table->Table[i].Address.si_family == AF_INET6) { + } else if (table->Table[i].Address.si_family == AF_INET6 && + !ares_ai_has_family(AF_INET6, *nodes)) { status = ares_append_ai_node(table->Table[i].Address.si_family, port, 0, &table->Table[i].Address.Ipv6.sin6_addr, nodes); @@ -195,8 +212,7 @@ ares_status_t ares_addrinfo_localhost(const char *name, unsigned short port, const struct ares_addrinfo_hints *hints, struct ares_addrinfo *ai) { - struct ares_addrinfo_node *nodes = NULL; - ares_status_t status; + ares_status_t status; /* Validate family */ switch (hints->ai_family) { @@ -208,26 +224,22 @@ ares_status_t ares_addrinfo_localhost(const char *name, unsigned short port, return ARES_EBADFAMILY; /* LCOV_EXCL_LINE: DefensiveCoding */ } + if (ai->name != NULL) { + ares_free(ai->name); + } ai->name = ares_strdup(name); - if (!ai->name) { - goto enomem; /* LCOV_EXCL_LINE: OutOfMemory */ + if (ai->name == NULL) { + status = ARES_ENOMEM; + goto done; /* LCOV_EXCL_LINE: OutOfMemory */ } - status = ares_system_loopback_addrs(hints->ai_family, port, &nodes); - - if (status == ARES_ENOTFOUND) { - status = ares_default_loopback_addrs(hints->ai_family, port, &nodes); + status = ares_system_loopback_addrs(hints->ai_family, port, &ai->nodes); + if (status != ARES_SUCCESS && status != ARES_ENOTFOUND) { + goto done; } - ares_addrinfo_cat_nodes(&ai->nodes, nodes); + status = ares_default_loopback_addrs(hints->ai_family, port, &ai->nodes); +done: return status; - -/* LCOV_EXCL_START: OutOfMemory */ -enomem: - ares_freeaddrinfo_nodes(nodes); - ares_free(ai->name); - ai->name = NULL; - return ARES_ENOMEM; - /* LCOV_EXCL_STOP */ } diff --git a/source/lib/c-ares-1.34.4/src/lib/ares_android.c b/source/lib/c-ares-1.34.6/src/lib/ares_android.c similarity index 100% rename from source/lib/c-ares-1.34.4/src/lib/ares_android.c rename to source/lib/c-ares-1.34.6/src/lib/ares_android.c diff --git a/source/lib/c-ares-1.34.4/src/lib/ares_android.h b/source/lib/c-ares-1.34.6/src/lib/ares_android.h similarity index 100% rename from source/lib/c-ares-1.34.4/src/lib/ares_android.h rename to source/lib/c-ares-1.34.6/src/lib/ares_android.h diff --git a/source/lib/c-ares-1.34.4/src/lib/ares_cancel.c b/source/lib/c-ares-1.34.6/src/lib/ares_cancel.c similarity index 100% rename from source/lib/c-ares-1.34.4/src/lib/ares_cancel.c rename to source/lib/c-ares-1.34.6/src/lib/ares_cancel.c diff --git a/source/lib/c-ares-1.34.4/src/lib/ares_close_sockets.c b/source/lib/c-ares-1.34.6/src/lib/ares_close_sockets.c similarity index 99% rename from source/lib/c-ares-1.34.4/src/lib/ares_close_sockets.c rename to source/lib/c-ares-1.34.6/src/lib/ares_close_sockets.c index fd3bf3c4..347f43e6 100644 --- a/source/lib/c-ares-1.34.4/src/lib/ares_close_sockets.c +++ b/source/lib/c-ares-1.34.6/src/lib/ares_close_sockets.c @@ -37,7 +37,7 @@ static void ares_requeue_queries(ares_conn_t *conn, ares_tvnow(&now); while ((query = ares_llist_first_val(conn->queries_to_conn)) != NULL) { - ares_requeue_query(query, &now, requeue_status, ARES_TRUE, NULL); + ares_requeue_query(query, &now, requeue_status, ARES_TRUE, NULL, NULL); } } diff --git a/source/lib/c-ares-1.34.4/src/lib/ares_config.h.cmake b/source/lib/c-ares-1.34.6/src/lib/ares_config.h.cmake similarity index 99% rename from source/lib/c-ares-1.34.4/src/lib/ares_config.h.cmake rename to source/lib/c-ares-1.34.6/src/lib/ares_config.h.cmake index 51744fe1..cff18187 100644 --- a/source/lib/c-ares-1.34.4/src/lib/ares_config.h.cmake +++ b/source/lib/c-ares-1.34.6/src/lib/ares_config.h.cmake @@ -145,6 +145,9 @@ /* Define to 1 if you have the `if_nametoindex' function. */ #cmakedefine HAVE_IF_NAMETOINDEX 1 +/* Define to 1 if you have the `GetBestRoute2' function. */ +#cmakedefine HAVE_GETBESTROUTE2 1 + /* Define to 1 if you have the `ConvertInterfaceIndexToLuid' function. */ #cmakedefine HAVE_CONVERTINTERFACEINDEXTOLUID 1 diff --git a/source/lib/c-ares-1.34.4/src/lib/ares_config.h.in b/source/lib/c-ares-1.34.6/src/lib/ares_config.h.in similarity index 99% rename from source/lib/c-ares-1.34.4/src/lib/ares_config.h.in rename to source/lib/c-ares-1.34.6/src/lib/ares_config.h.in index a62e1708..c56e426e 100644 --- a/source/lib/c-ares-1.34.4/src/lib/ares_config.h.in +++ b/source/lib/c-ares-1.34.6/src/lib/ares_config.h.in @@ -87,6 +87,9 @@ /* define if the compiler supports basic C++14 syntax */ #undef HAVE_CXX14 +/* define if the compiler supports basic C++17 syntax */ +#undef HAVE_CXX17 + /* Define to 1 if you have the header file. */ #undef HAVE_DLFCN_H @@ -105,6 +108,9 @@ /* fcntl() with O_NONBLOCK support */ #undef HAVE_FCNTL_O_NONBLOCK +/* Define to 1 if you have `GetBestRoute2` */ +#undef HAVE_GETBESTROUTE2 + /* Define to 1 if you have `getenv` */ #undef HAVE_GETENV diff --git a/source/lib/c-ares-1.34.4/src/lib/ares_conn.c b/source/lib/c-ares-1.34.6/src/lib/ares_conn.c similarity index 100% rename from source/lib/c-ares-1.34.4/src/lib/ares_conn.c rename to source/lib/c-ares-1.34.6/src/lib/ares_conn.c diff --git a/source/lib/c-ares-1.34.4/src/lib/ares_conn.h b/source/lib/c-ares-1.34.6/src/lib/ares_conn.h similarity index 100% rename from source/lib/c-ares-1.34.4/src/lib/ares_conn.h rename to source/lib/c-ares-1.34.6/src/lib/ares_conn.h diff --git a/source/lib/c-ares-1.34.4/src/lib/ares_cookie.c b/source/lib/c-ares-1.34.6/src/lib/ares_cookie.c similarity index 99% rename from source/lib/c-ares-1.34.4/src/lib/ares_cookie.c rename to source/lib/c-ares-1.34.6/src/lib/ares_cookie.c index f31c74e7..509e1205 100644 --- a/source/lib/c-ares-1.34.4/src/lib/ares_cookie.c +++ b/source/lib/c-ares-1.34.6/src/lib/ares_cookie.c @@ -115,7 +115,7 @@ * - If `cookie.unsupported_ts` evaluates less than * `COOKIE_UNSUPPORTED_TIMEOUT` * - Ensure there is no EDNS cookie opt (10) set (shouldn't be unless - * requestor had put this themselves), then **skip any remaining + * requester had put this themselves), then **skip any remaining * processing** as we don't want to try to send cookies. * - Otherwise: * - clear all cookie settings, set `cookie.state = INITIAL`. @@ -369,7 +369,8 @@ ares_status_t ares_cookie_apply(ares_dns_record_t *dnsrec, ares_conn_t *conn, ares_status_t ares_cookie_validate(ares_query_t *query, const ares_dns_record_t *dnsresp, - ares_conn_t *conn, const ares_timeval_t *now) + ares_conn_t *conn, const ares_timeval_t *now, + ares_array_t **requeue) { ares_server_t *server = conn->server; ares_cookie_t *cookie = &server->cookie; @@ -427,7 +428,8 @@ ares_status_t ares_cookie_validate(ares_query_t *query, /* Resend the request, hopefully it will work the next time as we should * have recorded a server cookie */ ares_requeue_query(query, now, ARES_SUCCESS, - ARES_FALSE /* Don't increment try count */, NULL); + ARES_FALSE /* Don't increment try count */, NULL, + requeue); /* Parent needs to drop this response */ return ARES_EBADRESP; diff --git a/source/lib/c-ares-1.34.4/src/lib/ares_data.c b/source/lib/c-ares-1.34.6/src/lib/ares_data.c similarity index 100% rename from source/lib/c-ares-1.34.4/src/lib/ares_data.c rename to source/lib/c-ares-1.34.6/src/lib/ares_data.c diff --git a/source/lib/c-ares-1.34.4/src/lib/ares_data.h b/source/lib/c-ares-1.34.6/src/lib/ares_data.h similarity index 100% rename from source/lib/c-ares-1.34.4/src/lib/ares_data.h rename to source/lib/c-ares-1.34.6/src/lib/ares_data.h diff --git a/source/lib/c-ares-1.34.4/src/lib/ares_destroy.c b/source/lib/c-ares-1.34.6/src/lib/ares_destroy.c similarity index 100% rename from source/lib/c-ares-1.34.4/src/lib/ares_destroy.c rename to source/lib/c-ares-1.34.6/src/lib/ares_destroy.c diff --git a/source/lib/c-ares-1.34.4/src/lib/ares_free_hostent.c b/source/lib/c-ares-1.34.6/src/lib/ares_free_hostent.c similarity index 91% rename from source/lib/c-ares-1.34.4/src/lib/ares_free_hostent.c rename to source/lib/c-ares-1.34.6/src/lib/ares_free_hostent.c index bf203723..dfcbdf49 100644 --- a/source/lib/c-ares-1.34.4/src/lib/ares_free_hostent.c +++ b/source/lib/c-ares-1.34.6/src/lib/ares_free_hostent.c @@ -44,9 +44,10 @@ void ares_free_hostent(struct hostent *host) } ares_free(host->h_aliases); if (host->h_addr_list) { - ares_free( - host->h_addr_list[0]); /* no matter if there is one or many entries, - there is only one malloc for all of them */ + size_t i; + for (i=0; host->h_addr_list[i] != NULL; i++) { + ares_free(host->h_addr_list[i]); + } ares_free(host->h_addr_list); } ares_free(host); diff --git a/source/lib/c-ares-1.34.4/src/lib/ares_free_string.c b/source/lib/c-ares-1.34.6/src/lib/ares_free_string.c similarity index 100% rename from source/lib/c-ares-1.34.4/src/lib/ares_free_string.c rename to source/lib/c-ares-1.34.6/src/lib/ares_free_string.c diff --git a/source/lib/c-ares-1.34.4/src/lib/ares_freeaddrinfo.c b/source/lib/c-ares-1.34.6/src/lib/ares_freeaddrinfo.c similarity index 100% rename from source/lib/c-ares-1.34.4/src/lib/ares_freeaddrinfo.c rename to source/lib/c-ares-1.34.6/src/lib/ares_freeaddrinfo.c diff --git a/source/lib/c-ares-1.34.4/src/lib/ares_getaddrinfo.c b/source/lib/c-ares-1.34.6/src/lib/ares_getaddrinfo.c similarity index 96% rename from source/lib/c-ares-1.34.4/src/lib/ares_getaddrinfo.c rename to source/lib/c-ares-1.34.6/src/lib/ares_getaddrinfo.c index 32791dc3..6009de36 100644 --- a/source/lib/c-ares-1.34.4/src/lib/ares_getaddrinfo.c +++ b/source/lib/c-ares-1.34.6/src/lib/ares_getaddrinfo.c @@ -418,9 +418,13 @@ static ares_status_t file_lookup(struct host_query *hquery) * SHOULD recognize localhost names as special and SHOULD always return the * IP loopback address for address queries". * We will also ignore ALL errors when trying to resolve localhost, such - * as permissions errors reading /etc/hosts or a malformed /etc/hosts */ - if (status != ARES_SUCCESS && status != ARES_ENOMEM && - ares_is_localhost(hquery->name)) { + * as permissions errors reading /etc/hosts or a malformed /etc/hosts. + * + * Also, just because the query itself returned success from /etc/hosts + * lookup doesn't mean it returned everything it needed to for all requested + * address families. As long as we're not on a critical out of memory + * condition pass it through to fill in any other address classes. */ + if (status != ARES_ENOMEM && ares_is_localhost(hquery->name)) { return ares_addrinfo_localhost(hquery->name, hquery->port, &hquery->hints, hquery->ai); } @@ -571,6 +575,23 @@ static void host_callback(void *arg, ares_status_t status, size_t timeouts, /* at this point we keep on waiting for the next query to finish */ } +static ares_bool_t numeric_service_to_port(const char *service, + unsigned short *port) +{ + char *end; + unsigned long val; + + errno = 0; + val = strtoul(service, &end, 10); + + if (errno == 0 && *end == '\0' && val <= 65535) { + *port = (unsigned short)val; + return ARES_TRUE; + } + + return ARES_FALSE; +} + static void ares_getaddrinfo_int(ares_channel_t *channel, const char *name, const char *service, const struct ares_addrinfo_hints *hints, @@ -602,25 +623,17 @@ static void ares_getaddrinfo_int(ares_channel_t *channel, const char *name, if (service) { if (hints->ai_flags & ARES_AI_NUMERICSERV) { - unsigned long val; - errno = 0; - val = strtoul(service, NULL, 0); - if ((val == 0 && errno != 0) || val > 65535) { + if (!numeric_service_to_port(service, &port)) { callback(arg, ARES_ESERVICE, 0, NULL); return; } - port = (unsigned short)val; } else { port = lookup_service(service, 0); if (!port) { - unsigned long val; - errno = 0; - val = strtoul(service, NULL, 0); - if ((val == 0 && errno != 0) || val > 65535) { + if (!numeric_service_to_port(service, &port)) { callback(arg, ARES_ESERVICE, 0, NULL); return; } - port = (unsigned short)val; } } } diff --git a/source/lib/c-ares-1.34.4/src/lib/ares_getenv.c b/source/lib/c-ares-1.34.6/src/lib/ares_getenv.c similarity index 100% rename from source/lib/c-ares-1.34.4/src/lib/ares_getenv.c rename to source/lib/c-ares-1.34.6/src/lib/ares_getenv.c diff --git a/source/lib/c-ares-1.34.4/src/lib/ares_getenv.h b/source/lib/c-ares-1.34.6/src/lib/ares_getenv.h similarity index 100% rename from source/lib/c-ares-1.34.4/src/lib/ares_getenv.h rename to source/lib/c-ares-1.34.6/src/lib/ares_getenv.h diff --git a/source/lib/c-ares-1.34.4/src/lib/ares_gethostbyaddr.c b/source/lib/c-ares-1.34.6/src/lib/ares_gethostbyaddr.c similarity index 99% rename from source/lib/c-ares-1.34.4/src/lib/ares_gethostbyaddr.c rename to source/lib/c-ares-1.34.6/src/lib/ares_gethostbyaddr.c index a7acf3c4..69c509ab 100644 --- a/source/lib/c-ares-1.34.4/src/lib/ares_gethostbyaddr.c +++ b/source/lib/c-ares-1.34.6/src/lib/ares_gethostbyaddr.c @@ -120,7 +120,7 @@ static void next_lookup(struct addr_query *aquery) { const char *p; ares_status_t status; - struct hostent *host; + struct hostent *host = NULL; char *name; for (p = aquery->remaining_lookups; *p; p++) { diff --git a/source/lib/c-ares-1.34.4/src/lib/ares_gethostbyname.c b/source/lib/c-ares-1.34.6/src/lib/ares_gethostbyname.c similarity index 96% rename from source/lib/c-ares-1.34.4/src/lib/ares_gethostbyname.c rename to source/lib/c-ares-1.34.6/src/lib/ares_gethostbyname.c index 56de7295..d451b468 100644 --- a/source/lib/c-ares-1.34.4/src/lib/ares_gethostbyname.c +++ b/source/lib/c-ares-1.34.6/src/lib/ares_gethostbyname.c @@ -287,6 +287,8 @@ static ares_status_t ares_gethostbyname_file_int(ares_channel_t *channel, return ARES_ENOTFOUND; } + *host = NULL; + /* Per RFC 7686, reject queries for ".onion" domain names with NXDOMAIN. */ if (ares_is_onion_domain(name)) { return ARES_ENOTFOUND; @@ -307,9 +309,13 @@ static ares_status_t ares_gethostbyname_file_int(ares_channel_t *channel, * SHOULD recognize localhost names as special and SHOULD always return the * IP loopback address for address queries". * We will also ignore ALL errors when trying to resolve localhost, such - * as permissions errors reading /etc/hosts or a malformed /etc/hosts */ - if (status != ARES_SUCCESS && status != ARES_ENOMEM && - ares_is_localhost(name)) { + * as permissions errors reading /etc/hosts or a malformed /etc/hosts. + * + * Also, just because the query itself returned success from /etc/hosts + * lookup doesn't mean it returned everything it needed to for all requested + * address families. As long as we're not on a critical out of memory + * condition pass it through to fill in any other address classes. */ + if (status != ARES_ENOMEM && ares_is_localhost(name)) { return ares_hostent_localhost(name, family, host); } diff --git a/source/lib/c-ares-1.34.4/src/lib/ares_getnameinfo.c b/source/lib/c-ares-1.34.6/src/lib/ares_getnameinfo.c similarity index 100% rename from source/lib/c-ares-1.34.4/src/lib/ares_getnameinfo.c rename to source/lib/c-ares-1.34.6/src/lib/ares_getnameinfo.c diff --git a/source/lib/c-ares-1.34.4/src/lib/ares_hosts_file.c b/source/lib/c-ares-1.34.6/src/lib/ares_hosts_file.c similarity index 98% rename from source/lib/c-ares-1.34.4/src/lib/ares_hosts_file.c rename to source/lib/c-ares-1.34.6/src/lib/ares_hosts_file.c index 0439b8e1..c0fb8076 100644 --- a/source/lib/c-ares-1.34.4/src/lib/ares_hosts_file.c +++ b/source/lib/c-ares-1.34.6/src/lib/ares_hosts_file.c @@ -41,6 +41,11 @@ #endif #include +#ifdef USE_WINSOCK +# define DATABASEPATH "DatabasePath" +# define WIN_PATH_HOSTS "\\hosts" +#endif + /* HOSTS FILE PROCESSING OVERVIEW * ============================== * The hosts file on the system contains static entries to be processed locally @@ -845,7 +850,7 @@ ares_status_t ares_hosts_entry_to_addrinfo(const ares_hosts_entry_t *entry, ares_bool_t want_cnames, struct ares_addrinfo *ai) { - ares_status_t status; + ares_status_t status = ARES_ENOTFOUND; struct ares_addrinfo_cname *cnames = NULL; struct ares_addrinfo_node *ainodes = NULL; ares_llist_node_t *node; @@ -860,6 +865,7 @@ ares_status_t ares_hosts_entry_to_addrinfo(const ares_hosts_entry_t *entry, } if (name != NULL) { + ares_free(ai->name); ai->name = ares_strdup(name); if (ai->name == NULL) { status = ARES_ENOMEM; /* LCOV_EXCL_LINE: OutOfMemory */ @@ -888,6 +894,11 @@ ares_status_t ares_hosts_entry_to_addrinfo(const ares_hosts_entry_t *entry, } } + /* Might be ARES_ENOTFOUND here if no ips matched requested address family */ + if (status != ARES_SUCCESS) { + goto done; + } + if (want_cnames) { status = ares_hosts_ai_append_cnames(entry, &cnames); if (status != ARES_SUCCESS) { diff --git a/source/lib/c-ares-1.34.4/src/lib/ares_inet_net_pton.h b/source/lib/c-ares-1.34.6/src/lib/ares_inet_net_pton.h similarity index 100% rename from source/lib/c-ares-1.34.4/src/lib/ares_inet_net_pton.h rename to source/lib/c-ares-1.34.6/src/lib/ares_inet_net_pton.h diff --git a/source/lib/c-ares-1.34.4/src/lib/ares_init.c b/source/lib/c-ares-1.34.6/src/lib/ares_init.c similarity index 92% rename from source/lib/c-ares-1.34.4/src/lib/ares_init.c rename to source/lib/c-ares-1.34.6/src/lib/ares_init.c index ae78262a..dc094c4d 100644 --- a/source/lib/c-ares-1.34.4/src/lib/ares_init.c +++ b/source/lib/c-ares-1.34.6/src/lib/ares_init.c @@ -271,6 +271,8 @@ int ares_init_options(ares_channel_t **channelptr, goto done; } + ares_set_socket_functions_def(channel); + /* Initialize Server List */ channel->servers = ares_slist_create(channel->rand_state, server_sort_cb, server_destroy_cb); @@ -346,8 +348,6 @@ int ares_init_options(ares_channel_t **channelptr, goto done; } - ares_set_socket_functions_def(channel); - /* Initialize the event thread */ if (channel->optmask & ARES_OPT_EVENT_THREAD) { ares_event_thread_t *e = NULL; @@ -362,7 +362,8 @@ int ares_init_options(ares_channel_t **channelptr, e = channel->sock_state_cb_data; status = ares_event_configchg_init(&e->configchg, e); if (status != ARES_SUCCESS && status != ARES_ENOTIMP) { - goto done; /* LCOV_EXCL_LINE: UntestablePath */ + DEBUGF(fprintf(stderr, "Error: ares_event_configchg_init failed: %s\n", + ares_strerror(status))); } status = ARES_SUCCESS; } @@ -484,16 +485,20 @@ int ares_dup(ares_channel_t **dest, const ares_channel_t *src) ares_channel_lock(src); /* Now clone the options that ares_save_options() doesn't support, but are * user-provided */ - (*dest)->sock_create_cb = src->sock_create_cb; - (*dest)->sock_create_cb_data = src->sock_create_cb_data; - (*dest)->sock_config_cb = src->sock_config_cb; - (*dest)->sock_config_cb_data = src->sock_config_cb_data; + (*dest)->sock_create_cb = src->sock_create_cb; + (*dest)->sock_create_cb_data = src->sock_create_cb_data; + (*dest)->sock_config_cb = src->sock_config_cb; + (*dest)->sock_config_cb_data = src->sock_config_cb_data; memcpy(&(*dest)->sock_funcs, &(src->sock_funcs), sizeof((*dest)->sock_funcs)); - (*dest)->sock_func_cb_data = src->sock_func_cb_data; - (*dest)->legacy_sock_funcs = src->legacy_sock_funcs; - (*dest)->legacy_sock_funcs_cb_data = src->legacy_sock_funcs_cb_data; - (*dest)->server_state_cb = src->server_state_cb; - (*dest)->server_state_cb_data = src->server_state_cb_data; + (*dest)->sock_func_cb_data = src->sock_func_cb_data; + (*dest)->legacy_sock_funcs = src->legacy_sock_funcs; + (*dest)->legacy_sock_funcs_cb_data = src->legacy_sock_funcs_cb_data; + (*dest)->server_state_cb = src->server_state_cb; + (*dest)->server_state_cb_data = src->server_state_cb_data; + (*dest)->notify_pending_write_cb = src->notify_pending_write_cb; + (*dest)->notify_pending_write_cb_data = src->notify_pending_write_cb_data; + (*dest)->query_enqueue_cb = src->query_enqueue_cb; + (*dest)->query_enqueue_cb_data = src->query_enqueue_cb_data; ares_strcpy((*dest)->local_dev_name, src->local_dev_name, sizeof((*dest)->local_dev_name)); @@ -598,3 +603,15 @@ int ares_set_sortlist(ares_channel_t *channel, const char *sortstr) ares_channel_unlock(channel); return (int)status; } + +void ares_set_query_enqueue_cb(ares_channel_t *channel, + ares_query_enqueue_cb callback, + void *user_data) +{ + if (channel == NULL) { + return; + } + + channel->query_enqueue_cb = callback; + channel->query_enqueue_cb_data = user_data; +} diff --git a/source/lib/c-ares-1.34.4/src/lib/ares_ipv6.h b/source/lib/c-ares-1.34.6/src/lib/ares_ipv6.h similarity index 89% rename from source/lib/c-ares-1.34.4/src/lib/ares_ipv6.h rename to source/lib/c-ares-1.34.6/src/lib/ares_ipv6.h index 5da341b0..d2007cc2 100644 --- a/source/lib/c-ares-1.34.4/src/lib/ares_ipv6.h +++ b/source/lib/c-ares-1.34.6/src/lib/ares_ipv6.h @@ -90,6 +90,16 @@ struct addrinfo { # define NS_INT16SZ 2 #endif +/* Windows XP Compatibility with later MSVC/Mingw versions */ +#if defined(_WIN32) +# if !defined(IF_MAX_STRING_SIZE) +# define IF_MAX_STRING_SIZE 256 /* =256 in */ +# endif +# if !defined(NDIS_IF_MAX_STRING_SIZE) +# define NDIS_IF_MAX_STRING_SIZE IF_MAX_STRING_SIZE /* =256 in */ +# endif +#endif + #ifndef IF_NAMESIZE # ifdef IFNAMSIZ # define IF_NAMESIZE IFNAMSIZ diff --git a/source/lib/c-ares-1.34.4/src/lib/ares_library_init.c b/source/lib/c-ares-1.34.6/src/lib/ares_library_init.c similarity index 100% rename from source/lib/c-ares-1.34.4/src/lib/ares_library_init.c rename to source/lib/c-ares-1.34.6/src/lib/ares_library_init.c diff --git a/source/lib/c-ares-1.34.4/src/lib/ares_metrics.c b/source/lib/c-ares-1.34.6/src/lib/ares_metrics.c similarity index 99% rename from source/lib/c-ares-1.34.4/src/lib/ares_metrics.c rename to source/lib/c-ares-1.34.6/src/lib/ares_metrics.c index 13e34dec..deb3b7fe 100644 --- a/source/lib/c-ares-1.34.4/src/lib/ares_metrics.c +++ b/source/lib/c-ares-1.34.6/src/lib/ares_metrics.c @@ -197,7 +197,7 @@ void ares_metrics_record(const ares_query_t *query, ares_server_t *server, } if (query_ms > server->metrics[i].latency_max_ms) { - server->metrics[i].latency_min_ms = query_ms; + server->metrics[i].latency_max_ms = query_ms; } server->metrics[i].total_count++; diff --git a/source/lib/c-ares-1.34.4/src/lib/ares_options.c b/source/lib/c-ares-1.34.6/src/lib/ares_options.c similarity index 100% rename from source/lib/c-ares-1.34.4/src/lib/ares_options.c rename to source/lib/c-ares-1.34.6/src/lib/ares_options.c diff --git a/source/lib/c-ares-1.34.4/src/lib/ares_parse_into_addrinfo.c b/source/lib/c-ares-1.34.6/src/lib/ares_parse_into_addrinfo.c similarity index 100% rename from source/lib/c-ares-1.34.4/src/lib/ares_parse_into_addrinfo.c rename to source/lib/c-ares-1.34.6/src/lib/ares_parse_into_addrinfo.c diff --git a/source/lib/c-ares-1.34.4/src/lib/ares_private.h b/source/lib/c-ares-1.34.6/src/lib/ares_private.h similarity index 96% rename from source/lib/c-ares-1.34.4/src/lib/ares_private.h rename to source/lib/c-ares-1.34.6/src/lib/ares_private.h index e6d44e8b..d6bd426d 100644 --- a/source/lib/c-ares-1.34.4/src/lib/ares_private.h +++ b/source/lib/c-ares-1.34.6/src/lib/ares_private.h @@ -83,21 +83,7 @@ #define CARES_INADDR_CAST(type, var) ((type)((const void *)var)) #if defined(USE_WINSOCK) - -# define WIN_NS_9X "System\\CurrentControlSet\\Services\\VxD\\MSTCP" # define WIN_NS_NT_KEY "System\\CurrentControlSet\\Services\\Tcpip\\Parameters" -# define WIN_DNSCLIENT "Software\\Policies\\Microsoft\\System\\DNSClient" -# define WIN_NT_DNSCLIENT \ - "Software\\Policies\\Microsoft\\Windows NT\\DNSClient" -# define NAMESERVER "NameServer" -# define DHCPNAMESERVER "DhcpNameServer" -# define DATABASEPATH "DatabasePath" -# define WIN_PATH_HOSTS "\\hosts" -# define SEARCHLIST_KEY "SearchList" -# define PRIMARYDNSSUFFIX_KEY "PrimaryDNSSuffix" -# define INTERFACES_KEY "Interfaces" -# define DOMAIN_KEY "Domain" -# define DHCPDOMAIN_KEY "DhcpDomain" # define PATH_RESOLV_CONF "" #elif defined(WATT32) @@ -145,6 +131,8 @@ W32_FUNC const char *_w32_GetHostsFile(void); #define DEFAULT_SERVER_RETRY_CHANCE 10 #define DEFAULT_SERVER_RETRY_DELAY 5000 +typedef void (*ares_query_enqueue_cb)(void *data); + struct ares_query; typedef struct ares_query ares_query_t; @@ -269,6 +257,9 @@ struct ares_channeldata { void *notify_pending_write_cb_data; ares_bool_t notify_pending_write; + ares_query_enqueue_cb query_enqueue_cb; + void *query_enqueue_cb_data; + /* Path for resolv.conf file, configurable via ares_options */ char *resolvconf_path; @@ -321,7 +312,8 @@ ares_status_t ares_send_query(ares_server_t *requested_server /* Optional */, ares_status_t ares_requeue_query(ares_query_t *query, const ares_timeval_t *now, ares_status_t status, ares_bool_t inc_try_count, - const ares_dns_record_t *dnsrec); + ares_dns_record_t *dnsrec, + ares_array_t **requeue); /*! Count the number of labels (dots+1) in a domain */ size_t ares_name_label_cnt(const char *name); @@ -366,6 +358,9 @@ ares_status_t ares_init_servers_state(ares_channel_t *channel); ares_status_t ares_init_by_options(ares_channel_t *channel, const struct ares_options *options, int optmask); +void ares_set_query_enqueue_cb(ares_channel_t *channel, + ares_query_enqueue_cb callback, + void *user_data); ares_status_t ares_init_by_sysconfig(ares_channel_t *channel); void ares_set_socket_functions_def(ares_channel_t *channel); @@ -455,8 +450,10 @@ ares_status_t ares_parse_ptr_reply_dnsrec(const ares_dns_record_t *dnsrec, const void *addr, int addrlen, int family, struct hostent **host); +/* host address must be valid or NULL as will create or append */ ares_status_t ares_addrinfo2hostent(const struct ares_addrinfo *ai, int family, struct hostent **host); + ares_status_t ares_addrinfo2addrttl(const struct ares_addrinfo *ai, int family, size_t req_naddrttls, struct ares_addrttl *addrttls, @@ -591,10 +588,10 @@ ares_status_t ares_qcache_create(ares_rand_state *rand_state, unsigned int max_ttl, ares_qcache_t **cache_out); void ares_qcache_flush(ares_qcache_t *cache); -ares_status_t ares_qcache_insert(ares_channel_t *channel, - const ares_timeval_t *now, - const ares_query_t *query, - ares_dns_record_t *dnsrec); +ares_status_t ares_qcache_insert(ares_channel_t *channel, + const ares_timeval_t *now, + const ares_query_t *query, + const ares_dns_record_t *dnsrec); ares_status_t ares_qcache_fetch(ares_channel_t *channel, const ares_timeval_t *now, const ares_dns_record_t *dnsrec, @@ -610,7 +607,8 @@ ares_status_t ares_cookie_apply(ares_dns_record_t *dnsrec, ares_conn_t *conn, ares_status_t ares_cookie_validate(ares_query_t *query, const ares_dns_record_t *dnsresp, ares_conn_t *conn, - const ares_timeval_t *now); + const ares_timeval_t *now, + ares_array_t **requeue); ares_status_t ares_channel_threading_init(ares_channel_t *channel); void ares_channel_threading_destroy(ares_channel_t *channel); diff --git a/source/lib/c-ares-1.34.4/src/lib/ares_process.c b/source/lib/c-ares-1.34.6/src/lib/ares_process.c similarity index 87% rename from source/lib/c-ares-1.34.4/src/lib/ares_process.c rename to source/lib/c-ares-1.34.6/src/lib/ares_process.c index 3d186ea9..93529684 100644 --- a/source/lib/c-ares-1.34.4/src/lib/ares_process.c +++ b/source/lib/c-ares-1.34.6/src/lib/ares_process.c @@ -56,14 +56,16 @@ static ares_status_t process_timeouts(ares_channel_t *channel, static ares_status_t process_answer(ares_channel_t *channel, const unsigned char *abuf, size_t alen, ares_conn_t *conn, - const ares_timeval_t *now); + const ares_timeval_t *now, + ares_array_t **requeue); static void handle_conn_error(ares_conn_t *conn, ares_bool_t critical_failure, ares_status_t failure_status); static ares_bool_t same_questions(const ares_query_t *query, const ares_dns_record_t *arec); static void end_query(ares_channel_t *channel, ares_server_t *server, ares_query_t *query, ares_status_t status, - const ares_dns_record_t *dnsrec); + ares_dns_record_t *dnsrec, + ares_array_t **requeue); static void ares_query_remove_from_conn(ares_query_t *query) { @@ -229,11 +231,14 @@ static ares_status_t ares_process_fds_nolock(ares_channel_t *channel, } if (!(flags & ARES_PROCESS_FLAG_SKIP_NON_FD)) { - ares_check_cleanup_conns(channel); status = process_timeouts(channel, &now); if (status == ARES_ENOMEM) { goto done; } + + /* Cleanup should be done after processing timeouts as it may invalidate + * connections */ + ares_check_cleanup_conns(channel); } done: @@ -510,10 +515,69 @@ static ares_status_t read_conn_packets(ares_conn_t *conn) return ARES_SUCCESS; } +typedef enum { + REQUEUE_REQUEUE = 1, + REQUEUE_ENDQUERY = 2 +} requeue_type_t; + +/* Simple data structure to store a query that needs to be requeued with + * optional server */ +typedef struct { + requeue_type_t type; /* type of entry, requeue or endquery */ + unsigned short qid; /* query id */ + ares_server_t *server; /* requeue only: optional */ + ares_status_t status; /* endquery only */ + ares_dns_record_t *dnsrec; /* endquery only: optional */ +} ares_requeue_t; + +static ares_status_t ares_append_requeue_int(ares_array_t **requeue, + requeue_type_t type, + ares_query_t *query, + ares_server_t *server, + ares_status_t status, + ares_dns_record_t *dnsrec) +{ + ares_requeue_t entry; + + if (*requeue == NULL) { + *requeue = ares_array_create(sizeof(ares_requeue_t), NULL); + if (*requeue == NULL) { + return ARES_ENOMEM; + } + } + + ares_query_remove_from_conn(query); + + entry.type = type; + entry.qid = query->qid; + entry.server = server; + entry.status = status; + entry.dnsrec = dnsrec; + return ares_array_insertdata_last(*requeue, &entry); +} + +static ares_status_t ares_append_requeue(ares_array_t **requeue, + ares_query_t *query, + ares_server_t *server) +{ + return ares_append_requeue_int(requeue, REQUEUE_REQUEUE, query, server, 0, + NULL); +} + +static ares_status_t ares_append_endqueue(ares_array_t **requeue, + ares_query_t *query, + ares_status_t status, + ares_dns_record_t *dnsrec) +{ + return ares_append_requeue_int(requeue, REQUEUE_ENDQUERY, query, NULL, status, + dnsrec); +} + static ares_status_t read_answers(ares_conn_t *conn, const ares_timeval_t *now) { ares_status_t status; - ares_channel_t *channel = conn->server->channel; + ares_channel_t *channel = conn->server->channel; + ares_array_t *requeue = NULL; /* Process all queued answers */ while (1) { @@ -550,15 +614,55 @@ static ares_status_t read_answers(ares_conn_t *conn, const ares_timeval_t *now) data_len -= 2; /* We finished reading this answer; process it */ - status = process_answer(channel, data, data_len, conn, now); + status = process_answer(channel, data, data_len, conn, now, &requeue); if (status != ARES_SUCCESS) { handle_conn_error(conn, ARES_TRUE, status); - return status; + goto cleanup; } /* Since we processed the answer, clear the tag so space can be reclaimed */ ares_buf_tag_clear(conn->in_buf); } + +cleanup: + + /* Flush requeue */ + while (ares_array_len(requeue) > 0) { + ares_query_t *query; + ares_requeue_t entry; + ares_status_t internal_status; + + internal_status = ares_array_claim_at(&entry, sizeof(entry), requeue, 0); + if (internal_status != ARES_SUCCESS) { + break; + } + + query = ares_htable_szvp_get_direct(channel->queries_by_qid, entry.qid); + + if (entry.type == REQUEUE_REQUEUE) { + /* query disappeared */ + if (query == NULL) { + continue; + } + internal_status = ares_send_query(entry.server, query, now); + /* We only care about ARES_ENOMEM */ + if (internal_status == ARES_ENOMEM) { + status = ARES_ENOMEM; + } + } else { /* REQUEUE_ENDQUERY */ + if (query != NULL) { + query->callback(query->arg, entry.status, query->timeouts, entry.dnsrec); + ares_free_query(query); + } + ares_dns_record_destroy(entry.dnsrec); + } + } + /* Don't forget to send notification if queue emptied */ + if (requeue != NULL) { + ares_queue_notify_empty(channel); + } + ares_array_destroy(requeue); + return status; } @@ -611,7 +715,8 @@ static ares_status_t process_timeouts(ares_channel_t *channel, conn = query->conn; server_increment_failures(conn->server, query->using_tcp); - status = ares_requeue_query(query, now, ARES_ETIMEOUT, ARES_TRUE, NULL); + status = ares_requeue_query(query, now, ARES_ETIMEOUT, ARES_TRUE, NULL, + NULL); if (status == ARES_ENOMEM) { goto done; } @@ -701,7 +806,8 @@ static ares_bool_t issue_might_be_edns(const ares_dns_record_t *req, static ares_status_t process_answer(ares_channel_t *channel, const unsigned char *abuf, size_t alen, ares_conn_t *conn, - const ares_timeval_t *now) + const ares_timeval_t *now, + ares_array_t **requeue) { ares_query_t *query; /* Cache these as once ares_send_query() gets called, it may end up @@ -745,7 +851,8 @@ static ares_status_t process_answer(ares_channel_t *channel, /* Validate DNS cookie in response. This function may need to requeue the * query. */ - if (ares_cookie_validate(query, rdnsrec, conn, now) != ARES_SUCCESS) { + if (ares_cookie_validate(query, rdnsrec, conn, now, requeue) + != ARES_SUCCESS) { /* Drop response and return */ status = ARES_SUCCESS; goto cleanup; @@ -764,13 +871,12 @@ static ares_status_t process_answer(ares_channel_t *channel, if (issue_might_be_edns(query->query, rdnsrec)) { status = rewrite_without_edns(query); if (status != ARES_SUCCESS) { - end_query(channel, server, query, status, NULL); + end_query(channel, server, query, status, NULL, NULL); goto cleanup; } - /* Send to same server */ - ares_send_query(server, query, now); - status = ARES_SUCCESS; + /* Requeue to same server */ + status = ares_append_requeue(requeue, query, server); goto cleanup; } @@ -782,8 +888,9 @@ static ares_status_t process_answer(ares_channel_t *channel, !(conn->flags & ARES_CONN_FLAG_TCP) && !(channel->flags & ARES_FLAG_IGNTC)) { query->using_tcp = ARES_TRUE; - ares_send_query(NULL, query, now); - status = ARES_SUCCESS; /* Switched to TCP is ok */ + status = ares_append_requeue(requeue, query, NULL); + /* Status will reflect success except on memory error, which is good since + * requeuing to TCP is ok */ goto cleanup; } @@ -809,23 +916,26 @@ static ares_status_t process_answer(ares_channel_t *channel, } server_increment_failures(server, query->using_tcp); - ares_requeue_query(query, now, status, ARES_TRUE, rdnsrec); - - /* Should any of these cause a connection termination? - * Maybe SERVER_FAILURE? */ - status = ARES_SUCCESS; + status = ares_requeue_query(query, now, status, ARES_TRUE, rdnsrec, + requeue); + rdnsrec = NULL; /* Free'd by ares_requeue_query() */ + + if (status != ARES_ENOMEM) { + /* Should any of these cause a connection termination? + * Maybe SERVER_FAILURE? */ + status = ARES_SUCCESS; + } goto cleanup; } } /* If cache insertion was successful, it took ownership. We ignore * other cache insertion failures. */ - if (ares_qcache_insert(channel, now, query, rdnsrec) == ARES_SUCCESS) { - is_cached = ARES_TRUE; - } + ares_qcache_insert(channel, now, query, rdnsrec); server_set_good(server, query->using_tcp); - end_query(channel, server, query, ARES_SUCCESS, rdnsrec); + end_query(channel, server, query, ARES_SUCCESS, rdnsrec, requeue); + rdnsrec = NULL; /* Free'd by the requeue */ status = ARES_SUCCESS; @@ -854,10 +964,14 @@ static void handle_conn_error(ares_conn_t *conn, ares_bool_t critical_failure, ares_close_connection(conn, failure_status); } +/* Requeue query will normally call ares_send_query() but in some circumstances + * this needs to be delayed, so if requeue is not NULL, it will add the query + * to the queue instead */ ares_status_t ares_requeue_query(ares_query_t *query, const ares_timeval_t *now, ares_status_t status, ares_bool_t inc_try_count, - const ares_dns_record_t *dnsrec) + ares_dns_record_t *dnsrec, + ares_array_t **requeue) { ares_channel_t *channel = query->channel; size_t max_tries = ares_slist_len(channel->servers) * channel->tries; @@ -873,6 +987,10 @@ ares_status_t ares_requeue_query(ares_query_t *query, const ares_timeval_t *now, } if (query->try_count < max_tries && !query->no_retries) { + ares_dns_record_destroy(dnsrec); + if (requeue != NULL) { + return ares_append_requeue(requeue, query, NULL); + } return ares_send_query(NULL, query, now); } @@ -881,7 +999,7 @@ ares_status_t ares_requeue_query(ares_query_t *query, const ares_timeval_t *now, query->error_status = ARES_ETIMEOUT; } - end_query(channel, NULL, query, query->error_status, dnsrec); + end_query(channel, NULL, query, query->error_status, dnsrec, requeue); return ARES_ETIMEOUT; } @@ -1087,6 +1205,12 @@ static ares_conn_t *ares_fetch_connection(const ares_channel_t *channel, return NULL; } + /* If the associated server has failures, don't use it. It should be cleaned + * up later. */ + if (conn->server->consec_failures > 0) { + return NULL; + } + /* Used too many times */ if (channel->udp_max_queries > 0 && conn->total_queries >= channel->udp_max_queries) { @@ -1148,7 +1272,6 @@ ares_status_t ares_send_query(ares_server_t *requested_server, ares_status_t status; ares_bool_t probe_downed_server = ARES_TRUE; - /* Choose the server to send the query to */ if (requested_server != NULL) { server = requested_server; @@ -1163,7 +1286,7 @@ ares_status_t ares_send_query(ares_server_t *requested_server, } if (server == NULL) { - end_query(channel, server, query, ARES_ENOSERVER /* ? */, NULL); + end_query(channel, server, query, ARES_ENOSERVER /* ? */, NULL, NULL); return ARES_ENOSERVER; } @@ -1187,11 +1310,11 @@ ares_status_t ares_send_query(ares_server_t *requested_server, case ARES_ECONNREFUSED: case ARES_EBADFAMILY: server_increment_failures(server, query->using_tcp); - return ares_requeue_query(query, now, status, ARES_TRUE, NULL); + return ares_requeue_query(query, now, status, ARES_TRUE, NULL, NULL); /* Anything else is not retryable, likely ENOMEM */ default: - end_query(channel, server, query, status, NULL); + end_query(channel, server, query, status, NULL, NULL); return status; } } @@ -1205,7 +1328,7 @@ ares_status_t ares_send_query(ares_server_t *requested_server, case ARES_ENOMEM: /* Not retryable */ - end_query(channel, server, query, status, NULL); + end_query(channel, server, query, status, NULL, NULL); return status; /* These conditions are retryable as they are server-specific @@ -1213,7 +1336,7 @@ ares_status_t ares_send_query(ares_server_t *requested_server, case ARES_ECONNREFUSED: case ARES_EBADFAMILY: handle_conn_error(conn, ARES_TRUE, status); - status = ares_requeue_query(query, now, status, ARES_TRUE, NULL); + status = ares_requeue_query(query, now, status, ARES_TRUE, NULL, NULL); if (status == ARES_ETIMEOUT) { status = ARES_ECONNREFUSED; } @@ -1221,7 +1344,7 @@ ares_status_t ares_send_query(ares_server_t *requested_server, default: server_increment_failures(server, query->using_tcp); - status = ares_requeue_query(query, now, status, ARES_TRUE, NULL); + status = ares_requeue_query(query, now, status, ARES_TRUE, NULL, NULL); return status; } @@ -1237,7 +1360,7 @@ ares_status_t ares_send_query(ares_server_t *requested_server, ares_slist_insert(channel->queries_by_timeout, query); if (!query->node_queries_by_timeout) { /* LCOV_EXCL_START: OutOfMemory */ - end_query(channel, server, query, ARES_ENOMEM, NULL); + end_query(channel, server, query, ARES_ENOMEM, NULL, NULL); return ARES_ENOMEM; /* LCOV_EXCL_STOP */ } @@ -1250,7 +1373,7 @@ ares_status_t ares_send_query(ares_server_t *requested_server, if (query->node_queries_to_conn == NULL) { /* LCOV_EXCL_START: OutOfMemory */ - end_query(channel, server, query, ARES_ENOMEM, NULL); + end_query(channel, server, query, ARES_ENOMEM, NULL, NULL); return ARES_ENOMEM; /* LCOV_EXCL_STOP */ } @@ -1264,6 +1387,10 @@ ares_status_t ares_send_query(ares_server_t *requested_server, ares_probe_failed_server(channel, server, query); } + if (channel->query_enqueue_cb) { + channel->query_enqueue_cb(channel->query_enqueue_cb_data); + } + return ARES_SUCCESS; } @@ -1338,7 +1465,7 @@ static void ares_detach_query(ares_query_t *query) static void end_query(ares_channel_t *channel, ares_server_t *server, ares_query_t *query, ares_status_t status, - const ares_dns_record_t *dnsrec) + ares_dns_record_t *dnsrec, ares_array_t **requeue) { /* If we were probing for the server to come back online, lets mark it as * no longer being probed */ @@ -1348,6 +1475,12 @@ static void end_query(ares_channel_t *channel, ares_server_t *server, ares_metrics_record(query, server, status, dnsrec); + /* Delay calling the query callback */ + if (requeue != NULL) { + ares_append_endqueue(requeue, query, status, dnsrec); + return; + } + /* Invoke the callback. */ query->callback(query->arg, status, query->timeouts, dnsrec); ares_free_query(query); diff --git a/source/lib/c-ares-1.34.4/src/lib/ares_qcache.c b/source/lib/c-ares-1.34.6/src/lib/ares_qcache.c similarity index 95% rename from source/lib/c-ares-1.34.4/src/lib/ares_qcache.c rename to source/lib/c-ares-1.34.6/src/lib/ares_qcache.c index 97c0a913..4050ff54 100644 --- a/source/lib/c-ares-1.34.4/src/lib/ares_qcache.c +++ b/source/lib/c-ares-1.34.6/src/lib/ares_qcache.c @@ -421,10 +421,20 @@ ares_status_t ares_qcache_fetch(ares_channel_t *channel, return status; } -ares_status_t ares_qcache_insert(ares_channel_t *channel, - const ares_timeval_t *now, - const ares_query_t *query, - ares_dns_record_t *dnsrec) +ares_status_t ares_qcache_insert(ares_channel_t *channel, + const ares_timeval_t *now, + const ares_query_t *query, + const ares_dns_record_t *dnsrec) { - return ares_qcache_insert_int(channel->qcache, dnsrec, query->query, now); + ares_dns_record_t *dupdns = ares_dns_record_duplicate(dnsrec); + ares_status_t status; + + if (dupdns == NULL) { + return ARES_ENOMEM; + } + status = ares_qcache_insert_int(channel->qcache, dupdns, query->query, now); + if (status != ARES_SUCCESS) { + ares_dns_record_destroy(dupdns); + } + return status; } diff --git a/source/lib/c-ares-1.34.4/src/lib/ares_query.c b/source/lib/c-ares-1.34.6/src/lib/ares_query.c similarity index 100% rename from source/lib/c-ares-1.34.4/src/lib/ares_query.c rename to source/lib/c-ares-1.34.6/src/lib/ares_query.c diff --git a/source/lib/c-ares-1.34.4/src/lib/ares_search.c b/source/lib/c-ares-1.34.6/src/lib/ares_search.c similarity index 100% rename from source/lib/c-ares-1.34.4/src/lib/ares_search.c rename to source/lib/c-ares-1.34.6/src/lib/ares_search.c diff --git a/source/lib/c-ares-1.34.4/src/lib/ares_send.c b/source/lib/c-ares-1.34.6/src/lib/ares_send.c similarity index 100% rename from source/lib/c-ares-1.34.4/src/lib/ares_send.c rename to source/lib/c-ares-1.34.6/src/lib/ares_send.c diff --git a/source/lib/c-ares-1.34.4/src/lib/ares_set_socket_functions.c b/source/lib/c-ares-1.34.6/src/lib/ares_set_socket_functions.c similarity index 98% rename from source/lib/c-ares-1.34.4/src/lib/ares_set_socket_functions.c rename to source/lib/c-ares-1.34.6/src/lib/ares_set_socket_functions.c index 7216ffa9..9994e81d 100644 --- a/source/lib/c-ares-1.34.4/src/lib/ares_set_socket_functions.c +++ b/source/lib/c-ares-1.34.6/src/lib/ares_set_socket_functions.c @@ -61,7 +61,7 @@ # define TFO_USE_SENDTO 0 # define TFO_USE_CONNECTX 0 # define TFO_CLIENT_SOCKOPT TCP_FASTOPEN_CONNECT -#elif defined(__FreeBSD__) && defined(TCP_FASTOPEN) +#elif (defined(__MidnightBSD__) || defined(__FreeBSD__)) && defined(TCP_FASTOPEN) # define TFO_SUPPORTED 1 # define TFO_SKIP_CONNECT 1 # define TFO_USE_SENDTO 1 @@ -127,6 +127,8 @@ ares_status_t channel->sock_funcs.asendto = funcs->asendto; channel->sock_funcs.agetsockname = funcs->agetsockname; channel->sock_funcs.abind = funcs->abind; + channel->sock_funcs.aif_nametoindex = funcs->aif_nametoindex; + channel->sock_funcs.aif_indextoname = funcs->aif_indextoname; } /* Implement newer versions here ...*/ diff --git a/source/lib/c-ares-1.34.4/src/lib/ares_setup.h b/source/lib/c-ares-1.34.6/src/lib/ares_setup.h similarity index 100% rename from source/lib/c-ares-1.34.4/src/lib/ares_setup.h rename to source/lib/c-ares-1.34.6/src/lib/ares_setup.h diff --git a/source/lib/c-ares-1.34.4/src/lib/ares_socket.c b/source/lib/c-ares-1.34.6/src/lib/ares_socket.c similarity index 100% rename from source/lib/c-ares-1.34.4/src/lib/ares_socket.c rename to source/lib/c-ares-1.34.6/src/lib/ares_socket.c diff --git a/source/lib/c-ares-1.34.4/src/lib/ares_socket.h b/source/lib/c-ares-1.34.6/src/lib/ares_socket.h similarity index 100% rename from source/lib/c-ares-1.34.4/src/lib/ares_socket.h rename to source/lib/c-ares-1.34.6/src/lib/ares_socket.h diff --git a/source/lib/c-ares-1.34.4/src/lib/ares_sortaddrinfo.c b/source/lib/c-ares-1.34.6/src/lib/ares_sortaddrinfo.c similarity index 100% rename from source/lib/c-ares-1.34.4/src/lib/ares_sortaddrinfo.c rename to source/lib/c-ares-1.34.6/src/lib/ares_sortaddrinfo.c diff --git a/source/lib/c-ares-1.34.4/src/lib/ares_strerror.c b/source/lib/c-ares-1.34.6/src/lib/ares_strerror.c similarity index 100% rename from source/lib/c-ares-1.34.4/src/lib/ares_strerror.c rename to source/lib/c-ares-1.34.6/src/lib/ares_strerror.c diff --git a/source/lib/c-ares-1.34.4/src/lib/ares_sysconfig.c b/source/lib/c-ares-1.34.6/src/lib/ares_sysconfig.c similarity index 100% rename from source/lib/c-ares-1.34.4/src/lib/ares_sysconfig.c rename to source/lib/c-ares-1.34.6/src/lib/ares_sysconfig.c diff --git a/source/lib/c-ares-1.34.4/src/lib/ares_sysconfig_files.c b/source/lib/c-ares-1.34.6/src/lib/ares_sysconfig_files.c similarity index 100% rename from source/lib/c-ares-1.34.4/src/lib/ares_sysconfig_files.c rename to source/lib/c-ares-1.34.6/src/lib/ares_sysconfig_files.c diff --git a/source/lib/c-ares-1.34.4/src/lib/ares_sysconfig_mac.c b/source/lib/c-ares-1.34.6/src/lib/ares_sysconfig_mac.c similarity index 100% rename from source/lib/c-ares-1.34.4/src/lib/ares_sysconfig_mac.c rename to source/lib/c-ares-1.34.6/src/lib/ares_sysconfig_mac.c diff --git a/source/lib/c-ares-1.34.4/src/lib/ares_sysconfig_win.c b/source/lib/c-ares-1.34.6/src/lib/ares_sysconfig_win.c similarity index 88% rename from source/lib/c-ares-1.34.4/src/lib/ares_sysconfig_win.c rename to source/lib/c-ares-1.34.6/src/lib/ares_sysconfig_win.c index f6e07f92..35f4bd8e 100644 --- a/source/lib/c-ares-1.34.4/src/lib/ares_sysconfig_win.c +++ b/source/lib/c-ares-1.34.6/src/lib/ares_sysconfig_win.c @@ -55,6 +55,19 @@ #include "ares_inet_net_pton.h" #if defined(USE_WINSOCK) + +# define WIN_NS_9X "System\\CurrentControlSet\\Services\\VxD\\MSTCP" +# define WIN_DNSCLIENT "Software\\Policies\\Microsoft\\System\\DNSClient" +# define WIN_NT_DNSCLIENT \ + "Software\\Policies\\Microsoft\\Windows NT\\DNSClient" +# define NAMESERVER "NameServer" +# define DHCPNAMESERVER "DhcpNameServer" +# define SEARCHLIST_KEY L"SearchList" +# define PRIMARYDNSSUFFIX_KEY L"PrimaryDNSSuffix" +# define INTERFACES_KEY "Interfaces" +# define DOMAIN_KEY L"Domain" +# define DHCPDOMAIN_KEY L"DhcpDomain" + /* * get_REG_SZ() * @@ -69,38 +82,48 @@ * * Supported on Windows NT 3.5 and newer. */ -static ares_bool_t get_REG_SZ(HKEY hKey, const char *leafKeyName, char **outptr) +static ares_bool_t get_REG_SZ(HKEY hKey, const WCHAR *leafKeyName, char **outptr) { - DWORD size = 0; - int res; + DWORD size = 0; + int res; + int len; + WCHAR *val = NULL; *outptr = NULL; /* Find out size of string stored in registry */ - res = RegQueryValueExA(hKey, leafKeyName, 0, NULL, NULL, &size); + res = RegQueryValueExW(hKey, leafKeyName, 0, NULL, NULL, &size); if ((res != ERROR_SUCCESS && res != ERROR_MORE_DATA) || !size) { return ARES_FALSE; } /* Allocate buffer of indicated size plus one given that string might have been stored without null termination */ - *outptr = ares_malloc(size + 1); - if (!*outptr) { + val = ares_malloc_zero(size + sizeof(WCHAR)); + if (val == NULL) { return ARES_FALSE; } /* Get the value for real */ - res = RegQueryValueExA(hKey, leafKeyName, 0, NULL, (unsigned char *)*outptr, - &size); - if ((res != ERROR_SUCCESS) || (size == 1)) { + res = RegQueryValueExW(hKey, leafKeyName, 0, NULL, (BYTE *)val, &size); + if (res != ERROR_SUCCESS || size == 1) { + ares_free(val); + return ARES_FALSE; + } + + /* Convert to UTF8 */ + len = WideCharToMultiByte(CP_UTF8, 0, val, -1, NULL, 0, NULL, NULL); + if (len == 0) { + return ARES_FALSE; + } + *outptr = ares_malloc_zero((size_t)len + 1); + if (WideCharToMultiByte(CP_UTF8, 0, val, -1, *outptr, len, NULL, NULL) + == 0) { ares_free(*outptr); *outptr = NULL; return ARES_FALSE; } - /* Null terminate buffer always */ - *(*outptr + size) = '\0'; - return ARES_TRUE; } @@ -135,6 +158,14 @@ static void commajoin(char **dst, const char *src) commanjoin(dst, src, ares_strlen(src)); } +static void commajoin_asciionly(char **dst, const char *src) +{ + if (!ares_str_isprint(src, ares_strlen(src))) { + return; + } + commanjoin(dst, src, ares_strlen(src)); +} + /* A structure to hold the string form of IPv4 and IPv6 addresses so we can * sort them by a metric. */ @@ -176,6 +207,7 @@ static int compareAddresses(const void *arg1, const void *arg2) return 0; } +#if defined(HAVE_GETBESTROUTE2) && !defined(__WATCOMC__) /* There can be multiple routes to "the Internet". And there can be different * DNS servers associated with each of the interfaces that offer those routes. * We have to assume that any DNS server can serve any request. But, some DNS @@ -213,18 +245,6 @@ static ULONG getBestRouteMetric(IF_LUID * const luid, /* Can't be const :( */ const SOCKADDR_INET * const dest, const ULONG interfaceMetric) { - /* On this interface, get the best route to that destination. */ -# if defined(__WATCOMC__) - /* OpenWatcom's builtin Windows SDK does not have a definition for - * MIB_IPFORWARD_ROW2, and also does not allow the usage of SOCKADDR_INET - * as a variable. Let's work around this by returning the worst possible - * metric, but only when using the OpenWatcom compiler. - * It may be worth investigating using a different version of the Windows - * SDK with OpenWatcom in the future, though this may be fixed in OpenWatcom - * 2.0. - */ - return (ULONG)-1; -# else MIB_IPFORWARD_ROW2 row; SOCKADDR_INET ignored; if (GetBestRoute2(/* The interface to use. The index is ignored since we are @@ -257,8 +277,8 @@ static ULONG getBestRouteMetric(IF_LUID * const luid, /* Can't be const :( */ * which describes the combination as a "sum". */ return row.Metric + interfaceMetric; -# endif /* __WATCOMC__ */ } +#endif /* * get_DNS_Windows() @@ -379,9 +399,21 @@ static ares_bool_t get_DNS_Windows(char **outptr) addressesSize = newSize; } +# if defined(HAVE_GETBESTROUTE2) && !defined(__WATCOMC__) + /* OpenWatcom's builtin Windows SDK does not have a definition for + * MIB_IPFORWARD_ROW2, and also does not allow the usage of SOCKADDR_INET + * as a variable. Let's work around this by returning the worst possible + * metric, but only when using the OpenWatcom compiler. + * It may be worth investigating using a different version of the Windows + * SDK with OpenWatcom in the future, though this may be fixed in OpenWatcom + * 2.0. + */ addresses[addressesIndex].metric = getBestRouteMetric( &ipaaEntry->Luid, (SOCKADDR_INET *)((void *)(namesrvr.sa)), ipaaEntry->Ipv4Metric); +# else + addresses[addressesIndex].metric = (ULONG)-1; +# endif /* Record insertion index to make qsort stable */ addresses[addressesIndex].orig_idx = addressesIndex; @@ -423,9 +455,13 @@ static ares_bool_t get_DNS_Windows(char **outptr) ll_scope = ipaaEntry->Ipv6IfIndex; } +# if defined(HAVE_GETBESTROUTE2) && !defined(__WATCOMC__) addresses[addressesIndex].metric = getBestRouteMetric( &ipaaEntry->Luid, (SOCKADDR_INET *)((void *)(namesrvr.sa)), ipaaEntry->Ipv6Metric); +# else + addresses[addressesIndex].metric = (ULONG)-1; +# endif /* Record insertion index to make qsort stable */ addresses[addressesIndex].orig_idx = addressesIndex; @@ -518,7 +554,7 @@ static ares_bool_t get_SuffixList_Windows(char **outptr) ERROR_SUCCESS) { get_REG_SZ(hKey, SEARCHLIST_KEY, outptr); if (get_REG_SZ(hKey, DOMAIN_KEY, &p)) { - commajoin(outptr, p); + commajoin_asciionly(outptr, p); ares_free(p); p = NULL; } @@ -528,7 +564,7 @@ static ares_bool_t get_SuffixList_Windows(char **outptr) if (RegOpenKeyExA(HKEY_LOCAL_MACHINE, WIN_NT_DNSCLIENT, 0, KEY_READ, &hKey) == ERROR_SUCCESS) { if (get_REG_SZ(hKey, SEARCHLIST_KEY, &p)) { - commajoin(outptr, p); + commajoin_asciionly(outptr, p); ares_free(p); p = NULL; } @@ -540,7 +576,7 @@ static ares_bool_t get_SuffixList_Windows(char **outptr) if (RegOpenKeyExA(HKEY_LOCAL_MACHINE, WIN_DNSCLIENT, 0, KEY_READ, &hKey) == ERROR_SUCCESS) { if (get_REG_SZ(hKey, PRIMARYDNSSUFFIX_KEY, &p)) { - commajoin(outptr, p); + commajoin_asciionly(outptr, p); ares_free(p); p = NULL; } @@ -562,17 +598,17 @@ static ares_bool_t get_SuffixList_Windows(char **outptr) } /* p can be comma separated (SearchList) */ if (get_REG_SZ(hKeyEnum, SEARCHLIST_KEY, &p)) { - commajoin(outptr, p); + commajoin_asciionly(outptr, p); ares_free(p); p = NULL; } if (get_REG_SZ(hKeyEnum, DOMAIN_KEY, &p)) { - commajoin(outptr, p); + commajoin_asciionly(outptr, p); ares_free(p); p = NULL; } if (get_REG_SZ(hKeyEnum, DHCPDOMAIN_KEY, &p)) { - commajoin(outptr, p); + commajoin_asciionly(outptr, p); ares_free(p); p = NULL; } diff --git a/source/lib/c-ares-1.34.4/src/lib/ares_timeout.c b/source/lib/c-ares-1.34.6/src/lib/ares_timeout.c similarity index 100% rename from source/lib/c-ares-1.34.4/src/lib/ares_timeout.c rename to source/lib/c-ares-1.34.6/src/lib/ares_timeout.c diff --git a/source/lib/c-ares-1.34.4/src/lib/ares_update_servers.c b/source/lib/c-ares-1.34.6/src/lib/ares_update_servers.c similarity index 100% rename from source/lib/c-ares-1.34.4/src/lib/ares_update_servers.c rename to source/lib/c-ares-1.34.6/src/lib/ares_update_servers.c diff --git a/source/lib/c-ares-1.34.4/src/lib/ares_version.c b/source/lib/c-ares-1.34.6/src/lib/ares_version.c similarity index 100% rename from source/lib/c-ares-1.34.4/src/lib/ares_version.c rename to source/lib/c-ares-1.34.6/src/lib/ares_version.c diff --git a/source/lib/c-ares-1.34.4/src/lib/cares.rc b/source/lib/c-ares-1.34.6/src/lib/cares.rc similarity index 97% rename from source/lib/c-ares-1.34.4/src/lib/cares.rc rename to source/lib/c-ares-1.34.6/src/lib/cares.rc index 9275b079..49fa71ce 100644 --- a/source/lib/c-ares-1.34.4/src/lib/cares.rc +++ b/source/lib/c-ares-1.34.6/src/lib/cares.rc @@ -63,7 +63,7 @@ BEGIN #endif VALUE "ProductName", "The c-ares library\0" VALUE "ProductVersion", ARES_VERSION_STR "\0" - VALUE "LegalCopyright", " " ARES_COPYRIGHT "\0" + VALUE "LegalCopyright", "Copyright (c) " ARES_COPYRIGHT "\0" VALUE "License", "https://c-ares.org/license.html\0" END END diff --git a/source/lib/c-ares-1.34.4/src/lib/config-dos.h b/source/lib/c-ares-1.34.6/src/lib/config-dos.h similarity index 100% rename from source/lib/c-ares-1.34.4/src/lib/config-dos.h rename to source/lib/c-ares-1.34.6/src/lib/config-dos.h diff --git a/source/lib/c-ares-1.34.4/src/lib/config-win32.h b/source/lib/c-ares-1.34.6/src/lib/config-win32.h similarity index 98% rename from source/lib/c-ares-1.34.4/src/lib/config-win32.h rename to source/lib/c-ares-1.34.6/src/lib/config-win32.h index be233a2f..fc533c75 100644 --- a/source/lib/c-ares-1.34.4/src/lib/config-win32.h +++ b/source/lib/c-ares-1.34.6/src/lib/config-win32.h @@ -237,8 +237,10 @@ # undef HAVE_NETIOAPI_H #endif -/* Threading support enabled */ -#define CARES_THREADS 1 +/* Threading support enabled for Vista+ */ +#if !defined(_WIN32_WINNT) || _WIN32_WINNT >= 0x0600 +# define CARES_THREADS 1 +#endif /* ---------------------------------------------------------------- */ /* TYPEDEF REPLACEMENTS */ @@ -370,6 +372,8 @@ # define HAVE_CONVERTINTERFACELUIDTONAMEA 1 /* Define to 1 if you have the `NotifyIpInterfaceChange' function. */ # define HAVE_NOTIFYIPINTERFACECHANGE 1 +/* Define to 1 if you have the `GetBestRoute2` function */ +# define HAVE_GETBESTROUTE2 1 #endif /* ---------------------------------------------------------------- */ diff --git a/source/lib/c-ares-1.34.4/src/lib/dsa/ares_array.c b/source/lib/c-ares-1.34.6/src/lib/dsa/ares_array.c similarity index 100% rename from source/lib/c-ares-1.34.4/src/lib/dsa/ares_array.c rename to source/lib/c-ares-1.34.6/src/lib/dsa/ares_array.c diff --git a/source/lib/c-ares-1.34.4/src/lib/dsa/ares_htable.c b/source/lib/c-ares-1.34.6/src/lib/dsa/ares_htable.c similarity index 100% rename from source/lib/c-ares-1.34.4/src/lib/dsa/ares_htable.c rename to source/lib/c-ares-1.34.6/src/lib/dsa/ares_htable.c diff --git a/source/lib/c-ares-1.34.4/src/lib/dsa/ares_htable.h b/source/lib/c-ares-1.34.6/src/lib/dsa/ares_htable.h similarity index 100% rename from source/lib/c-ares-1.34.4/src/lib/dsa/ares_htable.h rename to source/lib/c-ares-1.34.6/src/lib/dsa/ares_htable.h diff --git a/source/lib/c-ares-1.34.4/src/lib/dsa/ares_htable_asvp.c b/source/lib/c-ares-1.34.6/src/lib/dsa/ares_htable_asvp.c similarity index 100% rename from source/lib/c-ares-1.34.4/src/lib/dsa/ares_htable_asvp.c rename to source/lib/c-ares-1.34.6/src/lib/dsa/ares_htable_asvp.c diff --git a/source/lib/c-ares-1.34.4/src/lib/dsa/ares_htable_dict.c b/source/lib/c-ares-1.34.6/src/lib/dsa/ares_htable_dict.c similarity index 100% rename from source/lib/c-ares-1.34.4/src/lib/dsa/ares_htable_dict.c rename to source/lib/c-ares-1.34.6/src/lib/dsa/ares_htable_dict.c diff --git a/source/lib/c-ares-1.34.4/src/lib/dsa/ares_htable_strvp.c b/source/lib/c-ares-1.34.6/src/lib/dsa/ares_htable_strvp.c similarity index 100% rename from source/lib/c-ares-1.34.4/src/lib/dsa/ares_htable_strvp.c rename to source/lib/c-ares-1.34.6/src/lib/dsa/ares_htable_strvp.c diff --git a/source/lib/c-ares-1.34.4/src/lib/dsa/ares_htable_szvp.c b/source/lib/c-ares-1.34.6/src/lib/dsa/ares_htable_szvp.c similarity index 100% rename from source/lib/c-ares-1.34.4/src/lib/dsa/ares_htable_szvp.c rename to source/lib/c-ares-1.34.6/src/lib/dsa/ares_htable_szvp.c diff --git a/source/lib/c-ares-1.34.4/src/lib/dsa/ares_htable_vpstr.c b/source/lib/c-ares-1.34.6/src/lib/dsa/ares_htable_vpstr.c similarity index 100% rename from source/lib/c-ares-1.34.4/src/lib/dsa/ares_htable_vpstr.c rename to source/lib/c-ares-1.34.6/src/lib/dsa/ares_htable_vpstr.c diff --git a/source/lib/c-ares-1.34.4/src/lib/dsa/ares_htable_vpvp.c b/source/lib/c-ares-1.34.6/src/lib/dsa/ares_htable_vpvp.c similarity index 100% rename from source/lib/c-ares-1.34.4/src/lib/dsa/ares_htable_vpvp.c rename to source/lib/c-ares-1.34.6/src/lib/dsa/ares_htable_vpvp.c diff --git a/source/lib/c-ares-1.34.4/src/lib/dsa/ares_llist.c b/source/lib/c-ares-1.34.6/src/lib/dsa/ares_llist.c similarity index 100% rename from source/lib/c-ares-1.34.4/src/lib/dsa/ares_llist.c rename to source/lib/c-ares-1.34.6/src/lib/dsa/ares_llist.c diff --git a/source/lib/c-ares-1.34.4/src/lib/dsa/ares_slist.c b/source/lib/c-ares-1.34.6/src/lib/dsa/ares_slist.c similarity index 100% rename from source/lib/c-ares-1.34.4/src/lib/dsa/ares_slist.c rename to source/lib/c-ares-1.34.6/src/lib/dsa/ares_slist.c diff --git a/source/lib/c-ares-1.34.4/src/lib/dsa/ares_slist.h b/source/lib/c-ares-1.34.6/src/lib/dsa/ares_slist.h similarity index 100% rename from source/lib/c-ares-1.34.4/src/lib/dsa/ares_slist.h rename to source/lib/c-ares-1.34.6/src/lib/dsa/ares_slist.h diff --git a/source/lib/c-ares-1.34.4/src/lib/event/ares_event.h b/source/lib/c-ares-1.34.6/src/lib/event/ares_event.h similarity index 97% rename from source/lib/c-ares-1.34.4/src/lib/event/ares_event.h rename to source/lib/c-ares-1.34.6/src/lib/event/ares_event.h index 36cd10dc..bf298dfb 100644 --- a/source/lib/c-ares-1.34.4/src/lib/event/ares_event.h +++ b/source/lib/c-ares-1.34.6/src/lib/event/ares_event.h @@ -159,30 +159,33 @@ ares_status_t ares_event_update(ares_event_t **event, ares_event_thread_t *e, ares_event_signal_cb_t signal_cb); -#ifdef HAVE_PIPE +#ifdef CARES_THREADS +# ifdef HAVE_PIPE ares_event_t *ares_pipeevent_create(ares_event_thread_t *e); -#endif +# endif -#ifdef HAVE_POLL +# ifdef HAVE_POLL extern const ares_event_sys_t ares_evsys_poll; -#endif +# endif -#ifdef HAVE_KQUEUE +# ifdef HAVE_KQUEUE extern const ares_event_sys_t ares_evsys_kqueue; -#endif +# endif -#ifdef HAVE_EPOLL +# ifdef HAVE_EPOLL extern const ares_event_sys_t ares_evsys_epoll; -#endif +# endif -#ifdef _WIN32 +# ifdef _WIN32 extern const ares_event_sys_t ares_evsys_win32; -#endif +# endif /* All systems have select(), but not all have a way to wake, so we require * pipe() to wake the select() */ -#ifdef HAVE_PIPE +# ifdef HAVE_PIPE extern const ares_event_sys_t ares_evsys_select; +# endif + #endif #endif diff --git a/source/lib/c-ares-1.34.4/src/lib/event/ares_event_configchg.c b/source/lib/c-ares-1.34.6/src/lib/event/ares_event_configchg.c similarity index 97% rename from source/lib/c-ares-1.34.4/src/lib/event/ares_event_configchg.c rename to source/lib/c-ares-1.34.6/src/lib/event/ares_event_configchg.c index 5ecc6888..add72957 100644 --- a/source/lib/c-ares-1.34.4/src/lib/event/ares_event_configchg.c +++ b/source/lib/c-ares-1.34.6/src/lib/event/ares_event_configchg.c @@ -26,7 +26,7 @@ #include "ares_private.h" #include "ares_event.h" -#ifdef __ANDROID__ +#if defined(__ANDROID__) && defined(CARES_THREADS) ares_status_t ares_event_configchg_init(ares_event_configchg_t **configchg, ares_event_thread_t *e) @@ -43,7 +43,7 @@ void ares_event_configchg_destroy(ares_event_configchg_t *configchg) (void)configchg; } -#elif defined(__linux__) +#elif defined(__linux__) && defined(CARES_THREADS) # include @@ -174,7 +174,7 @@ ares_status_t ares_event_configchg_init(ares_event_configchg_t **configchg, return status; } -#elif defined(USE_WINSOCK) +#elif defined(USE_WINSOCK) && defined(CARES_THREADS) # include # include @@ -379,10 +379,11 @@ ares_status_t ares_event_configchg_init(ares_event_configchg_t **configchg, return status; } -#elif defined(__APPLE__) +#elif defined(__APPLE__) && defined(CARES_THREADS) # include # include +# include # include # include # include @@ -531,7 +532,7 @@ ares_status_t ares_event_configchg_init(ares_event_configchg_t **configchg, return status; } -#elif defined(HAVE_STAT) && !defined(_WIN32) +#elif defined(HAVE_STAT) && !defined(_WIN32) && defined(CARES_THREADS) # ifdef HAVE_SYS_TYPES_H # include # endif @@ -665,6 +666,12 @@ ares_status_t ares_event_configchg_init(ares_event_configchg_t **configchg, goto done; } + c->lock = ares_thread_mutex_create(); + if (c->lock == NULL) { + status = ARES_ENOMEM; + goto done; + } + c->resolvconf_path = c->e->channel->resolvconf_path; if (c->resolvconf_path == NULL) { c->resolvconf_path = PATH_RESOLV_CONF; @@ -722,6 +729,8 @@ void ares_event_configchg_destroy(ares_event_configchg_t *configchg) ares_status_t ares_event_configchg_init(ares_event_configchg_t **configchg, ares_event_thread_t *e) { + (void)configchg; + (void)e; /* No ability */ return ARES_ENOTIMP; } @@ -729,6 +738,7 @@ ares_status_t ares_event_configchg_init(ares_event_configchg_t **configchg, void ares_event_configchg_destroy(ares_event_configchg_t *configchg) { /* No-op */ + (void)configchg; } #endif diff --git a/source/lib/c-ares-1.34.4/src/lib/event/ares_event_epoll.c b/source/lib/c-ares-1.34.6/src/lib/event/ares_event_epoll.c similarity index 99% rename from source/lib/c-ares-1.34.4/src/lib/event/ares_event_epoll.c rename to source/lib/c-ares-1.34.6/src/lib/event/ares_event_epoll.c index 538c38b4..d451c86a 100644 --- a/source/lib/c-ares-1.34.4/src/lib/event/ares_event_epoll.c +++ b/source/lib/c-ares-1.34.6/src/lib/event/ares_event_epoll.c @@ -26,6 +26,8 @@ #include "ares_private.h" #include "ares_event.h" +#if defined(HAVE_EPOLL) && defined(CARES_THREADS) + #ifdef HAVE_SYS_EPOLL_H # include #endif @@ -33,8 +35,6 @@ # include #endif -#ifdef HAVE_EPOLL - typedef struct { int epoll_fd; } ares_evsys_epoll_t; diff --git a/source/lib/c-ares-1.34.4/src/lib/event/ares_event_kqueue.c b/source/lib/c-ares-1.34.6/src/lib/event/ares_event_kqueue.c similarity index 99% rename from source/lib/c-ares-1.34.4/src/lib/event/ares_event_kqueue.c rename to source/lib/c-ares-1.34.6/src/lib/event/ares_event_kqueue.c index dbbd0dbd..00cdcbe9 100644 --- a/source/lib/c-ares-1.34.4/src/lib/event/ares_event_kqueue.c +++ b/source/lib/c-ares-1.34.6/src/lib/event/ares_event_kqueue.c @@ -26,6 +26,8 @@ #include "ares_private.h" #include "ares_event.h" +#if defined(HAVE_KQUEUE) && defined(CARES_THREADS) + #ifdef HAVE_SYS_TYPES_H # include #endif @@ -39,8 +41,6 @@ # include #endif -#ifdef HAVE_KQUEUE - typedef struct { int kqueue_fd; struct kevent *changelist; diff --git a/source/lib/c-ares-1.34.4/src/lib/event/ares_event_poll.c b/source/lib/c-ares-1.34.6/src/lib/event/ares_event_poll.c similarity index 98% rename from source/lib/c-ares-1.34.4/src/lib/event/ares_event_poll.c rename to source/lib/c-ares-1.34.6/src/lib/event/ares_event_poll.c index c6ab4b62..28e3c096 100644 --- a/source/lib/c-ares-1.34.4/src/lib/event/ares_event_poll.c +++ b/source/lib/c-ares-1.34.6/src/lib/event/ares_event_poll.c @@ -25,12 +25,13 @@ */ #include "ares_private.h" #include "ares_event.h" + +#if defined(HAVE_POLL) && defined(CARES_THREADS) + #ifdef HAVE_POLL_H # include #endif -#if defined(HAVE_POLL) - static ares_bool_t ares_evsys_poll_init(ares_event_thread_t *e) { e->ev_signal = ares_pipeevent_create(e); diff --git a/source/lib/c-ares-1.34.4/src/lib/event/ares_event_select.c b/source/lib/c-ares-1.34.6/src/lib/event/ares_event_select.c similarity index 98% rename from source/lib/c-ares-1.34.4/src/lib/event/ares_event_select.c rename to source/lib/c-ares-1.34.6/src/lib/event/ares_event_select.c index 4d7c085d..df758b5a 100644 --- a/source/lib/c-ares-1.34.4/src/lib/event/ares_event_select.c +++ b/source/lib/c-ares-1.34.6/src/lib/event/ares_event_select.c @@ -31,13 +31,14 @@ #include "ares_private.h" #include "ares_event.h" -#ifdef HAVE_SYS_SELECT_H -# include -#endif /* All systems have select(), but not all have a way to wake, so we require * pipe() to wake the select() */ -#if defined(HAVE_PIPE) +#if defined(HAVE_PIPE) && defined(CARES_THREADS) + +#ifdef HAVE_SYS_SELECT_H +# include +#endif static ares_bool_t ares_evsys_select_init(ares_event_thread_t *e) { diff --git a/source/lib/c-ares-1.34.4/src/lib/event/ares_event_thread.c b/source/lib/c-ares-1.34.6/src/lib/event/ares_event_thread.c similarity index 97% rename from source/lib/c-ares-1.34.4/src/lib/event/ares_event_thread.c rename to source/lib/c-ares-1.34.6/src/lib/event/ares_event_thread.c index d59b7880..35e86377 100644 --- a/source/lib/c-ares-1.34.4/src/lib/event/ares_event_thread.c +++ b/source/lib/c-ares-1.34.6/src/lib/event/ares_event_thread.c @@ -26,6 +26,7 @@ #include "ares_private.h" #include "ares_event.h" +#ifdef CARES_THREADS static void ares_event_destroy_cb(void *arg) { ares_event_t *event = arg; @@ -234,6 +235,12 @@ static void notifywrite_cb(void *data) ares_event_thread_wake(e); } +static void notifyenqueue_cb(void *data) +{ + ares_event_thread_t *e = data; + ares_event_thread_wake(e); +} + static void ares_event_process_updates(ares_event_thread_t *e) { ares_llist_node_t *node; @@ -414,6 +421,7 @@ void ares_event_thread_destroy(ares_channel_t *channel) channel->sock_state_cb = NULL; channel->notify_pending_write_cb = NULL; channel->notify_pending_write_cb_data = NULL; + ares_set_query_enqueue_cb(channel, NULL, NULL); } static const ares_event_sys_t *ares_event_fetch_sys(ares_evsys_t evsys) @@ -520,6 +528,7 @@ ares_status_t ares_event_thread_init(ares_channel_t *channel) channel->sock_state_cb_data = e; channel->notify_pending_write_cb = notifywrite_cb; channel->notify_pending_write_cb_data = e; + ares_set_query_enqueue_cb(channel, notifyenqueue_cb, e); if (!e->ev_sys->init(e)) { /* LCOV_EXCL_START: UntestablePath */ @@ -549,3 +558,18 @@ ares_status_t ares_event_thread_init(ares_channel_t *channel) return ARES_SUCCESS; } + +#else + +ares_status_t ares_event_thread_init(ares_channel_t *channel) +{ + (void)channel; + return ARES_ENOTIMP; +} + +void ares_event_thread_destroy(ares_channel_t *channel) +{ + (void)channel; +} + +#endif diff --git a/source/lib/c-ares-1.34.4/src/lib/event/ares_event_wake_pipe.c b/source/lib/c-ares-1.34.6/src/lib/event/ares_event_wake_pipe.c similarity index 96% rename from source/lib/c-ares-1.34.4/src/lib/event/ares_event_wake_pipe.c rename to source/lib/c-ares-1.34.6/src/lib/event/ares_event_wake_pipe.c index d3b166a3..cd1534bb 100644 --- a/source/lib/c-ares-1.34.4/src/lib/event/ares_event_wake_pipe.c +++ b/source/lib/c-ares-1.34.6/src/lib/event/ares_event_wake_pipe.c @@ -25,14 +25,16 @@ */ #include "ares_private.h" #include "ares_event.h" -#ifdef HAVE_UNISTD_H -# include -#endif -#ifdef HAVE_FCNTL_H -# include -#endif -#ifdef HAVE_PIPE +#if defined(HAVE_PIPE) && defined(CARES_THREADS) + +# ifdef HAVE_UNISTD_H +# include +# endif +# ifdef HAVE_FCNTL_H +# include +# endif + typedef struct { int filedes[2]; } ares_pipeevent_t; diff --git a/source/lib/c-ares-1.34.4/src/lib/event/ares_event_win32.c b/source/lib/c-ares-1.34.6/src/lib/event/ares_event_win32.c similarity index 99% rename from source/lib/c-ares-1.34.4/src/lib/event/ares_event_win32.c rename to source/lib/c-ares-1.34.6/src/lib/event/ares_event_win32.c index 1531b6d8..d7d1d657 100644 --- a/source/lib/c-ares-1.34.4/src/lib/event/ares_event_win32.c +++ b/source/lib/c-ares-1.34.6/src/lib/event/ares_event_win32.c @@ -37,12 +37,14 @@ #include "ares_private.h" #include "ares_event.h" #include "ares_event_win32.h" + + +#if defined(USE_WINSOCK) && defined(CARES_THREADS) + #ifdef HAVE_LIMITS_H # include #endif -#if defined(USE_WINSOCK) - /* IMPLEMENTATION NOTES * ==================== * @@ -667,7 +669,7 @@ static ares_bool_t ares_evsys_win32_afd_cancel(ares_evsys_win32_eventdata_t *ed) /* NtCancelIoFileEx() may return STATUS_NOT_FOUND if the operation completed * just before calling NtCancelIoFileEx(), but we have not yet received the - * notifiction (but it should be queued for the next IOCP event). */ + * notification (but it should be queued for the next IOCP event). */ if (status == STATUS_SUCCESS || status == STATUS_NOT_FOUND) { return ARES_TRUE; } diff --git a/source/lib/c-ares-1.34.4/src/lib/event/ares_event_win32.h b/source/lib/c-ares-1.34.6/src/lib/event/ares_event_win32.h similarity index 100% rename from source/lib/c-ares-1.34.4/src/lib/event/ares_event_win32.h rename to source/lib/c-ares-1.34.6/src/lib/event/ares_event_win32.h diff --git a/source/lib/c-ares-1.34.4/src/lib/include/ares_array.h b/source/lib/c-ares-1.34.6/src/lib/include/ares_array.h similarity index 100% rename from source/lib/c-ares-1.34.4/src/lib/include/ares_array.h rename to source/lib/c-ares-1.34.6/src/lib/include/ares_array.h diff --git a/source/lib/c-ares-1.34.4/src/lib/include/ares_buf.h b/source/lib/c-ares-1.34.6/src/lib/include/ares_buf.h similarity index 100% rename from source/lib/c-ares-1.34.4/src/lib/include/ares_buf.h rename to source/lib/c-ares-1.34.6/src/lib/include/ares_buf.h diff --git a/source/lib/c-ares-1.34.4/src/lib/include/ares_htable_asvp.h b/source/lib/c-ares-1.34.6/src/lib/include/ares_htable_asvp.h similarity index 100% rename from source/lib/c-ares-1.34.4/src/lib/include/ares_htable_asvp.h rename to source/lib/c-ares-1.34.6/src/lib/include/ares_htable_asvp.h diff --git a/source/lib/c-ares-1.34.4/src/lib/include/ares_htable_dict.h b/source/lib/c-ares-1.34.6/src/lib/include/ares_htable_dict.h similarity index 100% rename from source/lib/c-ares-1.34.4/src/lib/include/ares_htable_dict.h rename to source/lib/c-ares-1.34.6/src/lib/include/ares_htable_dict.h diff --git a/source/lib/c-ares-1.34.4/src/lib/include/ares_htable_strvp.h b/source/lib/c-ares-1.34.6/src/lib/include/ares_htable_strvp.h similarity index 100% rename from source/lib/c-ares-1.34.4/src/lib/include/ares_htable_strvp.h rename to source/lib/c-ares-1.34.6/src/lib/include/ares_htable_strvp.h diff --git a/source/lib/c-ares-1.34.4/src/lib/include/ares_htable_szvp.h b/source/lib/c-ares-1.34.6/src/lib/include/ares_htable_szvp.h similarity index 100% rename from source/lib/c-ares-1.34.4/src/lib/include/ares_htable_szvp.h rename to source/lib/c-ares-1.34.6/src/lib/include/ares_htable_szvp.h diff --git a/source/lib/c-ares-1.34.4/src/lib/include/ares_htable_vpstr.h b/source/lib/c-ares-1.34.6/src/lib/include/ares_htable_vpstr.h similarity index 100% rename from source/lib/c-ares-1.34.4/src/lib/include/ares_htable_vpstr.h rename to source/lib/c-ares-1.34.6/src/lib/include/ares_htable_vpstr.h diff --git a/source/lib/c-ares-1.34.4/src/lib/include/ares_htable_vpvp.h b/source/lib/c-ares-1.34.6/src/lib/include/ares_htable_vpvp.h similarity index 100% rename from source/lib/c-ares-1.34.4/src/lib/include/ares_htable_vpvp.h rename to source/lib/c-ares-1.34.6/src/lib/include/ares_htable_vpvp.h diff --git a/source/lib/c-ares-1.34.4/src/lib/include/ares_llist.h b/source/lib/c-ares-1.34.6/src/lib/include/ares_llist.h similarity index 100% rename from source/lib/c-ares-1.34.4/src/lib/include/ares_llist.h rename to source/lib/c-ares-1.34.6/src/lib/include/ares_llist.h diff --git a/source/lib/c-ares-1.34.4/src/lib/include/ares_mem.h b/source/lib/c-ares-1.34.6/src/lib/include/ares_mem.h similarity index 100% rename from source/lib/c-ares-1.34.4/src/lib/include/ares_mem.h rename to source/lib/c-ares-1.34.6/src/lib/include/ares_mem.h diff --git a/source/lib/c-ares-1.34.4/src/lib/include/ares_str.h b/source/lib/c-ares-1.34.6/src/lib/include/ares_str.h similarity index 100% rename from source/lib/c-ares-1.34.4/src/lib/include/ares_str.h rename to source/lib/c-ares-1.34.6/src/lib/include/ares_str.h diff --git a/source/lib/c-ares-1.34.4/src/lib/inet_net_pton.c b/source/lib/c-ares-1.34.6/src/lib/inet_net_pton.c similarity index 100% rename from source/lib/c-ares-1.34.4/src/lib/inet_net_pton.c rename to source/lib/c-ares-1.34.6/src/lib/inet_net_pton.c diff --git a/source/lib/c-ares-1.34.4/src/lib/inet_ntop.c b/source/lib/c-ares-1.34.6/src/lib/inet_ntop.c similarity index 100% rename from source/lib/c-ares-1.34.4/src/lib/inet_ntop.c rename to source/lib/c-ares-1.34.6/src/lib/inet_ntop.c diff --git a/source/lib/c-ares-1.34.4/src/lib/legacy/ares_create_query.c b/source/lib/c-ares-1.34.6/src/lib/legacy/ares_create_query.c similarity index 100% rename from source/lib/c-ares-1.34.4/src/lib/legacy/ares_create_query.c rename to source/lib/c-ares-1.34.6/src/lib/legacy/ares_create_query.c diff --git a/source/lib/c-ares-1.34.4/src/lib/legacy/ares_expand_name.c b/source/lib/c-ares-1.34.6/src/lib/legacy/ares_expand_name.c similarity index 100% rename from source/lib/c-ares-1.34.4/src/lib/legacy/ares_expand_name.c rename to source/lib/c-ares-1.34.6/src/lib/legacy/ares_expand_name.c diff --git a/source/lib/c-ares-1.34.4/src/lib/legacy/ares_expand_string.c b/source/lib/c-ares-1.34.6/src/lib/legacy/ares_expand_string.c similarity index 100% rename from source/lib/c-ares-1.34.4/src/lib/legacy/ares_expand_string.c rename to source/lib/c-ares-1.34.6/src/lib/legacy/ares_expand_string.c diff --git a/source/lib/c-ares-1.34.4/src/lib/legacy/ares_fds.c b/source/lib/c-ares-1.34.6/src/lib/legacy/ares_fds.c similarity index 100% rename from source/lib/c-ares-1.34.4/src/lib/legacy/ares_fds.c rename to source/lib/c-ares-1.34.6/src/lib/legacy/ares_fds.c diff --git a/source/lib/c-ares-1.34.4/src/lib/legacy/ares_getsock.c b/source/lib/c-ares-1.34.6/src/lib/legacy/ares_getsock.c similarity index 100% rename from source/lib/c-ares-1.34.4/src/lib/legacy/ares_getsock.c rename to source/lib/c-ares-1.34.6/src/lib/legacy/ares_getsock.c diff --git a/source/lib/c-ares-1.34.4/src/lib/legacy/ares_parse_a_reply.c b/source/lib/c-ares-1.34.6/src/lib/legacy/ares_parse_a_reply.c similarity index 99% rename from source/lib/c-ares-1.34.4/src/lib/legacy/ares_parse_a_reply.c rename to source/lib/c-ares-1.34.6/src/lib/legacy/ares_parse_a_reply.c index 870aaccf..9fd4a07a 100644 --- a/source/lib/c-ares-1.34.4/src/lib/legacy/ares_parse_a_reply.c +++ b/source/lib/c-ares-1.34.6/src/lib/legacy/ares_parse_a_reply.c @@ -77,6 +77,7 @@ int ares_parse_a_reply(const unsigned char *abuf, int alen, } if (host != NULL) { + *host = NULL; status = ares_addrinfo2hostent(&ai, AF_INET, host); if (status != ARES_SUCCESS && status != ARES_ENODATA) { goto fail; /* LCOV_EXCL_LINE: DefensiveCoding */ diff --git a/source/lib/c-ares-1.34.4/src/lib/legacy/ares_parse_aaaa_reply.c b/source/lib/c-ares-1.34.6/src/lib/legacy/ares_parse_aaaa_reply.c similarity index 99% rename from source/lib/c-ares-1.34.4/src/lib/legacy/ares_parse_aaaa_reply.c rename to source/lib/c-ares-1.34.6/src/lib/legacy/ares_parse_aaaa_reply.c index 278642f0..4c177ec9 100644 --- a/source/lib/c-ares-1.34.4/src/lib/legacy/ares_parse_aaaa_reply.c +++ b/source/lib/c-ares-1.34.6/src/lib/legacy/ares_parse_aaaa_reply.c @@ -80,6 +80,7 @@ int ares_parse_aaaa_reply(const unsigned char *abuf, int alen, } if (host != NULL) { + *host = NULL; status = ares_addrinfo2hostent(&ai, AF_INET6, host); if (status != ARES_SUCCESS && status != ARES_ENODATA) { goto fail; /* LCOV_EXCL_LINE: DefensiveCoding */ diff --git a/source/lib/c-ares-1.34.4/src/lib/legacy/ares_parse_caa_reply.c b/source/lib/c-ares-1.34.6/src/lib/legacy/ares_parse_caa_reply.c similarity index 100% rename from source/lib/c-ares-1.34.4/src/lib/legacy/ares_parse_caa_reply.c rename to source/lib/c-ares-1.34.6/src/lib/legacy/ares_parse_caa_reply.c diff --git a/source/lib/c-ares-1.34.4/src/lib/legacy/ares_parse_mx_reply.c b/source/lib/c-ares-1.34.6/src/lib/legacy/ares_parse_mx_reply.c similarity index 100% rename from source/lib/c-ares-1.34.4/src/lib/legacy/ares_parse_mx_reply.c rename to source/lib/c-ares-1.34.6/src/lib/legacy/ares_parse_mx_reply.c diff --git a/source/lib/c-ares-1.34.4/src/lib/legacy/ares_parse_naptr_reply.c b/source/lib/c-ares-1.34.6/src/lib/legacy/ares_parse_naptr_reply.c similarity index 100% rename from source/lib/c-ares-1.34.4/src/lib/legacy/ares_parse_naptr_reply.c rename to source/lib/c-ares-1.34.6/src/lib/legacy/ares_parse_naptr_reply.c diff --git a/source/lib/c-ares-1.34.4/src/lib/legacy/ares_parse_ns_reply.c b/source/lib/c-ares-1.34.6/src/lib/legacy/ares_parse_ns_reply.c similarity index 100% rename from source/lib/c-ares-1.34.4/src/lib/legacy/ares_parse_ns_reply.c rename to source/lib/c-ares-1.34.6/src/lib/legacy/ares_parse_ns_reply.c diff --git a/source/lib/c-ares-1.34.4/src/lib/legacy/ares_parse_ptr_reply.c b/source/lib/c-ares-1.34.6/src/lib/legacy/ares_parse_ptr_reply.c similarity index 100% rename from source/lib/c-ares-1.34.4/src/lib/legacy/ares_parse_ptr_reply.c rename to source/lib/c-ares-1.34.6/src/lib/legacy/ares_parse_ptr_reply.c diff --git a/source/lib/c-ares-1.34.4/src/lib/legacy/ares_parse_soa_reply.c b/source/lib/c-ares-1.34.6/src/lib/legacy/ares_parse_soa_reply.c similarity index 100% rename from source/lib/c-ares-1.34.4/src/lib/legacy/ares_parse_soa_reply.c rename to source/lib/c-ares-1.34.6/src/lib/legacy/ares_parse_soa_reply.c diff --git a/source/lib/c-ares-1.34.4/src/lib/legacy/ares_parse_srv_reply.c b/source/lib/c-ares-1.34.6/src/lib/legacy/ares_parse_srv_reply.c similarity index 100% rename from source/lib/c-ares-1.34.4/src/lib/legacy/ares_parse_srv_reply.c rename to source/lib/c-ares-1.34.6/src/lib/legacy/ares_parse_srv_reply.c diff --git a/source/lib/c-ares-1.34.4/src/lib/legacy/ares_parse_txt_reply.c b/source/lib/c-ares-1.34.6/src/lib/legacy/ares_parse_txt_reply.c similarity index 100% rename from source/lib/c-ares-1.34.4/src/lib/legacy/ares_parse_txt_reply.c rename to source/lib/c-ares-1.34.6/src/lib/legacy/ares_parse_txt_reply.c diff --git a/source/lib/c-ares-1.34.4/src/lib/legacy/ares_parse_uri_reply.c b/source/lib/c-ares-1.34.6/src/lib/legacy/ares_parse_uri_reply.c similarity index 100% rename from source/lib/c-ares-1.34.4/src/lib/legacy/ares_parse_uri_reply.c rename to source/lib/c-ares-1.34.6/src/lib/legacy/ares_parse_uri_reply.c diff --git a/source/lib/c-ares-1.34.4/src/lib/record/ares_dns_mapping.c b/source/lib/c-ares-1.34.6/src/lib/record/ares_dns_mapping.c similarity index 100% rename from source/lib/c-ares-1.34.4/src/lib/record/ares_dns_mapping.c rename to source/lib/c-ares-1.34.6/src/lib/record/ares_dns_mapping.c diff --git a/source/lib/c-ares-1.34.4/src/lib/record/ares_dns_multistring.c b/source/lib/c-ares-1.34.6/src/lib/record/ares_dns_multistring.c similarity index 100% rename from source/lib/c-ares-1.34.4/src/lib/record/ares_dns_multistring.c rename to source/lib/c-ares-1.34.6/src/lib/record/ares_dns_multistring.c diff --git a/source/lib/c-ares-1.34.4/src/lib/record/ares_dns_multistring.h b/source/lib/c-ares-1.34.6/src/lib/record/ares_dns_multistring.h similarity index 100% rename from source/lib/c-ares-1.34.4/src/lib/record/ares_dns_multistring.h rename to source/lib/c-ares-1.34.6/src/lib/record/ares_dns_multistring.h diff --git a/source/lib/c-ares-1.34.4/src/lib/record/ares_dns_name.c b/source/lib/c-ares-1.34.6/src/lib/record/ares_dns_name.c similarity index 100% rename from source/lib/c-ares-1.34.4/src/lib/record/ares_dns_name.c rename to source/lib/c-ares-1.34.6/src/lib/record/ares_dns_name.c diff --git a/source/lib/c-ares-1.34.4/src/lib/record/ares_dns_parse.c b/source/lib/c-ares-1.34.6/src/lib/record/ares_dns_parse.c similarity index 100% rename from source/lib/c-ares-1.34.4/src/lib/record/ares_dns_parse.c rename to source/lib/c-ares-1.34.6/src/lib/record/ares_dns_parse.c diff --git a/source/lib/c-ares-1.34.4/src/lib/record/ares_dns_private.h b/source/lib/c-ares-1.34.6/src/lib/record/ares_dns_private.h similarity index 100% rename from source/lib/c-ares-1.34.4/src/lib/record/ares_dns_private.h rename to source/lib/c-ares-1.34.6/src/lib/record/ares_dns_private.h diff --git a/source/lib/c-ares-1.34.4/src/lib/record/ares_dns_record.c b/source/lib/c-ares-1.34.6/src/lib/record/ares_dns_record.c similarity index 100% rename from source/lib/c-ares-1.34.4/src/lib/record/ares_dns_record.c rename to source/lib/c-ares-1.34.6/src/lib/record/ares_dns_record.c diff --git a/source/lib/c-ares-1.34.4/src/lib/record/ares_dns_write.c b/source/lib/c-ares-1.34.6/src/lib/record/ares_dns_write.c similarity index 100% rename from source/lib/c-ares-1.34.4/src/lib/record/ares_dns_write.c rename to source/lib/c-ares-1.34.6/src/lib/record/ares_dns_write.c diff --git a/source/lib/c-ares-1.34.4/src/lib/str/ares_buf.c b/source/lib/c-ares-1.34.6/src/lib/str/ares_buf.c similarity index 100% rename from source/lib/c-ares-1.34.4/src/lib/str/ares_buf.c rename to source/lib/c-ares-1.34.6/src/lib/str/ares_buf.c diff --git a/source/lib/c-ares-1.34.4/src/lib/str/ares_str.c b/source/lib/c-ares-1.34.6/src/lib/str/ares_str.c similarity index 100% rename from source/lib/c-ares-1.34.4/src/lib/str/ares_str.c rename to source/lib/c-ares-1.34.6/src/lib/str/ares_str.c diff --git a/source/lib/c-ares-1.34.4/src/lib/str/ares_strsplit.c b/source/lib/c-ares-1.34.6/src/lib/str/ares_strsplit.c similarity index 100% rename from source/lib/c-ares-1.34.4/src/lib/str/ares_strsplit.c rename to source/lib/c-ares-1.34.6/src/lib/str/ares_strsplit.c diff --git a/source/lib/c-ares-1.34.4/src/lib/str/ares_strsplit.h b/source/lib/c-ares-1.34.6/src/lib/str/ares_strsplit.h similarity index 100% rename from source/lib/c-ares-1.34.4/src/lib/str/ares_strsplit.h rename to source/lib/c-ares-1.34.6/src/lib/str/ares_strsplit.h diff --git a/source/lib/c-ares-1.34.4/src/lib/thirdparty/apple/dnsinfo.h b/source/lib/c-ares-1.34.6/src/lib/thirdparty/apple/dnsinfo.h similarity index 100% rename from source/lib/c-ares-1.34.4/src/lib/thirdparty/apple/dnsinfo.h rename to source/lib/c-ares-1.34.6/src/lib/thirdparty/apple/dnsinfo.h diff --git a/source/lib/c-ares-1.34.4/src/lib/util/ares_iface_ips.c b/source/lib/c-ares-1.34.6/src/lib/util/ares_iface_ips.c similarity index 98% rename from source/lib/c-ares-1.34.4/src/lib/util/ares_iface_ips.c rename to source/lib/c-ares-1.34.6/src/lib/util/ares_iface_ips.c index 46cb291e..c5f507f8 100644 --- a/source/lib/c-ares-1.34.4/src/lib/util/ares_iface_ips.c +++ b/source/lib/c-ares-1.34.6/src/lib/util/ares_iface_ips.c @@ -431,8 +431,14 @@ static ares_status_t ares_iface_ips_enumerate(ares_iface_ips_t *ips, } status = ares_iface_ips_add(ips, addrflag, ifname, &addr, +#if _WIN32_WINNT >= 0x0600 ipaddr->OnLinkPrefixLength /* netmask */, - address->Ipv6IfIndex /* ll_scope */); +#else + ipaddr->Address.lpSockaddr->sa_family + == AF_INET?32:128, +#endif + address->Ipv6IfIndex /* ll_scope */ + ); if (status != ARES_SUCCESS) { goto done; diff --git a/source/lib/c-ares-1.34.4/src/lib/util/ares_iface_ips.h b/source/lib/c-ares-1.34.6/src/lib/util/ares_iface_ips.h similarity index 100% rename from source/lib/c-ares-1.34.4/src/lib/util/ares_iface_ips.h rename to source/lib/c-ares-1.34.6/src/lib/util/ares_iface_ips.h diff --git a/source/lib/c-ares-1.34.4/src/lib/util/ares_math.c b/source/lib/c-ares-1.34.6/src/lib/util/ares_math.c similarity index 100% rename from source/lib/c-ares-1.34.4/src/lib/util/ares_math.c rename to source/lib/c-ares-1.34.6/src/lib/util/ares_math.c diff --git a/source/lib/c-ares-1.34.4/src/lib/util/ares_math.h b/source/lib/c-ares-1.34.6/src/lib/util/ares_math.h similarity index 78% rename from source/lib/c-ares-1.34.4/src/lib/util/ares_math.h rename to source/lib/c-ares-1.34.6/src/lib/util/ares_math.h index 52fa1fac..3b60b00b 100644 --- a/source/lib/c-ares-1.34.4/src/lib/util/ares_math.h +++ b/source/lib/c-ares-1.34.6/src/lib/util/ares_math.h @@ -26,13 +26,21 @@ #ifndef __ARES_MATH_H #define __ARES_MATH_H -#ifdef _MSC_VER -typedef __int64 ares_int64_t; -typedef unsigned __int64 ares_uint64_t; -#else -typedef long long ares_int64_t; -typedef unsigned long long ares_uint64_t; -#endif +#include "ares_setup.h" + +#ifdef HAVE_STDINT_H +# include + typedef int64_t ares_int64_t; + typedef uint64_t ares_uint64_t; +#else /* HAVE_STDINT_H */ +# ifdef _MSC_VER + typedef __int64 ares_int64_t; + typedef unsigned __int64 ares_uint64_t; +# else + typedef long long ares_int64_t; + typedef unsigned long long ares_uint64_t; +# endif /* _MSC_VER */ +#endif /* HAVE_STDINT_H */ ares_bool_t ares_is_64bit(void); size_t ares_round_up_pow2(size_t n); diff --git a/source/lib/c-ares-1.34.4/src/lib/util/ares_rand.c b/source/lib/c-ares-1.34.6/src/lib/util/ares_rand.c similarity index 98% rename from source/lib/c-ares-1.34.4/src/lib/util/ares_rand.c rename to source/lib/c-ares-1.34.6/src/lib/util/ares_rand.c index 40899995..367dfa21 100644 --- a/source/lib/c-ares-1.34.4/src/lib/util/ares_rand.c +++ b/source/lib/c-ares-1.34.6/src/lib/util/ares_rand.c @@ -56,7 +56,7 @@ static unsigned int ares_u32_from_ptr(void *addr) { /* LCOV_EXCL_START: FallbackCode */ if (ares_is_64bit()) { - return (unsigned int)((((ares_uint64_t)addr >> 32) & 0xFFFFFFFF) | + return (unsigned int)((((ares_uint64_t)addr >> 32) & 0xFFFFFFFF) ^ ((ares_uint64_t)addr & 0xFFFFFFFF)); } return (unsigned int)((size_t)addr & 0xFFFFFFFF); @@ -94,12 +94,12 @@ static void ares_rc4_generate_key(ares_rand_rc4 *rc4_state, unsigned char *key, len += sizeof(data); ares_tvnow(&tv); - data = (unsigned int)((tv.sec | tv.usec) & 0xFFFFFFFF); + data = (unsigned int)((tv.sec ^ tv.usec) & 0xFFFFFFFF); memcpy(key + len, &data, sizeof(data)); len += sizeof(data); - srand(ares_u32_from_ptr(rc4_state) | ares_u32_from_ptr(&i) | - (unsigned int)((tv.sec | tv.usec) & 0xFFFFFFFF)); + srand(ares_u32_from_ptr(rc4_state) ^ ares_u32_from_ptr(&i) ^ + (unsigned int)((tv.sec ^ tv.usec) & 0xFFFFFFFF)); #endif for (i = len; i < key_len; i++) { diff --git a/source/lib/c-ares-1.34.4/src/lib/util/ares_rand.h b/source/lib/c-ares-1.34.6/src/lib/util/ares_rand.h similarity index 100% rename from source/lib/c-ares-1.34.4/src/lib/util/ares_rand.h rename to source/lib/c-ares-1.34.6/src/lib/util/ares_rand.h diff --git a/source/lib/c-ares-1.34.4/src/lib/util/ares_threads.c b/source/lib/c-ares-1.34.6/src/lib/util/ares_threads.c similarity index 100% rename from source/lib/c-ares-1.34.4/src/lib/util/ares_threads.c rename to source/lib/c-ares-1.34.6/src/lib/util/ares_threads.c diff --git a/source/lib/c-ares-1.34.4/src/lib/util/ares_threads.h b/source/lib/c-ares-1.34.6/src/lib/util/ares_threads.h similarity index 100% rename from source/lib/c-ares-1.34.4/src/lib/util/ares_threads.h rename to source/lib/c-ares-1.34.6/src/lib/util/ares_threads.h diff --git a/source/lib/c-ares-1.34.4/src/lib/util/ares_time.h b/source/lib/c-ares-1.34.6/src/lib/util/ares_time.h similarity index 100% rename from source/lib/c-ares-1.34.4/src/lib/util/ares_time.h rename to source/lib/c-ares-1.34.6/src/lib/util/ares_time.h diff --git a/source/lib/c-ares-1.34.4/src/lib/util/ares_timeval.c b/source/lib/c-ares-1.34.6/src/lib/util/ares_timeval.c similarity index 100% rename from source/lib/c-ares-1.34.4/src/lib/util/ares_timeval.c rename to source/lib/c-ares-1.34.6/src/lib/util/ares_timeval.c diff --git a/source/lib/c-ares-1.34.4/src/lib/util/ares_uri.c b/source/lib/c-ares-1.34.6/src/lib/util/ares_uri.c similarity index 99% rename from source/lib/c-ares-1.34.4/src/lib/util/ares_uri.c rename to source/lib/c-ares-1.34.6/src/lib/util/ares_uri.c index d656f834..04bad007 100644 --- a/source/lib/c-ares-1.34.4/src/lib/util/ares_uri.c +++ b/source/lib/c-ares-1.34.6/src/lib/util/ares_uri.c @@ -533,7 +533,7 @@ static char *ares_uri_path_normalize(const char *path) status = ares_buf_split_str_array(inpath, (const unsigned char *)"/", 1, ARES_BUF_SPLIT_TRIM, 0, &arr); if (status != ARES_SUCCESS) { - return NULL; + goto done; } for (i = 0; i < (ares_ssize_t)ares_array_len(arr); i++) { diff --git a/source/lib/c-ares-1.34.4/src/lib/util/ares_uri.h b/source/lib/c-ares-1.34.6/src/lib/util/ares_uri.h similarity index 99% rename from source/lib/c-ares-1.34.4/src/lib/util/ares_uri.h rename to source/lib/c-ares-1.34.6/src/lib/util/ares_uri.h index 6a703cba..2d8138fd 100644 --- a/source/lib/c-ares-1.34.4/src/lib/util/ares_uri.h +++ b/source/lib/c-ares-1.34.6/src/lib/util/ares_uri.h @@ -175,7 +175,7 @@ ares_status_t ares_uri_set_query_key(ares_uri_t *uri, const char *key, */ ares_status_t ares_uri_del_query_key(ares_uri_t *uri, const char *key); -/*! Retrieve the value associted with a query key. Keys are case-insensitive. +/*! Retrieve the value associated with a query key. Keys are case-insensitive. * * \param[in] uri Initialized URI object * \param[in] key Key to retrieve. diff --git a/source/lib/c-ares-1.34.4/src/lib/windows_port.c b/source/lib/c-ares-1.34.6/src/lib/windows_port.c similarity index 100% rename from source/lib/c-ares-1.34.4/src/lib/windows_port.c rename to source/lib/c-ares-1.34.6/src/lib/windows_port.c diff --git a/source/lib/c-ares-1.34.4/src/tools/CMakeLists.txt b/source/lib/c-ares-1.34.6/src/tools/CMakeLists.txt similarity index 100% rename from source/lib/c-ares-1.34.4/src/tools/CMakeLists.txt rename to source/lib/c-ares-1.34.6/src/tools/CMakeLists.txt diff --git a/source/lib/c-ares-1.34.4/src/tools/Makefile.am b/source/lib/c-ares-1.34.6/src/tools/Makefile.am similarity index 100% rename from source/lib/c-ares-1.34.4/src/tools/Makefile.am rename to source/lib/c-ares-1.34.6/src/tools/Makefile.am diff --git a/source/lib/c-ares-1.34.4/src/tools/Makefile.in b/source/lib/c-ares-1.34.6/src/tools/Makefile.in similarity index 99% rename from source/lib/c-ares-1.34.4/src/tools/Makefile.in rename to source/lib/c-ares-1.34.6/src/tools/Makefile.in index 19e99a25..b3f8333f 100644 --- a/source/lib/c-ares-1.34.4/src/tools/Makefile.in +++ b/source/lib/c-ares-1.34.6/src/tools/Makefile.in @@ -106,7 +106,6 @@ am__aclocal_m4_deps = $(top_srcdir)/m4/ares_check_user_namespace.m4 \ $(top_srcdir)/m4/ax_code_coverage.m4 \ $(top_srcdir)/m4/ax_compiler_vendor.m4 \ $(top_srcdir)/m4/ax_cxx_compile_stdcxx.m4 \ - $(top_srcdir)/m4/ax_cxx_compile_stdcxx_14.m4 \ $(top_srcdir)/m4/ax_file_escapes.m4 \ $(top_srcdir)/m4/ax_pthread.m4 \ $(top_srcdir)/m4/ax_require_defined.m4 \ @@ -264,14 +263,18 @@ EGREP = @EGREP@ ETAGS = @ETAGS@ EXEEXT = @EXEEXT@ FGREP = @FGREP@ +FILECMD = @FILECMD@ GCOV = @GCOV@ GENHTML = @GENHTML@ GMOCK112_CFLAGS = @GMOCK112_CFLAGS@ GMOCK112_LIBS = @GMOCK112_LIBS@ +GMOCK117_CFLAGS = @GMOCK117_CFLAGS@ +GMOCK117_LIBS = @GMOCK117_LIBS@ GMOCK_CFLAGS = @GMOCK_CFLAGS@ GMOCK_LIBS = @GMOCK_LIBS@ GREP = @GREP@ HAVE_CXX14 = @HAVE_CXX14@ +HAVE_CXX17 = @HAVE_CXX17@ INSTALL = @INSTALL@ INSTALL_DATA = @INSTALL_DATA@ INSTALL_PROGRAM = @INSTALL_PROGRAM@ diff --git a/source/lib/c-ares-1.34.4/src/tools/Makefile.inc b/source/lib/c-ares-1.34.6/src/tools/Makefile.inc similarity index 100% rename from source/lib/c-ares-1.34.4/src/tools/Makefile.inc rename to source/lib/c-ares-1.34.6/src/tools/Makefile.inc diff --git a/source/lib/c-ares-1.34.4/src/tools/adig.c b/source/lib/c-ares-1.34.6/src/tools/adig.c similarity index 100% rename from source/lib/c-ares-1.34.4/src/tools/adig.c rename to source/lib/c-ares-1.34.6/src/tools/adig.c diff --git a/source/lib/c-ares-1.34.4/src/tools/ahost.c b/source/lib/c-ares-1.34.6/src/tools/ahost.c similarity index 100% rename from source/lib/c-ares-1.34.4/src/tools/ahost.c rename to source/lib/c-ares-1.34.6/src/tools/ahost.c diff --git a/source/lib/c-ares-1.34.4/src/tools/ares_getopt.c b/source/lib/c-ares-1.34.6/src/tools/ares_getopt.c similarity index 100% rename from source/lib/c-ares-1.34.4/src/tools/ares_getopt.c rename to source/lib/c-ares-1.34.6/src/tools/ares_getopt.c diff --git a/source/lib/c-ares-1.34.4/src/tools/ares_getopt.h b/source/lib/c-ares-1.34.6/src/tools/ares_getopt.h similarity index 100% rename from source/lib/c-ares-1.34.4/src/tools/ares_getopt.h rename to source/lib/c-ares-1.34.6/src/tools/ares_getopt.h diff --git a/source/lib/c-ares-1.34.4/test/CMakeLists.txt b/source/lib/c-ares-1.34.6/test/CMakeLists.txt similarity index 97% rename from source/lib/c-ares-1.34.4/test/CMakeLists.txt rename to source/lib/c-ares-1.34.6/test/CMakeLists.txt index 13f637a8..88725b77 100644 --- a/source/lib/c-ares-1.34.4/test/CMakeLists.txt +++ b/source/lib/c-ares-1.34.6/test/CMakeLists.txt @@ -1,6 +1,8 @@ # Copyright (C) The c-ares project and its contributors # SPDX-License-Identifier: MIT +enable_language(CXX) + # Get rid of: warning C4530: C++ exception handler used, but unwind semantics are not enabled. Specify /EHsc IF (MSVC) SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /EHsc") @@ -50,6 +52,7 @@ IF (CMAKE_VERSION VERSION_LESS "3.23.0") ELSE () target_link_libraries(arestest PRIVATE caresinternal GTest::gmock) ENDIF() +target_compile_features(arestest PRIVATE cxx_std_14) target_compile_definitions(arestest PRIVATE CARES_NO_DEPRECATED) IF (CARES_BUILD_CONTAINER_TESTS) @@ -75,6 +78,7 @@ set_target_properties(aresfuzzname PROPERTIES COMPILE_PDB_NAME aresfuzzname.pdb) add_executable(dnsdump ${DUMPSOURCES}) target_compile_definitions(dnsdump PRIVATE CARES_NO_DEPRECATED) +target_compile_features(dnsdump PRIVATE cxx_std_14) target_link_libraries(dnsdump PRIVATE caresinternal) # Avoid "fatal error C1041: cannot open program database" due to multiple # targets trying to use the same PDB. /FS does NOT resolve this issue. diff --git a/source/lib/c-ares-1.34.4/test/Makefile.am b/source/lib/c-ares-1.34.6/test/Makefile.am similarity index 100% rename from source/lib/c-ares-1.34.4/test/Makefile.am rename to source/lib/c-ares-1.34.6/test/Makefile.am diff --git a/source/lib/c-ares-1.34.4/test/Makefile.in b/source/lib/c-ares-1.34.6/test/Makefile.in similarity index 99% rename from source/lib/c-ares-1.34.4/test/Makefile.in rename to source/lib/c-ares-1.34.6/test/Makefile.in index 773b5189..98f5df0f 100644 --- a/source/lib/c-ares-1.34.4/test/Makefile.in +++ b/source/lib/c-ares-1.34.6/test/Makefile.in @@ -108,7 +108,6 @@ am__aclocal_m4_deps = $(top_srcdir)/m4/ares_check_user_namespace.m4 \ $(top_srcdir)/m4/ax_code_coverage.m4 \ $(top_srcdir)/m4/ax_compiler_vendor.m4 \ $(top_srcdir)/m4/ax_cxx_compile_stdcxx.m4 \ - $(top_srcdir)/m4/ax_cxx_compile_stdcxx_14.m4 \ $(top_srcdir)/m4/ax_file_escapes.m4 \ $(top_srcdir)/m4/ax_pthread.m4 \ $(top_srcdir)/m4/ax_require_defined.m4 \ @@ -547,14 +546,18 @@ EGREP = @EGREP@ ETAGS = @ETAGS@ EXEEXT = @EXEEXT@ FGREP = @FGREP@ +FILECMD = @FILECMD@ GCOV = @GCOV@ GENHTML = @GENHTML@ GMOCK112_CFLAGS = @GMOCK112_CFLAGS@ GMOCK112_LIBS = @GMOCK112_LIBS@ +GMOCK117_CFLAGS = @GMOCK117_CFLAGS@ +GMOCK117_LIBS = @GMOCK117_LIBS@ GMOCK_CFLAGS = @GMOCK_CFLAGS@ GMOCK_LIBS = @GMOCK_LIBS@ GREP = @GREP@ HAVE_CXX14 = @HAVE_CXX14@ +HAVE_CXX17 = @HAVE_CXX17@ INSTALL = @INSTALL@ INSTALL_DATA = @INSTALL_DATA@ INSTALL_PROGRAM = @INSTALL_PROGRAM@ diff --git a/source/lib/c-ares-1.34.4/test/Makefile.inc b/source/lib/c-ares-1.34.6/test/Makefile.inc similarity index 100% rename from source/lib/c-ares-1.34.4/test/Makefile.inc rename to source/lib/c-ares-1.34.6/test/Makefile.inc diff --git a/source/lib/c-ares-1.34.4/test/Makefile.m32 b/source/lib/c-ares-1.34.6/test/Makefile.m32 similarity index 96% rename from source/lib/c-ares-1.34.4/test/Makefile.m32 rename to source/lib/c-ares-1.34.6/test/Makefile.m32 index 5227f98f..26ab0421 100644 --- a/source/lib/c-ares-1.34.4/test/Makefile.m32 +++ b/source/lib/c-ares-1.34.6/test/Makefile.m32 @@ -10,6 +10,8 @@ CXX = g++ CC = gcc LD = g++ +WIN32_WINNT ?= 0x0602 + ifeq "$(GTEST_ROOT)" "" $(error missing GTEST_ROOT) endif @@ -19,7 +21,7 @@ ARES_SRC_DIR = .. # Where to find the built c-ares static library ARES_BLD_DIR = .. ARESLIB = $(ARES_BLD_DIR)/src/lib/libcares.a -CPPFLAGS = -I$(ARES_SRC_DIR)/include -I$(ARES_SRC_DIR)/src/lib -I$(ARES_SRC_DIR)/src/lib/include -I$(GTEST_ROOT)/include -DCARES_STATICLIB -DCARES_NO_DEPRECATED +CPPFLAGS = -I$(ARES_SRC_DIR)/include -I$(ARES_SRC_DIR)/src/lib -I$(ARES_SRC_DIR)/src/lib/include -I$(GTEST_ROOT)/include -DCARES_STATICLIB -DCARES_NO_DEPRECATED -D_WIN32_WINNT=$(WIN32_WINNT) CXXFLAGS = -Wall $(PTHREAD_CFLAGS) -std=gnu++14 LDFLAGS = LDLIBS = -lws2_32 -liphlpapi diff --git a/source/lib/c-ares-1.34.4/test/Makefile.msvc b/source/lib/c-ares-1.34.6/test/Makefile.msvc similarity index 100% rename from source/lib/c-ares-1.34.4/test/Makefile.msvc rename to source/lib/c-ares-1.34.6/test/Makefile.msvc diff --git a/source/lib/c-ares-1.34.4/test/README.md b/source/lib/c-ares-1.34.6/test/README.md similarity index 100% rename from source/lib/c-ares-1.34.4/test/README.md rename to source/lib/c-ares-1.34.6/test/README.md diff --git a/source/lib/c-ares-1.34.4/test/ares-fuzz.c b/source/lib/c-ares-1.34.6/test/ares-fuzz.c similarity index 99% rename from source/lib/c-ares-1.34.4/test/ares-fuzz.c rename to source/lib/c-ares-1.34.6/test/ares-fuzz.c index 8203d48d..239f127d 100644 --- a/source/lib/c-ares-1.34.4/test/ares-fuzz.c +++ b/source/lib/c-ares-1.34.6/test/ares-fuzz.c @@ -33,7 +33,7 @@ #include #include #include -#ifdef WIN32 +#ifdef _WIN32 # include #else # include diff --git a/source/lib/c-ares-1.34.4/test/ares-test-ai.h b/source/lib/c-ares-1.34.6/test/ares-test-ai.h similarity index 100% rename from source/lib/c-ares-1.34.4/test/ares-test-ai.h rename to source/lib/c-ares-1.34.6/test/ares-test-ai.h diff --git a/source/lib/c-ares-1.34.4/test/ares-test-fuzz-name.c b/source/lib/c-ares-1.34.6/test/ares-test-fuzz-name.c similarity index 100% rename from source/lib/c-ares-1.34.4/test/ares-test-fuzz-name.c rename to source/lib/c-ares-1.34.6/test/ares-test-fuzz-name.c diff --git a/source/lib/c-ares-1.34.4/test/ares-test-fuzz.c b/source/lib/c-ares-1.34.6/test/ares-test-fuzz.c similarity index 100% rename from source/lib/c-ares-1.34.4/test/ares-test-fuzz.c rename to source/lib/c-ares-1.34.6/test/ares-test-fuzz.c diff --git a/source/lib/c-ares-1.34.4/test/ares-test-init.cc b/source/lib/c-ares-1.34.6/test/ares-test-init.cc similarity index 100% rename from source/lib/c-ares-1.34.4/test/ares-test-init.cc rename to source/lib/c-ares-1.34.6/test/ares-test-init.cc diff --git a/source/lib/c-ares-1.34.4/test/ares-test-internal.cc b/source/lib/c-ares-1.34.6/test/ares-test-internal.cc similarity index 99% rename from source/lib/c-ares-1.34.4/test/ares-test-internal.cc rename to source/lib/c-ares-1.34.6/test/ares-test-internal.cc index 3e3760bf..55f68326 100644 --- a/source/lib/c-ares-1.34.4/test/ares-test-internal.cc +++ b/source/lib/c-ares-1.34.6/test/ares-test-internal.cc @@ -299,6 +299,7 @@ TEST_F(LibraryTest, SlistMisuse) { EXPECT_EQ(NULL, ares_slist_node_claim(NULL)); } +#if !defined(_WIN32) || _WIN32_WINNT >= 0x0600 TEST_F(LibraryTest, IfaceIPs) { ares_status_t status; ares_iface_ips_t *ips = NULL; @@ -354,6 +355,7 @@ TEST_F(LibraryTest, IfaceIPs) { ares_iface_ips_destroy(ips); } +#endif TEST_F(LibraryTest, HtableMisuse) { EXPECT_EQ(NULL, ares_htable_create(NULL, NULL, NULL, NULL)); @@ -592,6 +594,18 @@ TEST_F(FileChannelTest, GetAddrInfoHostsIPV6) { EXPECT_EQ("{ipv6.com addr=[[0000:0000:0000:0000:0000:0000:0000:0001]]}", ss.str()); } +TEST_F(FileChannelTest, GetAddrInfoInvalidService) { + TempFile hostsfile("1.2.3.4 example.com"); + EnvValue with_env("CARES_HOSTS", hostsfile.filename()); + struct ares_addrinfo_hints hints{}; + AddrInfoResult result{}; + hints.ai_family = AF_INET; + hints.ai_flags = ARES_AI_CANONNAME | ARES_AI_ENVHOSTS | ARES_AI_NOSORT; + ares_getaddrinfo(channel_, "example.com", "invalid", &hints, AddrInfoCallback, &result); + Process(); + EXPECT_TRUE(result.done_); + EXPECT_EQ(result.status_, ARES_ESERVICE); +} TEST_F(FileChannelTest, GetAddrInfoAllocFail) { TempFile hostsfile("1.2.3.4 example.com alias1 alias2\n"); diff --git a/source/lib/c-ares-1.34.4/test/ares-test-live.cc b/source/lib/c-ares-1.34.6/test/ares-test-live.cc similarity index 89% rename from source/lib/c-ares-1.34.4/test/ares-test-live.cc rename to source/lib/c-ares-1.34.6/test/ares-test-live.cc index e23dadfe..557c485e 100644 --- a/source/lib/c-ares-1.34.4/test/ares-test-live.cc +++ b/source/lib/c-ares-1.34.6/test/ares-test-live.cc @@ -669,115 +669,7 @@ VIRT_NONVIRT_TEST_F(DefaultChannelTest, LiveGetNameInfoAllocFail) { EXPECT_EQ(ARES_ENOMEM, result.status_); } -VIRT_NONVIRT_TEST_F(DefaultChannelTest, GetSock) { - ares_socket_t socks[3] = {ARES_SOCKET_BAD, ARES_SOCKET_BAD, ARES_SOCKET_BAD}; - int bitmask = ares_getsock(channel_, socks, 3); - EXPECT_EQ(0, bitmask); - bitmask = ares_getsock(channel_, nullptr, 0); - EXPECT_EQ(0, bitmask); - - // Ask again with a pending query. - HostResult result; - ares_gethostbyname(channel_, "www.google.com.", AF_INET, HostCallback, &result); - bitmask = ares_getsock(channel_, socks, 3); - EXPECT_NE(0, bitmask); - - size_t sock_cnt = 0; - for (size_t i=0; i<3; i++) { - if (ARES_GETSOCK_READABLE(bitmask, i) || ARES_GETSOCK_WRITABLE(bitmask, i)) { - EXPECT_NE(ARES_SOCKET_BAD, socks[i]); - if (socks[i] != ARES_SOCKET_BAD) - sock_cnt++; - } - } - EXPECT_NE((size_t)0, sock_cnt); - - bitmask = ares_getsock(channel_, nullptr, 0); - EXPECT_EQ(0, bitmask); - - Process(); -} - -TEST_F(LibraryTest, GetTCPSock) { - ares_channel_t *channel; - struct ares_options opts; - memset(&opts, 0, sizeof(opts)); - opts.tcp_port = 53; - opts.flags = ARES_FLAG_USEVC; - int optmask = ARES_OPT_TCP_PORT | ARES_OPT_FLAGS; - EXPECT_EQ(ARES_SUCCESS, ares_init_options(&channel, &opts, optmask)); - EXPECT_NE(nullptr, channel); - - ares_socket_t socks[3] = {ARES_SOCKET_BAD, ARES_SOCKET_BAD, ARES_SOCKET_BAD}; - int bitmask = ares_getsock(channel, socks, 3); - EXPECT_EQ(0, bitmask); - bitmask = ares_getsock(channel, nullptr, 0); - EXPECT_EQ(0, bitmask); - - // Ask again with a pending query. - HostResult result; - ares_gethostbyname(channel, "www.google.com.", AF_INET, HostCallback, &result); - bitmask = ares_getsock(channel, socks, 3); - EXPECT_NE(0, bitmask); - - size_t sock_cnt = 0; - for (size_t i=0; i<3; i++) { - if (ARES_GETSOCK_READABLE(bitmask, i) || ARES_GETSOCK_WRITABLE(bitmask, i)) { - EXPECT_NE(ARES_SOCKET_BAD, socks[i]); - if (socks[i] != ARES_SOCKET_BAD) - sock_cnt++; - } - } - EXPECT_NE((size_t)0, sock_cnt); - - bitmask = ares_getsock(channel, nullptr, 0); - EXPECT_EQ(0, bitmask); - - ProcessWork(channel, NoExtraFDs, nullptr); - ares_destroy(channel); -} - -TEST_F(DefaultChannelTest, VerifySocketFunctionCallback) { - VirtualizeIO vio(channel_); - - auto my_functions = VirtualizeIO::default_functions; - size_t count = 0; - - my_functions.asocket = [](int af, int type, int protocol, void * p) -> ares_socket_t { - EXPECT_NE(nullptr, p); - (*reinterpret_cast(p))++; - return ::socket(af, type, protocol); - }; - - ares_set_socket_functions(channel_, &my_functions, &count); - - { - count = 0; - HostResult result; - ares_gethostbyname(channel_, "www.google.com.", AF_INET, HostCallback, &result); - Process(); - - EXPECT_TRUE(result.done_); - EXPECT_NE((size_t)0, count); - } - - { - count = 0; - ares_channel_t *copy; - EXPECT_EQ(ARES_SUCCESS, ares_dup(©, channel_)); - - HostResult result; - ares_gethostbyname(copy, "www.google.com.", AF_INET, HostCallback, &result); - - ProcessWork(copy, NoExtraFDs, nullptr); - - EXPECT_TRUE(result.done_); - ares_destroy(copy); - EXPECT_NE((size_t)0, count); - } - -} TEST_F(DefaultChannelTest, LiveSetServers) { struct ares_addr_node server1; diff --git a/source/lib/c-ares-1.34.4/test/ares-test-main.cc b/source/lib/c-ares-1.34.6/test/ares-test-main.cc similarity index 100% rename from source/lib/c-ares-1.34.4/test/ares-test-main.cc rename to source/lib/c-ares-1.34.6/test/ares-test-main.cc diff --git a/source/lib/c-ares-1.34.4/test/ares-test-misc.cc b/source/lib/c-ares-1.34.6/test/ares-test-misc.cc similarity index 100% rename from source/lib/c-ares-1.34.4/test/ares-test-misc.cc rename to source/lib/c-ares-1.34.6/test/ares-test-misc.cc diff --git a/source/lib/c-ares-1.34.4/test/ares-test-mock-ai.cc b/source/lib/c-ares-1.34.6/test/ares-test-mock-ai.cc similarity index 76% rename from source/lib/c-ares-1.34.4/test/ares-test-mock-ai.cc rename to source/lib/c-ares-1.34.6/test/ares-test-mock-ai.cc index a36918cd..ae45c455 100644 --- a/source/lib/c-ares-1.34.4/test/ares-test-mock-ai.cc +++ b/source/lib/c-ares-1.34.6/test/ares-test-mock-ai.cc @@ -713,6 +713,132 @@ TEST_P(MockChannelTestAI, FamilyUnspecified) { EXPECT_THAT(result.ai_, IncludesV6Address("2121:0000:0000:0000:0000:0000:0000:0303")); } + +TEST_P(MockChannelTestAI, TriggerResendThenConnFailSERVFAIL) { + // Set up the server response. The server always returns SERVFAIL. + DNSPacket badrsp4; + badrsp4.set_response().set_aa().set_rcode(SERVFAIL) + .add_question(new DNSQuestion("www.google.com", T_A)); + DNSPacket goodrsp4; + goodrsp4.set_response().set_aa() + .add_question(new DNSQuestion("www.google.com", T_A)) + .add_answer(new DNSARR("www.google.com", 0x0100, {0x01, 0x02, 0x03, 0x04})); + + DNSPacket goodrsp6; + goodrsp6.set_response().set_aa() + .add_question(new DNSQuestion("www.google.com", T_AAAA)) + .add_answer(new DNSAaaaRR("www.google.com", 100, + {0x21, 0x21, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, 0x03})); + + EXPECT_CALL(server_, OnRequest("www.google.com", T_A)) + .WillOnce(SetReplyAndFailSend(&server_, &badrsp4)) + .WillOnce(SetReply(&server_, &goodrsp4)); + + EXPECT_CALL(server_, OnRequest("www.google.com", T_AAAA)) + .WillRepeatedly(SetReply(&server_, &goodrsp6)); + + ares_socket_functions sock_funcs; + memset(&sock_funcs, 0, sizeof(sock_funcs)); + + sock_funcs.asendv = ares_sendv_fail; + + ares_set_socket_functions(channel_, &sock_funcs, NULL); + + AddrInfoResult result; + struct ares_addrinfo_hints hints = {0, 0, 0, 0}; + hints.ai_family = AF_UNSPEC; + hints.ai_flags = ARES_AI_NOSORT; + ares_getaddrinfo(channel_, "www.google.com.", NULL, &hints, + AddrInfoCallback, &result); + + Process(); + EXPECT_TRUE(result.done_); + EXPECT_TRUE(result.done_); + EXPECT_THAT(result.ai_, IncludesNumAddresses(2)); + EXPECT_THAT(result.ai_, IncludesV4Address("1.2.3.4")); + EXPECT_THAT(result.ai_, IncludesV6Address("2121:0000:0000:0000:0000:0000:0000:0303")); +} + +TEST_P(MockUDPChannelTestAI, TriggerResendThenConnFailEDNS) { + // Set up the server response to simulate an EDNS failure + DNSPacket badrsp4; + badrsp4.set_response().set_aa().set_rcode(FORMERR) + .add_question(new DNSQuestion("www.google.com", T_A)); + DNSPacket goodrsp4; + goodrsp4.set_response().set_aa() + .add_question(new DNSQuestion("www.google.com", T_A)) + .add_answer(new DNSARR("www.google.com", 0x0100, {0x01, 0x02, 0x03, 0x04})); + DNSPacket goodrsp6; + goodrsp6.set_response().set_aa() + .add_question(new DNSQuestion("www.google.com", T_AAAA)) + .add_answer(new DNSAaaaRR("www.google.com", 100, + {0x21, 0x21, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, 0x03})); + + EXPECT_CALL(server_, OnRequest("www.google.com", T_A)) + .WillOnce(SetReplyAndFailSend(&server_, &badrsp4)) + .WillOnce(SetReply(&server_, &goodrsp4)); + + EXPECT_CALL(server_, OnRequest("www.google.com", T_AAAA)) + .WillRepeatedly(SetReply(&server_, &goodrsp6)); + + ares_socket_functions sock_funcs; + memset(&sock_funcs, 0, sizeof(sock_funcs)); + + sock_funcs.asendv = ares_sendv_fail; + + ares_set_socket_functions(channel_, &sock_funcs, NULL); + + AddrInfoResult result; + struct ares_addrinfo_hints hints = {0, 0, 0, 0}; + hints.ai_family = AF_UNSPEC; + hints.ai_flags = ARES_AI_NOSORT; + ares_getaddrinfo(channel_, "www.google.com.", NULL, &hints, + AddrInfoCallback, &result); + + Process(); + EXPECT_TRUE(result.done_); + EXPECT_TRUE(result.done_); + EXPECT_THAT(result.ai_, IncludesNumAddresses(2)); + EXPECT_THAT(result.ai_, IncludesV4Address("1.2.3.4")); + EXPECT_THAT(result.ai_, IncludesV6Address("2121:0000:0000:0000:0000:0000:0000:0303")); +} + +TEST_P(MockUDPChannelTestAI, ConnectionRefusedOnSearchDomainRetry) { + DNSPacket badrsp4; + badrsp4.set_response().set_aa() + .add_question(new DNSQuestion("www.google.com", T_A)) + .set_rcode(NXDOMAIN); + + EXPECT_CALL(server_, OnRequest("www.google.com", T_A)) + .WillOnce(SetReplyAndFailSend(&server_, &badrsp4)); + + DNSPacket goodrsp4; + goodrsp4.set_response().set_aa() + .add_question(new DNSQuestion("www.google.com.first.com", T_A)) + .add_answer(new DNSARR("www.google.com.first.com", 0x0100, {0x01, 0x02, 0x03, 0x04})); + + EXPECT_CALL(server_, OnRequest("www.google.com.first.com", T_A)) + .WillOnce(SetReply(&server_, &goodrsp4)); + + ares_socket_functions sock_funcs; + memset(&sock_funcs, 0, sizeof(sock_funcs)); + + sock_funcs.asendv = ares_sendv_fail; + + ares_set_socket_functions(channel_, &sock_funcs, NULL); + + AddrInfoResult result; + struct ares_addrinfo_hints hints = {0, 0, 0, 0}; + hints.ai_family = AF_INET; + hints.ai_flags = ARES_AI_NOSORT; + ares_getaddrinfo(channel_, "www.google.com", NULL, &hints, + AddrInfoCallback, &result); + + Process(); +} + class MockEDNSChannelTestAI : public MockFlagsChannelOptsTestAI { public: MockEDNSChannelTestAI() : MockFlagsChannelOptsTestAI(ARES_FLAG_EDNS) {} @@ -979,6 +1105,182 @@ TEST_P(MockChannelTestAI, FamilyV4ServiceName) { EXPECT_EQ("{addr=[1.1.1.1:80], addr=[2.2.2.2:80]}", ss.str()); } +#ifdef HAVE_CONTAINER + +class ContainedMockChannelAISysConfig + : public MockChannelOptsTest, + public ::testing::WithParamInterface> { + public: + ContainedMockChannelAISysConfig() + : MockChannelOptsTest(1, GetParam().first, GetParam().second, true, nullptr, 0) {} +}; + +static NameContentList files_no_ndots = { + {"/etc/resolv.conf", "nameserver 1.2.3.4\n" // Will be replaced + "search example.com example.org\n" + "options edns0 trust-ad\n"}, // ndots:1 is default + {"/etc/hosts", "3.4.5.6 ahostname.com\n"}, + {"/etc/nsswitch.conf", "hosts: files dns\n"}}; + +/* These tests should still work even with /etc/hosts not having any localhost + * entries */ +CONTAINED_TEST_P(ContainedMockChannelAISysConfig, NoHostsLocalHostv4, + "myhostname", "mydomainname.org", files_no_ndots) { + AddrInfoResult result = {}; + struct ares_addrinfo_hints hints = {0, 0, 0, 0}; + hints.ai_family = AF_INET; + hints.ai_flags = ARES_AI_NOSORT; + ares_getaddrinfo(channel_, "localhost", NULL, &hints, AddrInfoCallback, &result); + Process(); + EXPECT_TRUE(result.done_); + EXPECT_EQ(result.status_, ARES_SUCCESS); + EXPECT_THAT(result.ai_, IncludesNumAddresses(1)); + EXPECT_THAT(result.ai_, IncludesV4Address("127.0.0.1")); + return HasFailure(); +} + +CONTAINED_TEST_P(ContainedMockChannelAISysConfig, NoHostsLocalHostv6, + "myhostname", "mydomainname.org", files_no_ndots) { + AddrInfoResult result = {}; + struct ares_addrinfo_hints hints = {0, 0, 0, 0}; + hints.ai_family = AF_INET6; + hints.ai_flags = ARES_AI_NOSORT; + ares_getaddrinfo(channel_, "localhost", NULL, &hints, AddrInfoCallback, &result); + Process(); + EXPECT_TRUE(result.done_); + EXPECT_EQ(result.status_, ARES_SUCCESS); + EXPECT_THAT(result.ai_, IncludesNumAddresses(1)); + EXPECT_THAT(result.ai_, IncludesV6Address("0000:0000:0000:0000:0000:0000:0000:0001")); + return HasFailure(); +} + +CONTAINED_TEST_P(ContainedMockChannelAISysConfig, NoHostsLocalHostUnspec, + "myhostname", "mydomainname.org", files_no_ndots) { + AddrInfoResult result = {}; + struct ares_addrinfo_hints hints = {0, 0, 0, 0}; + hints.ai_family = AF_UNSPEC; + hints.ai_flags = ARES_AI_NOSORT; + ares_getaddrinfo(channel_, "localhost", NULL, &hints, AddrInfoCallback, &result); + Process(); + EXPECT_TRUE(result.done_); + EXPECT_EQ(result.status_, ARES_SUCCESS); + EXPECT_THAT(result.ai_, IncludesNumAddresses(2)); + EXPECT_THAT(result.ai_, IncludesV4Address("127.0.0.1")); + EXPECT_THAT(result.ai_, IncludesV6Address("0000:0000:0000:0000:0000:0000:0000:0001")); + return HasFailure(); +} + + +/* Issue #946 says if a v4 localhost entry exists, but not a v6 entry, v6 + * isn't output correctly. */ +static NameContentList files_localhost_v4localhostonly = { + {"/etc/resolv.conf", "nameserver 1.2.3.4\n" // Will be replaced + "search example.com example.org\n" + "options edns0 trust-ad\n"}, // ndots:1 is default + {"/etc/hosts", "127.0.0.1 localhost\n"}, + {"/etc/nsswitch.conf", "hosts: files dns\n"}}; +CONTAINED_TEST_P(ContainedMockChannelAISysConfig, v4OnlyLocalHostv4, + "myhostname", "mydomainname.org", files_localhost_v4localhostonly) { + AddrInfoResult result = {}; + struct ares_addrinfo_hints hints = {0, 0, 0, 0}; + hints.ai_family = AF_INET; + hints.ai_flags = ARES_AI_NOSORT; + ares_getaddrinfo(channel_, "localhost", NULL, &hints, AddrInfoCallback, &result); + Process(); + EXPECT_TRUE(result.done_); + EXPECT_EQ(result.status_, ARES_SUCCESS); + EXPECT_THAT(result.ai_, IncludesNumAddresses(1)); + EXPECT_THAT(result.ai_, IncludesV4Address("127.0.0.1")); + return HasFailure(); +} + +CONTAINED_TEST_P(ContainedMockChannelAISysConfig, v4OnlyLocalHostv6, + "myhostname", "mydomainname.org", files_localhost_v4localhostonly) { + AddrInfoResult result = {}; + struct ares_addrinfo_hints hints = {0, 0, 0, 0}; + hints.ai_family = AF_INET6; + hints.ai_flags = ARES_AI_NOSORT; + ares_getaddrinfo(channel_, "localhost", NULL, &hints, AddrInfoCallback, &result); + Process(); + EXPECT_TRUE(result.done_); + EXPECT_EQ(result.status_, ARES_SUCCESS); + EXPECT_THAT(result.ai_, IncludesNumAddresses(1)); + EXPECT_THAT(result.ai_, IncludesV6Address("0000:0000:0000:0000:0000:0000:0000:0001")); + return HasFailure(); +} + +CONTAINED_TEST_P(ContainedMockChannelAISysConfig, v4OnlyLocalHostUnspec, + "myhostname", "mydomainname.org", files_localhost_v4localhostonly) { + AddrInfoResult result = {}; + struct ares_addrinfo_hints hints = {0, 0, 0, 0}; + hints.ai_family = AF_UNSPEC; + hints.ai_flags = ARES_AI_NOSORT; + ares_getaddrinfo(channel_, "localhost", NULL, &hints, AddrInfoCallback, &result); + Process(); + EXPECT_TRUE(result.done_); + EXPECT_EQ(result.status_, ARES_SUCCESS); + EXPECT_THAT(result.ai_, IncludesNumAddresses(2)); + EXPECT_THAT(result.ai_, IncludesV4Address("127.0.0.1")); + EXPECT_THAT(result.ai_, IncludesV6Address("0000:0000:0000:0000:0000:0000:0000:0001")); + return HasFailure(); +} + + +static NameContentList files_localhost_v6localhostonly = { + {"/etc/resolv.conf", "nameserver 1.2.3.4\n" // Will be replaced + "search example.com example.org\n" + "options edns0 trust-ad\n"}, // ndots:1 is default + {"/etc/hosts", "::1 localhost\n"}, + {"/etc/nsswitch.conf", "hosts: files dns\n"}}; +CONTAINED_TEST_P(ContainedMockChannelAISysConfig, v6OnlyLocalHostv4, + "myhostname", "mydomainname.org", files_localhost_v6localhostonly) { + AddrInfoResult result = {}; + struct ares_addrinfo_hints hints = {0, 0, 0, 0}; + hints.ai_family = AF_INET; + hints.ai_flags = ARES_AI_NOSORT; + ares_getaddrinfo(channel_, "localhost", NULL, &hints, AddrInfoCallback, &result); + Process(); + EXPECT_TRUE(result.done_); + EXPECT_EQ(result.status_, ARES_SUCCESS); + EXPECT_THAT(result.ai_, IncludesNumAddresses(1)); + EXPECT_THAT(result.ai_, IncludesV4Address("127.0.0.1")); + return HasFailure(); +} + +CONTAINED_TEST_P(ContainedMockChannelAISysConfig, v6OnlyLocalHostv6, + "myhostname", "mydomainname.org", files_localhost_v6localhostonly) { + AddrInfoResult result = {}; + struct ares_addrinfo_hints hints = {0, 0, 0, 0}; + hints.ai_family = AF_INET6; + hints.ai_flags = ARES_AI_NOSORT; + ares_getaddrinfo(channel_, "localhost", NULL, &hints, AddrInfoCallback, &result); + Process(); + EXPECT_TRUE(result.done_); + EXPECT_EQ(result.status_, ARES_SUCCESS); + EXPECT_THAT(result.ai_, IncludesNumAddresses(1)); + EXPECT_THAT(result.ai_, IncludesV6Address("0000:0000:0000:0000:0000:0000:0000:0001")); + return HasFailure(); +} + +CONTAINED_TEST_P(ContainedMockChannelAISysConfig, v6OnlyLocalHostUnspec, + "myhostname", "mydomainname.org", files_localhost_v6localhostonly) { + AddrInfoResult result = {}; + struct ares_addrinfo_hints hints = {0, 0, 0, 0}; + hints.ai_family = AF_UNSPEC; + hints.ai_flags = ARES_AI_NOSORT; + ares_getaddrinfo(channel_, "localhost", NULL, &hints, AddrInfoCallback, &result); + Process(); + EXPECT_TRUE(result.done_); + EXPECT_EQ(result.status_, ARES_SUCCESS); + EXPECT_THAT(result.ai_, IncludesNumAddresses(2)); + EXPECT_THAT(result.ai_, IncludesV4Address("127.0.0.1")); + EXPECT_THAT(result.ai_, IncludesV6Address("0000:0000:0000:0000:0000:0000:0000:0001")); + return HasFailure(); +} + +INSTANTIATE_TEST_SUITE_P(AddressFamiliesAI, ContainedMockChannelAISysConfig, ::testing::ValuesIn(ares::test::families_modes), PrintFamilyMode); +#endif + INSTANTIATE_TEST_SUITE_P(AddressFamiliesAI, MockChannelTestAI, ::testing::ValuesIn(ares::test::families_modes), PrintFamilyMode); diff --git a/source/lib/c-ares-1.34.4/test/ares-test-mock-et.cc b/source/lib/c-ares-1.34.6/test/ares-test-mock-et.cc similarity index 95% rename from source/lib/c-ares-1.34.4/test/ares-test-mock-et.cc rename to source/lib/c-ares-1.34.6/test/ares-test-mock-et.cc index 22f80e89..b8cf5dd3 100644 --- a/source/lib/c-ares-1.34.4/test/ares-test-mock-et.cc +++ b/source/lib/c-ares-1.34.6/test/ares-test-mock-et.cc @@ -112,6 +112,114 @@ TEST_P(MockUDPEventThreadTest, BadLoopbackServerNoTimeouts) { } } +static int noop_close(ares_socket_t sock, void *user_data) +{ + (void)user_data; +#if defined(HAVE_CLOSESOCKET) + return closesocket(sock); +#elif defined(HAVE_CLOSESOCKET_CAMEL) + return CloseSocket(sock); +#elif defined(HAVE_CLOSE_S) + return close_s(sock); +#else + return close(sock); +#endif + return 0; +} + +static ares_socket_t noop_socket(int domain, int type, int protocol, + void *user_data) +{ + (void)user_data; + return socket(domain, type, protocol); +} + +static int noop_setsockopt(ares_socket_t sock, ares_socket_opt_t opt, + const void *val, ares_socklen_t val_size, + void *user_data) +{ + (void)sock; + (void)opt; + (void)val; + (void)val_size; + (void)user_data; + return 0; +} + +static int noop_connect(ares_socket_t sock, const struct sockaddr *address, + ares_socklen_t address_len, unsigned int flags, + void *user_data) +{ + (void)sock; + (void)address; + (void)address_len; + (void)flags; + (void)user_data; + + return 0; +} + +static ares_ssize_t noop_recvfrom(ares_socket_t sock, void *buffer, + size_t length, int flags, + struct sockaddr *address, + ares_socklen_t *address_len, + void *user_data) +{ + (void)sock; + (void)buffer; + (void)length; + (void)flags; + (void)address; + (void)address_len; + (void)user_data; + + errno = EAGAIN; + return 0; +} + +static ares_ssize_t noop_sendto(ares_socket_t sock, const void *buffer, + size_t length, int flags, + const struct sockaddr *address, + ares_socklen_t address_len, void *user_data) +{ + (void)sock; + (void)buffer; + (void)flags; + (void)address; + (void)address_len; + (void)user_data; + /* Eat all data */ + return (ares_ssize_t)length; +} + +// Issue #1000 Event Thread stall on temporarily downed server. +TEST_P(MockUDPEventThreadTest, DownServer) { + struct ares_socket_functions_ex noop_sock_funcs; + memset(&noop_sock_funcs, 0, sizeof(noop_sock_funcs)); + noop_sock_funcs.version = 1; + noop_sock_funcs.asocket = noop_socket; + noop_sock_funcs.aclose = noop_close; + noop_sock_funcs.asetsockopt = noop_setsockopt; + noop_sock_funcs.aconnect = noop_connect; + noop_sock_funcs.arecvfrom = noop_recvfrom; + noop_sock_funcs.asendto = noop_sendto; + ares_set_socket_functions_ex(channel_, &noop_sock_funcs, NULL); + + QueryResult result; + ares_query_dnsrec(channel_, "www.google.com", ARES_CLASS_IN, ARES_REC_TYPE_A, QueryCallback, &result, NULL); + // no need to call Process() since we're not actually connecting + ares_queue_wait_empty(channel_, -1); + EXPECT_TRUE(result.done_); + EXPECT_NE(0, result.timeouts_); + + // Issue states second query stalls + ares_query_dnsrec(channel_, "www.google.com", ARES_CLASS_IN, ARES_REC_TYPE_A, QueryCallback, &result, NULL); + // no need to call Process() since we're not actually connecting + ares_queue_wait_empty(channel_, -1); + EXPECT_TRUE(result.done_); + EXPECT_NE(0, result.timeouts_); +} + // UDP to TCP specific test TEST_P(MockUDPEventThreadTest, TruncationRetry) { DNSPacket rsptruncated; diff --git a/source/lib/c-ares-1.34.4/test/ares-test-mock.cc b/source/lib/c-ares-1.34.6/test/ares-test-mock.cc similarity index 93% rename from source/lib/c-ares-1.34.4/test/ares-test-mock.cc rename to source/lib/c-ares-1.34.6/test/ares-test-mock.cc index d53d04d0..4b60eee8 100644 --- a/source/lib/c-ares-1.34.4/test/ares-test-mock.cc +++ b/source/lib/c-ares-1.34.6/test/ares-test-mock.cc @@ -850,7 +850,7 @@ class ContainedMockChannelSysConfig : MockChannelOptsTest(1, GetParam().first, GetParam().second, true, nullptr, 0) {} }; -NameContentList files_no_ndots = { +static NameContentList files_no_ndots = { {"/etc/resolv.conf", "nameserver 1.2.3.4\n" // Will be replaced "search example.com example.org\n" "options edns0 trust-ad\n"}, // ndots:1 is default @@ -875,7 +875,8 @@ CONTAINED_TEST_P(ContainedMockChannelSysConfig, SysConfigNdotsDefault, return HasFailure(); } -NameContentList files_ndots0 = { + +static NameContentList files_ndots0 = { {"/etc/resolv.conf", "nameserver 1.2.3.4\n" // Will be replaced "search example.com example.org\n" "options edns0 trust-ad ndots:0\n"}, // ndots:1 is default @@ -1661,6 +1662,195 @@ TEST_P(MockChannelTest, GetHostByAddrDestroy) { EXPECT_EQ(0, result.timeouts_); } +TEST_P(MockChannelTest, TriggerResendThenConnFailSERVFAIL) { + // Set up the server response. The server always returns SERVFAIL. + DNSPacket badrsp; + badrsp.set_response().set_aa().set_rcode(SERVFAIL) + .add_question(new DNSQuestion("www.google.com", T_A)); + DNSPacket goodrsp; + goodrsp.set_response().set_aa() + .add_question(new DNSQuestion("www.google.com", T_A)) + .add_answer(new DNSARR("www.google.com", 0x0100, {0x01, 0x02, 0x03, 0x04})); + EXPECT_CALL(server_, OnRequest("www.google.com", T_A)) + .WillOnce(SetReplyAndFailSend(&server_, &badrsp)) + .WillOnce(SetReply(&server_, &goodrsp)); + + ares_socket_functions sock_funcs; + memset(&sock_funcs, 0, sizeof(sock_funcs)); + + sock_funcs.asendv = ares_sendv_fail; + + ares_set_socket_functions(channel_, &sock_funcs, NULL); + + HostResult result; + ares_gethostbyname(channel_, "www.google.com.", AF_INET, HostCallback, + &result); + Process(); + EXPECT_TRUE(result.done_); + std::stringstream ss; + ss << result.host_; + EXPECT_EQ("{'www.google.com' aliases=[] addrs=[1.2.3.4]}", ss.str()); +} + +TEST_P(MockUDPChannelTest, TriggerResendThenConnFailEDNS) { + // Set up the server response to simulate an EDNS failure + DNSPacket badrsp; + badrsp.set_response().set_aa().set_rcode(FORMERR) + .add_question(new DNSQuestion("www.google.com", T_A)); + DNSPacket goodrsp; + goodrsp.set_response().set_aa() + .add_question(new DNSQuestion("www.google.com", T_A)) + .add_answer(new DNSARR("www.google.com", 0x0100, {0x01, 0x02, 0x03, 0x04})); + EXPECT_CALL(server_, OnRequest("www.google.com", T_A)) + .WillOnce(SetReplyAndFailSend(&server_, &badrsp)) + .WillOnce(SetReply(&server_, &goodrsp)); + + ares_socket_functions sock_funcs; + memset(&sock_funcs, 0, sizeof(sock_funcs)); + + sock_funcs.asendv = ares_sendv_fail; + + ares_set_socket_functions(channel_, &sock_funcs, NULL); + + HostResult result; + ares_gethostbyname(channel_, "www.google.com.", AF_INET, HostCallback, + &result); + Process(); + EXPECT_TRUE(result.done_); + std::stringstream ss; + ss << result.host_; + EXPECT_EQ("{'www.google.com' aliases=[] addrs=[1.2.3.4]}", ss.str()); +} + +TEST_P(MockUDPChannelTest, GetSock) { + DNSPacket reply; + reply.set_response().set_aa() + .add_question(new DNSQuestion("www.google.com", T_A)) + .add_answer(new DNSARR("www.google.com", 0x0100, {0x01, 0x02, 0x03, 0x04})); + ON_CALL(server_, OnRequest("www.google.com", T_A)) + .WillByDefault(SetReply(&server_, &reply)); + + ares_socket_t socks[3] = {ARES_SOCKET_BAD, ARES_SOCKET_BAD, ARES_SOCKET_BAD}; + int bitmask; + + bitmask = ares_getsock(channel_, socks, 3); + EXPECT_EQ(0, bitmask); + bitmask = ares_getsock(channel_, nullptr, 0); + EXPECT_EQ(0, bitmask); + + // Ask again with a pending query. + HostResult result; + ares_gethostbyname(channel_, "www.google.com.", AF_INET, HostCallback, &result); + bitmask = ares_getsock(channel_, socks, 3); + EXPECT_NE(0, bitmask); + + size_t sock_cnt = 0; + for (size_t i=0; i<3; i++) { + if (ARES_GETSOCK_READABLE(bitmask, i) || ARES_GETSOCK_WRITABLE(bitmask, i)) { + EXPECT_NE(ARES_SOCKET_BAD, socks[i]); + if (socks[i] != ARES_SOCKET_BAD) + sock_cnt++; + } + } + EXPECT_NE((size_t)0, sock_cnt); + + Process(); + + bitmask = ares_getsock(channel_, nullptr, 0); + EXPECT_EQ(0, bitmask); +} + +TEST_P(MockTCPChannelTest, GetSock) { + DNSPacket reply; + reply.set_response().set_aa() + .add_question(new DNSQuestion("www.google.com", T_A)) + .add_answer(new DNSARR("www.google.com", 0x0100, {0x01, 0x02, 0x03, 0x04})); + ON_CALL(server_, OnRequest("www.google.com", T_A)) + .WillByDefault(SetReply(&server_, &reply)); + + ares_socket_t socks[3] = {ARES_SOCKET_BAD, ARES_SOCKET_BAD, ARES_SOCKET_BAD}; + int bitmask; + + bitmask = ares_getsock(channel_, socks, 3); + EXPECT_EQ(0, bitmask); + bitmask = ares_getsock(channel_, nullptr, 0); + EXPECT_EQ(0, bitmask); + + // Ask again with a pending query. + HostResult result; + ares_gethostbyname(channel_, "www.google.com.", AF_INET, HostCallback, &result); + bitmask = ares_getsock(channel_, socks, 3); + EXPECT_NE(0, bitmask); + + size_t sock_cnt = 0; + for (size_t i=0; i<3; i++) { + if (ARES_GETSOCK_READABLE(bitmask, i) || ARES_GETSOCK_WRITABLE(bitmask, i)) { + EXPECT_NE(ARES_SOCKET_BAD, socks[i]); + if (socks[i] != ARES_SOCKET_BAD) + sock_cnt++; + } + } + EXPECT_NE((size_t)0, sock_cnt); + + Process(); + + bitmask = ares_getsock(channel_, nullptr, 0); + EXPECT_EQ(0, bitmask); +} + + +TEST_P(MockChannelTest, VerifySocketFunctionCallback) { + ares_socket_functions sock_funcs; + memset(&sock_funcs, 0, sizeof(sock_funcs)); + + DNSPacket reply; + reply.set_response().set_aa() + .add_question(new DNSQuestion("www.google.com", T_A)) + .add_answer(new DNSARR("www.google.com", 0x0100, {0x01, 0x02, 0x03, 0x04})); + ON_CALL(server_, OnRequest("www.google.com", T_A)) + .WillByDefault(SetReply(&server_, &reply)); + + size_t count = 0; + + sock_funcs.asocket = [](int af, int type, int protocol, void * p) -> ares_socket_t { + EXPECT_NE(nullptr, p); + (*reinterpret_cast(p))++; + return ::socket(af, type, protocol); + }; + + ares_set_socket_functions(channel_, &sock_funcs, &count); + + { + count = 0; + HostResult result; + ares_gethostbyname(channel_, "www.google.com.", AF_INET, HostCallback, &result); + Process(); + + EXPECT_TRUE(result.done_); + EXPECT_EQ(ARES_SUCCESS, result.status_); + EXPECT_EQ(0, result.timeouts_); + EXPECT_NE((size_t)0, count); + } + + { + count = 0; + ares_channel_t *copy; + EXPECT_EQ(ARES_SUCCESS, ares_dup(©, channel_)); + + HostResult result; + ares_gethostbyname(copy, "www.google.com.", AF_INET, HostCallback, &result); + + ProcessAltChannel(copy); + + EXPECT_TRUE(result.done_); + ares_destroy(copy); + EXPECT_NE((size_t)0, count); + EXPECT_EQ(ARES_SUCCESS, result.status_); + EXPECT_EQ(0, result.timeouts_); + } + +} + static const unsigned char * fetch_server_cookie(const ares_dns_record_t *dnsrec, size_t *len) { diff --git a/source/lib/c-ares-1.34.4/test/ares-test-ns.cc b/source/lib/c-ares-1.34.6/test/ares-test-ns.cc similarity index 100% rename from source/lib/c-ares-1.34.4/test/ares-test-ns.cc rename to source/lib/c-ares-1.34.6/test/ares-test-ns.cc diff --git a/source/lib/c-ares-1.34.4/test/ares-test-parse-a.cc b/source/lib/c-ares-1.34.6/test/ares-test-parse-a.cc similarity index 100% rename from source/lib/c-ares-1.34.4/test/ares-test-parse-a.cc rename to source/lib/c-ares-1.34.6/test/ares-test-parse-a.cc diff --git a/source/lib/c-ares-1.34.4/test/ares-test-parse-aaaa.cc b/source/lib/c-ares-1.34.6/test/ares-test-parse-aaaa.cc similarity index 100% rename from source/lib/c-ares-1.34.4/test/ares-test-parse-aaaa.cc rename to source/lib/c-ares-1.34.6/test/ares-test-parse-aaaa.cc diff --git a/source/lib/c-ares-1.34.4/test/ares-test-parse-caa.cc b/source/lib/c-ares-1.34.6/test/ares-test-parse-caa.cc similarity index 100% rename from source/lib/c-ares-1.34.4/test/ares-test-parse-caa.cc rename to source/lib/c-ares-1.34.6/test/ares-test-parse-caa.cc diff --git a/source/lib/c-ares-1.34.4/test/ares-test-parse-mx.cc b/source/lib/c-ares-1.34.6/test/ares-test-parse-mx.cc similarity index 100% rename from source/lib/c-ares-1.34.4/test/ares-test-parse-mx.cc rename to source/lib/c-ares-1.34.6/test/ares-test-parse-mx.cc diff --git a/source/lib/c-ares-1.34.4/test/ares-test-parse-naptr.cc b/source/lib/c-ares-1.34.6/test/ares-test-parse-naptr.cc similarity index 100% rename from source/lib/c-ares-1.34.4/test/ares-test-parse-naptr.cc rename to source/lib/c-ares-1.34.6/test/ares-test-parse-naptr.cc diff --git a/source/lib/c-ares-1.34.4/test/ares-test-parse-ns.cc b/source/lib/c-ares-1.34.6/test/ares-test-parse-ns.cc similarity index 100% rename from source/lib/c-ares-1.34.4/test/ares-test-parse-ns.cc rename to source/lib/c-ares-1.34.6/test/ares-test-parse-ns.cc diff --git a/source/lib/c-ares-1.34.4/test/ares-test-parse-ptr.cc b/source/lib/c-ares-1.34.6/test/ares-test-parse-ptr.cc similarity index 100% rename from source/lib/c-ares-1.34.4/test/ares-test-parse-ptr.cc rename to source/lib/c-ares-1.34.6/test/ares-test-parse-ptr.cc diff --git a/source/lib/c-ares-1.34.4/test/ares-test-parse-soa-any.cc b/source/lib/c-ares-1.34.6/test/ares-test-parse-soa-any.cc similarity index 100% rename from source/lib/c-ares-1.34.4/test/ares-test-parse-soa-any.cc rename to source/lib/c-ares-1.34.6/test/ares-test-parse-soa-any.cc diff --git a/source/lib/c-ares-1.34.4/test/ares-test-parse-soa.cc b/source/lib/c-ares-1.34.6/test/ares-test-parse-soa.cc similarity index 100% rename from source/lib/c-ares-1.34.4/test/ares-test-parse-soa.cc rename to source/lib/c-ares-1.34.6/test/ares-test-parse-soa.cc diff --git a/source/lib/c-ares-1.34.4/test/ares-test-parse-srv.cc b/source/lib/c-ares-1.34.6/test/ares-test-parse-srv.cc similarity index 100% rename from source/lib/c-ares-1.34.4/test/ares-test-parse-srv.cc rename to source/lib/c-ares-1.34.6/test/ares-test-parse-srv.cc diff --git a/source/lib/c-ares-1.34.4/test/ares-test-parse-txt.cc b/source/lib/c-ares-1.34.6/test/ares-test-parse-txt.cc similarity index 100% rename from source/lib/c-ares-1.34.4/test/ares-test-parse-txt.cc rename to source/lib/c-ares-1.34.6/test/ares-test-parse-txt.cc diff --git a/source/lib/c-ares-1.34.4/test/ares-test-parse-uri.cc b/source/lib/c-ares-1.34.6/test/ares-test-parse-uri.cc similarity index 100% rename from source/lib/c-ares-1.34.4/test/ares-test-parse-uri.cc rename to source/lib/c-ares-1.34.6/test/ares-test-parse-uri.cc diff --git a/source/lib/c-ares-1.34.4/test/ares-test-parse.cc b/source/lib/c-ares-1.34.6/test/ares-test-parse.cc similarity index 100% rename from source/lib/c-ares-1.34.4/test/ares-test-parse.cc rename to source/lib/c-ares-1.34.6/test/ares-test-parse.cc diff --git a/source/lib/c-ares-1.34.4/test/ares-test.cc b/source/lib/c-ares-1.34.6/test/ares-test.cc similarity index 98% rename from source/lib/c-ares-1.34.4/test/ares-test.cc rename to source/lib/c-ares-1.34.6/test/ares-test.cc index 99ab0a00..076adbc5 100644 --- a/source/lib/c-ares-1.34.4/test/ares-test.cc +++ b/source/lib/c-ares-1.34.6/test/ares-test.cc @@ -250,6 +250,7 @@ std::vector> families_modes = both_families_both_modes; unsigned long long LibraryTest::fails_ = 0; std::map LibraryTest::size_fails_; std::mutex LibraryTest::lock_; +bool LibraryTest::failsend_ = false; void ares_sleep_time(unsigned int ms) { @@ -340,7 +341,29 @@ void ProcessWork(ares_channel_t *channel, } +void LibraryTest::SetFailSend() { + failsend_ = true; +} + // static +ares_ssize_t LibraryTest::ares_sendv_fail(ares_socket_t socket, const struct iovec *vec, int len, void *user_data) +{ + (void)user_data; + + if (failsend_) { +#ifdef USE_WINSOCK + WSASetLastError(WSAECONNREFUSED); +#else + errno = ECONNREFUSED; +#endif + failsend_ = false; + return -1; + } + + return send(socket, (const char *)vec[0].iov_base, vec[0].iov_len, 0); +} + + void LibraryTest::SetAllocFail(int nth) { lock_.lock(); assert(nth > 0); @@ -684,6 +707,7 @@ void MockServer::ProcessRequest(ares_socket_t fd, struct sockaddr_storage* addr, /* DNS 0x20 will mix case, do case-insensitive matching of name in request */ char lower_name[256]; int flags = 0; + arestest_strtolower(lower_name, name, sizeof(lower_name)); // Before processing, let gMock know the request is happening. @@ -745,10 +769,11 @@ void MockServer::ProcessRequest(ares_socket_t fd, struct sockaddr_storage* addr, #endif ares_ssize_t rc = (ares_ssize_t)sendto(fd, BYTE_CAST reply.data(), (SEND_TYPE_ARG3)reply.size(), flags, - (struct sockaddr *)addr, addrlen); + (struct sockaddr *)addr, addrlen); if (rc < static_cast(reply.size())) { std::cerr << "Failed to send full reply, rc=" << rc << std::endl; } + } // static @@ -888,14 +913,18 @@ void MockChannelOptsTest::ProcessFD(ares_socket_t fd) { } } -void MockChannelOptsTest::Process(unsigned int cancel_ms) { +void MockChannelOptsTest::ProcessAltChannel(ares_channel_t *chan, unsigned int cancel_ms) { using namespace std::placeholders; - ProcessWork(channel_, + ProcessWork(chan, std::bind(&MockChannelOptsTest::fds, this), std::bind(&MockChannelOptsTest::ProcessFD, this, _1), cancel_ms); } +void MockChannelOptsTest::Process(unsigned int cancel_ms) { + ProcessAltChannel(channel_, cancel_ms); +} + void MockEventThreadOptsTest::Process(unsigned int cancel_ms) { std::set fds; diff --git a/source/lib/c-ares-1.34.4/test/ares-test.h b/source/lib/c-ares-1.34.6/test/ares-test.h similarity index 97% rename from source/lib/c-ares-1.34.4/test/ares-test.h rename to source/lib/c-ares-1.34.6/test/ares-test.h index 61275921..1cab18fb 100644 --- a/source/lib/c-ares-1.34.4/test/ares-test.h +++ b/source/lib/c-ares-1.34.6/test/ares-test.h @@ -61,6 +61,16 @@ # define sclose(x) close(x) #endif +#ifndef HAVE_WRITEV +extern "C" { +/* Structure for scatter/gather I/O. */ +struct iovec { + void *iov_base; /* Pointer to data. */ + size_t iov_len; /* Length of data. */ +}; +}; +#endif + namespace ares { typedef unsigned char byte; @@ -140,11 +150,17 @@ class LibraryTest : public ::testing::Test { static void *arealloc(void *ptr, size_t size); static void afree(void *ptr); + static void SetFailSend(void); + static ares_ssize_t ares_sendv_fail(ares_socket_t socket, const struct iovec *vec, int len, + void *user_data); + + private: static bool ShouldAllocFail(size_t size); static unsigned long long fails_; static std::map size_fails_; static std::mutex lock_; + static bool failsend_; }; // Test fixture that uses a default channel. @@ -328,6 +344,7 @@ class MockChannelOptsTest : public LibraryTest { // Process all pending work on ares-owned and mock-server-owned file // descriptors. + void ProcessAltChannel(ares_channel_t *chan, unsigned int cancel_ms = 0); void Process(unsigned int cancel_ms = 0); protected: @@ -450,6 +467,12 @@ ACTION_P2(SetReplyData, mockserver, data) mockserver->SetReplyData(data); } +ACTION_P2(SetReplyAndFailSend, mockserver, reply) +{ + mockserver->SetReply(reply); + LibraryTest::SetFailSend(); +} + ACTION_P2(SetReply, mockserver, reply) { mockserver->SetReply(reply); diff --git a/source/lib/c-ares-1.34.4/test/ares_queryloop.c b/source/lib/c-ares-1.34.6/test/ares_queryloop.c similarity index 100% rename from source/lib/c-ares-1.34.4/test/ares_queryloop.c rename to source/lib/c-ares-1.34.6/test/ares_queryloop.c diff --git a/source/lib/c-ares-1.34.4/test/dns-dump.cc b/source/lib/c-ares-1.34.6/test/dns-dump.cc similarity index 100% rename from source/lib/c-ares-1.34.4/test/dns-dump.cc rename to source/lib/c-ares-1.34.6/test/dns-dump.cc diff --git a/source/lib/c-ares-1.34.4/test/dns-proto-test.cc b/source/lib/c-ares-1.34.6/test/dns-proto-test.cc similarity index 100% rename from source/lib/c-ares-1.34.4/test/dns-proto-test.cc rename to source/lib/c-ares-1.34.6/test/dns-proto-test.cc diff --git a/source/lib/c-ares-1.34.4/test/dns-proto.cc b/source/lib/c-ares-1.34.6/test/dns-proto.cc similarity index 100% rename from source/lib/c-ares-1.34.4/test/dns-proto.cc rename to source/lib/c-ares-1.34.6/test/dns-proto.cc diff --git a/source/lib/c-ares-1.34.4/test/dns-proto.h b/source/lib/c-ares-1.34.6/test/dns-proto.h similarity index 100% rename from source/lib/c-ares-1.34.4/test/dns-proto.h rename to source/lib/c-ares-1.34.6/test/dns-proto.h diff --git a/source/lib/c-ares-1.34.4/test/fuzzcheck.sh b/source/lib/c-ares-1.34.6/test/fuzzcheck.sh similarity index 100% rename from source/lib/c-ares-1.34.4/test/fuzzcheck.sh rename to source/lib/c-ares-1.34.6/test/fuzzcheck.sh diff --git a/source/lib/c-ares-1.34.4/test/fuzzinput/004a216d3cff18b0c5c6b68b807f1529 b/source/lib/c-ares-1.34.6/test/fuzzinput/004a216d3cff18b0c5c6b68b807f1529 similarity index 100% rename from source/lib/c-ares-1.34.4/test/fuzzinput/004a216d3cff18b0c5c6b68b807f1529 rename to source/lib/c-ares-1.34.6/test/fuzzinput/004a216d3cff18b0c5c6b68b807f1529 diff --git a/source/lib/c-ares-1.34.4/test/fuzzinput/00539467ca159b36aea95e61f9729115 b/source/lib/c-ares-1.34.6/test/fuzzinput/00539467ca159b36aea95e61f9729115 similarity index 100% rename from source/lib/c-ares-1.34.4/test/fuzzinput/00539467ca159b36aea95e61f9729115 rename to source/lib/c-ares-1.34.6/test/fuzzinput/00539467ca159b36aea95e61f9729115 diff --git a/source/lib/c-ares-1.34.4/test/fuzzinput/00e846db8f43f2f507cd1666ed5a753e b/source/lib/c-ares-1.34.6/test/fuzzinput/00e846db8f43f2f507cd1666ed5a753e similarity index 100% rename from source/lib/c-ares-1.34.4/test/fuzzinput/00e846db8f43f2f507cd1666ed5a753e rename to source/lib/c-ares-1.34.6/test/fuzzinput/00e846db8f43f2f507cd1666ed5a753e diff --git a/source/lib/c-ares-1.34.4/test/fuzzinput/0177b7566f08c013699eaea9a77abeb3 b/source/lib/c-ares-1.34.6/test/fuzzinput/0177b7566f08c013699eaea9a77abeb3 similarity index 100% rename from source/lib/c-ares-1.34.4/test/fuzzinput/0177b7566f08c013699eaea9a77abeb3 rename to source/lib/c-ares-1.34.6/test/fuzzinput/0177b7566f08c013699eaea9a77abeb3 diff --git a/source/lib/c-ares-1.34.4/test/fuzzinput/020a4fa317715bfdb236ed13751e6b65 b/source/lib/c-ares-1.34.6/test/fuzzinput/020a4fa317715bfdb236ed13751e6b65 similarity index 100% rename from source/lib/c-ares-1.34.4/test/fuzzinput/020a4fa317715bfdb236ed13751e6b65 rename to source/lib/c-ares-1.34.6/test/fuzzinput/020a4fa317715bfdb236ed13751e6b65 diff --git a/source/lib/c-ares-1.34.4/test/fuzzinput/0310f2e81bea31f4fe3f330872a877dd b/source/lib/c-ares-1.34.6/test/fuzzinput/0310f2e81bea31f4fe3f330872a877dd similarity index 100% rename from source/lib/c-ares-1.34.4/test/fuzzinput/0310f2e81bea31f4fe3f330872a877dd rename to source/lib/c-ares-1.34.6/test/fuzzinput/0310f2e81bea31f4fe3f330872a877dd diff --git a/source/lib/c-ares-1.34.4/test/fuzzinput/0449be67df1730b2d0887d412a9b7cc4 b/source/lib/c-ares-1.34.6/test/fuzzinput/0449be67df1730b2d0887d412a9b7cc4 similarity index 100% rename from source/lib/c-ares-1.34.4/test/fuzzinput/0449be67df1730b2d0887d412a9b7cc4 rename to source/lib/c-ares-1.34.6/test/fuzzinput/0449be67df1730b2d0887d412a9b7cc4 diff --git a/source/lib/c-ares-1.34.4/test/fuzzinput/0449dd14f7aa94bf0d716bfe09b287a8 b/source/lib/c-ares-1.34.6/test/fuzzinput/0449dd14f7aa94bf0d716bfe09b287a8 similarity index 100% rename from source/lib/c-ares-1.34.4/test/fuzzinput/0449dd14f7aa94bf0d716bfe09b287a8 rename to source/lib/c-ares-1.34.6/test/fuzzinput/0449dd14f7aa94bf0d716bfe09b287a8 diff --git a/source/lib/c-ares-1.34.4/test/fuzzinput/04c93cdf7208979aa4df80a3a0d5a2d8 b/source/lib/c-ares-1.34.6/test/fuzzinput/04c93cdf7208979aa4df80a3a0d5a2d8 similarity index 100% rename from source/lib/c-ares-1.34.4/test/fuzzinput/04c93cdf7208979aa4df80a3a0d5a2d8 rename to source/lib/c-ares-1.34.6/test/fuzzinput/04c93cdf7208979aa4df80a3a0d5a2d8 diff --git a/source/lib/c-ares-1.34.4/test/fuzzinput/0567e7171e08e75f3f91c4ca74c17adc b/source/lib/c-ares-1.34.6/test/fuzzinput/0567e7171e08e75f3f91c4ca74c17adc similarity index 100% rename from source/lib/c-ares-1.34.4/test/fuzzinput/0567e7171e08e75f3f91c4ca74c17adc rename to source/lib/c-ares-1.34.6/test/fuzzinput/0567e7171e08e75f3f91c4ca74c17adc diff --git a/source/lib/c-ares-1.34.4/test/fuzzinput/05ba948578a397e9cbc6a7b3e78622fa b/source/lib/c-ares-1.34.6/test/fuzzinput/05ba948578a397e9cbc6a7b3e78622fa similarity index 100% rename from source/lib/c-ares-1.34.4/test/fuzzinput/05ba948578a397e9cbc6a7b3e78622fa rename to source/lib/c-ares-1.34.6/test/fuzzinput/05ba948578a397e9cbc6a7b3e78622fa diff --git a/source/lib/c-ares-1.34.4/test/fuzzinput/060afe5ed25f3e2e86167e545f27edca b/source/lib/c-ares-1.34.6/test/fuzzinput/060afe5ed25f3e2e86167e545f27edca similarity index 100% rename from source/lib/c-ares-1.34.4/test/fuzzinput/060afe5ed25f3e2e86167e545f27edca rename to source/lib/c-ares-1.34.6/test/fuzzinput/060afe5ed25f3e2e86167e545f27edca diff --git a/source/lib/c-ares-1.34.4/test/fuzzinput/06d47d3681493f1b1d41236f460d896f b/source/lib/c-ares-1.34.6/test/fuzzinput/06d47d3681493f1b1d41236f460d896f similarity index 100% rename from source/lib/c-ares-1.34.4/test/fuzzinput/06d47d3681493f1b1d41236f460d896f rename to source/lib/c-ares-1.34.6/test/fuzzinput/06d47d3681493f1b1d41236f460d896f diff --git a/source/lib/c-ares-1.34.4/test/fuzzinput/0724a810b0e131c2fddb6de9003b9064 b/source/lib/c-ares-1.34.6/test/fuzzinput/0724a810b0e131c2fddb6de9003b9064 similarity index 100% rename from source/lib/c-ares-1.34.4/test/fuzzinput/0724a810b0e131c2fddb6de9003b9064 rename to source/lib/c-ares-1.34.6/test/fuzzinput/0724a810b0e131c2fddb6de9003b9064 diff --git a/source/lib/c-ares-1.34.4/test/fuzzinput/0b5279148826f5b962bcf1896bdb4ede b/source/lib/c-ares-1.34.6/test/fuzzinput/0b5279148826f5b962bcf1896bdb4ede similarity index 100% rename from source/lib/c-ares-1.34.4/test/fuzzinput/0b5279148826f5b962bcf1896bdb4ede rename to source/lib/c-ares-1.34.6/test/fuzzinput/0b5279148826f5b962bcf1896bdb4ede diff --git a/source/lib/c-ares-1.34.4/test/fuzzinput/114048c0f6b10bdc67ce9166405d195e b/source/lib/c-ares-1.34.6/test/fuzzinput/114048c0f6b10bdc67ce9166405d195e similarity index 100% rename from source/lib/c-ares-1.34.4/test/fuzzinput/114048c0f6b10bdc67ce9166405d195e rename to source/lib/c-ares-1.34.6/test/fuzzinput/114048c0f6b10bdc67ce9166405d195e diff --git a/source/lib/c-ares-1.34.4/test/fuzzinput/11b8464a0ef8735d202955c34c36b0c7 b/source/lib/c-ares-1.34.6/test/fuzzinput/11b8464a0ef8735d202955c34c36b0c7 similarity index 100% rename from source/lib/c-ares-1.34.4/test/fuzzinput/11b8464a0ef8735d202955c34c36b0c7 rename to source/lib/c-ares-1.34.6/test/fuzzinput/11b8464a0ef8735d202955c34c36b0c7 diff --git a/source/lib/c-ares-1.34.4/test/fuzzinput/11cb626f1668c7b41954ce7d768fe528 b/source/lib/c-ares-1.34.6/test/fuzzinput/11cb626f1668c7b41954ce7d768fe528 similarity index 100% rename from source/lib/c-ares-1.34.4/test/fuzzinput/11cb626f1668c7b41954ce7d768fe528 rename to source/lib/c-ares-1.34.6/test/fuzzinput/11cb626f1668c7b41954ce7d768fe528 diff --git a/source/lib/c-ares-1.34.4/test/fuzzinput/14b133bf18125b75a1976fa63a1df6b7 b/source/lib/c-ares-1.34.6/test/fuzzinput/14b133bf18125b75a1976fa63a1df6b7 similarity index 100% rename from source/lib/c-ares-1.34.4/test/fuzzinput/14b133bf18125b75a1976fa63a1df6b7 rename to source/lib/c-ares-1.34.6/test/fuzzinput/14b133bf18125b75a1976fa63a1df6b7 diff --git a/source/lib/c-ares-1.34.4/test/fuzzinput/153c6b3afa8faa03c8bc28f936a6d4cf b/source/lib/c-ares-1.34.6/test/fuzzinput/153c6b3afa8faa03c8bc28f936a6d4cf similarity index 100% rename from source/lib/c-ares-1.34.4/test/fuzzinput/153c6b3afa8faa03c8bc28f936a6d4cf rename to source/lib/c-ares-1.34.6/test/fuzzinput/153c6b3afa8faa03c8bc28f936a6d4cf diff --git a/source/lib/c-ares-1.34.4/test/fuzzinput/182cad2a342ed7317b7c21a5d17020d1 b/source/lib/c-ares-1.34.6/test/fuzzinput/182cad2a342ed7317b7c21a5d17020d1 similarity index 100% rename from source/lib/c-ares-1.34.4/test/fuzzinput/182cad2a342ed7317b7c21a5d17020d1 rename to source/lib/c-ares-1.34.6/test/fuzzinput/182cad2a342ed7317b7c21a5d17020d1 diff --git a/source/lib/c-ares-1.34.4/test/fuzzinput/1c61a61bb7057b52c5b15188345a5238 b/source/lib/c-ares-1.34.6/test/fuzzinput/1c61a61bb7057b52c5b15188345a5238 similarity index 100% rename from source/lib/c-ares-1.34.4/test/fuzzinput/1c61a61bb7057b52c5b15188345a5238 rename to source/lib/c-ares-1.34.6/test/fuzzinput/1c61a61bb7057b52c5b15188345a5238 diff --git a/source/lib/c-ares-1.34.4/test/fuzzinput/1dbe2cf62ed2e4fa1c3cb473f08710b5 b/source/lib/c-ares-1.34.6/test/fuzzinput/1dbe2cf62ed2e4fa1c3cb473f08710b5 similarity index 100% rename from source/lib/c-ares-1.34.4/test/fuzzinput/1dbe2cf62ed2e4fa1c3cb473f08710b5 rename to source/lib/c-ares-1.34.6/test/fuzzinput/1dbe2cf62ed2e4fa1c3cb473f08710b5 diff --git a/source/lib/c-ares-1.34.4/test/fuzzinput/21199be504fcfece5c7096ee0dbba507 b/source/lib/c-ares-1.34.6/test/fuzzinput/21199be504fcfece5c7096ee0dbba507 similarity index 100% rename from source/lib/c-ares-1.34.4/test/fuzzinput/21199be504fcfece5c7096ee0dbba507 rename to source/lib/c-ares-1.34.6/test/fuzzinput/21199be504fcfece5c7096ee0dbba507 diff --git a/source/lib/c-ares-1.34.4/test/fuzzinput/21891480074b5635dbbe7137bdcabccd b/source/lib/c-ares-1.34.6/test/fuzzinput/21891480074b5635dbbe7137bdcabccd similarity index 100% rename from source/lib/c-ares-1.34.4/test/fuzzinput/21891480074b5635dbbe7137bdcabccd rename to source/lib/c-ares-1.34.6/test/fuzzinput/21891480074b5635dbbe7137bdcabccd diff --git a/source/lib/c-ares-1.34.4/test/fuzzinput/233aea42e15aa73e131eefabf16088c9 b/source/lib/c-ares-1.34.6/test/fuzzinput/233aea42e15aa73e131eefabf16088c9 similarity index 100% rename from source/lib/c-ares-1.34.4/test/fuzzinput/233aea42e15aa73e131eefabf16088c9 rename to source/lib/c-ares-1.34.6/test/fuzzinput/233aea42e15aa73e131eefabf16088c9 diff --git a/source/lib/c-ares-1.34.4/test/fuzzinput/24660d4e7ac7aa21d600ea7a3d198bbb b/source/lib/c-ares-1.34.6/test/fuzzinput/24660d4e7ac7aa21d600ea7a3d198bbb similarity index 100% rename from source/lib/c-ares-1.34.4/test/fuzzinput/24660d4e7ac7aa21d600ea7a3d198bbb rename to source/lib/c-ares-1.34.6/test/fuzzinput/24660d4e7ac7aa21d600ea7a3d198bbb diff --git a/source/lib/c-ares-1.34.4/test/fuzzinput/25589deb55c08429345f289d1c9b0254 b/source/lib/c-ares-1.34.6/test/fuzzinput/25589deb55c08429345f289d1c9b0254 similarity index 100% rename from source/lib/c-ares-1.34.4/test/fuzzinput/25589deb55c08429345f289d1c9b0254 rename to source/lib/c-ares-1.34.6/test/fuzzinput/25589deb55c08429345f289d1c9b0254 diff --git a/source/lib/c-ares-1.34.4/test/fuzzinput/2573bd823e4da11f727a17f8e1f35c26 b/source/lib/c-ares-1.34.6/test/fuzzinput/2573bd823e4da11f727a17f8e1f35c26 similarity index 100% rename from source/lib/c-ares-1.34.4/test/fuzzinput/2573bd823e4da11f727a17f8e1f35c26 rename to source/lib/c-ares-1.34.6/test/fuzzinput/2573bd823e4da11f727a17f8e1f35c26 diff --git a/source/lib/c-ares-1.34.4/test/fuzzinput/276f12da56866273e76059ad0e7be97e b/source/lib/c-ares-1.34.6/test/fuzzinput/276f12da56866273e76059ad0e7be97e similarity index 100% rename from source/lib/c-ares-1.34.4/test/fuzzinput/276f12da56866273e76059ad0e7be97e rename to source/lib/c-ares-1.34.6/test/fuzzinput/276f12da56866273e76059ad0e7be97e diff --git a/source/lib/c-ares-1.34.4/test/fuzzinput/29198a2e380cb19babec9e02116d213e b/source/lib/c-ares-1.34.6/test/fuzzinput/29198a2e380cb19babec9e02116d213e similarity index 100% rename from source/lib/c-ares-1.34.4/test/fuzzinput/29198a2e380cb19babec9e02116d213e rename to source/lib/c-ares-1.34.6/test/fuzzinput/29198a2e380cb19babec9e02116d213e diff --git a/source/lib/c-ares-1.34.4/test/fuzzinput/2c94ba9434b1a1b9396fc5364f101363 b/source/lib/c-ares-1.34.6/test/fuzzinput/2c94ba9434b1a1b9396fc5364f101363 similarity index 100% rename from source/lib/c-ares-1.34.4/test/fuzzinput/2c94ba9434b1a1b9396fc5364f101363 rename to source/lib/c-ares-1.34.6/test/fuzzinput/2c94ba9434b1a1b9396fc5364f101363 diff --git a/source/lib/c-ares-1.34.4/test/fuzzinput/2d578c357dc2f5e02dc55cddb30641d1 b/source/lib/c-ares-1.34.6/test/fuzzinput/2d578c357dc2f5e02dc55cddb30641d1 similarity index 100% rename from source/lib/c-ares-1.34.4/test/fuzzinput/2d578c357dc2f5e02dc55cddb30641d1 rename to source/lib/c-ares-1.34.6/test/fuzzinput/2d578c357dc2f5e02dc55cddb30641d1 diff --git a/source/lib/c-ares-1.34.4/test/fuzzinput/2dff6cc5a223e67fde9e5e79af456992 b/source/lib/c-ares-1.34.6/test/fuzzinput/2dff6cc5a223e67fde9e5e79af456992 similarity index 100% rename from source/lib/c-ares-1.34.4/test/fuzzinput/2dff6cc5a223e67fde9e5e79af456992 rename to source/lib/c-ares-1.34.6/test/fuzzinput/2dff6cc5a223e67fde9e5e79af456992 diff --git a/source/lib/c-ares-1.34.4/test/fuzzinput/2f103b1f9477f2d8934bd84328d51c75 b/source/lib/c-ares-1.34.6/test/fuzzinput/2f103b1f9477f2d8934bd84328d51c75 similarity index 100% rename from source/lib/c-ares-1.34.4/test/fuzzinput/2f103b1f9477f2d8934bd84328d51c75 rename to source/lib/c-ares-1.34.6/test/fuzzinput/2f103b1f9477f2d8934bd84328d51c75 diff --git a/source/lib/c-ares-1.34.4/test/fuzzinput/31cd3a8413de13d9624adbb1613784bf b/source/lib/c-ares-1.34.6/test/fuzzinput/31cd3a8413de13d9624adbb1613784bf similarity index 100% rename from source/lib/c-ares-1.34.4/test/fuzzinput/31cd3a8413de13d9624adbb1613784bf rename to source/lib/c-ares-1.34.6/test/fuzzinput/31cd3a8413de13d9624adbb1613784bf diff --git a/source/lib/c-ares-1.34.4/test/fuzzinput/36415bdf1d180098fe6234b4186e69f3 b/source/lib/c-ares-1.34.6/test/fuzzinput/36415bdf1d180098fe6234b4186e69f3 similarity index 100% rename from source/lib/c-ares-1.34.4/test/fuzzinput/36415bdf1d180098fe6234b4186e69f3 rename to source/lib/c-ares-1.34.6/test/fuzzinput/36415bdf1d180098fe6234b4186e69f3 diff --git a/source/lib/c-ares-1.34.4/test/fuzzinput/3a04a80f0242e8dff0cd732e7c4767da b/source/lib/c-ares-1.34.6/test/fuzzinput/3a04a80f0242e8dff0cd732e7c4767da similarity index 100% rename from source/lib/c-ares-1.34.4/test/fuzzinput/3a04a80f0242e8dff0cd732e7c4767da rename to source/lib/c-ares-1.34.6/test/fuzzinput/3a04a80f0242e8dff0cd732e7c4767da diff --git a/source/lib/c-ares-1.34.4/test/fuzzinput/44d0f973b7b0fb3e4a07770c943dcd5a b/source/lib/c-ares-1.34.6/test/fuzzinput/44d0f973b7b0fb3e4a07770c943dcd5a similarity index 100% rename from source/lib/c-ares-1.34.4/test/fuzzinput/44d0f973b7b0fb3e4a07770c943dcd5a rename to source/lib/c-ares-1.34.6/test/fuzzinput/44d0f973b7b0fb3e4a07770c943dcd5a diff --git a/source/lib/c-ares-1.34.4/test/fuzzinput/50bc00daa0ddcd6cfb2b5d9f62c81f47 b/source/lib/c-ares-1.34.6/test/fuzzinput/50bc00daa0ddcd6cfb2b5d9f62c81f47 similarity index 100% rename from source/lib/c-ares-1.34.4/test/fuzzinput/50bc00daa0ddcd6cfb2b5d9f62c81f47 rename to source/lib/c-ares-1.34.6/test/fuzzinput/50bc00daa0ddcd6cfb2b5d9f62c81f47 diff --git a/source/lib/c-ares-1.34.4/test/fuzzinput/51ed2d1fb77b3078b54e94e85606b7df b/source/lib/c-ares-1.34.6/test/fuzzinput/51ed2d1fb77b3078b54e94e85606b7df similarity index 100% rename from source/lib/c-ares-1.34.4/test/fuzzinput/51ed2d1fb77b3078b54e94e85606b7df rename to source/lib/c-ares-1.34.6/test/fuzzinput/51ed2d1fb77b3078b54e94e85606b7df diff --git a/source/lib/c-ares-1.34.4/test/fuzzinput/5c5e0e899cf2e7d053a9e45fb76f6e5a b/source/lib/c-ares-1.34.6/test/fuzzinput/5c5e0e899cf2e7d053a9e45fb76f6e5a similarity index 100% rename from source/lib/c-ares-1.34.4/test/fuzzinput/5c5e0e899cf2e7d053a9e45fb76f6e5a rename to source/lib/c-ares-1.34.6/test/fuzzinput/5c5e0e899cf2e7d053a9e45fb76f6e5a diff --git a/source/lib/c-ares-1.34.4/test/fuzzinput/70152ed033f139443fbfb1b858bb3b1b b/source/lib/c-ares-1.34.6/test/fuzzinput/70152ed033f139443fbfb1b858bb3b1b similarity index 100% rename from source/lib/c-ares-1.34.4/test/fuzzinput/70152ed033f139443fbfb1b858bb3b1b rename to source/lib/c-ares-1.34.6/test/fuzzinput/70152ed033f139443fbfb1b858bb3b1b diff --git a/source/lib/c-ares-1.34.4/test/fuzzinput/7030ca2b24e5a7f9dd8f62096a48eb33 b/source/lib/c-ares-1.34.6/test/fuzzinput/7030ca2b24e5a7f9dd8f62096a48eb33 similarity index 100% rename from source/lib/c-ares-1.34.4/test/fuzzinput/7030ca2b24e5a7f9dd8f62096a48eb33 rename to source/lib/c-ares-1.34.6/test/fuzzinput/7030ca2b24e5a7f9dd8f62096a48eb33 diff --git a/source/lib/c-ares-1.34.4/test/fuzzinput/71eec1a0ef2d25bb9e2ef17f23be7e9e b/source/lib/c-ares-1.34.6/test/fuzzinput/71eec1a0ef2d25bb9e2ef17f23be7e9e similarity index 100% rename from source/lib/c-ares-1.34.4/test/fuzzinput/71eec1a0ef2d25bb9e2ef17f23be7e9e rename to source/lib/c-ares-1.34.6/test/fuzzinput/71eec1a0ef2d25bb9e2ef17f23be7e9e diff --git a/source/lib/c-ares-1.34.4/test/fuzzinput/7a6b0177210ea4ef40b254daf99393c5 b/source/lib/c-ares-1.34.6/test/fuzzinput/7a6b0177210ea4ef40b254daf99393c5 similarity index 100% rename from source/lib/c-ares-1.34.4/test/fuzzinput/7a6b0177210ea4ef40b254daf99393c5 rename to source/lib/c-ares-1.34.6/test/fuzzinput/7a6b0177210ea4ef40b254daf99393c5 diff --git a/source/lib/c-ares-1.34.4/test/fuzzinput/7f1567733711ffb61839621af0cbfa33 b/source/lib/c-ares-1.34.6/test/fuzzinput/7f1567733711ffb61839621af0cbfa33 similarity index 100% rename from source/lib/c-ares-1.34.4/test/fuzzinput/7f1567733711ffb61839621af0cbfa33 rename to source/lib/c-ares-1.34.6/test/fuzzinput/7f1567733711ffb61839621af0cbfa33 diff --git a/source/lib/c-ares-1.34.4/test/fuzzinput/850c6d57c5bb7be8205fc2438d14d7e5 b/source/lib/c-ares-1.34.6/test/fuzzinput/850c6d57c5bb7be8205fc2438d14d7e5 similarity index 100% rename from source/lib/c-ares-1.34.4/test/fuzzinput/850c6d57c5bb7be8205fc2438d14d7e5 rename to source/lib/c-ares-1.34.6/test/fuzzinput/850c6d57c5bb7be8205fc2438d14d7e5 diff --git a/source/lib/c-ares-1.34.4/test/fuzzinput/a5c8cd2784a5792b9e91c2d7895b3b34 b/source/lib/c-ares-1.34.6/test/fuzzinput/a5c8cd2784a5792b9e91c2d7895b3b34 similarity index 100% rename from source/lib/c-ares-1.34.4/test/fuzzinput/a5c8cd2784a5792b9e91c2d7895b3b34 rename to source/lib/c-ares-1.34.6/test/fuzzinput/a5c8cd2784a5792b9e91c2d7895b3b34 diff --git a/source/lib/c-ares-1.34.4/test/fuzzinput/a9135cdc7151d023300ff194bad90af9 b/source/lib/c-ares-1.34.6/test/fuzzinput/a9135cdc7151d023300ff194bad90af9 similarity index 100% rename from source/lib/c-ares-1.34.4/test/fuzzinput/a9135cdc7151d023300ff194bad90af9 rename to source/lib/c-ares-1.34.6/test/fuzzinput/a9135cdc7151d023300ff194bad90af9 diff --git a/source/lib/c-ares-1.34.4/test/fuzzinput/af2597e8ac7dec1e8b4a47518312912a b/source/lib/c-ares-1.34.6/test/fuzzinput/af2597e8ac7dec1e8b4a47518312912a similarity index 100% rename from source/lib/c-ares-1.34.4/test/fuzzinput/af2597e8ac7dec1e8b4a47518312912a rename to source/lib/c-ares-1.34.6/test/fuzzinput/af2597e8ac7dec1e8b4a47518312912a diff --git a/source/lib/c-ares-1.34.4/test/fuzzinput/answer_a b/source/lib/c-ares-1.34.6/test/fuzzinput/answer_a similarity index 100% rename from source/lib/c-ares-1.34.4/test/fuzzinput/answer_a rename to source/lib/c-ares-1.34.6/test/fuzzinput/answer_a diff --git a/source/lib/c-ares-1.34.4/test/fuzzinput/answer_aaaa b/source/lib/c-ares-1.34.6/test/fuzzinput/answer_aaaa similarity index 100% rename from source/lib/c-ares-1.34.4/test/fuzzinput/answer_aaaa rename to source/lib/c-ares-1.34.6/test/fuzzinput/answer_aaaa diff --git a/source/lib/c-ares-1.34.4/test/fuzzinput/b3f53ef826b831bb09dd25c7f5960249 b/source/lib/c-ares-1.34.6/test/fuzzinput/b3f53ef826b831bb09dd25c7f5960249 similarity index 100% rename from source/lib/c-ares-1.34.4/test/fuzzinput/b3f53ef826b831bb09dd25c7f5960249 rename to source/lib/c-ares-1.34.6/test/fuzzinput/b3f53ef826b831bb09dd25c7f5960249 diff --git a/source/lib/c-ares-1.34.4/test/fuzzinput/cda0f8751f5c4993974c2b549d29bcc8 b/source/lib/c-ares-1.34.6/test/fuzzinput/cda0f8751f5c4993974c2b549d29bcc8 similarity index 100% rename from source/lib/c-ares-1.34.4/test/fuzzinput/cda0f8751f5c4993974c2b549d29bcc8 rename to source/lib/c-ares-1.34.6/test/fuzzinput/cda0f8751f5c4993974c2b549d29bcc8 diff --git a/source/lib/c-ares-1.34.4/test/fuzzinput/ce6c26c0e469339873d0e7f616ab0945 b/source/lib/c-ares-1.34.6/test/fuzzinput/ce6c26c0e469339873d0e7f616ab0945 similarity index 100% rename from source/lib/c-ares-1.34.4/test/fuzzinput/ce6c26c0e469339873d0e7f616ab0945 rename to source/lib/c-ares-1.34.6/test/fuzzinput/ce6c26c0e469339873d0e7f616ab0945 diff --git a/source/lib/c-ares-1.34.4/test/fuzzinput/clusterfuzz-5637790584012800 b/source/lib/c-ares-1.34.6/test/fuzzinput/clusterfuzz-5637790584012800 similarity index 100% rename from source/lib/c-ares-1.34.4/test/fuzzinput/clusterfuzz-5637790584012800 rename to source/lib/c-ares-1.34.6/test/fuzzinput/clusterfuzz-5637790584012800 diff --git a/source/lib/c-ares-1.34.4/test/fuzzinput/clusterfuzz-5650695891451904 b/source/lib/c-ares-1.34.6/test/fuzzinput/clusterfuzz-5650695891451904 similarity index 100% rename from source/lib/c-ares-1.34.4/test/fuzzinput/clusterfuzz-5650695891451904 rename to source/lib/c-ares-1.34.6/test/fuzzinput/clusterfuzz-5650695891451904 diff --git a/source/lib/c-ares-1.34.4/test/fuzzinput/clusterfuzz-5651369832218624 b/source/lib/c-ares-1.34.6/test/fuzzinput/clusterfuzz-5651369832218624 similarity index 100% rename from source/lib/c-ares-1.34.4/test/fuzzinput/clusterfuzz-5651369832218624 rename to source/lib/c-ares-1.34.6/test/fuzzinput/clusterfuzz-5651369832218624 diff --git a/source/lib/c-ares-1.34.4/test/fuzzinput/clusterfuzz-5674462260756480 b/source/lib/c-ares-1.34.6/test/fuzzinput/clusterfuzz-5674462260756480 similarity index 100% rename from source/lib/c-ares-1.34.4/test/fuzzinput/clusterfuzz-5674462260756480 rename to source/lib/c-ares-1.34.6/test/fuzzinput/clusterfuzz-5674462260756480 diff --git a/source/lib/c-ares-1.34.4/test/fuzzinput/clusterfuzz-5680630672654336 b/source/lib/c-ares-1.34.6/test/fuzzinput/clusterfuzz-5680630672654336 similarity index 100% rename from source/lib/c-ares-1.34.4/test/fuzzinput/clusterfuzz-5680630672654336 rename to source/lib/c-ares-1.34.6/test/fuzzinput/clusterfuzz-5680630672654336 diff --git a/source/lib/c-ares-1.34.4/test/fuzzinput/clusterfuzz-5683497160671232 b/source/lib/c-ares-1.34.6/test/fuzzinput/clusterfuzz-5683497160671232 similarity index 100% rename from source/lib/c-ares-1.34.4/test/fuzzinput/clusterfuzz-5683497160671232 rename to source/lib/c-ares-1.34.6/test/fuzzinput/clusterfuzz-5683497160671232 diff --git a/source/lib/c-ares-1.34.4/test/fuzzinput/clusterfuzz-5687310655422464 b/source/lib/c-ares-1.34.6/test/fuzzinput/clusterfuzz-5687310655422464 similarity index 100% rename from source/lib/c-ares-1.34.4/test/fuzzinput/clusterfuzz-5687310655422464 rename to source/lib/c-ares-1.34.6/test/fuzzinput/clusterfuzz-5687310655422464 diff --git a/source/lib/c-ares-1.34.4/test/fuzzinput/clusterfuzz-5695341573177344 b/source/lib/c-ares-1.34.6/test/fuzzinput/clusterfuzz-5695341573177344 similarity index 100% rename from source/lib/c-ares-1.34.4/test/fuzzinput/clusterfuzz-5695341573177344 rename to source/lib/c-ares-1.34.6/test/fuzzinput/clusterfuzz-5695341573177344 diff --git a/source/lib/c-ares-1.34.4/test/fuzzinput/clusterfuzz-5697835103682560 b/source/lib/c-ares-1.34.6/test/fuzzinput/clusterfuzz-5697835103682560 similarity index 100% rename from source/lib/c-ares-1.34.4/test/fuzzinput/clusterfuzz-5697835103682560 rename to source/lib/c-ares-1.34.6/test/fuzzinput/clusterfuzz-5697835103682560 diff --git a/source/lib/c-ares-1.34.4/test/fuzzinput/clusterfuzz-5728518081609728 b/source/lib/c-ares-1.34.6/test/fuzzinput/clusterfuzz-5728518081609728 similarity index 100% rename from source/lib/c-ares-1.34.4/test/fuzzinput/clusterfuzz-5728518081609728 rename to source/lib/c-ares-1.34.6/test/fuzzinput/clusterfuzz-5728518081609728 diff --git a/source/lib/c-ares-1.34.4/test/fuzzinput/clusterfuzz-5732960017317888 b/source/lib/c-ares-1.34.6/test/fuzzinput/clusterfuzz-5732960017317888 similarity index 100% rename from source/lib/c-ares-1.34.4/test/fuzzinput/clusterfuzz-5732960017317888 rename to source/lib/c-ares-1.34.6/test/fuzzinput/clusterfuzz-5732960017317888 diff --git a/source/lib/c-ares-1.34.4/test/fuzzinput/e4dd7e7c2dd4ed7c2e17a6af5d04f9c9 b/source/lib/c-ares-1.34.6/test/fuzzinput/e4dd7e7c2dd4ed7c2e17a6af5d04f9c9 similarity index 100% rename from source/lib/c-ares-1.34.4/test/fuzzinput/e4dd7e7c2dd4ed7c2e17a6af5d04f9c9 rename to source/lib/c-ares-1.34.6/test/fuzzinput/e4dd7e7c2dd4ed7c2e17a6af5d04f9c9 diff --git a/source/lib/c-ares-1.34.4/test/fuzzinput/ed50ed8ee36230a5a69746ad830437e5 b/source/lib/c-ares-1.34.6/test/fuzzinput/ed50ed8ee36230a5a69746ad830437e5 similarity index 100% rename from source/lib/c-ares-1.34.4/test/fuzzinput/ed50ed8ee36230a5a69746ad830437e5 rename to source/lib/c-ares-1.34.6/test/fuzzinput/ed50ed8ee36230a5a69746ad830437e5 diff --git a/source/lib/c-ares-1.34.4/test/fuzzinput/f1b900d50806021953321c3b604ee497 b/source/lib/c-ares-1.34.6/test/fuzzinput/f1b900d50806021953321c3b604ee497 similarity index 100% rename from source/lib/c-ares-1.34.4/test/fuzzinput/f1b900d50806021953321c3b604ee497 rename to source/lib/c-ares-1.34.6/test/fuzzinput/f1b900d50806021953321c3b604ee497 diff --git a/source/lib/c-ares-1.34.4/test/fuzzinput/f6606f624be8c628328cea01d2cd07a9 b/source/lib/c-ares-1.34.6/test/fuzzinput/f6606f624be8c628328cea01d2cd07a9 similarity index 100% rename from source/lib/c-ares-1.34.4/test/fuzzinput/f6606f624be8c628328cea01d2cd07a9 rename to source/lib/c-ares-1.34.6/test/fuzzinput/f6606f624be8c628328cea01d2cd07a9 diff --git a/source/lib/c-ares-1.34.4/test/fuzzinput/f89f6c8176b564a7dd646f14305573ce b/source/lib/c-ares-1.34.6/test/fuzzinput/f89f6c8176b564a7dd646f14305573ce similarity index 100% rename from source/lib/c-ares-1.34.4/test/fuzzinput/f89f6c8176b564a7dd646f14305573ce rename to source/lib/c-ares-1.34.6/test/fuzzinput/f89f6c8176b564a7dd646f14305573ce diff --git a/source/lib/c-ares-1.34.4/test/fuzzinput/f9ad508d2dbd08d3aaaabc7d1174677d b/source/lib/c-ares-1.34.6/test/fuzzinput/f9ad508d2dbd08d3aaaabc7d1174677d similarity index 100% rename from source/lib/c-ares-1.34.4/test/fuzzinput/f9ad508d2dbd08d3aaaabc7d1174677d rename to source/lib/c-ares-1.34.6/test/fuzzinput/f9ad508d2dbd08d3aaaabc7d1174677d diff --git a/source/lib/c-ares-1.34.4/test/fuzzinput/multi-indir b/source/lib/c-ares-1.34.6/test/fuzzinput/multi-indir similarity index 100% rename from source/lib/c-ares-1.34.4/test/fuzzinput/multi-indir rename to source/lib/c-ares-1.34.6/test/fuzzinput/multi-indir diff --git a/source/lib/c-ares-1.34.4/test/fuzznames/name01 b/source/lib/c-ares-1.34.6/test/fuzznames/name01 similarity index 100% rename from source/lib/c-ares-1.34.4/test/fuzznames/name01 rename to source/lib/c-ares-1.34.6/test/fuzznames/name01 diff --git a/source/lib/c-ares-1.34.4/test/fuzznames/name02 b/source/lib/c-ares-1.34.6/test/fuzznames/name02 similarity index 100% rename from source/lib/c-ares-1.34.4/test/fuzznames/name02 rename to source/lib/c-ares-1.34.6/test/fuzznames/name02 diff --git a/source/lib/c-ares-1.34.4/test/fuzznames/name03 b/source/lib/c-ares-1.34.6/test/fuzznames/name03 similarity index 100% rename from source/lib/c-ares-1.34.4/test/fuzznames/name03 rename to source/lib/c-ares-1.34.6/test/fuzznames/name03 diff --git a/source/lib/c-ares-1.34.4/test/fuzznames/name04 b/source/lib/c-ares-1.34.6/test/fuzznames/name04 similarity index 100% rename from source/lib/c-ares-1.34.4/test/fuzznames/name04 rename to source/lib/c-ares-1.34.6/test/fuzznames/name04 diff --git a/source/lib/c-ares-1.34.4/test/fuzznames/name05 b/source/lib/c-ares-1.34.6/test/fuzznames/name05 similarity index 100% rename from source/lib/c-ares-1.34.4/test/fuzznames/name05 rename to source/lib/c-ares-1.34.6/test/fuzznames/name05 diff --git a/source/lib/c-ares-1.34.4/test/fuzznames/name06 b/source/lib/c-ares-1.34.6/test/fuzznames/name06 similarity index 100% rename from source/lib/c-ares-1.34.4/test/fuzznames/name06 rename to source/lib/c-ares-1.34.6/test/fuzznames/name06 diff --git a/source/lib/c-ares-1.34.4/test/fuzznames/name07 b/source/lib/c-ares-1.34.6/test/fuzznames/name07 similarity index 100% rename from source/lib/c-ares-1.34.4/test/fuzznames/name07 rename to source/lib/c-ares-1.34.6/test/fuzznames/name07 diff --git a/source/lib/c-ares-1.34.4/test/fuzznames/name08 b/source/lib/c-ares-1.34.6/test/fuzznames/name08 similarity index 100% rename from source/lib/c-ares-1.34.4/test/fuzznames/name08 rename to source/lib/c-ares-1.34.6/test/fuzznames/name08 diff --git a/source/lib/c-ares-1.34.4/test/fuzznames/name09 b/source/lib/c-ares-1.34.6/test/fuzznames/name09 similarity index 100% rename from source/lib/c-ares-1.34.4/test/fuzznames/name09 rename to source/lib/c-ares-1.34.6/test/fuzznames/name09 diff --git a/source/lib/c-ares-1.34.4/test/fuzznames/uri1 b/source/lib/c-ares-1.34.6/test/fuzznames/uri1 similarity index 100% rename from source/lib/c-ares-1.34.4/test/fuzznames/uri1 rename to source/lib/c-ares-1.34.6/test/fuzznames/uri1 diff --git a/source/lib/c-ares-1.34.4/test/fuzznames/uri10 b/source/lib/c-ares-1.34.6/test/fuzznames/uri10 similarity index 100% rename from source/lib/c-ares-1.34.4/test/fuzznames/uri10 rename to source/lib/c-ares-1.34.6/test/fuzznames/uri10 diff --git a/source/lib/c-ares-1.34.4/test/fuzznames/uri11 b/source/lib/c-ares-1.34.6/test/fuzznames/uri11 similarity index 100% rename from source/lib/c-ares-1.34.4/test/fuzznames/uri11 rename to source/lib/c-ares-1.34.6/test/fuzznames/uri11 diff --git a/source/lib/c-ares-1.34.4/test/fuzznames/uri12 b/source/lib/c-ares-1.34.6/test/fuzznames/uri12 similarity index 100% rename from source/lib/c-ares-1.34.4/test/fuzznames/uri12 rename to source/lib/c-ares-1.34.6/test/fuzznames/uri12 diff --git a/source/lib/c-ares-1.34.4/test/fuzznames/uri13 b/source/lib/c-ares-1.34.6/test/fuzznames/uri13 similarity index 100% rename from source/lib/c-ares-1.34.4/test/fuzznames/uri13 rename to source/lib/c-ares-1.34.6/test/fuzznames/uri13 diff --git a/source/lib/c-ares-1.34.4/test/fuzznames/uri14 b/source/lib/c-ares-1.34.6/test/fuzznames/uri14 similarity index 100% rename from source/lib/c-ares-1.34.4/test/fuzznames/uri14 rename to source/lib/c-ares-1.34.6/test/fuzznames/uri14 diff --git a/source/lib/c-ares-1.34.4/test/fuzznames/uri16 b/source/lib/c-ares-1.34.6/test/fuzznames/uri16 similarity index 100% rename from source/lib/c-ares-1.34.4/test/fuzznames/uri16 rename to source/lib/c-ares-1.34.6/test/fuzznames/uri16 diff --git a/source/lib/c-ares-1.34.4/test/fuzznames/uri17 b/source/lib/c-ares-1.34.6/test/fuzznames/uri17 similarity index 100% rename from source/lib/c-ares-1.34.4/test/fuzznames/uri17 rename to source/lib/c-ares-1.34.6/test/fuzznames/uri17 diff --git a/source/lib/c-ares-1.34.4/test/fuzznames/uri18 b/source/lib/c-ares-1.34.6/test/fuzznames/uri18 similarity index 100% rename from source/lib/c-ares-1.34.4/test/fuzznames/uri18 rename to source/lib/c-ares-1.34.6/test/fuzznames/uri18 diff --git a/source/lib/c-ares-1.34.4/test/fuzznames/uri19 b/source/lib/c-ares-1.34.6/test/fuzznames/uri19 similarity index 100% rename from source/lib/c-ares-1.34.4/test/fuzznames/uri19 rename to source/lib/c-ares-1.34.6/test/fuzznames/uri19 diff --git a/source/lib/c-ares-1.34.4/test/fuzznames/uri2 b/source/lib/c-ares-1.34.6/test/fuzznames/uri2 similarity index 100% rename from source/lib/c-ares-1.34.4/test/fuzznames/uri2 rename to source/lib/c-ares-1.34.6/test/fuzznames/uri2 diff --git a/source/lib/c-ares-1.34.4/test/fuzznames/uri20 b/source/lib/c-ares-1.34.6/test/fuzznames/uri20 similarity index 100% rename from source/lib/c-ares-1.34.4/test/fuzznames/uri20 rename to source/lib/c-ares-1.34.6/test/fuzznames/uri20 diff --git a/source/lib/c-ares-1.34.4/test/fuzznames/uri21 b/source/lib/c-ares-1.34.6/test/fuzznames/uri21 similarity index 100% rename from source/lib/c-ares-1.34.4/test/fuzznames/uri21 rename to source/lib/c-ares-1.34.6/test/fuzznames/uri21 diff --git a/source/lib/c-ares-1.34.4/test/fuzznames/uri22 b/source/lib/c-ares-1.34.6/test/fuzznames/uri22 similarity index 100% rename from source/lib/c-ares-1.34.4/test/fuzznames/uri22 rename to source/lib/c-ares-1.34.6/test/fuzznames/uri22 diff --git a/source/lib/c-ares-1.34.4/test/fuzznames/uri23 b/source/lib/c-ares-1.34.6/test/fuzznames/uri23 similarity index 100% rename from source/lib/c-ares-1.34.4/test/fuzznames/uri23 rename to source/lib/c-ares-1.34.6/test/fuzznames/uri23 diff --git a/source/lib/c-ares-1.34.4/test/fuzznames/uri24 b/source/lib/c-ares-1.34.6/test/fuzznames/uri24 similarity index 100% rename from source/lib/c-ares-1.34.4/test/fuzznames/uri24 rename to source/lib/c-ares-1.34.6/test/fuzznames/uri24 diff --git a/source/lib/c-ares-1.34.4/test/fuzznames/uri25 b/source/lib/c-ares-1.34.6/test/fuzznames/uri25 similarity index 100% rename from source/lib/c-ares-1.34.4/test/fuzznames/uri25 rename to source/lib/c-ares-1.34.6/test/fuzznames/uri25 diff --git a/source/lib/c-ares-1.34.4/test/fuzznames/uri26 b/source/lib/c-ares-1.34.6/test/fuzznames/uri26 similarity index 100% rename from source/lib/c-ares-1.34.4/test/fuzznames/uri26 rename to source/lib/c-ares-1.34.6/test/fuzznames/uri26 diff --git a/source/lib/c-ares-1.34.4/test/fuzznames/uri27 b/source/lib/c-ares-1.34.6/test/fuzznames/uri27 similarity index 100% rename from source/lib/c-ares-1.34.4/test/fuzznames/uri27 rename to source/lib/c-ares-1.34.6/test/fuzznames/uri27 diff --git a/source/lib/c-ares-1.34.4/test/fuzznames/uri28 b/source/lib/c-ares-1.34.6/test/fuzznames/uri28 similarity index 100% rename from source/lib/c-ares-1.34.4/test/fuzznames/uri28 rename to source/lib/c-ares-1.34.6/test/fuzznames/uri28 diff --git a/source/lib/c-ares-1.34.4/test/fuzznames/uri29 b/source/lib/c-ares-1.34.6/test/fuzznames/uri29 similarity index 100% rename from source/lib/c-ares-1.34.4/test/fuzznames/uri29 rename to source/lib/c-ares-1.34.6/test/fuzznames/uri29 diff --git a/source/lib/c-ares-1.34.4/test/fuzznames/uri3 b/source/lib/c-ares-1.34.6/test/fuzznames/uri3 similarity index 100% rename from source/lib/c-ares-1.34.4/test/fuzznames/uri3 rename to source/lib/c-ares-1.34.6/test/fuzznames/uri3 diff --git a/source/lib/c-ares-1.34.4/test/fuzznames/uri30 b/source/lib/c-ares-1.34.6/test/fuzznames/uri30 similarity index 100% rename from source/lib/c-ares-1.34.4/test/fuzznames/uri30 rename to source/lib/c-ares-1.34.6/test/fuzznames/uri30 diff --git a/source/lib/c-ares-1.34.4/test/fuzznames/uri31 b/source/lib/c-ares-1.34.6/test/fuzznames/uri31 similarity index 100% rename from source/lib/c-ares-1.34.4/test/fuzznames/uri31 rename to source/lib/c-ares-1.34.6/test/fuzznames/uri31 diff --git a/source/lib/c-ares-1.34.4/test/fuzznames/uri32 b/source/lib/c-ares-1.34.6/test/fuzznames/uri32 similarity index 100% rename from source/lib/c-ares-1.34.4/test/fuzznames/uri32 rename to source/lib/c-ares-1.34.6/test/fuzznames/uri32 diff --git a/source/lib/c-ares-1.34.4/test/fuzznames/uri33 b/source/lib/c-ares-1.34.6/test/fuzznames/uri33 similarity index 100% rename from source/lib/c-ares-1.34.4/test/fuzznames/uri33 rename to source/lib/c-ares-1.34.6/test/fuzznames/uri33 diff --git a/source/lib/c-ares-1.34.4/test/fuzznames/uri34 b/source/lib/c-ares-1.34.6/test/fuzznames/uri34 similarity index 100% rename from source/lib/c-ares-1.34.4/test/fuzznames/uri34 rename to source/lib/c-ares-1.34.6/test/fuzznames/uri34 diff --git a/source/lib/c-ares-1.34.4/test/fuzznames/uri35 b/source/lib/c-ares-1.34.6/test/fuzznames/uri35 similarity index 100% rename from source/lib/c-ares-1.34.4/test/fuzznames/uri35 rename to source/lib/c-ares-1.34.6/test/fuzznames/uri35 diff --git a/source/lib/c-ares-1.34.4/test/fuzznames/uri36 b/source/lib/c-ares-1.34.6/test/fuzznames/uri36 similarity index 100% rename from source/lib/c-ares-1.34.4/test/fuzznames/uri36 rename to source/lib/c-ares-1.34.6/test/fuzznames/uri36 diff --git a/source/lib/c-ares-1.34.4/test/fuzznames/uri37 b/source/lib/c-ares-1.34.6/test/fuzznames/uri37 similarity index 100% rename from source/lib/c-ares-1.34.4/test/fuzznames/uri37 rename to source/lib/c-ares-1.34.6/test/fuzznames/uri37 diff --git a/source/lib/c-ares-1.34.4/test/fuzznames/uri4 b/source/lib/c-ares-1.34.6/test/fuzznames/uri4 similarity index 100% rename from source/lib/c-ares-1.34.4/test/fuzznames/uri4 rename to source/lib/c-ares-1.34.6/test/fuzznames/uri4 diff --git a/source/lib/c-ares-1.34.4/test/fuzznames/uri5 b/source/lib/c-ares-1.34.6/test/fuzznames/uri5 similarity index 100% rename from source/lib/c-ares-1.34.4/test/fuzznames/uri5 rename to source/lib/c-ares-1.34.6/test/fuzznames/uri5 diff --git a/source/lib/c-ares-1.34.4/test/fuzznames/uri6 b/source/lib/c-ares-1.34.6/test/fuzznames/uri6 similarity index 100% rename from source/lib/c-ares-1.34.4/test/fuzznames/uri6 rename to source/lib/c-ares-1.34.6/test/fuzznames/uri6 diff --git a/source/lib/c-ares-1.34.4/test/fuzznames/uri7 b/source/lib/c-ares-1.34.6/test/fuzznames/uri7 similarity index 100% rename from source/lib/c-ares-1.34.4/test/fuzznames/uri7 rename to source/lib/c-ares-1.34.6/test/fuzznames/uri7 diff --git a/source/lib/c-ares-1.34.4/test/fuzznames/uri8 b/source/lib/c-ares-1.34.6/test/fuzznames/uri8 similarity index 100% rename from source/lib/c-ares-1.34.4/test/fuzznames/uri8 rename to source/lib/c-ares-1.34.6/test/fuzznames/uri8 diff --git a/source/lib/c-ares-1.34.4/test/fuzznames/uri9 b/source/lib/c-ares-1.34.6/test/fuzznames/uri9 similarity index 100% rename from source/lib/c-ares-1.34.4/test/fuzznames/uri9 rename to source/lib/c-ares-1.34.6/test/fuzznames/uri9 From 7880d0f5b33af2e91bc0bb8ed9f96028471f6010 Mon Sep 17 00:00:00 2001 From: Patrick Stephens Date: Sun, 3 May 2026 12:35:11 +0100 Subject: [PATCH 157/213] [upstream] lib: monkey: upgrade to v1.8.6 Upstream-Ref: https://github.com/fluent/fluent-bit/commit/3683ce3a43eb4b0e5d2254d03993c86f145af699 Cherry-picked from Fluent Bit v4.2.4 --- source/lib/monkey/CMakeLists.txt | 2 +- source/lib/monkey/mk_core/mk_event_epoll.c | 5 +--- source/lib/monkey/mk_core/mk_event_kqueue.c | 26 +++++++++++-------- source/lib/monkey/mk_core/mk_event_libevent.c | 5 +--- source/lib/monkey/mk_core/mk_event_poll.c | 5 +--- source/lib/monkey/mk_core/mk_event_select.c | 5 +--- 6 files changed, 20 insertions(+), 28 deletions(-) diff --git a/source/lib/monkey/CMakeLists.txt b/source/lib/monkey/CMakeLists.txt index 3fbace59..9797f474 100644 --- a/source/lib/monkey/CMakeLists.txt +++ b/source/lib/monkey/CMakeLists.txt @@ -23,7 +23,7 @@ endif() # Monkey Version set(MK_VERSION_MAJOR 1) set(MK_VERSION_MINOR 8) -set(MK_VERSION_PATCH 5) +set(MK_VERSION_PATCH 6) set(MK_VERSION_STR "${MK_VERSION_MAJOR}.${MK_VERSION_MINOR}.${MK_VERSION_PATCH}") # Output paths diff --git a/source/lib/monkey/mk_core/mk_event_epoll.c b/source/lib/monkey/mk_core/mk_event_epoll.c index 6ebb90ab..5a2701f4 100644 --- a/source/lib/monkey/mk_core/mk_event_epoll.c +++ b/source/lib/monkey/mk_core/mk_event_epoll.c @@ -398,14 +398,11 @@ static inline int _mk_event_channel_destroy(struct mk_event_ctx *ctx, } ret = _mk_event_del(ctx, event); - if (ret != 0) { - return ret; - } close(r_fd); close(w_fd); - return 0; + return ret; } static inline int _mk_event_inject(struct mk_event_loop *loop, diff --git a/source/lib/monkey/mk_core/mk_event_kqueue.c b/source/lib/monkey/mk_core/mk_event_kqueue.c index 46b4f352..a6475514 100644 --- a/source/lib/monkey/mk_core/mk_event_kqueue.c +++ b/source/lib/monkey/mk_core/mk_event_kqueue.c @@ -74,8 +74,9 @@ static inline int _mk_event_add(struct mk_event_ctx *ctx, int fd, int ret; int set = MK_FALSE; struct mk_event *event; - struct kevent ke = {0, 0, 0, 0, 0, 0}; + struct kevent ke; + EV_SET(&ke, 0, 0, 0, 0, 0, 0); mk_bug(ctx == NULL); mk_bug(data == NULL); @@ -142,8 +143,9 @@ static inline int _mk_event_add(struct mk_event_ctx *ctx, int fd, static inline int _mk_event_del(struct mk_event_ctx *ctx, struct mk_event *event) { int ret; - struct kevent ke = {0, 0, 0, 0, 0, 0}; + struct kevent ke; + EV_SET(&ke, 0, 0, 0, 0, 0, 0); mk_bug(ctx == NULL); mk_bug(event == NULL); @@ -208,12 +210,16 @@ static inline int _mk_event_timeout_create(struct mk_event_ctx *ctx, event->priority = MK_EVENT_PRIORITY_DEFAULT; mk_list_entry_init(&event->_priority_head); -#if defined(NOTE_SECONDS) && !defined(__APPLE__) - /* FreeBSD or LINUX_KQUEUE defined */ - /* TODO : high resolution interval support. */ +#if defined(NOTE_NSECONDS) + /* The modern FreeBSD & NetBSD & OpenBSD & macOS have a high-resolution + event timer. */ + EV_SET(&ke, fd, EVFILT_TIMER, EV_ADD, NOTE_NSECONDS, + (sec * 1000000000) + nsec, event); +#elif defined(NOTE_SECONDS) && !defined(__APPLE__) + /* LINUX_KQUEUE defined */ EV_SET(&ke, fd, EVFILT_TIMER, EV_ADD, NOTE_SECONDS, sec, event); #else - /* Other BSD have no NOTE_SECONDS & specify milliseconds */ + /* Keep backward compatibility; use the millisecond-resolution event timer. */ /* Also, on macOS, NOTE_SECONDS has severe side effect that cause * performance degradation. */ EV_SET(&ke, fd, EVFILT_TIMER, EV_ADD, 0, (sec * 1000) + (nsec / 1000000) , event); @@ -239,8 +245,9 @@ static inline int _mk_event_timeout_destroy(struct mk_event_ctx *ctx, void *data { int ret; struct mk_event *event; - struct kevent ke = {0, 0, 0, 0, 0, 0}; + struct kevent ke; + EV_SET(&ke, 0, 0, 0, 0, 0, 0); if (data == NULL) { return 0; } @@ -316,14 +323,11 @@ static inline int _mk_event_channel_destroy(struct mk_event_ctx *ctx, } ret = _mk_event_del(ctx, event); - if (ret != 0) { - return ret; - } close(r_fd); close(w_fd); - return 0; + return ret; } static inline int _mk_event_inject(struct mk_event_loop *loop, diff --git a/source/lib/monkey/mk_core/mk_event_libevent.c b/source/lib/monkey/mk_core/mk_event_libevent.c index b155159e..4f1403f4 100644 --- a/source/lib/monkey/mk_core/mk_event_libevent.c +++ b/source/lib/monkey/mk_core/mk_event_libevent.c @@ -404,14 +404,11 @@ static inline int _mk_event_channel_destroy(struct mk_event_ctx *ctx, } ret = _mk_event_del(ctx, event); - if (ret != 0) { - return ret; - } evutil_closesocket(r_fd); evutil_closesocket(w_fd); - return 0; + return ret; } static inline int _mk_event_inject(struct mk_event_loop *loop, diff --git a/source/lib/monkey/mk_core/mk_event_poll.c b/source/lib/monkey/mk_core/mk_event_poll.c index 8378c6f5..63d394d3 100644 --- a/source/lib/monkey/mk_core/mk_event_poll.c +++ b/source/lib/monkey/mk_core/mk_event_poll.c @@ -332,14 +332,11 @@ static inline int _mk_event_channel_destroy(struct mk_event_ctx *ctx, } ret = _mk_event_del(ctx, event); - if (ret != 0) { - return ret; - } close(r_fd); close(w_fd); - return 0; + return ret; } static inline int _mk_event_inject(struct mk_event_loop *loop, diff --git a/source/lib/monkey/mk_core/mk_event_select.c b/source/lib/monkey/mk_core/mk_event_select.c index f23dafc7..2f6297f6 100644 --- a/source/lib/monkey/mk_core/mk_event_select.c +++ b/source/lib/monkey/mk_core/mk_event_select.c @@ -345,14 +345,11 @@ static inline int _mk_event_channel_destroy(struct mk_event_ctx *ctx, } ret = _mk_event_del(ctx, event); - if (ret != 0) { - return ret; - } close(r_fd); close(w_fd); - return 0; + return ret; } static inline int _mk_event_inject(struct mk_event_loop *loop, From 31879663bb5b7f1bfe2e5f2e59777335a41f52eb Mon Sep 17 00:00:00 2001 From: Patrick Stephens Date: Sun, 3 May 2026 12:35:11 +0100 Subject: [PATCH 158/213] [upstream] lib: cmetrics: upgrade to v1.0.6 Upstream-Ref: https://github.com/fluent/fluent-bit/commit/96a31e30750e90e95be90943ead72ee0b4597fc1 Cherry-picked from Fluent Bit v4.2.4 --- .../lib/cmetrics/.github/workflows/build.yaml | 13 +- .../lib/cmetrics/.github/workflows/lint.yaml | 4 +- .../cmetrics/.github/workflows/packages.yaml | 14 +- source/lib/cmetrics/CMakeLists.txt | 2 +- source/lib/cmetrics/src/cmt_cat.c | 40 +++- source/lib/cmetrics/src/cmt_histogram.c | 2 +- source/lib/cmetrics/tests/cat.c | 224 ++++++++++++++++++ 7 files changed, 274 insertions(+), 25 deletions(-) diff --git a/source/lib/cmetrics/.github/workflows/build.yaml b/source/lib/cmetrics/.github/workflows/build.yaml index 305a0133..40f8432a 100644 --- a/source/lib/cmetrics/.github/workflows/build.yaml +++ b/source/lib/cmetrics/.github/workflows/build.yaml @@ -15,11 +15,11 @@ jobs: strategy: fail-fast: false matrix: - os: [windows-latest, windows-2019] + os: [windows-latest, windows-2022] permissions: contents: read steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v6 with: submodules: true @@ -90,6 +90,9 @@ jobs: steps: - name: Set up base image dependencies run: | + # Update sources to use archive.debian.org (Buster reached end-of-life) + sed -i 's/deb.debian.org/archive.debian.org/g' /etc/apt/sources.list + sed -i 's/security.debian.org/archive.debian.org/g' /etc/apt/sources.list apt-get update apt-get install -y build-essential wget make gcc g++ git libcurl4-openssl-dev @@ -136,7 +139,7 @@ jobs: permissions: contents: read steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v6 with: submodules: true @@ -170,7 +173,7 @@ jobs: permissions: contents: read steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v6 with: submodules: true @@ -200,7 +203,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Check out the repository - uses: actions/checkout@v5 + uses: actions/checkout@v6 with: submodules: true diff --git a/source/lib/cmetrics/.github/workflows/lint.yaml b/source/lib/cmetrics/.github/workflows/lint.yaml index 7b23e8bb..b0e49d65 100644 --- a/source/lib/cmetrics/.github/workflows/lint.yaml +++ b/source/lib/cmetrics/.github/workflows/lint.yaml @@ -10,7 +10,7 @@ jobs: permissions: contents: read steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v6 - uses: ludeeus/action-shellcheck@master actionlint: @@ -19,7 +19,7 @@ jobs: permissions: contents: read steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v6 - run: | echo "::add-matcher::.github/actionlint-matcher.json" bash <(curl https://raw.githubusercontent.com/rhysd/actionlint/main/scripts/download-actionlint.bash) diff --git a/source/lib/cmetrics/.github/workflows/packages.yaml b/source/lib/cmetrics/.github/workflows/packages.yaml index ec02c4a2..ee4fd0a4 100644 --- a/source/lib/cmetrics/.github/workflows/packages.yaml +++ b/source/lib/cmetrics/.github/workflows/packages.yaml @@ -18,7 +18,7 @@ jobs: matrix: format: [ rpm, deb ] steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v6 with: submodules: true @@ -39,7 +39,7 @@ jobs: echo ${{ matrix.format }} | awk '{print toupper($0)}' | xargs -I{} cpack -G {} - name: Store the master package artifacts - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v6 with: name: ${{ matrix.format }}-arm64 path: | @@ -54,7 +54,7 @@ jobs: runs-on: [ ubuntu-latest ] steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v6 with: submodules: true @@ -64,7 +64,7 @@ jobs: echo ${{ matrix.format }} | awk '{print toupper($0)}' | xargs -I{} cpack -G {} - name: Store the master package artifacts - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v6 with: name: ${{ matrix.format }}-amd64 path: | @@ -81,7 +81,7 @@ jobs: ext: pkg runs-on: macos-14 steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v6 with: submodules: true @@ -91,7 +91,7 @@ jobs: echo ${{ matrix.config.format }} | xargs -I{} cpack -G {} - name: Store the master package artifacts - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v6 with: name: ${{ matrix.config.format }}-${{matrix.config.arch}} path: | @@ -109,7 +109,7 @@ jobs: contents: write steps: - name: Download all artefacts - uses: actions/download-artifact@v5 + uses: actions/download-artifact@v7 with: path: artifacts/ diff --git a/source/lib/cmetrics/CMakeLists.txt b/source/lib/cmetrics/CMakeLists.txt index 5421f8c8..a5572c30 100644 --- a/source/lib/cmetrics/CMakeLists.txt +++ b/source/lib/cmetrics/CMakeLists.txt @@ -6,7 +6,7 @@ set(CMAKE_EXPORT_COMPILE_COMMANDS ON) # CMetrics Version set(CMT_VERSION_MAJOR 1) set(CMT_VERSION_MINOR 0) -set(CMT_VERSION_PATCH 5) +set(CMT_VERSION_PATCH 6) set(CMT_VERSION_STR "${CMT_VERSION_MAJOR}.${CMT_VERSION_MINOR}.${CMT_VERSION_PATCH}") # Include helpers diff --git a/source/lib/cmetrics/src/cmt_cat.c b/source/lib/cmetrics/src/cmt_cat.c index dcd941bd..595225bb 100644 --- a/source/lib/cmetrics/src/cmt_cat.c +++ b/source/lib/cmetrics/src/cmt_cat.c @@ -96,19 +96,38 @@ static int copy_label_values(struct cmt_metric *metric, char **out) return i; } -static inline int cat_histogram_values(struct cmt_metric *metric_dst, struct cmt_histogram *histogram, - struct cmt_metric *metric_src) +static inline int cat_histogram_values(struct cmt_metric *metric_dst, struct cmt_histogram *histogram_src, + struct cmt_metric *metric_src, struct cmt_histogram *histogram_dst) { int i; + size_t bucket_count_src; + size_t bucket_count_dst; + /* Validate source histogram buckets exist */ + if (!metric_src->hist_buckets) { + /* Source has no bucket data, nothing to concatenate */ + return 0; + } + + bucket_count_src = histogram_src->buckets->count; + bucket_count_dst = histogram_dst->buckets->count; + + /* Validate that source and destination have matching bucket structures */ + if (bucket_count_src != bucket_count_dst) { + /* Histogram bucket structures don't match - cannot concatenate */ + return -1; + } + + /* Allocate destination buckets if needed */ if (!metric_dst->hist_buckets) { - metric_dst->hist_buckets = calloc(1, sizeof(uint64_t) * (histogram->buckets->count + 1)); + metric_dst->hist_buckets = calloc(1, sizeof(uint64_t) * (bucket_count_dst + 1)); if (!metric_dst->hist_buckets) { return -1; } } - for (i = 0; i < histogram->buckets->count; i++) { + /* Concatenate bucket values including +Inf bucket at index bucket_count_dst */ + for (i = 0; i <= bucket_count_dst; i++) { /* histogram buckets are always integers, no need to convert them */ metric_dst->hist_buckets[i] += metric_src->hist_buckets[i]; } @@ -165,7 +184,8 @@ int cmt_cat_copy_map(struct cmt_opts *opts, struct cmt_map *dst, struct cmt_map struct cmt_metric *metric_dst; struct cmt_metric *metric_src; struct cmt_summary *summary; - struct cmt_histogram *histogram; + struct cmt_histogram *histogram_src; + struct cmt_histogram *histogram_dst; /* Handle static metric (no labels case) */ if (src->metric_static_set) { @@ -176,8 +196,9 @@ int cmt_cat_copy_map(struct cmt_opts *opts, struct cmt_map *dst, struct cmt_map metric_src = &src->metric; if (src->type == CMT_HISTOGRAM) { - histogram = (struct cmt_histogram *) src->parent; - ret = cat_histogram_values(metric_dst, histogram, metric_src); + histogram_src = (struct cmt_histogram *) src->parent; + histogram_dst = (struct cmt_histogram *) dst->parent; + ret = cat_histogram_values(metric_dst, histogram_src, metric_src, histogram_dst); if (ret == -1) { return -1; } @@ -214,8 +235,9 @@ int cmt_cat_copy_map(struct cmt_opts *opts, struct cmt_map *dst, struct cmt_map } if (src->type == CMT_HISTOGRAM) { - histogram = (struct cmt_histogram *) src->parent; - ret = cat_histogram_values(metric_dst, histogram, metric_src); + histogram_src = (struct cmt_histogram *) src->parent; + histogram_dst = (struct cmt_histogram *) dst->parent; + ret = cat_histogram_values(metric_dst, histogram_src, metric_src, histogram_dst); if (ret == -1) { return -1; } diff --git a/source/lib/cmetrics/src/cmt_histogram.c b/source/lib/cmetrics/src/cmt_histogram.c index df4a5c8f..61efa6e0 100644 --- a/source/lib/cmetrics/src/cmt_histogram.c +++ b/source/lib/cmetrics/src/cmt_histogram.c @@ -36,7 +36,7 @@ struct cmt_histogram_buckets *cmt_histogram_buckets_create_size(double *bkts, si } /* besides buckets set by the user, we add an implicit bucket for +inf */ - upper_bounds = calloc(1, sizeof(double) * count + 1); + upper_bounds = calloc(1, sizeof(double) * (count + 1)); if (!upper_bounds) { cmt_errno(); return NULL; diff --git a/source/lib/cmetrics/tests/cat.c b/source/lib/cmetrics/tests/cat.c index a41d7945..3ef59fca 100644 --- a/source/lib/cmetrics/tests/cat.c +++ b/source/lib/cmetrics/tests/cat.c @@ -391,8 +391,232 @@ void test_duplicate_metrics() } +void test_histogram_empty_concatenation() +{ + int ret; + struct cmt *cmt1; + struct cmt *cmt2; + struct cmt_histogram *h; + struct cmt_histogram_buckets *buckets; + + /* Test concatenating an empty histogram (no observations, NULL hist_buckets) */ + cmt1 = cmt_create(); + TEST_CHECK(cmt1 != NULL); + + buckets = cmt_histogram_buckets_create(11, + 0.005, 0.01, 0.025, 0.05, + 0.1, 0.25, 0.5, 1.0, 2.5, + 5.0, 10.0); + TEST_CHECK(buckets != NULL); + + /* Create histogram but never observe - hist_buckets will be NULL */ + h = cmt_histogram_create(cmt1, + "test", "histogram", "empty", "Empty histogram test", + buckets, + 0, NULL); + TEST_CHECK(h != NULL); + + /* Create destination context */ + cmt2 = cmt_create(); + TEST_CHECK(cmt2 != NULL); + + /* Concatenate empty histogram - should handle NULL hist_buckets gracefully */ + ret = cmt_cat(cmt2, cmt1); + TEST_CHECK(ret == 0); + + cmt_destroy(cmt1); + cmt_destroy(cmt2); +} + +void test_histogram_mismatched_buckets() +{ + int ret; + int i; + double val; + uint64_t ts; + struct cmt *cmt1; + struct cmt *cmt2; + struct cmt_histogram *h1; + struct cmt_histogram *h2; + struct cmt_histogram_buckets *buckets1; + struct cmt_histogram_buckets *buckets2; + + /* Test concatenating histograms with different bucket structures */ + cmt1 = cmt_create(); + TEST_CHECK(cmt1 != NULL); + + /* Create histogram with 11 buckets */ + buckets1 = cmt_histogram_buckets_create(11, + 0.005, 0.01, 0.025, 0.05, + 0.1, 0.25, 0.5, 1.0, 2.5, + 5.0, 10.0); + TEST_CHECK(buckets1 != NULL); + + h1 = cmt_histogram_create(cmt1, + "test", "histogram", "mismatch", "Mismatched buckets test", + buckets1, + 0, NULL); + TEST_CHECK(h1 != NULL); + + ts = cfl_time_now(); + for (i = 0; i < sizeof(hist_observe_values)/(sizeof(double)); i++) { + val = hist_observe_values[i]; + cmt_histogram_observe(h1, ts, val, 0, NULL); + } + + /* Create second context with different bucket structure */ + cmt2 = cmt_create(); + TEST_CHECK(cmt2 != NULL); + + /* Create histogram with 5 buckets (different structure) */ + buckets2 = cmt_histogram_buckets_create(5, + 0.1, 0.5, 1.0, 5.0, 10.0); + TEST_CHECK(buckets2 != NULL); + + h2 = cmt_histogram_create(cmt2, + "test", "histogram", "mismatch", "Mismatched buckets test", + buckets2, + 0, NULL); + TEST_CHECK(h2 != NULL); + + ts = cfl_time_now(); + for (i = 0; i < sizeof(hist_observe_values)/(sizeof(double)); i++) { + val = hist_observe_values[i]; + cmt_histogram_observe(h2, ts, val, 0, NULL); + } + + /* Try to concatenate - should fail due to bucket mismatch */ + ret = cmt_cat(cmt1, cmt2); + TEST_CHECK(ret == -1); + + cmt_destroy(cmt1); + cmt_destroy(cmt2); +} + +void test_histogram_empty_to_populated() +{ + int ret; + int i; + double val; + uint64_t ts; + struct cmt *cmt1; + struct cmt *cmt2; + struct cmt_histogram *h; + struct cmt_histogram_buckets *buckets1; + struct cmt_histogram_buckets *buckets2; + + /* Test concatenating empty histogram to one with data */ + cmt1 = cmt_create(); + TEST_CHECK(cmt1 != NULL); + + buckets1 = cmt_histogram_buckets_create(11, + 0.005, 0.01, 0.025, 0.05, + 0.1, 0.25, 0.5, 1.0, 2.5, + 5.0, 10.0); + TEST_CHECK(buckets1 != NULL); + + /* Create empty histogram (no observations) */ + h = cmt_histogram_create(cmt1, + "test", "histogram", "empty_to_full", "Empty to populated test", + buckets1, + 0, NULL); + TEST_CHECK(h != NULL); + + /* Create second context with populated histogram */ + cmt2 = cmt_create(); + TEST_CHECK(cmt2 != NULL); + + buckets2 = cmt_histogram_buckets_create(11, + 0.005, 0.01, 0.025, 0.05, + 0.1, 0.25, 0.5, 1.0, 2.5, + 5.0, 10.0); + TEST_CHECK(buckets2 != NULL); + + h = cmt_histogram_create(cmt2, + "test", "histogram", "empty_to_full", "Empty to populated test", + buckets2, + 0, NULL); + TEST_CHECK(h != NULL); + + ts = cfl_time_now(); + for (i = 0; i < sizeof(hist_observe_values)/(sizeof(double)); i++) { + val = hist_observe_values[i]; + cmt_histogram_observe(h, ts, val, 0, NULL); + } + + /* Concatenate empty to populated - should succeed */ + ret = cmt_cat(cmt1, cmt2); + TEST_CHECK(ret == 0); + + cmt_destroy(cmt1); + cmt_destroy(cmt2); +} + +void test_histogram_populated_to_empty() +{ + int ret; + int i; + double val; + uint64_t ts; + struct cmt *cmt1; + struct cmt *cmt2; + struct cmt_histogram *h; + struct cmt_histogram_buckets *buckets1; + struct cmt_histogram_buckets *buckets2; + + /* Test concatenating populated histogram to empty one */ + cmt1 = cmt_create(); + TEST_CHECK(cmt1 != NULL); + + buckets1 = cmt_histogram_buckets_create(11, + 0.005, 0.01, 0.025, 0.05, + 0.1, 0.25, 0.5, 1.0, 2.5, + 5.0, 10.0); + TEST_CHECK(buckets1 != NULL); + + h = cmt_histogram_create(cmt1, + "test", "histogram", "full_to_empty", "Populated to empty test", + buckets1, + 0, NULL); + TEST_CHECK(h != NULL); + + ts = cfl_time_now(); + for (i = 0; i < sizeof(hist_observe_values)/(sizeof(double)); i++) { + val = hist_observe_values[i]; + cmt_histogram_observe(h, ts, val, 0, NULL); + } + + /* Create second context with empty histogram */ + cmt2 = cmt_create(); + TEST_CHECK(cmt2 != NULL); + + buckets2 = cmt_histogram_buckets_create(11, + 0.005, 0.01, 0.025, 0.05, + 0.1, 0.25, 0.5, 1.0, 2.5, + 5.0, 10.0); + TEST_CHECK(buckets2 != NULL); + + /* Create empty histogram (no observations) */ + h = cmt_histogram_create(cmt2, + "test", "histogram", "full_to_empty", "Populated to empty test", + buckets2, + 0, NULL); + TEST_CHECK(h != NULL); + + /* Concatenate populated to empty - should succeed */ + ret = cmt_cat(cmt1, cmt2); + TEST_CHECK(ret == 0); + + cmt_destroy(cmt1); + cmt_destroy(cmt2); +} + TEST_LIST = { {"cat", test_cat}, {"duplicate_metrics", test_duplicate_metrics}, + {"histogram_empty_concatenation", test_histogram_empty_concatenation}, + {"histogram_mismatched_buckets", test_histogram_mismatched_buckets}, + {"histogram_empty_to_populated", test_histogram_empty_to_populated}, + {"histogram_populated_to_empty", test_histogram_populated_to_empty}, { 0 } }; From cbd93a15cadb366ef526fc936b1988cfd25be1ce Mon Sep 17 00:00:00 2001 From: Patrick Stephens Date: Sun, 3 May 2026 12:35:11 +0100 Subject: [PATCH 159/213] [upstream] lib: luajit: upgrade to 7152e154 (20251223) Upstream-Ref: https://github.com/fluent/fluent-bit/commit/ddfef360d7f3ac5268942c47ccc9b01864424a05 Cherry-picked from Fluent Bit v4.2.4 --- source/cmake/libraries.cmake | 2 +- source/lib/luajit-04dca791/.relver | 1 - source/lib/luajit-04dca791/configure | 0 .../.gitattributes | 0 .../.gitignore | 0 source/lib/luajit-7152e154/.relver | 1 + .../COPYRIGHT | 2 +- .../Makefile | 13 +- .../README | 2 +- .../doc/bluequad-print.css | 2 +- .../doc/bluequad.css | 2 +- .../doc/contact.html | 6 +- .../doc/ext_buffer.html | 4 +- .../doc/ext_c_api.html | 4 +- .../doc/ext_ffi.html | 4 +- .../doc/ext_ffi_api.html | 4 +- .../doc/ext_ffi_semantics.html | 60 ++-- .../doc/ext_ffi_tutorial.html | 4 +- .../doc/ext_jit.html | 4 +- .../doc/ext_profiler.html | 4 +- .../doc/extensions.html | 4 +- .../doc/img/contact.png | Bin .../doc/install.html | 35 ++- .../doc/luajit.html | 6 +- .../doc/running.html | 8 +- .../dynasm/dasm_arm.h | 2 +- .../dynasm/dasm_arm.lua | 2 +- .../dynasm/dasm_arm64.h | 2 +- .../dynasm/dasm_arm64.lua | 24 +- .../dynasm/dasm_mips.h | 2 +- .../dynasm/dasm_mips.lua | 2 +- .../dynasm/dasm_mips64.lua | 2 +- .../dynasm/dasm_ppc.h | 2 +- .../dynasm/dasm_ppc.lua | 2 +- .../dynasm/dasm_proto.h | 2 +- .../dynasm/dasm_x64.lua | 2 +- .../dynasm/dasm_x86.h | 2 +- .../dynasm/dasm_x86.lua | 2 +- .../dynasm/dynasm.lua | 4 +- .../etc/luajit.1 | 2 +- .../etc/luajit.pc | 0 .../src/.gitignore | 0 .../src/Makefile | 20 +- .../src/Makefile.dep | 4 +- .../src/host/.gitignore | 0 .../src/host/README | 0 .../src/host/buildvm.c | 2 +- .../src/host/buildvm.h | 2 +- .../src/host/buildvm_asm.c | 6 +- .../src/host/buildvm_fold.c | 2 +- .../src/host/buildvm_lib.c | 2 +- .../src/host/buildvm_libbc.h | 0 .../src/host/buildvm_peobj.c | 2 +- .../src/host/genlibbc.lua | 2 +- .../src/host/genminilua.lua | 2 +- .../src/host/genversion.lua | 2 +- .../src/host/minilua.c | 0 .../src/jit/.gitignore | 0 .../src/jit/bc.lua | 2 +- .../src/jit/bcsave.lua | 34 ++- .../src/jit/dis_arm.lua | 2 +- .../src/jit/dis_arm64.lua | 38 ++- .../src/jit/dis_arm64be.lua | 2 +- .../src/jit/dis_mips.lua | 2 +- .../src/jit/dis_mips64.lua | 2 +- .../src/jit/dis_mips64el.lua | 2 +- .../src/jit/dis_mips64r6.lua | 2 +- .../src/jit/dis_mips64r6el.lua | 2 +- .../src/jit/dis_mipsel.lua | 2 +- .../src/jit/dis_ppc.lua | 2 +- .../src/jit/dis_x64.lua | 2 +- .../src/jit/dis_x86.lua | 22 +- .../src/jit/dump.lua | 2 +- .../src/jit/p.lua | 6 +- .../src/jit/v.lua | 2 +- .../src/jit/zone.lua | 2 +- .../src/lauxlib.h | 0 .../src/lib_aux.c | 2 +- .../src/lib_base.c | 6 +- .../src/lib_bit.c | 4 +- .../src/lib_buffer.c | 2 +- .../src/lib_debug.c | 2 +- .../src/lib_ffi.c | 2 +- .../src/lib_init.c | 2 +- .../src/lib_io.c | 13 +- .../src/lib_jit.c | 26 +- .../src/lib_math.c | 2 +- .../src/lib_os.c | 10 +- .../src/lib_package.c | 2 +- .../src/lib_string.c | 2 +- .../src/lib_table.c | 2 +- .../src/lj_alloc.c | 0 .../src/lj_alloc.h | 0 .../src/lj_api.c | 26 +- .../src/lj_arch.h | 40 ++- .../src/lj_asm.c | 36 ++- .../src/lj_asm.h | 2 +- .../src/lj_asm_arm.h | 85 ++++-- .../src/lj_asm_arm64.h | 81 ++++-- .../src/lj_asm_mips.h | 143 ++++----- .../src/lj_asm_ppc.h | 97 ++++--- .../src/lj_asm_x86.h | 138 +++++---- .../src/lj_assert.c | 2 +- .../src/lj_bc.c | 2 +- .../src/lj_bc.h | 7 +- .../src/lj_bcdump.h | 2 +- .../src/lj_bcread.c | 14 +- .../src/lj_bcwrite.c | 21 +- .../src/lj_buf.c | 2 +- .../src/lj_buf.h | 2 +- .../src/lj_carith.c | 6 +- .../src/lj_carith.h | 2 +- .../src/lj_ccall.c | 41 ++- .../src/lj_ccall.h | 2 +- .../src/lj_ccallback.c | 59 +++- .../src/lj_ccallback.h | 2 +- .../src/lj_cconv.c | 14 +- .../src/lj_cconv.h | 2 +- .../src/lj_cdata.c | 9 +- .../src/lj_cdata.h | 2 +- .../src/lj_char.c | 0 .../src/lj_char.h | 0 .../src/lj_clib.c | 2 +- .../src/lj_clib.h | 2 +- .../src/lj_cparse.c | 2 +- .../src/lj_cparse.h | 2 +- .../src/lj_crecord.c | 101 ++++--- .../src/lj_crecord.h | 2 +- .../src/lj_ctype.c | 5 +- .../src/lj_ctype.h | 4 +- .../src/lj_debug.c | 3 +- .../src/lj_debug.h | 2 +- .../src/lj_def.h | 4 +- .../src/lj_dispatch.c | 2 +- .../src/lj_dispatch.h | 2 +- .../src/lj_emit_arm.h | 13 +- .../src/lj_emit_arm64.h | 40 ++- .../src/lj_emit_mips.h | 5 +- .../src/lj_emit_ppc.h | 5 +- .../src/lj_emit_x86.h | 22 +- .../src/lj_err.c | 18 +- .../src/lj_err.h | 2 +- .../src/lj_errmsg.h | 2 +- .../src/lj_ff.h | 2 +- .../src/lj_ffrecord.c | 16 +- .../src/lj_ffrecord.h | 2 +- .../src/lj_frame.h | 2 +- .../src/lj_func.c | 2 +- .../src/lj_func.h | 2 +- .../src/lj_gc.c | 32 +- .../src/lj_gc.h | 2 +- .../src/lj_gdbjit.c | 2 +- .../src/lj_gdbjit.h | 2 +- .../src/lj_ir.c | 25 +- .../src/lj_ir.h | 2 +- .../src/lj_ircall.h | 34 +-- .../src/lj_iropt.h | 2 +- .../src/lj_jit.h | 48 ++- .../src/lj_lex.c | 2 +- .../src/lj_lex.h | 2 +- .../src/lj_lib.c | 4 +- .../src/lj_lib.h | 2 +- .../src/lj_load.c | 22 +- .../src/lj_mcode.c | 274 +++++++++++------- .../src/lj_mcode.h | 2 +- .../src/lj_meta.c | 5 +- .../src/lj_meta.h | 2 +- .../src/lj_obj.c | 2 +- .../src/lj_obj.h | 97 ++++--- .../src/lj_opt_dce.c | 2 +- .../src/lj_opt_fold.c | 95 +++--- .../src/lj_opt_loop.c | 2 +- .../src/lj_opt_mem.c | 6 +- .../src/lj_opt_narrow.c | 41 ++- .../src/lj_opt_sink.c | 2 +- .../src/lj_opt_split.c | 17 +- .../src/lj_parse.c | 63 ++-- .../src/lj_parse.h | 2 +- .../src/lj_prng.c | 4 +- .../src/lj_prng.h | 2 +- .../src/lj_profile.c | 2 +- .../src/lj_profile.h | 2 +- .../src/lj_record.c | 127 +++++--- .../src/lj_record.h | 2 +- .../src/lj_serialize.c | 2 +- .../src/lj_serialize.h | 2 +- .../src/lj_snap.c | 9 +- .../src/lj_snap.h | 2 +- .../src/lj_state.c | 14 +- .../src/lj_state.h | 2 +- .../src/lj_str.c | 2 +- .../src/lj_str.h | 2 +- .../src/lj_strfmt.c | 13 +- .../src/lj_strfmt.h | 2 +- .../src/lj_strfmt_num.c | 45 ++- .../src/lj_strscan.c | 10 +- .../src/lj_strscan.h | 2 +- .../src/lj_tab.c | 36 +-- .../src/lj_tab.h | 5 +- .../src/lj_target.h | 2 +- .../src/lj_target_arm.h | 4 +- .../src/lj_target_arm64.h | 9 +- .../src/lj_target_mips.h | 2 +- .../src/lj_target_ppc.h | 3 +- .../src/lj_target_x86.h | 6 +- .../src/lj_trace.c | 155 +++++----- .../src/lj_trace.h | 2 +- .../src/lj_traceerr.h | 2 +- .../src/lj_udata.c | 2 +- .../src/lj_udata.h | 2 +- .../src/lj_vm.h | 24 +- .../src/lj_vmevent.c | 7 +- .../src/lj_vmevent.h | 24 +- .../src/lj_vmmath.c | 4 +- .../src/ljamalg.c | 2 +- .../src/lua.h | 0 .../src/lua.hpp | 0 .../src/luaconf.h | 12 +- .../src/luajit.c | 25 +- .../src/luajit_rolling.h | 4 +- .../src/lualib.h | 2 +- .../src/msvcbuild.bat | 35 ++- .../src/nxbuild.bat | 0 .../src/ps4build.bat | 0 .../src/ps5build.bat | 0 .../src/psvitabuild.bat | 0 .../src/vm_arm.dasc | 134 ++++++++- .../src/vm_arm64.dasc | 133 ++++++++- .../src/vm_mips.dasc | 121 +++++++- .../src/vm_mips64.dasc | 149 +++++++++- .../src/vm_ppc.dasc | 150 +++++++++- .../src/vm_x64.dasc | 127 +++++++- .../src/vm_x86.dasc | 95 +++++- .../src/xb1build.bat | 0 .../src/xedkbuild.bat | 0 235 files changed, 2667 insertions(+), 1232 deletions(-) delete mode 100644 source/lib/luajit-04dca791/.relver delete mode 100755 source/lib/luajit-04dca791/configure rename source/lib/{luajit-04dca791 => luajit-7152e154}/.gitattributes (100%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/.gitignore (100%) create mode 100644 source/lib/luajit-7152e154/.relver rename source/lib/{luajit-04dca791 => luajit-7152e154}/COPYRIGHT (98%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/Makefile (94%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/README (89%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/doc/bluequad-print.css (98%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/doc/bluequad.css (99%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/doc/contact.html (95%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/doc/ext_buffer.html (99%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/doc/ext_c_api.html (98%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/doc/ext_ffi.html (99%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/doc/ext_ffi_api.html (99%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/doc/ext_ffi_semantics.html (97%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/doc/ext_ffi_tutorial.html (99%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/doc/ext_jit.html (98%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/doc/ext_profiler.html (99%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/doc/extensions.html (99%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/doc/img/contact.png (100%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/doc/install.html (96%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/doc/luajit.html (97%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/doc/running.html (97%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/dynasm/dasm_arm.h (99%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/dynasm/dasm_arm.lua (99%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/dynasm/dasm_arm64.h (99%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/dynasm/dasm_arm64.lua (98%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/dynasm/dasm_mips.h (99%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/dynasm/dasm_mips.lua (99%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/dynasm/dasm_mips64.lua (89%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/dynasm/dasm_ppc.h (99%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/dynasm/dasm_ppc.lua (99%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/dynasm/dasm_proto.h (97%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/dynasm/dasm_x64.lua (89%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/dynasm/dasm_x86.h (99%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/dynasm/dasm_x86.lua (99%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/dynasm/dynasm.lua (99%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/etc/luajit.1 (97%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/etc/luajit.pc (100%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/src/.gitignore (100%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/src/Makefile (97%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/src/Makefile.dep (99%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/src/host/.gitignore (100%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/src/host/README (100%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/src/host/buildvm.c (99%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/src/host/buildvm.h (97%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/src/host/buildvm_asm.c (97%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/src/host/buildvm_fold.c (99%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/src/host/buildvm_lib.c (99%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/src/host/buildvm_libbc.h (100%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/src/host/buildvm_peobj.c (99%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/src/host/genlibbc.lua (99%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/src/host/genminilua.lua (99%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/src/host/genversion.lua (96%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/src/host/minilua.c (100%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/src/jit/.gitignore (100%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/src/jit/bc.lua (98%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/src/jit/bcsave.lua (95%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/src/jit/dis_arm.lua (99%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/src/jit/dis_arm64.lua (96%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/src/jit/dis_arm64be.lua (90%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/src/jit/dis_mips.lua (99%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/src/jit/dis_mips64.lua (91%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/src/jit/dis_mips64el.lua (91%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/src/jit/dis_mips64r6.lua (91%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/src/jit/dis_mips64r6el.lua (91%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/src/jit/dis_mipsel.lua (91%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/src/jit/dis_ppc.lua (99%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/src/jit/dis_x64.lua (91%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/src/jit/dis_x86.lua (98%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/src/jit/dump.lua (99%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/src/jit/p.lua (98%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/src/jit/v.lua (98%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/src/jit/zone.lua (94%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/src/lauxlib.h (100%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/src/lib_aux.c (99%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/src/lib_base.c (98%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/src/lib_bit.c (97%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/src/lib_buffer.c (99%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/src/lib_debug.c (99%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/src/lib_ffi.c (99%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/src/lib_init.c (95%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/src/lib_io.c (97%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/src/lib_jit.c (95%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/src/lib_math.c (98%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/src/lib_os.c (95%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/src/lib_package.c (99%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/src/lib_string.c (99%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/src/lib_table.c (99%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/src/lj_alloc.c (100%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/src/lj_alloc.h (100%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/src/lj_api.c (98%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/src/lj_arch.h (92%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/src/lj_asm.c (99%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/src/lj_asm.h (82%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/src/lj_asm_arm.h (96%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/src/lj_asm_arm64.h (96%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/src/lj_asm_mips.h (96%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/src/lj_asm_ppc.h (97%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/src/lj_asm_x86.h (96%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/src/lj_assert.c (89%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/src/lj_bc.c (75%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/src/lj_bc.h (97%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/src/lj_bcdump.h (96%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/src/lj_bcread.c (96%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/src/lj_bcwrite.c (96%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/src/lj_buf.c (99%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/src/lj_buf.h (98%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/src/lj_carith.c (98%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/src/lj_carith.h (94%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/src/lj_ccall.c (97%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/src/lj_ccall.h (98%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/src/lj_ccallback.c (92%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/src/lj_ccallback.h (89%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/src/lj_cconv.c (98%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/src/lj_cconv.h (96%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/src/lj_cdata.c (97%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/src/lj_cdata.h (97%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/src/lj_char.c (100%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/src/lj_char.h (100%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/src/lj_clib.c (99%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/src/lj_clib.h (91%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/src/lj_cparse.c (99%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/src/lj_cparse.h (97%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/src/lj_crecord.c (95%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/src/lj_crecord.h (96%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/src/lj_ctype.c (99%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/src/lj_ctype.h (99%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/src/lj_debug.c (99%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/src/lj_debug.h (96%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/src/lj_def.h (98%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/src/lj_dispatch.c (99%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/src/lj_dispatch.h (98%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/src/lj_emit_arm.h (96%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/src/lj_emit_arm64.h (93%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/src/lj_emit_mips.h (98%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/src/lj_emit_ppc.h (97%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/src/lj_emit_x86.h (97%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/src/lj_err.c (99%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/src/lj_err.h (96%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/src/lj_errmsg.h (99%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/src/lj_ff.h (82%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/src/lj_ffrecord.c (99%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/src/lj_ffrecord.h (89%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/src/lj_frame.h (99%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/src/lj_func.c (98%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/src/lj_func.h (91%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/src/lj_gc.c (97%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/src/lj_gc.h (98%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/src/lj_gdbjit.c (99%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/src/lj_gdbjit.h (86%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/src/lj_ir.c (96%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/src/lj_ir.h (99%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/src/lj_ircall.h (91%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/src/lj_iropt.h (98%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/src/lj_jit.h (96%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/src/lj_lex.c (99%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/src/lj_lex.h (98%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/src/lj_lib.c (98%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/src/lj_lib.h (98%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/src/lj_load.c (93%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/src/lj_mcode.c (54%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/src/lj_mcode.h (90%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/src/lj_meta.c (99%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/src/lj_meta.h (95%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/src/lj_obj.c (95%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/src/lj_obj.h (92%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/src/lj_opt_dce.c (96%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/src/lj_opt_fold.c (97%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/src/lj_opt_loop.c (99%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/src/lj_opt_mem.c (99%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/src/lj_opt_narrow.c (96%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/src/lj_opt_sink.c (99%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/src/lj_opt_split.c (97%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/src/lj_parse.c (98%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/src/lj_parse.h (84%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/src/lj_prng.c (98%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/src/lj_prng.h (90%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/src/lj_profile.c (99%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/src/lj_profile.h (85%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/src/lj_record.c (97%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/src/lj_record.h (96%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/src/lj_serialize.c (99%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/src/lj_serialize.h (92%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/src/lj_snap.c (99%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/src/lj_snap.h (93%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/src/lj_state.c (96%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/src/lj_state.h (94%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/src/lj_str.c (99%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/src/lj_str.h (93%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/src/lj_strfmt.c (98%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/src/lj_strfmt.h (98%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/src/lj_strfmt_num.c (93%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/src/lj_strscan.c (98%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/src/lj_strscan.h (94%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/src/lj_tab.c (96%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/src/lj_tab.h (95%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/src/lj_target.h (98%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/src/lj_target_arm.h (98%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/src/lj_target_arm64.h (97%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/src/lj_target_mips.h (99%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/src/lj_target_ppc.h (98%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/src/lj_target_x86.h (98%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/src/lj_trace.c (91%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/src/lj_trace.h (96%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/src/lj_traceerr.h (96%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/src/lj_udata.c (96%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/src/lj_udata.h (83%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/src/lj_vm.h (82%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/src/lj_vmevent.c (88%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/src/lj_vmevent.h (62%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/src/lj_vmmath.c (96%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/src/ljamalg.c (96%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/src/lua.h (100%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/src/lua.hpp (100%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/src/luaconf.h (92%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/src/luajit.c (96%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/src/luajit_rolling.h (96%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/src/lualib.h (94%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/src/msvcbuild.bat (79%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/src/nxbuild.bat (100%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/src/ps4build.bat (100%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/src/ps5build.bat (100%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/src/psvitabuild.bat (100%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/src/vm_arm.dasc (97%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/src/vm_arm64.dasc (97%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/src/vm_mips.dasc (97%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/src/vm_mips64.dasc (97%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/src/vm_ppc.dasc (97%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/src/vm_x64.dasc (98%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/src/vm_x86.dasc (98%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/src/xb1build.bat (100%) rename source/lib/{luajit-04dca791 => luajit-7152e154}/src/xedkbuild.bat (100%) diff --git a/source/cmake/libraries.cmake b/source/cmake/libraries.cmake index c73c81f1..d48905fd 100644 --- a/source/cmake/libraries.cmake +++ b/source/cmake/libraries.cmake @@ -11,7 +11,7 @@ set(FLB_PATH_LIB_MSGPACK "lib/msgpack-c") set(FLB_PATH_LIB_NGHTTP2 "lib/nghttp2-1.65.0") set(FLB_PATH_LIB_AVRO "lib/avro") set(FLB_PATH_LIB_CHUNKIO "lib/chunkio") -set(FLB_PATH_LIB_LUAJIT "lib/luajit-04dca791") +set(FLB_PATH_LIB_LUAJIT "lib/luajit-7152e154") set(FLB_PATH_LIB_MONKEY "lib/monkey") set(FLB_PATH_LIB_JSMN "lib/jsmn") set(FLB_PATH_LIB_SQLITE "lib/sqlite-amalgamation-3510100") diff --git a/source/lib/luajit-04dca791/.relver b/source/lib/luajit-04dca791/.relver deleted file mode 100644 index 37baebc8..00000000 --- a/source/lib/luajit-04dca791/.relver +++ /dev/null @@ -1 +0,0 @@ -1720049189 diff --git a/source/lib/luajit-04dca791/configure b/source/lib/luajit-04dca791/configure deleted file mode 100755 index e69de29b..00000000 diff --git a/source/lib/luajit-04dca791/.gitattributes b/source/lib/luajit-7152e154/.gitattributes similarity index 100% rename from source/lib/luajit-04dca791/.gitattributes rename to source/lib/luajit-7152e154/.gitattributes diff --git a/source/lib/luajit-04dca791/.gitignore b/source/lib/luajit-7152e154/.gitignore similarity index 100% rename from source/lib/luajit-04dca791/.gitignore rename to source/lib/luajit-7152e154/.gitignore diff --git a/source/lib/luajit-7152e154/.relver b/source/lib/luajit-7152e154/.relver new file mode 100644 index 00000000..c0dbef3f --- /dev/null +++ b/source/lib/luajit-7152e154/.relver @@ -0,0 +1 @@ +1765228720 diff --git a/source/lib/luajit-04dca791/COPYRIGHT b/source/lib/luajit-7152e154/COPYRIGHT similarity index 98% rename from source/lib/luajit-04dca791/COPYRIGHT rename to source/lib/luajit-7152e154/COPYRIGHT index d7620314..a192ae49 100644 --- a/source/lib/luajit-04dca791/COPYRIGHT +++ b/source/lib/luajit-7152e154/COPYRIGHT @@ -1,7 +1,7 @@ =============================================================================== LuaJIT -- a Just-In-Time Compiler for Lua. https://luajit.org/ -Copyright (C) 2005-2023 Mike Pall. All rights reserved. +Copyright (C) 2005-2025 Mike Pall. All rights reserved. Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/source/lib/luajit-04dca791/Makefile b/source/lib/luajit-7152e154/Makefile similarity index 94% rename from source/lib/luajit-04dca791/Makefile rename to source/lib/luajit-7152e154/Makefile index 6ae2c49d..c41b3345 100644 --- a/source/lib/luajit-04dca791/Makefile +++ b/source/lib/luajit-7152e154/Makefile @@ -10,7 +10,7 @@ # For MSVC, please follow the instructions given in src/msvcbuild.bat. # For MinGW and Cygwin, cd to src and run make with the Makefile there. # -# Copyright (C) 2005-2023 Mike Pall. See Copyright Notice in luajit.h +# Copyright (C) 2005-2025 Mike Pall. See Copyright Notice in luajit.h ############################################################################## MAJVER= 2 @@ -37,12 +37,13 @@ export MULTILIB= lib DPREFIX= $(DESTDIR)$(PREFIX) INSTALL_BIN= $(DPREFIX)/bin INSTALL_LIB= $(DPREFIX)/$(MULTILIB) -INSTALL_SHARE= $(DPREFIX)/share +INSTALL_SHARE_= $(PREFIX)/share +INSTALL_SHARE= $(DESTDIR)$(INSTALL_SHARE_) INSTALL_DEFINC= $(DPREFIX)/include/luajit-$(MMVERSION) INSTALL_INC= $(INSTALL_DEFINC) -INSTALL_LJLIBD= $(INSTALL_SHARE)/luajit-$(MMVERSION) -INSTALL_JITLIB= $(INSTALL_LJLIBD)/jit +export INSTALL_LJLIBD= $(INSTALL_SHARE_)/luajit-$(MMVERSION) +INSTALL_JITLIB= $(DESTDIR)$(INSTALL_LJLIBD)/jit INSTALL_LMODD= $(INSTALL_SHARE)/lua INSTALL_LMOD= $(INSTALL_LMODD)/$(ABIVER) INSTALL_CMODD= $(INSTALL_LIB)/lua @@ -71,7 +72,7 @@ INSTALL_PC= $(INSTALL_PKGCONFIG)/$(INSTALL_PCNAME) INSTALL_DIRS= $(INSTALL_BIN) $(INSTALL_LIB) $(INSTALL_INC) $(INSTALL_MAN) \ $(INSTALL_PKGCONFIG) $(INSTALL_JITLIB) $(INSTALL_LMOD) $(INSTALL_CMOD) -UNINSTALL_DIRS= $(INSTALL_JITLIB) $(INSTALL_LJLIBD) $(INSTALL_INC) \ +UNINSTALL_DIRS= $(INSTALL_JITLIB) $(DESTDIR)$(INSTALL_LJLIBD) $(INSTALL_INC) \ $(INSTALL_LMOD) $(INSTALL_LMODD) $(INSTALL_CMOD) $(INSTALL_CMODD) RM= rm -f @@ -109,7 +110,7 @@ else endif TARGET_SYS?= $(HOST_SYS) -ifeq (Darwin,$(TARGET_SYS)) +ifneq (,$(filter $(TARGET_SYS),Darwin iOS)) INSTALL_SONAME= $(INSTALL_DYLIBNAME) INSTALL_SOSHORT1= $(INSTALL_DYLIBSHORT1) INSTALL_SOSHORT2= $(INSTALL_DYLIBSHORT2) diff --git a/source/lib/luajit-04dca791/README b/source/lib/luajit-7152e154/README similarity index 89% rename from source/lib/luajit-04dca791/README rename to source/lib/luajit-7152e154/README index e4a69265..201f1b72 100644 --- a/source/lib/luajit-04dca791/README +++ b/source/lib/luajit-7152e154/README @@ -5,7 +5,7 @@ LuaJIT is a Just-In-Time (JIT) compiler for the Lua programming language. Project Homepage: https://luajit.org/ -LuaJIT is Copyright (C) 2005-2023 Mike Pall. +LuaJIT is Copyright (C) 2005-2025 Mike Pall. LuaJIT is free software, released under the MIT license. See full Copyright Notice in the COPYRIGHT file or in luajit.h. diff --git a/source/lib/luajit-04dca791/doc/bluequad-print.css b/source/lib/luajit-7152e154/doc/bluequad-print.css similarity index 98% rename from source/lib/luajit-04dca791/doc/bluequad-print.css rename to source/lib/luajit-7152e154/doc/bluequad-print.css index 4a139278..5bfda5d3 100644 --- a/source/lib/luajit-04dca791/doc/bluequad-print.css +++ b/source/lib/luajit-7152e154/doc/bluequad-print.css @@ -1,4 +1,4 @@ -/* Copyright (C) 2004-2023 Mike Pall. +/* Copyright (C) 2004-2025 Mike Pall. * * You are welcome to use the general ideas of this design for your own sites. * But please do not steal the stylesheet, the layout or the color scheme. diff --git a/source/lib/luajit-04dca791/doc/bluequad.css b/source/lib/luajit-7152e154/doc/bluequad.css similarity index 99% rename from source/lib/luajit-04dca791/doc/bluequad.css rename to source/lib/luajit-7152e154/doc/bluequad.css index 7399f625..5334a759 100644 --- a/source/lib/luajit-04dca791/doc/bluequad.css +++ b/source/lib/luajit-7152e154/doc/bluequad.css @@ -1,4 +1,4 @@ -/* Copyright (C) 2004-2023 Mike Pall. +/* Copyright (C) 2004-2025 Mike Pall. * * You are welcome to use the general ideas of this design for your own sites. * But please do not steal the stylesheet, the layout or the color scheme. diff --git a/source/lib/luajit-04dca791/doc/contact.html b/source/lib/luajit-7152e154/doc/contact.html similarity index 95% rename from source/lib/luajit-04dca791/doc/contact.html rename to source/lib/luajit-7152e154/doc/contact.html index cc4d8c72..d8d34a69 100644 --- a/source/lib/luajit-04dca791/doc/contact.html +++ b/source/lib/luajit-7152e154/doc/contact.html @@ -3,7 +3,7 @@ Contact - + @@ -94,7 +94,7 @@

          Contact

          Copyright

          All documentation is -Copyright © 2005-2023 Mike Pall. +Copyright © 2005-2025 Mike Pall.

          @@ -102,7 +102,7 @@

          Copyright

        • Integerrounddouble, float
          double, floattrunc int32_tnarrow(u)int8_t, (u)int16_t
          double, floattrunc int64_tnarrow *(u)int8_t, (u)int16_t, (u)int32_t
          double, floattrunc(u)int32_t, (u)int64_t
          double, floattruncint64_t
          double, floattrunc uint64_t ∪ int64_t →reinterpret *uint64_t
          double, floatroundfloat, double
          Numbern == 0 → 0, otherwise 1bool
          boolfalse → 0, true → 1Number
          Complex numberconvert real partNumber
          Numberconvert real part, imag = 0Complex number
          Numberconvert real part, imag = 0Complex number
          Complex numberconvert real and imag partComplex number
          Numberconvert scalar and replicateVector
          Vectorcopy (same size)Vector
          struct/uniontake base address (compat)Pointer
          Arraytake base address (compat)Pointer
          Arraytake base address (compat)Pointer
          Functiontake function addressFunction pointer
          Numberconvert via uintptr_t (cast)Pointer
          Pointerconvert address (compat/cast)Pointer
          Pointerconvert address (cast)Integer
          Pointerconvert address (compat/cast)Pointer
          Pointerconvert address (cast)Integer
          Arrayconvert base address (cast)Integer
          Arraycopy (compat)Array
          struct/unioncopy (identical type)struct/union

          @@ -384,6 +386,24 @@

          Conversions between C types

          Conversions not listed above will raise an error. E.g. it's not possible to convert a pointer to a complex number or vice versa.

          +

          +* Some conversions from double have a larger defined range to +allow for mixed-signedness conversions, which are common in C code. +E.g. initializing an int32_t field with 0xffffffff +or initializing an uint32_t or uint64_t field with +-1. Under strict conversion rules, these assignments would +give undefined results, since Lua numbers are doubles. The extended +ranges make these conversions defined. Lua numbers that are even +outside that range give an architecture-specific result. +

          +

          +Please note that doubles do not have the precision to represent the +whole signed or unsigned 64 bit integer range. Beware of large hex +constants in particular: e.g. 0xffffffffffffffff is a double +rounded up to 0x1p64 during parsing. This will not +convert to a defined 64 bit integer value. Use the 64 bit literal +syntax instead, i.e. 0xffffffffffffffffULL. +

          Conversions for vararg C function arguments

          @@ -1259,7 +1279,7 @@

          Current Status

        • src/msvcbuild.bat has settings for compiling LuaJIT with MSVC (Visual Studio).
        • @@ -195,10 +195,8 @@

          Installing LuaJIT

          Windows Systems

          Prerequisites

          -Either install one of the open source SDKs -(» MinGW or -» Cygwin), which come with a modified -GCC plus the required development headers. +Either install the open source SDK » MinGW, +which comes with a modified GCC plus the required development headers. Or install Microsoft's Visual Studio (MSVC).

          Building with MSVC

          @@ -217,9 +215,9 @@

          Building with MSVC

          For an x64 to ARM64 cross-build run this first: vcvarsall.bat x64_arm64

          -

          Building with MinGW or Cygwin

          +

          Building with MinGW

          -Open a command prompt window and make sure the MinGW or Cygwin programs +Open a command prompt window and make sure the MinGW programs are in your path. Then cd to the directory of the git repository. Then run this command for MinGW:

          @@ -227,12 +225,6 @@

          Building with MinGW or Cygwin

          mingw32-make

          -Or this command for Cygwin: -

          -
          -make
          -
          -

          Then follow the installation instructions below.

          Installing LuaJIT

          @@ -249,6 +241,19 @@

          Installing LuaJIT

          directory where luajit.exe is installed (see src/luaconf.h).

          +

          +The final directory layout should look like this: +

          +
          +├── luajit.exe
          +├── lua51.dll
          +├── <- put your own classic Lua/C API modules (*.dll) here
          +└── lua
          +    ├── <- put your own Lua modules (*.lua) here
          +    └── jit
          +        ├── bc.lua
          +        └── (etc …)
          +

          Cross-compiling LuaJIT

          @@ -572,7 +577,7 @@

          Hints for Distribution Maintainers

          recunroll2Min. unroll factor for true recursion
          sizemcode32Size of each machine code area in KBytes (Windows: 64K)
          sizemcode64Size of each machine code area in KBytes
          maxmcode512Max. total size of all machine code areas in KBytes
          maxmcode2048Max. total size of all machine code areas in KBytes