Skip to content

[FEATURE] Remove unnecessary filter for DataHistogram aggregation #4873

@LantaoJin

Description

@LantaoJin

Is your feature request related to a problem?
big5 query

source = big5
| where `@timestamp` >= '2023-01-01 00:00:00' and `@timestamp` < '2023-01-03 00:00:00'
| stats count() by span(`@timestamp`, 1m)

The current plan is

{
  "from": 0,
  "size": 0,
  "timeout": "1m",
  "query": {
    "bool": {
      "must": [
        {
          "range": {
            "@timestamp": {
              "from": "2023-01-01T00:00:00.000Z",
              "to": "2023-01-03T00:00:00.000Z",
              "include_lower": true,
              "include_upper": false,
              "format": "date_time",
              "boost": 1
            }
          }
        },
        {
          "exists": {
            "field": "@timestamp",
            "boost": 1
          }
        }
      ],
      "adjust_pure_negative": true,
      "boost": 1
    }
  },
  "_source": {
    "includes": [
      "@timestamp"
    ],
    "excludes": []
  },
  "aggregations": {
    "composite_buckets": {
      "composite": {
        "size": 10000,
        "sources": [
          {
            "span(`@timestamp`,1m)": {
              "date_histogram": {
                "field": "@timestamp",
                "missing_bucket": false,
                "order": "asc",
                "fixed_interval": "1m"
              }
            }
          }
        ]
      }
    }
  }
}

The expected plan is

{
  "from": 0,
  "size": 0,
  "timeout": "1m",
  "query": {
    "bool": {
      "range": {
      "@timestamp": {
        "from": "2023-01-01T00:00:00.000Z",
        "to": "2023-01-03T00:00:00.000Z",
        "include_lower": true,
        "include_upper": false,
        "format": "date_time",
        "boost": 1
      }
    }
  },
  "_source": {
    "includes": [
      "@timestamp"
    ],
    "excludes": []
  },
  "aggregations": {
    "composite_buckets": {
      "composite": {
        "size": 10000,
        "sources": [
          {
            "span(`@timestamp`,1m)": {
              "date_histogram": {
                "field": "@timestamp",
                "missing_bucket": false,
                "order": "asc",
                "fixed_interval": "1m"
              }
            }
          }
        ]
      }
    }
  }
}

The execution time reduced from 500ms to 20ms, which has 25x time improved.

Metadata

Metadata

Assignees

Labels

Type

No type
No fields configured for issues without a type.

Projects

Status

Done

Milestone

No milestone

Relationships

None yet

Development

No branches or pull requests

Issue actions