From 8ab0c89c0f9a6cafc306ae2ad3eb9cc4a33d67ce Mon Sep 17 00:00:00 2001 From: Yuanchun Shen Date: Mon, 28 Jul 2025 21:35:42 +0800 Subject: [PATCH 1/4] Add reverse op for compare ip to support pushdown Signed-off-by: Yuanchun Shen --- .../sql/calcite/utils/PPLReturnTypes.java | 2 + .../utils/UserDefinedFunctionUtils.java | 1 + .../function/PPLBuiltinOperators.java | 28 ++++++-- .../function/UserDefinedFunctionBuilder.java | 31 ++++++++- .../function/udf/ip/CompareIpFunction.java | 67 ++++++++++++++++--- .../function/udf/ip/IPFunction.java | 5 +- .../opensearch/request/PredicateAnalyzer.java | 15 +++++ 7 files changed, 131 insertions(+), 18 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/calcite/utils/PPLReturnTypes.java b/core/src/main/java/org/opensearch/sql/calcite/utils/PPLReturnTypes.java index 42a2853bee9..c3b3f0f4f8f 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/utils/PPLReturnTypes.java +++ b/core/src/main/java/org/opensearch/sql/calcite/utils/PPLReturnTypes.java @@ -24,6 +24,8 @@ private PPLReturnTypes() {} ReturnTypes.explicit(UserDefinedFunctionUtils.NULLABLE_TIME_UDT); public static final SqlReturnTypeInference TIMESTAMP_FORCE_NULLABLE = ReturnTypes.explicit(UserDefinedFunctionUtils.NULLABLE_TIMESTAMP_UDT); + public static final SqlReturnTypeInference IP_FORCE_NULLABLE = + ReturnTypes.explicit(UserDefinedFunctionUtils.NULLABLE_IP_UDT); public static SqlReturnTypeInference INTEGER_FORCE_NULLABLE = ReturnTypes.INTEGER.andThen(SqlTypeTransforms.FORCE_NULLABLE); public static SqlReturnTypeInference STRING_FORCE_NULLABLE = diff --git a/core/src/main/java/org/opensearch/sql/calcite/utils/UserDefinedFunctionUtils.java b/core/src/main/java/org/opensearch/sql/calcite/utils/UserDefinedFunctionUtils.java index 22d70a2fd89..50ea112f95d 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/utils/UserDefinedFunctionUtils.java +++ b/core/src/main/java/org/opensearch/sql/calcite/utils/UserDefinedFunctionUtils.java @@ -54,6 +54,7 @@ public class UserDefinedFunctionUtils { TYPE_FACTORY.createUDT(ExprUDT.EXPR_TIMESTAMP, true); public static final RelDataType NULLABLE_STRING = TYPE_FACTORY.createTypeWithNullability(TYPE_FACTORY.createSqlType(SqlTypeName.VARCHAR), true); + public static final RelDataType NULLABLE_IP_UDT = TYPE_FACTORY.createUDT(EXPR_IP, true); public static RelDataType nullablePatternAggList = createArrayType( diff --git a/core/src/main/java/org/opensearch/sql/expression/function/PPLBuiltinOperators.java b/core/src/main/java/org/opensearch/sql/expression/function/PPLBuiltinOperators.java index 257f0c04668..cd1f27ac951 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/PPLBuiltinOperators.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/PPLBuiltinOperators.java @@ -12,6 +12,7 @@ import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; import java.util.List; +import java.util.concurrent.atomic.AtomicReference; import java.util.function.Supplier; import org.apache.calcite.adapter.enumerable.NullPolicy; import org.apache.calcite.adapter.enumerable.RexImpTable; @@ -109,12 +110,19 @@ public class PPLBuiltinOperators extends ReflectiveSqlOperatorTable { // IP comparing functions public static final SqlOperator NOT_EQUALS_IP = - CompareIpFunction.notEquals().toUDF("NOT_EQUALS_IP"); - public static final SqlOperator EQUALS_IP = CompareIpFunction.equals().toUDF("EQUALS_IP"); - public static final SqlOperator GREATER_IP = CompareIpFunction.greater().toUDF("GREATER_IP"); - public static final SqlOperator GTE_IP = CompareIpFunction.greaterOrEquals().toUDF("GTE_IP"); - public static final SqlOperator LESS_IP = CompareIpFunction.less().toUDF("LESS_IP"); - public static final SqlOperator LTE_IP = CompareIpFunction.lessOrEquals().toUDF("LTE_IP"); + CompareIpFunction.notEquals() + .withReverse(lookupOperator("NOT_EQUALS_IP")) + .toUDF("NOT_EQUALS_IP"); + public static final SqlOperator EQUALS_IP = + CompareIpFunction.equals().withReverse(lookupOperator("EQUALS_IP")).toUDF("EQUALS_IP"); + public static final SqlOperator GREATER_IP = + CompareIpFunction.greater().withReverse(lookupOperator("LESS_IP")).toUDF("GREATER_IP"); + public static final SqlOperator GTE_IP = + CompareIpFunction.greaterOrEquals().withReverse(lookupOperator("LTE_IP")).toUDF("GTE_IP"); + public static final SqlOperator LESS_IP = + CompareIpFunction.less().withReverse(lookupOperator("GREATER_IP")).toUDF("LESS_IP"); + public static final SqlOperator LTE_IP = + CompareIpFunction.lessOrEquals().withReverse(lookupOperator("GTE_IP")).toUDF("LTE_IP"); // Condition function public static final SqlOperator EARLIEST = new EarliestFunction().toUDF("EARLIEST"); @@ -382,4 +390,12 @@ private static Expression invokeCalciteImplementor( method.setAccessible(true); return (Expression) method.invoke(rexCallImplementor, translator, call, List.of(field)); } + + private static Supplier lookupOperator(String name) { + return () -> { + AtomicReference ref = new AtomicReference<>(); + INSTANCE.get().lookUpOperators(name, false, ref::set); + return ref.get(); + }; + } } diff --git a/core/src/main/java/org/opensearch/sql/expression/function/UserDefinedFunctionBuilder.java b/core/src/main/java/org/opensearch/sql/expression/function/UserDefinedFunctionBuilder.java index 30dd644fc76..426ff1a00d2 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/UserDefinedFunctionBuilder.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/UserDefinedFunctionBuilder.java @@ -6,13 +6,17 @@ package org.opensearch.sql.expression.function; import java.util.Collections; +import java.util.function.Supplier; import org.apache.calcite.schema.ImplementableFunction; import org.apache.calcite.sql.SqlIdentifier; import org.apache.calcite.sql.SqlKind; +import org.apache.calcite.sql.SqlOperator; +import org.apache.calcite.sql.SqlSyntax; import org.apache.calcite.sql.parser.SqlParserPos; import org.apache.calcite.sql.type.InferTypes; import org.apache.calcite.sql.type.SqlReturnTypeInference; import org.apache.calcite.sql.validate.SqlUserDefinedFunction; +import org.checkerframework.checker.nullness.qual.Nullable; /** * The interface helps to construct a SqlUserDefinedFunction @@ -32,6 +36,18 @@ public interface UserDefinedFunctionBuilder { UDFOperandMetadata getOperandMetadata(); + default SqlKind getKind() { + return SqlKind.OTHER_FUNCTION; + } + + default SqlSyntax getSqlSyntax() { + return SqlSyntax.FUNCTION; + } + + default Supplier getReverse() { + return null; + } + default SqlUserDefinedFunction toUDF(String functionName) { return toUDF(functionName, true); } @@ -50,7 +66,7 @@ default SqlUserDefinedFunction toUDF(String functionName, boolean isDeterministi new SqlIdentifier(Collections.singletonList(functionName), null, SqlParserPos.ZERO, null); return new SqlUserDefinedFunction( udfLtrimIdentifier, - SqlKind.OTHER_FUNCTION, + getKind(), getReturnTypeInference(), InferTypes.ANY_NULLABLE, getOperandMetadata(), @@ -59,6 +75,19 @@ default SqlUserDefinedFunction toUDF(String functionName, boolean isDeterministi public boolean isDeterministic() { return isDeterministic; } + + @Override + public @Nullable SqlOperator reverse() { + if (getReverse() == null) { + return null; + } + return getReverse().get(); + } + + @Override + public SqlSyntax getSyntax() { + return getSqlSyntax(); + } }; } } diff --git a/core/src/main/java/org/opensearch/sql/expression/function/udf/ip/CompareIpFunction.java b/core/src/main/java/org/opensearch/sql/expression/function/udf/ip/CompareIpFunction.java index a580a5f18f4..89a2394ac45 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/udf/ip/CompareIpFunction.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/udf/ip/CompareIpFunction.java @@ -6,6 +6,7 @@ package org.opensearch.sql.expression.function.udf.ip; import java.util.List; +import java.util.function.Supplier; import org.apache.calcite.adapter.enumerable.NotNullImplementor; import org.apache.calcite.adapter.enumerable.NullPolicy; import org.apache.calcite.adapter.enumerable.RexToLixTranslator; @@ -13,6 +14,9 @@ import org.apache.calcite.linq4j.tree.Expression; import org.apache.calcite.linq4j.tree.Expressions; import org.apache.calcite.rex.RexCall; +import org.apache.calcite.sql.SqlKind; +import org.apache.calcite.sql.SqlOperator; +import org.apache.calcite.sql.SqlSyntax; import org.apache.calcite.sql.type.ReturnTypes; import org.apache.calcite.sql.type.SqlReturnTypeInference; import org.opensearch.sql.data.model.ExprIpValue; @@ -32,33 +36,80 @@ * */ public class CompareIpFunction extends ImplementorUDF { + private Supplier reverse; private CompareIpFunction(ComparisonType comparisonType) { super(new CompareImplementor(comparisonType), NullPolicy.ANY); + reverse = null; + } + + @Override + public SqlSyntax getSqlSyntax() { + return SqlSyntax.BINARY; } public static CompareIpFunction less() { - return new CompareIpFunction(ComparisonType.LESS); + return new CompareIpFunction(ComparisonType.LESS) { + @Override + public SqlKind getKind() { + return SqlKind.LESS_THAN; + } + }; + } + + @Override + public Supplier getReverse() { + return reverse; + } + + public CompareIpFunction withReverse(Supplier supplier) { + this.reverse = supplier; + return this; } public static CompareIpFunction greater() { - return new CompareIpFunction(ComparisonType.GREATER); + return new CompareIpFunction(ComparisonType.GREATER) { + @Override + public SqlKind getKind() { + return SqlKind.GREATER_THAN; + } + }; } public static CompareIpFunction lessOrEquals() { - return new CompareIpFunction(ComparisonType.LESS_OR_EQUAL); + return new CompareIpFunction(ComparisonType.LESS_OR_EQUAL) { + @Override + public SqlKind getKind() { + return SqlKind.LESS_THAN_OR_EQUAL; + } + }; } public static CompareIpFunction greaterOrEquals() { - return new CompareIpFunction(ComparisonType.GREATER_OR_EQUAL); + return new CompareIpFunction(ComparisonType.GREATER_OR_EQUAL) { + @Override + public SqlKind getKind() { + return SqlKind.GREATER_THAN_OR_EQUAL; + } + }; } public static CompareIpFunction equals() { - return new CompareIpFunction(ComparisonType.EQUALS); + return new CompareIpFunction(ComparisonType.EQUALS) { + @Override + public SqlKind getKind() { + return SqlKind.EQUALS; + } + }; } public static CompareIpFunction notEquals() { - return new CompareIpFunction(ComparisonType.NOT_EQUALS); + return new CompareIpFunction(ComparisonType.NOT_EQUALS) { + @Override + public SqlKind getKind() { + return SqlKind.NOT_EQUALS; + } + }; } @Override @@ -88,10 +139,10 @@ public Expression implement( translatedOperands.get(0), translatedOperands.get(1)); - return generateComparisonExpression(compareResult, comparisonType); + return evalCompareResult(compareResult, comparisonType); } - private static Expression generateComparisonExpression( + private static Expression evalCompareResult( Expression compareResult, ComparisonType comparisonType) { final ConstantExpression zero = Expressions.constant(0); return switch (comparisonType) { diff --git a/core/src/main/java/org/opensearch/sql/expression/function/udf/ip/IPFunction.java b/core/src/main/java/org/opensearch/sql/expression/function/udf/ip/IPFunction.java index e8143b9b384..baf6b8a37e1 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/udf/ip/IPFunction.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/udf/ip/IPFunction.java @@ -12,9 +12,9 @@ import org.apache.calcite.linq4j.tree.Expression; import org.apache.calcite.linq4j.tree.Expressions; import org.apache.calcite.rex.RexCall; -import org.apache.calcite.sql.type.ReturnTypes; import org.apache.calcite.sql.type.SqlReturnTypeInference; import org.opensearch.sql.calcite.utils.OpenSearchTypeFactory; +import org.opensearch.sql.calcite.utils.PPLReturnTypes; import org.opensearch.sql.data.model.ExprIpValue; import org.opensearch.sql.data.type.ExprCoreType; import org.opensearch.sql.data.type.ExprType; @@ -46,8 +46,7 @@ public UDFOperandMetadata getOperandMetadata() { @Override public SqlReturnTypeInference getReturnTypeInference() { - return ReturnTypes.explicit( - OpenSearchTypeFactory.TYPE_FACTORY.createUDT(OpenSearchTypeFactory.ExprUDT.EXPR_IP, true)); + return PPLReturnTypes.IP_FORCE_NULLABLE; } public static class CastImplementor diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/request/PredicateAnalyzer.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/request/PredicateAnalyzer.java index 58f5b4cf585..cf3e8155bfd 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/request/PredicateAnalyzer.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/request/PredicateAnalyzer.java @@ -84,8 +84,10 @@ import org.opensearch.index.query.ScriptQueryBuilder; import org.opensearch.script.Script; import org.opensearch.sql.calcite.plan.OpenSearchConstants; +import org.opensearch.sql.calcite.type.ExprIPType; import org.opensearch.sql.calcite.type.ExprSqlType; import org.opensearch.sql.calcite.utils.OpenSearchTypeFactory.ExprUDT; +import org.opensearch.sql.data.model.ExprIpValue; import org.opensearch.sql.data.model.ExprTimestampValue; import org.opensearch.sql.data.type.ExprCoreType; import org.opensearch.sql.data.type.ExprType; @@ -335,6 +337,8 @@ public Expression visitCall(RexCall call) { } }; case FUNCTION: + if (call.getOperator().getName().equalsIgnoreCase("IP")) {} + return visitRelevanceFunc(call); default: String message = @@ -1348,6 +1352,11 @@ private static String timestampValueForPushDown(String value) { // https://github.com/opensearch-project/sql/pull/3442 } + private static String ipValueForPushDown(String value) { + ExprIpValue exprIpValue = new ExprIpValue(value); + return exprIpValue.value(); + } + public static class ScriptQueryExpression extends QueryExpression { private final String code; private RexNode analyzedNode; @@ -1539,6 +1548,8 @@ Object value() { return timestampValueForPushDown(RexLiteral.stringValue(literal)); } else if (isString()) { return RexLiteral.stringValue(literal); + } else if (isIp()) { + return ipValueForPushDown(RexLiteral.stringValue(literal)); } else { return rawValue(); } @@ -1575,6 +1586,10 @@ public boolean isTimestamp() { return false; } + public boolean isIp() { + return literal.getType() instanceof ExprIPType; + } + long longValue() { return ((Number) literal.getValue()).longValue(); } From 9f42c12db4bf4bcded6b72f8b2877cd20ecf739d Mon Sep 17 00:00:00 2001 From: Yuanchun Shen Date: Wed, 30 Jul 2025 17:07:06 +0800 Subject: [PATCH 2/4] Pushdown ip comparison Signed-off-by: Yuanchun Shen --- .../utils/UserDefinedFunctionUtils.java | 1 + .../function/PPLBuiltinOperators.java | 3 ++- .../org/opensearch/sql/ppl/ExplainIT.java | 22 +++++++++++++++---- .../calcite/explain_filter_compare_ip.json | 2 +- .../explain_filter_compare_ipv6_swapped.json | 6 +++++ .../explain_filter_compare_ip.json | 2 +- .../explain_filter_compare_ipv6_swapped.json | 6 +++++ .../explain_filter_compare_ipv6_swapped.json | 17 ++++++++++++++ .../opensearch/request/PredicateAnalyzer.java | 16 +++++++++++--- 9 files changed, 65 insertions(+), 10 deletions(-) create mode 100644 integ-test/src/test/resources/expectedOutput/calcite/explain_filter_compare_ipv6_swapped.json create mode 100644 integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_filter_compare_ipv6_swapped.json create mode 100644 integ-test/src/test/resources/expectedOutput/ppl/explain_filter_compare_ipv6_swapped.json diff --git a/core/src/main/java/org/opensearch/sql/calcite/utils/UserDefinedFunctionUtils.java b/core/src/main/java/org/opensearch/sql/calcite/utils/UserDefinedFunctionUtils.java index 50ea112f95d..e4b29ebcb47 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/utils/UserDefinedFunctionUtils.java +++ b/core/src/main/java/org/opensearch/sql/calcite/utils/UserDefinedFunctionUtils.java @@ -77,6 +77,7 @@ public class UserDefinedFunctionUtils { ImmutableSet.of("match", "match_phrase", "match_bool_prefix", "match_phrase_prefix"); public static Set MULTI_FIELDS_RELEVANCE_FUNCTION_SET = ImmutableSet.of("simple_query_string", "query_string", "multi_match"); + public static String IP_FUNCTION_NAME = "IP"; public static RelBuilder.AggCall TransferUserDefinedAggFunction( Class UDAF, diff --git a/core/src/main/java/org/opensearch/sql/expression/function/PPLBuiltinOperators.java b/core/src/main/java/org/opensearch/sql/expression/function/PPLBuiltinOperators.java index cd1f27ac951..bd2541a1d73 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/PPLBuiltinOperators.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/PPLBuiltinOperators.java @@ -285,7 +285,8 @@ public class PPLBuiltinOperators extends ReflectiveSqlOperatorTable { .toUDF("TIME"); // IP cast function - public static final SqlOperator IP = new IPFunction().toUDF("IP"); + public static final SqlOperator IP = + new IPFunction().toUDF(UserDefinedFunctionUtils.IP_FUNCTION_NAME); public static final SqlOperator TIME_TO_SEC = adaptExprMethodToUDF( DateTimeFunctions.class, diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/ExplainIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/ExplainIT.java index 8d398b82832..fd8b42a288d 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/ExplainIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/ExplainIT.java @@ -13,6 +13,7 @@ import java.io.IOException; import java.util.Locale; +import org.junit.Assume; import org.junit.Ignore; import org.junit.jupiter.api.Test; import org.opensearch.client.ResponseException; @@ -93,11 +94,9 @@ public void testFilterByCompareStringTimePushDownExplain() throws IOException { @Test public void testFilterByCompareIPCoercion() throws IOException { - // Should automatically cast the string literal to IP. - // TODO: Push down IP comparison as range query with Calcite - String expected = loadExpectedPlan("explain_filter_compare_ip.json"); + // Should automatically cast the string literal to IP and pushdown it as a range query assertJsonEqualsIgnoreId( - expected, + loadExpectedPlan("explain_filter_compare_ip.json"), explainQueryToString( String.format( Locale.ROOT, @@ -105,6 +104,21 @@ public void testFilterByCompareIPCoercion() throws IOException { TEST_INDEX_WEBLOGS))); } + @Test + public void testFilterByCompareIpv6Swapped() throws IOException { + // Ignored in v2: the serialized string is unstable because of function properties + Assume.assumeTrue(isCalciteEnabled()); + // Test swapping ip and string. In v2, this is pushed down as script; + // with Calcite, it will still be pushed down as a range query + assertJsonEqualsIgnoreId( + loadExpectedPlan("explain_filter_compare_ipv6_swapped.json"), + explainQueryToString( + String.format( + Locale.ROOT, + "source=%s | where '::ffff:1234' <= host | fields host", + TEST_INDEX_WEBLOGS))); + } + @Test public void testWeekArgumentCoercion() throws IOException { String expected = loadExpectedPlan("explain_week_argument_coercion.json"); diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_compare_ip.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_compare_ip.json index ae49a08b653..316ef78c583 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_compare_ip.json +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_compare_ip.json @@ -1,6 +1,6 @@ { "calcite": { "logical": "LogicalProject(host=[$0])\n LogicalFilter(condition=[GREATER_IP($0, IP('1.1.1.1':VARCHAR))])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_weblogs]])\n", - "physical": "CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_weblogs]], PushDownContext=[[PROJECT->[host], SCRIPT->GREATER_IP($0, IP('1.1.1.1':VARCHAR))], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"timeout\":\"1m\",\"query\":{\"script\":{\"script\":{\"source\":\"{\\\"langType\\\":\\\"calcite\\\",\\\"script\\\":\\\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQAensKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJPVEhFUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJuYW1lIjogImhvc3QiCiAgICB9CiAgXSwKICAibnVsbGFibGUiOiBmYWxzZQp9dAAEZXhwcnQDfXsKICAib3AiOiB7CiAgICAibmFtZSI6ICJHUkVBVEVSX0lQIiwKICAgICJraW5kIjogIk9USEVSX0ZVTkNUSU9OIiwKICAgICJzeW50YXgiOiAiRlVOQ1RJT04iCiAgfSwKICAib3BlcmFuZHMiOiBbCiAgICB7CiAgICAgICJpbnB1dCI6IDAsCiAgICAgICJuYW1lIjogIiQwIgogICAgfSwKICAgIHsKICAgICAgIm9wIjogewogICAgICAgICJuYW1lIjogIklQIiwKICAgICAgICAia2luZCI6ICJPVEhFUl9GVU5DVElPTiIsCiAgICAgICAgInN5bnRheCI6ICJGVU5DVElPTiIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJsaXRlcmFsIjogIjEuMS4xLjEiLAogICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAgICAgICAibnVsbGFibGUiOiBmYWxzZSwKICAgICAgICAgICAgInByZWNpc2lvbiI6IC0xCiAgICAgICAgICB9CiAgICAgICAgfQogICAgICBdLAogICAgICAiY2xhc3MiOiAib3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uZnVuY3Rpb24uVXNlckRlZmluZWRGdW5jdGlvbkJ1aWxkZXIkMSIsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIk9USEVSIiwKICAgICAgICAibnVsbGFibGUiOiB0cnVlCiAgICAgIH0sCiAgICAgICJkZXRlcm1pbmlzdGljIjogdHJ1ZSwKICAgICAgImR5bmFtaWMiOiBmYWxzZQogICAgfQogIF0sCiAgImNsYXNzIjogIm9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLlVzZXJEZWZpbmVkRnVuY3Rpb25CdWlsZGVyJDEiLAogICJ0eXBlIjogewogICAgInR5cGUiOiAiQk9PTEVBTiIsCiAgICAibnVsbGFibGUiOiB0cnVlCiAgfSwKICAiZGV0ZXJtaW5pc3RpYyI6IHRydWUsCiAgImR5bmFtaWMiOiBmYWxzZQp9dAAKZmllbGRUeXBlc3NyABFqYXZhLnV0aWwuSGFzaE1hcAUH2sHDFmDRAwACRgAKbG9hZEZhY3RvckkACXRocmVzaG9sZHhwP0AAAAAAAAx3CAAAABAAAAABdAAEaG9zdH5yAClvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS50eXBlLkV4cHJDb3JlVHlwZQAAAAAAAAAAEgAAeHIADmphdmEubGFuZy5FbnVtAAAAAAAAAAASAAB4cHQAAklQeHg=\\\"}\",\"lang\":\"opensearch_compounded_script\",\"params\":{\"utcTimestamp\":1753756416891521000}},\"boost\":1.0}},\"_source\":{\"includes\":[\"host\"],\"excludes\":[]},\"sort\":[{\"_doc\":{\"order\":\"asc\"}}]}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n" + "physical": "CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_weblogs]], PushDownContext=[[PROJECT->[host], FILTER->GREATER_IP($0, IP('1.1.1.1':VARCHAR))], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"timeout\":\"1m\",\"query\":{\"range\":{\"host\":{\"from\":\"1.1.1.1\",\"to\":null,\"include_lower\":false,\"include_upper\":true,\"boost\":1.0}}},\"_source\":{\"includes\":[\"host\"],\"excludes\":[]},\"sort\":[{\"_doc\":{\"order\":\"asc\"}}]}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n" } } diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_compare_ipv6_swapped.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_compare_ipv6_swapped.json new file mode 100644 index 00000000000..f7a56d47689 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_compare_ipv6_swapped.json @@ -0,0 +1,6 @@ +{ + "calcite": { + "logical": "LogicalProject(host=[$0])\n LogicalFilter(condition=[LTE_IP(IP('::ffff:1234':VARCHAR), $0)])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_weblogs]])\n", + "physical": "CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_weblogs]], PushDownContext=[[PROJECT->[host], FILTER->LTE_IP(IP('::ffff:1234':VARCHAR), $0)], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"timeout\":\"1m\",\"query\":{\"range\":{\"host\":{\"from\":\"::ffff:1234\",\"to\":null,\"include_lower\":true,\"include_upper\":true,\"boost\":1.0}}},\"_source\":{\"includes\":[\"host\"],\"excludes\":[]},\"sort\":[{\"_doc\":{\"order\":\"asc\"}}]}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n" + } +} diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_filter_compare_ip.json b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_filter_compare_ip.json index 4d275fcd4ab..84fa7a344d8 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_filter_compare_ip.json +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_filter_compare_ip.json @@ -3,4 +3,4 @@ "logical": "LogicalProject(host=[$0])\n LogicalFilter(condition=[GREATER_IP($0, IP('1.1.1.1':VARCHAR))])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_weblogs]])\n", "physical": "EnumerableCalc(expr#0..11=[{inputs}], expr#12=['1.1.1.1':VARCHAR], expr#13=[IP($t12)], expr#14=[GREATER_IP($t0, $t13)], host=[$t0], $condition=[$t14])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_weblogs]])\n" } -} \ No newline at end of file +} diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_filter_compare_ipv6_swapped.json b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_filter_compare_ipv6_swapped.json new file mode 100644 index 00000000000..cd11ff4dd2b --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_filter_compare_ipv6_swapped.json @@ -0,0 +1,6 @@ +{ + "calcite": { + "logical": "LogicalProject(host=[$0])\n LogicalFilter(condition=[LTE_IP(IP('::ffff:1234':VARCHAR), $0)])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_weblogs]])\n", + "physical": "EnumerableCalc(expr#0..11=[{inputs}], expr#12=['::ffff:1234':VARCHAR], expr#13=[IP($t12)], expr#14=[LTE_IP($t13, $t0)], host=[$t0], $condition=[$t14])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_weblogs]])\n" + } +} diff --git a/integ-test/src/test/resources/expectedOutput/ppl/explain_filter_compare_ipv6_swapped.json b/integ-test/src/test/resources/expectedOutput/ppl/explain_filter_compare_ipv6_swapped.json new file mode 100644 index 00000000000..120e8330c17 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/ppl/explain_filter_compare_ipv6_swapped.json @@ -0,0 +1,17 @@ +{ + "root": { + "name": "ProjectOperator", + "description": { + "fields": "[host]" + }, + "children": [ + { + "name": "OpenSearchIndexScan", + "description": { + "request": "OpenSearchQueryRequest(indexName=opensearch-sql_test_index_weblogs, sourceBuilder={\"from\":0,\"size\":10000,\"timeout\":\"1m\",\"query\":{\"script\":{\"script\":{\"source\":\"{\\\"langType\\\":\\\"v2\\\",\\\"script\\\":\\\"rO0ABXNyADRvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5mdW5jdGlvbi5GdW5jdGlvbkRTTCQzHWCy3iOeynUCAAVMAA12YWwkYXJndW1lbnRzdAAQTGphdmEvdXRpbC9MaXN0O0wADHZhbCRmdW5jdGlvbnQAQExvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9TZXJpYWxpemFibGVUcmlGdW5jdGlvbjtMABB2YWwkZnVuY3Rpb25OYW1ldAA1TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uTmFtZTtMABZ2YWwkZnVuY3Rpb25Qcm9wZXJ0aWVzdAA7TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMAA52YWwkcmV0dXJuVHlwZXQAJ0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS90eXBlL0V4cHJUeXBlO3hyADBvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5GdW5jdGlvbkV4cHJlc3Npb26yKjDT3HVqewIAAkwACWFyZ3VtZW50c3EAfgABTAAMZnVuY3Rpb25OYW1lcQB+AAN4cHNyABNqYXZhLnV0aWwuQXJyYXlMaXN0eIHSHZnHYZ0DAAFJAARzaXpleHAAAAACdwQAAAACc3IANG9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uRFNMJDI9znTUIQE9bAIABUwADXZhbCRhcmd1bWVudHNxAH4AAUwADHZhbCRmdW5jdGlvbnQAP0xvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9TZXJpYWxpemFibGVCaUZ1bmN0aW9uO0wAEHZhbCRmdW5jdGlvbk5hbWVxAH4AA0wAFnZhbCRmdW5jdGlvblByb3BlcnRpZXNxAH4ABEwADnZhbCRyZXR1cm5UeXBlcQB+AAV4cQB+AAZzcQB+AAgAAAABdwQAAAABc3IAL29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLkxpdGVyYWxFeHByZXNzaW9uRUIt8IzHgiQCAAFMAAlleHByVmFsdWV0AClMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3hwc3IALW9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLm1vZGVsLkV4cHJTdHJpbmdWYWx1ZQBBMiVziQ4TAgABTAAFdmFsdWV0ABJMamF2YS9sYW5nL1N0cmluZzt4cgAvb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEubW9kZWwuQWJzdHJhY3RFeHByVmFsdWXJa7V2BhREigIAAHhwdAALOjpmZmZmOjEyMzR4c3IAM29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uTmFtZQuoOE3O9meXAgABTAAMZnVuY3Rpb25OYW1lcQB+ABJ4cHQACmNhc3RfdG9faXBxAH4ADXNyACFqYXZhLmxhbmcuaW52b2tlLlNlcmlhbGl6ZWRMYW1iZGFvYdCULCk2hQIACkkADmltcGxNZXRob2RLaW5kWwAMY2FwdHVyZWRBcmdzdAATW0xqYXZhL2xhbmcvT2JqZWN0O0wADmNhcHR1cmluZ0NsYXNzdAARTGphdmEvbGFuZy9DbGFzcztMABhmdW5jdGlvbmFsSW50ZXJmYWNlQ2xhc3NxAH4AEkwAHWZ1bmN0aW9uYWxJbnRlcmZhY2VNZXRob2ROYW1lcQB+ABJMACJmdW5jdGlvbmFsSW50ZXJmYWNlTWV0aG9kU2lnbmF0dXJlcQB+ABJMAAlpbXBsQ2xhc3NxAH4AEkwADmltcGxNZXRob2ROYW1lcQB+ABJMABNpbXBsTWV0aG9kU2lnbmF0dXJlcQB+ABJMABZpbnN0YW50aWF0ZWRNZXRob2RUeXBlcQB+ABJ4cAAAAAZ1cgATW0xqYXZhLmxhbmcuT2JqZWN0O5DOWJ8QcylsAgAAeHAAAAABc3EAfgAZAAAABnVxAH4AHQAAAAFzcQB+ABkAAAAGdXEAfgAdAAAAAHZyAEBvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5vcGVyYXRvci5jb252ZXJ0LlR5cGVDYXN0T3BlcmF0b3JzAAAAAAAAAAAAAAB4cHQAO29yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL1NlcmlhbGl6YWJsZUZ1bmN0aW9udAAFYXBwbHl0ACYoTGphdmEvbGFuZy9PYmplY3Q7KUxqYXZhL2xhbmcvT2JqZWN0O3QAQG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL29wZXJhdG9yL2NvbnZlcnQvVHlwZUNhc3RPcGVyYXRvcnN0ABpsYW1iZGEkY2FzdFRvSXAkMTJjN2RjNDgkMXQAVChMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlOylMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3EAfgAqdnIAMm9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uRFNMAAAAAAAAAAAAAAB4cHEAfgAlcQB+ACZxAH4AJ3QAMm9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uRFNMdAAlbGFtYmRhJG51bGxNaXNzaW5nSGFuZGxpbmckODc4MDY5YzgkMXQAkShMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlRnVuY3Rpb247TG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTspTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTtxAH4AKnEAfgAsdAA9b3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbnEAfgAmdAA4KExqYXZhL2xhbmcvT2JqZWN0O0xqYXZhL2xhbmcvT2JqZWN0OylMamF2YS9sYW5nL09iamVjdDtxAH4ALXQAFmxhbWJkYSRpbXBsJDhkNTg2Y2RjJDF0AMwoTG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL1NlcmlhbGl6YWJsZUZ1bmN0aW9uO0xvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9GdW5jdGlvblByb3BlcnRpZXM7TG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTspTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTt0AI8oTG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlOylMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3EAfgAXc3IAOW9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uUHJvcGVydGllc888WWObqPmzAgADTAANY3VycmVudFpvbmVJZHQAEkxqYXZhL3RpbWUvWm9uZUlkO0wACm5vd0luc3RhbnR0ABNMamF2YS90aW1lL0luc3RhbnQ7TAAJcXVlcnlUeXBldAAnTG9yZy9vcGVuc2VhcmNoL3NxbC9leGVjdXRvci9RdWVyeVR5cGU7eHBzcgANamF2YS50aW1lLlNlcpVdhLobIkiyDAAAeHB3AggAeHNxAH4AOncNAgAAAABoifJmKvyT4Hh+cgAlb3JnLm9wZW5zZWFyY2guc3FsLmV4ZWN1dG9yLlF1ZXJ5VHlwZQAAAAAAAAAAEgAAeHIADmphdmEubGFuZy5FbnVtAAAAAAAAAAASAAB4cHQAA1BQTH5yAClvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS50eXBlLkV4cHJDb3JlVHlwZQAAAAAAAAAAEgAAeHEAfgA+dAACSVBzcgAxb3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uUmVmZXJlbmNlRXhwcmVzc2lvbqtE71wSB4XWAgAETAAEYXR0cnEAfgASTAAFcGF0aHNxAH4AAUwAB3Jhd1BhdGhxAH4AEkwABHR5cGVxAH4ABXhwdAAEaG9zdHNyABpqYXZhLnV0aWwuQXJyYXlzJEFycmF5TGlzdNmkPL7NiAbSAgABWwABYXEAfgAaeHB1cgATW0xqYXZhLmxhbmcuU3RyaW5nO63SVufpHXtHAgAAeHAAAAABcQB+AEZxAH4ARnEAfgBCeHNxAH4AFnQAAjw9cQB+AAlzcQB+ABkAAAAGdXEAfgAdAAAAAXNxAH4AGQAAAAZ1cQB+AB0AAAABc3EAfgAZAAAABnVxAH4AHQAAAAB2cgBJb3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24ub3BlcmF0b3IucHJlZGljYXRlLkJpbmFyeVByZWRpY2F0ZU9wZXJhdG9ycwAAAAAAAAAAAAAAeHBxAH4AMHEAfgAmcQB+ADF0AElvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9vcGVyYXRvci9wcmVkaWNhdGUvQmluYXJ5UHJlZGljYXRlT3BlcmF0b3JzdAAVbGFtYmRhJGx0ZSQ5NTA0OGZjMSQxdAB9KExvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7TG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTspTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTtxAH4AV3EAfgAscQB+ADBxAH4AJnEAfgAxcQB+AC10ACVsYW1iZGEkbnVsbE1pc3NpbmdIYW5kbGluZyRhNTAwNTI4MSQxdAC8KExvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9TZXJpYWxpemFibGVCaUZ1bmN0aW9uO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7TG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTspTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTtxAH4AV3EAfgAsdAA+b3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlVHJpRnVuY3Rpb25xAH4AJnQASihMamF2YS9sYW5nL09iamVjdDtMamF2YS9sYW5nL09iamVjdDtMamF2YS9sYW5nL09iamVjdDspTGphdmEvbGFuZy9PYmplY3Q7cQB+AC10ABZsYW1iZGEkaW1wbCRhMGZiMzRkNCQxdAD3KExvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9TZXJpYWxpemFibGVCaUZ1bmN0aW9uO0xvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9GdW5jdGlvblByb3BlcnRpZXM7TG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlOylMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3QAuChMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vRnVuY3Rpb25Qcm9wZXJ0aWVzO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7TG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTspTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTtxAH4AS3EAfgA5fnEAfgBBdAAHQk9PTEVBTg==\\\"}\",\"lang\":\"opensearch_compounded_script\"},\"boost\":1.0}},\"_source\":{\"includes\":[\"host\"],\"excludes\":[]},\"sort\":[{\"_doc\":{\"order\":\"asc\"}}]}, needClean=true, searchDone=false, pitId=*, cursorKeepAlive=1m, searchAfter=null, searchResponse=null)" + }, + "children": [] + } + ] + } +} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/request/PredicateAnalyzer.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/request/PredicateAnalyzer.java index cf3e8155bfd..219130b5c58 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/request/PredicateAnalyzer.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/request/PredicateAnalyzer.java @@ -87,6 +87,7 @@ import org.opensearch.sql.calcite.type.ExprIPType; import org.opensearch.sql.calcite.type.ExprSqlType; import org.opensearch.sql.calcite.utils.OpenSearchTypeFactory.ExprUDT; +import org.opensearch.sql.calcite.utils.UserDefinedFunctionUtils; import org.opensearch.sql.data.model.ExprIpValue; import org.opensearch.sql.data.model.ExprTimestampValue; import org.opensearch.sql.data.type.ExprCoreType; @@ -337,9 +338,14 @@ public Expression visitCall(RexCall call) { } }; case FUNCTION: - if (call.getOperator().getName().equalsIgnoreCase("IP")) {} - - return visitRelevanceFunc(call); + String functionName = call.getOperator().getName().toLowerCase(Locale.ROOT); + if (functionName.equalsIgnoreCase(UserDefinedFunctionUtils.IP_FUNCTION_NAME)) { + return visitIpFunction(call); + } else if (SINGLE_FIELD_RELEVANCE_FUNCTION_SET.contains(functionName) + || MULTI_FIELDS_RELEVANCE_FUNCTION_SET.contains(functionName)) { + return visitRelevanceFunc(call); + } + // fall through default: String message = format(Locale.ROOT, "Unsupported syntax [%s] for call: [%s]", syntax, call); @@ -388,6 +394,10 @@ private QueryExpression visitRelevanceFunc(RexCall call) { format(Locale.ROOT, "Unsupported search relevance function: [%s]", funcName)); } + private LiteralExpression visitIpFunction(RexCall call) { + return new LiteralExpression((RexLiteral) call.getOperands().getFirst()); + } + @FunctionalInterface private interface SingleFieldRelevanceFunctionHandler { QueryExpression apply(NamedFieldExpression field, String query, Map opts); From 3f7c6a6253ae72be1d121b10638bc21ce6e3fdb3 Mon Sep 17 00:00:00 2001 From: Yuanchun Shen Date: Tue, 5 Aug 2025 11:29:10 +0800 Subject: [PATCH 3/4] Refactor CompareIpFunction to use SqlKind directly Signed-off-by: Yuanchun Shen --- .../function/PPLBuiltinOperators.java | 15 ++- .../function/udf/ip/CompareIpFunction.java | 108 ++++++++---------- 2 files changed, 62 insertions(+), 61 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/expression/function/PPLBuiltinOperators.java b/core/src/main/java/org/opensearch/sql/expression/function/PPLBuiltinOperators.java index bd2541a1d73..4a13124fd91 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/PPLBuiltinOperators.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/PPLBuiltinOperators.java @@ -392,10 +392,23 @@ private static Expression invokeCalciteImplementor( return (Expression) method.invoke(rexCallImplementor, translator, call, List.of(field)); } + /** + * Creates a lazy supplier for looking up a SQL operator by name. + * + *

This method enables operators to reference each other, even when they have circular + * dependencies. Instead of looking up the operator immediately, it returns a supplier that will + * perform the lookup only when needed. + * + *

For example, {@code LESS_IP} needs to reference {@code GREATER_IP} as its reverse, and vice + * versa. Using this lazy approach, both can be defined before either is used. + * + * @param name The name of the operator to look up + * @return A supplier that will look up the operator when called + */ private static Supplier lookupOperator(String name) { return () -> { AtomicReference ref = new AtomicReference<>(); - INSTANCE.get().lookUpOperators(name, false, ref::set); + instance().lookUpOperators(name, false, ref::set); return ref.get(); }; } diff --git a/core/src/main/java/org/opensearch/sql/expression/function/udf/ip/CompareIpFunction.java b/core/src/main/java/org/opensearch/sql/expression/function/udf/ip/CompareIpFunction.java index 89a2394ac45..70cbadbeee5 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/udf/ip/CompareIpFunction.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/udf/ip/CompareIpFunction.java @@ -6,6 +6,7 @@ package org.opensearch.sql.expression.function.udf.ip; import java.util.List; +import java.util.Locale; import java.util.function.Supplier; import org.apache.calcite.adapter.enumerable.NotNullImplementor; import org.apache.calcite.adapter.enumerable.NullPolicy; @@ -36,11 +37,14 @@ * */ public class CompareIpFunction extends ImplementorUDF { + private final SqlKind kind; private Supplier reverse; - private CompareIpFunction(ComparisonType comparisonType) { - super(new CompareImplementor(comparisonType), NullPolicy.ANY); - reverse = null; + private CompareIpFunction(SqlKind kind) { + super(new CompareImplementor(kind), NullPolicy.ANY); + this.kind = kind; + // Will be set later + this.reverse = null; } @Override @@ -48,13 +52,9 @@ public SqlSyntax getSqlSyntax() { return SqlSyntax.BINARY; } - public static CompareIpFunction less() { - return new CompareIpFunction(ComparisonType.LESS) { - @Override - public SqlKind getKind() { - return SqlKind.LESS_THAN; - } - }; + @Override + public SqlKind getKind() { + return kind; } @Override @@ -62,54 +62,50 @@ public Supplier getReverse() { return reverse; } + /** + * Sets the reverse operator supplier for this comparison function. + * + *

This method is used to establish the reversed relationship between comparison operators + * (e.g., "less than" and "greater than"). When the query optimizer normalizes expressions, it may + * need to transform "b > a" to "a < b". + * + *

E.g. in the {@code hashCode} method of {@link org.apache.calcite.rex.RexNormalize}#L115, it + * always converts B [comparator] A to A [reverse_comparator] B if the ordinal of + * the reverse of the comparator is smaller. + * + *

IP comparison functions use this to inform the optimizer that ip_less_than is the reverse of + * ip_greater_than, allowing for proper query normalization. + * + * @param supplier The supplier that provides the reverse SQL operator + * @return This CompareIpFunction instance for method chaining + */ public CompareIpFunction withReverse(Supplier supplier) { this.reverse = supplier; return this; } + public static CompareIpFunction less() { + return new CompareIpFunction(SqlKind.LESS_THAN); + } + public static CompareIpFunction greater() { - return new CompareIpFunction(ComparisonType.GREATER) { - @Override - public SqlKind getKind() { - return SqlKind.GREATER_THAN; - } - }; + return new CompareIpFunction(SqlKind.GREATER_THAN); } public static CompareIpFunction lessOrEquals() { - return new CompareIpFunction(ComparisonType.LESS_OR_EQUAL) { - @Override - public SqlKind getKind() { - return SqlKind.LESS_THAN_OR_EQUAL; - } - }; + return new CompareIpFunction(SqlKind.LESS_THAN_OR_EQUAL); } public static CompareIpFunction greaterOrEquals() { - return new CompareIpFunction(ComparisonType.GREATER_OR_EQUAL) { - @Override - public SqlKind getKind() { - return SqlKind.GREATER_THAN_OR_EQUAL; - } - }; + return new CompareIpFunction(SqlKind.GREATER_THAN_OR_EQUAL); } public static CompareIpFunction equals() { - return new CompareIpFunction(ComparisonType.EQUALS) { - @Override - public SqlKind getKind() { - return SqlKind.EQUALS; - } - }; + return new CompareIpFunction(SqlKind.EQUALS); } public static CompareIpFunction notEquals() { - return new CompareIpFunction(ComparisonType.NOT_EQUALS) { - @Override - public SqlKind getKind() { - return SqlKind.NOT_EQUALS; - } - }; + return new CompareIpFunction(SqlKind.NOT_EQUALS); } @Override @@ -123,10 +119,10 @@ public UDFOperandMetadata getOperandMetadata() { } public static class CompareImplementor implements NotNullImplementor { - private final ComparisonType comparisonType; + private final SqlKind compareType; - public CompareImplementor(ComparisonType comparisonType) { - this.comparisonType = comparisonType; + public CompareImplementor(SqlKind compareType) { + this.compareType = compareType; } @Override @@ -139,19 +135,20 @@ public Expression implement( translatedOperands.get(0), translatedOperands.get(1)); - return evalCompareResult(compareResult, comparisonType); + return evalCompareResult(compareResult, compareType); } - private static Expression evalCompareResult( - Expression compareResult, ComparisonType comparisonType) { + private static Expression evalCompareResult(Expression compareResult, SqlKind compareType) { final ConstantExpression zero = Expressions.constant(0); - return switch (comparisonType) { + return switch (compareType) { case EQUALS -> Expressions.equal(compareResult, zero); case NOT_EQUALS -> Expressions.notEqual(compareResult, zero); - case LESS -> Expressions.lessThan(compareResult, zero); - case LESS_OR_EQUAL -> Expressions.lessThanOrEqual(compareResult, zero); - case GREATER -> Expressions.greaterThan(compareResult, zero); - case GREATER_OR_EQUAL -> Expressions.greaterThanOrEqual(compareResult, zero); + case LESS_THAN -> Expressions.lessThan(compareResult, zero); + case LESS_THAN_OR_EQUAL -> Expressions.lessThanOrEqual(compareResult, zero); + case GREATER_THAN -> Expressions.greaterThan(compareResult, zero); + case GREATER_THAN_OR_EQUAL -> Expressions.greaterThanOrEqual(compareResult, zero); + default -> throw new UnsupportedOperationException( + String.format(Locale.ROOT, "Unsupported compare type: %s", compareType)); }; } @@ -170,13 +167,4 @@ private static ExprIpValue toExprIpValue(Object obj) { throw new IllegalArgumentException("Invalid IP type: " + obj); } } - - public enum ComparisonType { - EQUALS, - NOT_EQUALS, - LESS, - LESS_OR_EQUAL, - GREATER, - GREATER_OR_EQUAL - } } From a61ac6182d1e2e0629545b4479563d852dcd7ac2 Mon Sep 17 00:00:00 2001 From: Yuanchun Shen Date: Mon, 11 Aug 2025 18:07:59 +0800 Subject: [PATCH 4/4] Simplify the overriding of reverse() for IP comparators Signed-off-by: Yuanchun Shen --- .../function/PPLBuiltinOperators.java | 41 ++------- .../function/UserDefinedFunctionBuilder.java | 31 +------ .../function/udf/ip/CompareIpFunction.java | 86 ++++++++++--------- 3 files changed, 52 insertions(+), 106 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/expression/function/PPLBuiltinOperators.java b/core/src/main/java/org/opensearch/sql/expression/function/PPLBuiltinOperators.java index 4a13124fd91..38f3eeaa889 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/PPLBuiltinOperators.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/PPLBuiltinOperators.java @@ -12,7 +12,6 @@ import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; import java.util.List; -import java.util.concurrent.atomic.AtomicReference; import java.util.function.Supplier; import org.apache.calcite.adapter.enumerable.NullPolicy; import org.apache.calcite.adapter.enumerable.RexImpTable; @@ -110,19 +109,12 @@ public class PPLBuiltinOperators extends ReflectiveSqlOperatorTable { // IP comparing functions public static final SqlOperator NOT_EQUALS_IP = - CompareIpFunction.notEquals() - .withReverse(lookupOperator("NOT_EQUALS_IP")) - .toUDF("NOT_EQUALS_IP"); - public static final SqlOperator EQUALS_IP = - CompareIpFunction.equals().withReverse(lookupOperator("EQUALS_IP")).toUDF("EQUALS_IP"); - public static final SqlOperator GREATER_IP = - CompareIpFunction.greater().withReverse(lookupOperator("LESS_IP")).toUDF("GREATER_IP"); - public static final SqlOperator GTE_IP = - CompareIpFunction.greaterOrEquals().withReverse(lookupOperator("LTE_IP")).toUDF("GTE_IP"); - public static final SqlOperator LESS_IP = - CompareIpFunction.less().withReverse(lookupOperator("GREATER_IP")).toUDF("LESS_IP"); - public static final SqlOperator LTE_IP = - CompareIpFunction.lessOrEquals().withReverse(lookupOperator("GTE_IP")).toUDF("LTE_IP"); + CompareIpFunction.notEquals().toUDF("NOT_EQUALS_IP"); + public static final SqlOperator EQUALS_IP = CompareIpFunction.equals().toUDF("EQUALS_IP"); + public static final SqlOperator GREATER_IP = CompareIpFunction.greater().toUDF("GREATER_IP"); + public static final SqlOperator GTE_IP = CompareIpFunction.greaterOrEquals().toUDF("GTE_IP"); + public static final SqlOperator LESS_IP = CompareIpFunction.less().toUDF("LESS_IP"); + public static final SqlOperator LTE_IP = CompareIpFunction.lessOrEquals().toUDF("LTE_IP"); // Condition function public static final SqlOperator EARLIEST = new EarliestFunction().toUDF("EARLIEST"); @@ -391,25 +383,4 @@ private static Expression invokeCalciteImplementor( method.setAccessible(true); return (Expression) method.invoke(rexCallImplementor, translator, call, List.of(field)); } - - /** - * Creates a lazy supplier for looking up a SQL operator by name. - * - *

This method enables operators to reference each other, even when they have circular - * dependencies. Instead of looking up the operator immediately, it returns a supplier that will - * perform the lookup only when needed. - * - *

For example, {@code LESS_IP} needs to reference {@code GREATER_IP} as its reverse, and vice - * versa. Using this lazy approach, both can be defined before either is used. - * - * @param name The name of the operator to look up - * @return A supplier that will look up the operator when called - */ - private static Supplier lookupOperator(String name) { - return () -> { - AtomicReference ref = new AtomicReference<>(); - instance().lookUpOperators(name, false, ref::set); - return ref.get(); - }; - } } diff --git a/core/src/main/java/org/opensearch/sql/expression/function/UserDefinedFunctionBuilder.java b/core/src/main/java/org/opensearch/sql/expression/function/UserDefinedFunctionBuilder.java index 426ff1a00d2..30dd644fc76 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/UserDefinedFunctionBuilder.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/UserDefinedFunctionBuilder.java @@ -6,17 +6,13 @@ package org.opensearch.sql.expression.function; import java.util.Collections; -import java.util.function.Supplier; import org.apache.calcite.schema.ImplementableFunction; import org.apache.calcite.sql.SqlIdentifier; import org.apache.calcite.sql.SqlKind; -import org.apache.calcite.sql.SqlOperator; -import org.apache.calcite.sql.SqlSyntax; import org.apache.calcite.sql.parser.SqlParserPos; import org.apache.calcite.sql.type.InferTypes; import org.apache.calcite.sql.type.SqlReturnTypeInference; import org.apache.calcite.sql.validate.SqlUserDefinedFunction; -import org.checkerframework.checker.nullness.qual.Nullable; /** * The interface helps to construct a SqlUserDefinedFunction @@ -36,18 +32,6 @@ public interface UserDefinedFunctionBuilder { UDFOperandMetadata getOperandMetadata(); - default SqlKind getKind() { - return SqlKind.OTHER_FUNCTION; - } - - default SqlSyntax getSqlSyntax() { - return SqlSyntax.FUNCTION; - } - - default Supplier getReverse() { - return null; - } - default SqlUserDefinedFunction toUDF(String functionName) { return toUDF(functionName, true); } @@ -66,7 +50,7 @@ default SqlUserDefinedFunction toUDF(String functionName, boolean isDeterministi new SqlIdentifier(Collections.singletonList(functionName), null, SqlParserPos.ZERO, null); return new SqlUserDefinedFunction( udfLtrimIdentifier, - getKind(), + SqlKind.OTHER_FUNCTION, getReturnTypeInference(), InferTypes.ANY_NULLABLE, getOperandMetadata(), @@ -75,19 +59,6 @@ default SqlUserDefinedFunction toUDF(String functionName, boolean isDeterministi public boolean isDeterministic() { return isDeterministic; } - - @Override - public @Nullable SqlOperator reverse() { - if (getReverse() == null) { - return null; - } - return getReverse().get(); - } - - @Override - public SqlSyntax getSyntax() { - return getSqlSyntax(); - } }; } } diff --git a/core/src/main/java/org/opensearch/sql/expression/function/udf/ip/CompareIpFunction.java b/core/src/main/java/org/opensearch/sql/expression/function/udf/ip/CompareIpFunction.java index 70cbadbeee5..96d59a0a704 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/udf/ip/CompareIpFunction.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/udf/ip/CompareIpFunction.java @@ -5,9 +5,9 @@ package org.opensearch.sql.expression.function.udf.ip; +import java.util.Collections; import java.util.List; import java.util.Locale; -import java.util.function.Supplier; import org.apache.calcite.adapter.enumerable.NotNullImplementor; import org.apache.calcite.adapter.enumerable.NullPolicy; import org.apache.calcite.adapter.enumerable.RexToLixTranslator; @@ -15,14 +15,20 @@ import org.apache.calcite.linq4j.tree.Expression; import org.apache.calcite.linq4j.tree.Expressions; import org.apache.calcite.rex.RexCall; +import org.apache.calcite.sql.SqlIdentifier; import org.apache.calcite.sql.SqlKind; import org.apache.calcite.sql.SqlOperator; import org.apache.calcite.sql.SqlSyntax; +import org.apache.calcite.sql.parser.SqlParserPos; +import org.apache.calcite.sql.type.InferTypes; import org.apache.calcite.sql.type.ReturnTypes; import org.apache.calcite.sql.type.SqlReturnTypeInference; +import org.apache.calcite.sql.validate.SqlUserDefinedFunction; +import org.checkerframework.checker.nullness.qual.Nullable; import org.opensearch.sql.data.model.ExprIpValue; import org.opensearch.sql.data.type.ExprCoreType; import org.opensearch.sql.expression.function.ImplementorUDF; +import org.opensearch.sql.expression.function.PPLBuiltinOperators; import org.opensearch.sql.expression.function.UDFOperandMetadata; /** @@ -38,50 +44,10 @@ */ public class CompareIpFunction extends ImplementorUDF { private final SqlKind kind; - private Supplier reverse; private CompareIpFunction(SqlKind kind) { super(new CompareImplementor(kind), NullPolicy.ANY); this.kind = kind; - // Will be set later - this.reverse = null; - } - - @Override - public SqlSyntax getSqlSyntax() { - return SqlSyntax.BINARY; - } - - @Override - public SqlKind getKind() { - return kind; - } - - @Override - public Supplier getReverse() { - return reverse; - } - - /** - * Sets the reverse operator supplier for this comparison function. - * - *

This method is used to establish the reversed relationship between comparison operators - * (e.g., "less than" and "greater than"). When the query optimizer normalizes expressions, it may - * need to transform "b > a" to "a < b". - * - *

E.g. in the {@code hashCode} method of {@link org.apache.calcite.rex.RexNormalize}#L115, it - * always converts B [comparator] A to A [reverse_comparator] B if the ordinal of - * the reverse of the comparator is smaller. - * - *

IP comparison functions use this to inform the optimizer that ip_less_than is the reverse of - * ip_greater_than, allowing for proper query normalization. - * - * @param supplier The supplier that provides the reverse SQL operator - * @return This CompareIpFunction instance for method chaining - */ - public CompareIpFunction withReverse(Supplier supplier) { - this.reverse = supplier; - return this; } public static CompareIpFunction less() { @@ -108,6 +74,44 @@ public static CompareIpFunction notEquals() { return new CompareIpFunction(SqlKind.NOT_EQUALS); } + @Override + public SqlUserDefinedFunction toUDF(String functionName, boolean isDeterministic) { + SqlIdentifier udfIdentifier = + new SqlIdentifier(Collections.singletonList(functionName), null, SqlParserPos.ZERO, null); + return new SqlUserDefinedFunction( + udfIdentifier, + kind, + getReturnTypeInference(), + InferTypes.ANY_NULLABLE, + getOperandMetadata(), + getFunction()) { + @Override + public boolean isDeterministic() { + return isDeterministic; + } + + @Override + public @Nullable SqlOperator reverse() { + return switch (kind) { + case LESS_THAN -> PPLBuiltinOperators.GREATER_IP; + case GREATER_THAN -> PPLBuiltinOperators.LESS_IP; + case LESS_THAN_OR_EQUAL -> PPLBuiltinOperators.GTE_IP; + case GREATER_THAN_OR_EQUAL -> PPLBuiltinOperators.LTE_IP; + case EQUALS -> PPLBuiltinOperators.EQUALS_IP; + case NOT_EQUALS -> PPLBuiltinOperators.NOT_EQUALS_IP; + default -> throw new IllegalArgumentException( + String.format( + Locale.ROOT, "CompareIpFunction is not supposed to be of kind: %s", kind)); + }; + } + + @Override + public SqlSyntax getSyntax() { + return SqlSyntax.BINARY; + } + }; + } + @Override public SqlReturnTypeInference getReturnTypeInference() { return ReturnTypes.BOOLEAN_FORCE_NULLABLE;