From 2ccd9d3d90214f59b7576aecd19d6d367e2d1460 Mon Sep 17 00:00:00 2001 From: xinyual Date: Wed, 5 Mar 2025 10:35:03 +0800 Subject: [PATCH 1/5] add string udfs Signed-off-by: xinyual --- .../calcite/utils/BuiltinFunctionUtils.java | 47 ++++++++++++++++++- 1 file changed, 45 insertions(+), 2 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/calcite/utils/BuiltinFunctionUtils.java b/core/src/main/java/org/opensearch/sql/calcite/utils/BuiltinFunctionUtils.java index c668cad4208..386eec8e717 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/utils/BuiltinFunctionUtils.java +++ b/core/src/main/java/org/opensearch/sql/calcite/utils/BuiltinFunctionUtils.java @@ -15,6 +15,7 @@ import org.apache.calcite.sql.SqlOperator; import org.apache.calcite.sql.fun.SqlLibraryOperators; import org.apache.calcite.sql.fun.SqlStdOperatorTable; +import org.apache.calcite.sql.fun.SqlTrimFunction; import org.apache.calcite.sql.type.ReturnTypes; import org.opensearch.sql.calcite.CalcitePlanContext; import org.opensearch.sql.calcite.udf.mathUDF.SqrtFunction; @@ -53,10 +54,28 @@ static SqlOperator translate(String op) { case "/": return SqlStdOperatorTable.DIVIDE; // Built-in String Functions + case "CONCAT": + return SqlLibraryOperators.CONCAT_FUNCTION; + case "CONCAT_WS": + return SqlLibraryOperators.CONCAT_WS; + case "LIKE": + return SqlLibraryOperators.ILIKE; + case "LTRIM", "RTRIM", "TRIM": + return SqlStdOperatorTable.TRIM; + case "LENGTH": + return SqlStdOperatorTable.CHAR_LENGTH; case "LOWER": return SqlStdOperatorTable.LOWER; - case "LIKE": - return SqlStdOperatorTable.LIKE; + case "POSITION": + return SqlStdOperatorTable.POSITION; + case "REVERSE": + return SqlLibraryOperators.REVERSE; + case "RIGHT": + return SqlLibraryOperators.RIGHT; + case "SUBSTRING": + return SqlStdOperatorTable.SUBSTRING; + case "UPPER": + return SqlStdOperatorTable.UPPER; // Built-in Math Functions case "ABS": return SqlStdOperatorTable.ABS; @@ -99,6 +118,30 @@ static SqlOperator translate(String op) { static List translateArgument( String op, List argList, CalcitePlanContext context) { switch (op.toUpperCase(Locale.ROOT)) { + case "TRIM": + List trimArgs = + new ArrayList<>( + List.of( + context.rexBuilder.makeFlag(SqlTrimFunction.Flag.BOTH), + context.rexBuilder.makeLiteral(" "))); + trimArgs.addAll(argList); + return trimArgs; + case "LTRIM": + List LTrimArgs = + new ArrayList<>( + List.of( + context.rexBuilder.makeFlag(SqlTrimFunction.Flag.LEADING), + context.rexBuilder.makeLiteral(" "))); + LTrimArgs.addAll(argList); + return LTrimArgs; + case "RTRIM": + List RTrimArgs = + new ArrayList<>( + List.of( + context.rexBuilder.makeFlag(SqlTrimFunction.Flag.TRAILING), + context.rexBuilder.makeLiteral(" "))); + RTrimArgs.addAll(argList); + return RTrimArgs; case "ATAN": List AtanArgs = new ArrayList<>(argList); if (AtanArgs.size() == 1) { From 9d267e7b4b99a2379e77efddcd2936921419eb93 Mon Sep 17 00:00:00 2001 From: xinyual Date: Thu, 6 Mar 2025 19:37:32 +0800 Subject: [PATCH 2/5] add it to string Signed-off-by: xinyual --- .../calcite/utils/BuiltinFunctionUtils.java | 24 ++-- integ-test/build.gradle | 4 +- .../CalcitePPLBuiltinFunctionIT.java | 122 ++++++++++++++++++ plugin/build.gradle | 2 +- 4 files changed, 137 insertions(+), 15 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/calcite/utils/BuiltinFunctionUtils.java b/core/src/main/java/org/opensearch/sql/calcite/utils/BuiltinFunctionUtils.java index 386eec8e717..71d224f626a 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/utils/BuiltinFunctionUtils.java +++ b/core/src/main/java/org/opensearch/sql/calcite/utils/BuiltinFunctionUtils.java @@ -120,26 +120,26 @@ static List translateArgument( switch (op.toUpperCase(Locale.ROOT)) { case "TRIM": List trimArgs = - new ArrayList<>( - List.of( - context.rexBuilder.makeFlag(SqlTrimFunction.Flag.BOTH), - context.rexBuilder.makeLiteral(" "))); + new ArrayList<>( + List.of( + context.rexBuilder.makeFlag(SqlTrimFunction.Flag.BOTH), + context.rexBuilder.makeLiteral(" "))); trimArgs.addAll(argList); return trimArgs; case "LTRIM": List LTrimArgs = - new ArrayList<>( - List.of( - context.rexBuilder.makeFlag(SqlTrimFunction.Flag.LEADING), - context.rexBuilder.makeLiteral(" "))); + new ArrayList<>( + List.of( + context.rexBuilder.makeFlag(SqlTrimFunction.Flag.LEADING), + context.rexBuilder.makeLiteral(" "))); LTrimArgs.addAll(argList); return LTrimArgs; case "RTRIM": List RTrimArgs = - new ArrayList<>( - List.of( - context.rexBuilder.makeFlag(SqlTrimFunction.Flag.TRAILING), - context.rexBuilder.makeLiteral(" "))); + new ArrayList<>( + List.of( + context.rexBuilder.makeFlag(SqlTrimFunction.Flag.TRAILING), + context.rexBuilder.makeLiteral(" "))); RTrimArgs.addAll(argList); return RTrimArgs; case "ATAN": diff --git a/integ-test/build.gradle b/integ-test/build.gradle index 56d54ccb6f6..cc59edd637c 100644 --- a/integ-test/build.gradle +++ b/integ-test/build.gradle @@ -44,7 +44,7 @@ apply plugin: 'java' apply plugin: 'io.freefair.lombok' apply plugin: 'com.wiredforcode.spawn' -String baseVersion = "2.17.0" +String baseVersion = "2.17.1" String bwcVersion = baseVersion + ".0"; String baseName = "sqlBwcCluster" String bwcFilePath = "src/test/resources/bwc/" @@ -199,7 +199,7 @@ dependencies { testCompileOnly 'org.apiguardian:apiguardian-api:1.1.2' // Needed for BWC tests - zipArchive group: 'org.opensearch.plugin', name:'opensearch-job-scheduler', version: "${opensearch_build}" + zipArchive group: 'org.opensearch.plugin', name:'opensearch-job-scheduler', version: "3.0.0.0-alpha1-SNAPSHOT" zipArchive group: 'org.opensearch.plugin', name:'opensearch-sql-plugin', version: "${bwcVersion}-SNAPSHOT" } diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalcitePPLBuiltinFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalcitePPLBuiltinFunctionIT.java index 112d6d4ff98..c211db5ffa2 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalcitePPLBuiltinFunctionIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalcitePPLBuiltinFunctionIT.java @@ -9,6 +9,11 @@ import static org.opensearch.sql.util.MatcherUtils.rows; import java.io.IOException; +import java.util.List; + +import org.hamcrest.Matcher; +import org.hamcrest.Matchers; +import org.json.JSONArray; import org.json.JSONObject; import org.junit.jupiter.api.Test; @@ -31,4 +36,121 @@ public void testSqrtAndPow() { verifyDataRows(actual, rows("Hello", 30)); } + + // Test + @Test + public void testConcat() { + JSONObject actual = + executeQuery( + String.format( + "source=%s | where name=concat('He', 'llo') | fields name, age", + TEST_INDEX_STATE_COUNTRY)); + + verifySchema(actual, schema("name", "string"), schema("age", "integer")); + + verifyDataRows(actual, rows("Hello", 30)); + } + + @Test + public void testConcatWithField() { + JSONObject actual = + executeQuery( + String.format( + "source=%s | where name=concat('Hello', state) | fields name, age", + TEST_INDEX_STATE_COUNTRY)); + + verifySchema(actual, schema("name", "string"), schema("age", "integer")); + + verifyDataRows(actual, (Matcher) List.of()); + } + + @Test + public void testConcatWs() { + JSONObject actual = + executeQuery( + String.format( + "source=%s | where name=concat('Hello', state) | fields name, age", + TEST_INDEX_STATE_COUNTRY)); + + verifySchema(actual, schema("name", "string"), schema("age", "integer")); + + verifyDataRows(actual, rows()); + } + + @Test + public void testLength() { + JSONObject actual = + executeQuery( + String.format( + "source=%s | where length(name) = 5 | fields name, age", TEST_INDEX_STATE_COUNTRY)); + + verifySchema(actual, schema("name", "string"), schema("age", "integer")); + + verifyDataRows(actual, rows("Hello", 30)); + } + + @Test + public void testLengthShouldBeInsensitive() { + JSONObject actual = + executeQuery( + String.format( + "source=%s | where leNgTh(name) = 5 | fields name, age", TEST_INDEX_STATE_COUNTRY)); + + verifySchema(actual, schema("name", "string"), schema("age", "integer")); + + verifyDataRows(actual, rows("Hello", 30)); + } + + @Test + public void testLower() { + JSONObject actual = + executeQuery( + String.format( + "source=%s | where lower(name) = 'hello' | fields name, age", + TEST_INDEX_STATE_COUNTRY)); + + verifySchema(actual, schema("name", "string"), schema("age", "integer")); + + verifyDataRows(actual, rows("Hello", 30)); + } + + @Test + public void testUpper() { + JSONObject actual = + executeQuery( + String.format( + "source=%s | where upper(name) = upper('hello') | fields name, age", + TEST_INDEX_STATE_COUNTRY)); + + verifySchema(actual, schema("name", "string"), schema("age", "integer")); + + verifyDataRows(actual, rows("Hello", 30)); + } + + @Test + public void testLike() { + JSONObject actual = + executeQuery( + String.format( + "source=%s | where like(name, '_ello%%') | fields name, age", + TEST_INDEX_STATE_COUNTRY)); + + verifySchema(actual, schema("name", "string"), schema("age", "integer")); + + verifyDataRows(actual, rows("Hello", 30)); + } + + @Test + public void testSubstring() { + JSONObject actual = + executeQuery( + String.format( + "source=%s | where substring(name, 2, 2) = 'el' | fields name, age", + TEST_INDEX_STATE_COUNTRY)); + + verifySchema(actual, schema("name", "string"), schema("age", "integer")); + + verifyDataRows(actual, rows("Hello", 30)); + } + } diff --git a/plugin/build.gradle b/plugin/build.gradle index 9df3d3dd483..b0b7a5a8053 100644 --- a/plugin/build.gradle +++ b/plugin/build.gradle @@ -170,7 +170,7 @@ dependencies { testImplementation group: 'org.mockito', name: 'mockito-junit-jupiter', version: "${versions.mockito}" testImplementation 'org.junit.jupiter:junit-jupiter:5.9.3' - zipArchive group: 'org.opensearch.plugin', name:'opensearch-job-scheduler', version: "${opensearch_build}" + zipArchive group: 'org.opensearch.plugin', name:'opensearch-job-scheduler', version: "3.0.0.0-alpha1-SNAPSHOT" } test { From 413233178078aa9a8d08dd2950356c0e747375b2 Mon Sep 17 00:00:00 2001 From: xinyual Date: Fri, 7 Mar 2025 11:38:17 +0800 Subject: [PATCH 3/5] add IT for string function Signed-off-by: xinyual --- .../CalcitePPLBuiltinFunctionIT.java | 151 ++++++++++++++++-- 1 file changed, 134 insertions(+), 17 deletions(-) diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalcitePPLBuiltinFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalcitePPLBuiltinFunctionIT.java index c211db5ffa2..2ed726fb4b7 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalcitePPLBuiltinFunctionIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalcitePPLBuiltinFunctionIT.java @@ -9,13 +9,9 @@ import static org.opensearch.sql.util.MatcherUtils.rows; import java.io.IOException; -import java.util.List; - -import org.hamcrest.Matcher; -import org.hamcrest.Matchers; -import org.json.JSONArray; import org.json.JSONObject; import org.junit.jupiter.api.Test; +import org.opensearch.client.Request; public class CalcitePPLBuiltinFunctionIT extends CalcitePPLIntegTestCase { @Override @@ -52,7 +48,12 @@ public void testConcat() { } @Test - public void testConcatWithField() { + public void testConcatWithField() throws IOException { + Request request1 = + new Request("PUT", "/opensearch-sql_test_index_state_country/_doc/5?refresh=true"); + request1.setJsonEntity( + "{\"name\":\"HelloWay\",\"age\":27,\"state\":\"Way\",\"country\":\"Canada\",\"year\":2023,\"month\":4}"); + client().performRequest(request1); JSONObject actual = executeQuery( String.format( @@ -61,20 +62,25 @@ public void testConcatWithField() { verifySchema(actual, schema("name", "string"), schema("age", "integer")); - verifyDataRows(actual, (Matcher) List.of()); + verifyDataRows(actual, rows("HelloWay", 27)); } @Test - public void testConcatWs() { + public void testConcatWs() throws IOException { + Request request1 = + new Request("PUT", "/opensearch-sql_test_index_state_country/_doc/5?refresh=true"); + request1.setJsonEntity( + "{\"name\":\"John,Way\",\"age\":27,\"state\":\"Way\",\"country\":\"Canada\",\"year\":2023,\"month\":4}"); + client().performRequest(request1); JSONObject actual = - executeQuery( - String.format( - "source=%s | where name=concat('Hello', state) | fields name, age", - TEST_INDEX_STATE_COUNTRY)); + executeQuery( + String.format( + "source=%s | where name=concat_ws(',', 'John', state) | fields name, age", + TEST_INDEX_STATE_COUNTRY)); verifySchema(actual, schema("name", "string"), schema("age", "integer")); - verifyDataRows(actual, rows()); + verifyDataRows(actual, rows("John,Way", 27)); } @Test @@ -143,14 +149,125 @@ public void testLike() { @Test public void testSubstring() { JSONObject actual = - executeQuery( - String.format( - "source=%s | where substring(name, 2, 2) = 'el' | fields name, age", - TEST_INDEX_STATE_COUNTRY)); + executeQuery( + String.format( + "source=%s | where substring(name, 2, 2) = 'el' | fields name, age", + TEST_INDEX_STATE_COUNTRY)); + + verifySchema(actual, schema("name", "string"), schema("age", "integer")); + + verifyDataRows(actual, rows("Hello", 30)); + } + + @Test + public void testPosition() { + JSONObject actual = + executeQuery( + String.format( + "source=%s | where position('He' in name) = 1 | fields name, age", + TEST_INDEX_STATE_COUNTRY)); + + verifySchema(actual, schema("name", "string"), schema("age", "integer")); + + verifyDataRows(actual, rows("Hello", 30)); + } + + @Test + public void testTrim() throws IOException { + prepareTrim(); + JSONObject actual = + executeQuery( + String.format( + "source=%s | where Trim(name) = 'Jim' | fields name, age", + TEST_INDEX_STATE_COUNTRY)); + + verifySchema(actual, schema("name", "string"), schema("age", "integer")); + + verifyDataRows(actual, rows(" Jim", 27), rows("Jim ", 57), rows(" Jim ", 70)); + } + + @Test + public void testRTrim() throws IOException { + prepareTrim(); + JSONObject actual = + executeQuery( + String.format( + "source=%s | where RTrim(name) = 'Jim' | fields name, age", + TEST_INDEX_STATE_COUNTRY)); + + verifySchema(actual, schema("name", "string"), schema("age", "integer")); + + verifyDataRows(actual, rows("Jim ", 57)); + } + + @Test + public void testLTrim() throws IOException { + prepareTrim(); + JSONObject actual = + executeQuery( + String.format( + "source=%s | where LTrim(name) = 'Jim' | fields name, age", + TEST_INDEX_STATE_COUNTRY)); + + verifySchema(actual, schema("name", "string"), schema("age", "integer")); + + verifyDataRows(actual, rows(" Jim", 27)); + } + + @Test + public void testReverse() throws IOException { + Request request1 = + new Request("PUT", "/opensearch-sql_test_index_state_country/_doc/5?refresh=true"); + request1.setJsonEntity( + "{\"name\":\"DeD\",\"age\":27,\"state\":\"B.C\",\"country\":\"Canada\",\"year\":2023,\"month\":4}"); + client().performRequest(request1); + JSONObject actual = + executeQuery( + String.format( + "source=%s | where Reverse(name) = name | fields name, age", + TEST_INDEX_STATE_COUNTRY)); + + verifySchema(actual, schema("name", "string"), schema("age", "integer")); + + verifyDataRows(actual, rows("DeD", 27)); + } + + @Test + public void testRight() throws IOException { + Request request1 = + new Request("PUT", "/opensearch-sql_test_index_state_country/_doc/5?refresh=true"); + request1.setJsonEntity( + "{\"name\":\"DeD\",\"age\":27,\"state\":\"B.C\",\"country\":\"Canada\",\"year\":2023,\"month\":4}"); + client().performRequest(request1); + JSONObject actual = + executeQuery( + String.format( + "source=%s | where right(name, 2) = 'lo' | fields name, age", + TEST_INDEX_STATE_COUNTRY)); verifySchema(actual, schema("name", "string"), schema("age", "integer")); verifyDataRows(actual, rows("Hello", 30)); } + private void prepareTrim() throws IOException { + Request request1 = + new Request("PUT", "/opensearch-sql_test_index_state_country/_doc/5?refresh=true"); + request1.setJsonEntity( + "{\"name\":\" " + + " Jim\",\"age\":27,\"state\":\"B.C\",\"country\":\"Canada\",\"year\":2023,\"month\":4}"); + client().performRequest(request1); + Request request2 = + new Request("PUT", "/opensearch-sql_test_index_state_country/_doc/6?refresh=true"); + request2.setJsonEntity( + "{\"name\":\"Jim " + + " \",\"age\":57,\"state\":\"B.C\",\"country\":\"Canada\",\"year\":2023,\"month\":4}"); + client().performRequest(request2); + Request request3 = + new Request("PUT", "/opensearch-sql_test_index_state_country/_doc/7?refresh=true"); + request3.setJsonEntity( + "{\"name\":\" Jim " + + " \",\"age\":70,\"state\":\"B.C\",\"country\":\"Canada\",\"year\":2023,\"month\":4}"); + client().performRequest(request3); + } } From a951d573a0dd57be9782fadbf467ad8f860a19f1 Mon Sep 17 00:00:00 2001 From: xinyual Date: Fri, 7 Mar 2025 17:20:07 +0800 Subject: [PATCH 4/5] remove change for local test Signed-off-by: xinyual --- integ-test/build.gradle | 4 ++-- plugin/build.gradle | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/integ-test/build.gradle b/integ-test/build.gradle index cc59edd637c..22821b5ca1d 100644 --- a/integ-test/build.gradle +++ b/integ-test/build.gradle @@ -44,7 +44,7 @@ apply plugin: 'java' apply plugin: 'io.freefair.lombok' apply plugin: 'com.wiredforcode.spawn' -String baseVersion = "2.17.1" +String baseVersion = "2.17.0" String bwcVersion = baseVersion + ".0"; String baseName = "sqlBwcCluster" String bwcFilePath = "src/test/resources/bwc/" @@ -199,7 +199,7 @@ dependencies { testCompileOnly 'org.apiguardian:apiguardian-api:1.1.2' // Needed for BWC tests - zipArchive group: 'org.opensearch.plugin', name:'opensearch-job-scheduler', version: "3.0.0.0-alpha1-SNAPSHOT" + zipArchive group: 'org.opensearch.plugin', name:'opensearch-job-scheduler', version:"${opensearch_build}" zipArchive group: 'org.opensearch.plugin', name:'opensearch-sql-plugin', version: "${bwcVersion}-SNAPSHOT" } diff --git a/plugin/build.gradle b/plugin/build.gradle index b0b7a5a8053..9df3d3dd483 100644 --- a/plugin/build.gradle +++ b/plugin/build.gradle @@ -170,7 +170,7 @@ dependencies { testImplementation group: 'org.mockito', name: 'mockito-junit-jupiter', version: "${versions.mockito}" testImplementation 'org.junit.jupiter:junit-jupiter:5.9.3' - zipArchive group: 'org.opensearch.plugin', name:'opensearch-job-scheduler', version: "3.0.0.0-alpha1-SNAPSHOT" + zipArchive group: 'org.opensearch.plugin', name:'opensearch-job-scheduler', version: "${opensearch_build}" } test { From dc2a6bd82091ea8d68e033a6680a82c0ab5dbb6d Mon Sep 17 00:00:00 2001 From: xinyual Date: Mon, 10 Mar 2025 10:39:53 +0800 Subject: [PATCH 5/5] revert change Signed-off-by: xinyual --- integ-test/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integ-test/build.gradle b/integ-test/build.gradle index 22821b5ca1d..56d54ccb6f6 100644 --- a/integ-test/build.gradle +++ b/integ-test/build.gradle @@ -199,7 +199,7 @@ dependencies { testCompileOnly 'org.apiguardian:apiguardian-api:1.1.2' // Needed for BWC tests - zipArchive group: 'org.opensearch.plugin', name:'opensearch-job-scheduler', version:"${opensearch_build}" + zipArchive group: 'org.opensearch.plugin', name:'opensearch-job-scheduler', version: "${opensearch_build}" zipArchive group: 'org.opensearch.plugin', name:'opensearch-sql-plugin', version: "${bwcVersion}-SNAPSHOT" }