diff --git a/DEPENDENCY-LICENSES b/DEPENDENCY-LICENSES index 7c806ee4a45..de2a0141484 100644 --- a/DEPENDENCY-LICENSES +++ b/DEPENDENCY-LICENSES @@ -6,7 +6,6 @@ List of third-party dependencies grouped by their license type. * Kryo (com.esotericsoftware:kryo:5.6.2 - https://github.com/EsotericSoftware/kryo/kryo) * MinLog (com.esotericsoftware:minlog:1.3.1 - https://github.com/EsotericSoftware/minlog) - * Protocol Buffers [Core] (com.google.protobuf:protobuf-java:3.3.0 - https://developers.google.com/protocol-buffers/protobuf-java/) * ReflectASM (com.esotericsoftware:reflectasm:1.11.9 - https://github.com/EsotericSoftware/reflectasm) AL 2.0, GPL v2, MPL 2.0 @@ -20,10 +19,8 @@ List of third-party dependencies grouped by their license type. Apache License, Version 2.0 * ActiveMQ :: Client (org.apache.activemq:activemq-client:6.1.7 - http://activemq.apache.org/activemq-client) - * Aggregate Designer Algorithm (net.hydromatic:aggdesigner-algorithm:6.0 - http://github.com/julianhyde/aggdesigner/aggdesigner-algorithm) * Annotations for Metrics (io.dropwizard.metrics:metrics-annotation:4.2.33 - https://metrics.dropwizard.io/metrics-annotation) * Apache Avro (org.apache.avro:avro:1.12.0 - https://avro.apache.org) - * Apache Calcite Avatica (org.apache.calcite.avatica:avatica-core:1.11.0 - https://calcite.apache.org/avatica/avatica-core) * Apache Commons BeanUtils (commons-beanutils:commons-beanutils:1.9.4 - https://commons.apache.org/proper/commons-beanutils/) * Apache Commons CLI (commons-cli:commons-cli:1.9.0 - https://commons.apache.org/proper/commons-cli/) * Apache Commons Codec (commons-codec:commons-codec:1.18.0 - https://commons.apache.org/proper/commons-codec/) @@ -32,7 +29,6 @@ List of third-party dependencies grouped by their license type. * Apache Commons Compress (org.apache.commons:commons-compress:1.27.1 - https://commons.apache.org/proper/commons-compress/) * Apache Commons Configuration (org.apache.commons:commons-configuration2:2.12.0 - https://commons.apache.org/proper/commons-configuration/) * Apache Commons Crypto (org.apache.commons:commons-crypto:1.1.0 - https://commons.apache.org/proper/commons-crypto/) - * Apache Commons CSV (org.apache.commons:commons-csv:1.14.0 - https://commons.apache.org/proper/commons-csv/) * Apache Commons Exec (org.apache.commons:commons-exec:1.5.0 - https://commons.apache.org/proper/commons-exec/) * Apache Commons FileUpload (commons-fileupload:commons-fileupload:1.6.0 - https://commons.apache.org/proper/commons-fileupload/) * Apache Commons IO (commons-io:commons-io:2.20.0 - https://commons.apache.org/proper/commons-io/) @@ -85,16 +81,11 @@ List of third-party dependencies grouped by their license type. * AutoService (com.google.auto.service:auto-service-annotations:1.1.1 - https://github.com/google/auto/tree/main/service) * AutoService Processor (com.google.auto.service:auto-service:1.1.1 - https://github.com/google/auto/tree/main/service) * Caffeine cache (com.github.ben-manes.caffeine:caffeine:3.2.2 - https://github.com/ben-manes/caffeine) - * Calcite Core (org.apache.calcite:calcite-core:1.16.0 - https://calcite.apache.org/calcite-core) - * Calcite Linq4j (org.apache.calcite:calcite-linq4j:1.16.0 - https://calcite.apache.org/calcite-linq4j) * CDI APIs (javax.enterprise:cdi-api:1.0 - http://www.seamframework.org/Weld/cdi-api) * chill-java (com.twitter:chill-java:0.9.5 - https://github.com/twitter/chill) * ClassMate (com.fasterxml:classmate:1.7.0 - https://github.com/FasterXML/java-classmate) * com.helger:profiler (com.helger:profiler:1.1.1 - https://github.com/phax/profiler) - * com.yahoo.datasketches:memory (com.yahoo.datasketches:memory:0.9.0 - https://datasketches.github.io/memory/) - * com.yahoo.datasketches:sketches-core (com.yahoo.datasketches:sketches-core:0.9.0 - https://datasketches.github.io/sketches-core/) * Commons Lang (commons-lang:commons-lang:2.6 - http://commons.apache.org/lang/) - * Commons Logging (commons-logging:commons-logging:1.1.3 - http://commons.apache.org/proper/commons-logging/) * Curator Client (org.apache.curator:curator-client:5.9.0 - https://curator.apache.org/curator-client) * Curator Framework (org.apache.curator:curator-framework:5.9.0 - https://curator.apache.org/curator-framework) * Curator Recipes (org.apache.curator:curator-recipes:5.9.0 - https://curator.apache.org/curator-recipes) @@ -113,8 +104,6 @@ List of third-party dependencies grouped by their license type. * Dropwizard Utility Classes (io.dropwizard:dropwizard-util:4.0.14 - http://www.dropwizard.io/4.0.14/dropwizard-bom/dropwizard-dependencies/dropwizard-parent/dropwizard-util) * Dropwizard Validation Support (io.dropwizard:dropwizard-validation:4.0.14 - http://www.dropwizard.io/4.0.14/dropwizard-bom/dropwizard-dependencies/dropwizard-parent/dropwizard-validation) * error-prone annotations (com.google.errorprone:error_prone_annotations:2.40.0 - https://errorprone.info/error_prone_annotations) - * Esri Geometry API for Java (com.esri.geometry:esri-geometry-api:2.0.0 - https://github.com/Esri/geometry-api-java) - * FindBugs-jsr305 (com.google.code.findbugs:jsr305:3.0.1 - http://findbugs.sourceforge.net/) * FindBugs-jsr305 (com.google.code.findbugs:jsr305:3.0.2 - http://findbugs.sourceforge.net/) * Google Guice - Core Library (com.google.inject:guice:4.2.1 - https://github.com/google/guice/guice) * Graphite Integration for Metrics (io.dropwizard.metrics:metrics-graphite:4.2.33 - https://metrics.dropwizard.io/metrics-graphite) @@ -346,9 +335,7 @@ List of third-party dependencies grouped by their license type. * asm (org.ow2.asm:asm:9.8 - http://asm.ow2.io/) * asm-commons (org.ow2.asm:asm-commons:9.8 - http://asm.ow2.io/) * asm-tree (org.ow2.asm:asm-tree:9.8 - http://asm.ow2.io/) - * Commons Compiler (org.codehaus.janino:commons-compiler:2.7.6 - http://docs.codehaus.org/display/JANINO/Home/commons-compiler) * dnsjava (dnsjava:dnsjava:3.6.1 - https://github.com/dnsjava/dnsjava) - * Janino (org.codehaus.janino:janino:2.7.6 - http://docs.codehaus.org/display/JANINO/Home/janino) * Protocol Buffer Java API (com.google.protobuf:protobuf-java:2.5.0 - http://code.google.com/p/protobuf) BSD License diff --git a/LICENSE-binary b/LICENSE-binary index 66f049914a8..38364df1564 100644 --- a/LICENSE-binary +++ b/LICENSE-binary @@ -647,7 +647,6 @@ The license texts of these dependencies can be found in the licenses directory. * Kryo (com.esotericsoftware:kryo:5.6.2 - https://github.com/EsotericSoftware/kryo/kryo) * MinLog (com.esotericsoftware:minlog:1.3.1 - https://github.com/EsotericSoftware/minlog) - * Protocol Buffers [Core] (com.google.protobuf:protobuf-java:3.3.0 - https://developers.google.com/protocol-buffers/protobuf-java/) * ReflectASM (com.esotericsoftware:reflectasm:1.11.9 - https://github.com/EsotericSoftware/reflectasm) Apache-2.0 @@ -669,9 +668,7 @@ The license texts of these dependencies can be found in the licenses directory. Apache License, Version 2.0 - * Aggregate Designer Algorithm (net.hydromatic:aggdesigner-algorithm:6.0 - http://github.com/julianhyde/aggdesigner/aggdesigner-algorithm) * Annotations for Metrics (io.dropwizard.metrics:metrics-annotation:4.2.33 - https://metrics.dropwizard.io/metrics-annotation) - * Apache Calcite Avatica (org.apache.calcite.avatica:avatica-core:1.11.0 - https://calcite.apache.org/avatica/avatica-core) * Apache Commons BeanUtils (commons-beanutils:commons-beanutils:1.9.4 - https://commons.apache.org/proper/commons-beanutils/) * Apache Commons CLI (commons-cli:commons-cli:1.9.0 - http://commons.apache.org/proper/commons-cli/) * Apache Commons Codec (commons-codec:commons-codec:1.18.0 - http://commons.apache.org/proper/commons-codec/) @@ -680,7 +677,6 @@ The license texts of these dependencies can be found in the licenses directory. * Apache Commons Compress (org.apache.commons:commons-compress:1.27.1 - https://commons.apache.org/proper/commons-compress/) * Apache Commons Configuration (org.apache.commons:commons-configuration2:2.12.0 - https://commons.apache.org/proper/commons-configuration/) * Apache Commons Crypto (org.apache.commons:commons-crypto:1.1.0 - https://commons.apache.org/proper/commons-crypto/) - * Apache Commons CSV (org.apache.commons:commons-csv:1.14.0 - http://commons.apache.org/proper/commons-csv/) * Apache Commons Exec (org.apache.commons:commons-exec:1.5.0 - http://commons.apache.org/proper/commons-exec/) * Apache Commons IO (commons-io:commons-io:2.20.0 - https://commons.apache.org/proper/commons-io/) * Apache Commons Logging (commons-logging:commons-logging:1.2 - http://commons.apache.org/proper/commons-logging/) @@ -717,15 +713,10 @@ The license texts of these dependencies can be found in the licenses directory. * Apache ZooKeeper - Server (org.apache.zookeeper:zookeeper:3.9.3 - http://zookeeper.apache.org/zookeeper) * ASM based accessors helper used by json-smart (net.minidev:accessors-smart:2.5.2 - https://urielch.github.io/) * Caffeine cache (com.github.ben-manes.caffeine:caffeine:3.2.2 - https://github.com/ben-manes/caffeine) - * Calcite Core (org.apache.calcite:calcite-core:1.16.0 - https://calcite.apache.org/calcite-core) - * Calcite Linq4j (org.apache.calcite:calcite-linq4j:1.16.0 - https://calcite.apache.org/calcite-linq4j) * chill-java (com.twitter:chill-java:0.9.5 - https://github.com/twitter/chill) * ClassMate (com.fasterxml:classmate:1.7.0 - https://github.com/FasterXML/java-classmate) * com.helger:profiler (com.helger:profiler:1.1.1 - https://github.com/phax/profiler) - * com.yahoo.datasketches:memory (com.yahoo.datasketches:memory:0.9.0 - https://datasketches.github.io/memory/) - * com.yahoo.datasketches:sketches-core (com.yahoo.datasketches:sketches-core:0.9.0 - https://datasketches.github.io/sketches-core/) * Commons Lang (commons-lang:commons-lang:2.6 - http://commons.apache.org/lang/) - * Commons Logging (commons-logging:commons-logging:1.1.3 - http://commons.apache.org/proper/commons-logging/) * Commons Math (org.apache.commons:commons-math3:3.6.1 - http://commons.apache.org/math/) * Curator Client (org.apache.curator:curator-client:5.9.0 - https://curator.apache.org/curator-client) * Curator Framework (org.apache.curator:curator-framework:5.9.0 - https://curator.apache.org/curator-framework) @@ -745,8 +736,6 @@ The license texts of these dependencies can be found in the licenses directory. * Dropwizard Utility Classes (io.dropwizard:dropwizard-util:4.0.14 - http://www.dropwizard.io/4.0.14/dropwizard-bom/dropwizard-dependencies/dropwizard-parent/dropwizard-util) * Dropwizard Validation Support (io.dropwizard:dropwizard-validation:4.0.14 - http://www.dropwizard.io/4.0.14/dropwizard-bom/dropwizard-dependencies/dropwizard-parent/dropwizard-validation) * error-prone annotations (com.google.errorprone:error_prone_annotations:2.40.0 - https://errorprone.info/error_prone_annotations) - * Esri Geometry API for Java (com.esri.geometry:esri-geometry-api:2.0.0 - https://github.com/Esri/geometry-api-java) - * FindBugs-jsr305 (com.google.code.findbugs:jsr305:3.0.1 - http://findbugs.sourceforge.net/) * FindBugs-jsr305 (com.google.code.findbugs:jsr305:3.0.2 - http://findbugs.sourceforge.net/) * Graphite Integration for Metrics (io.dropwizard.metrics:metrics-graphite:4.2.33 - https://metrics.dropwizard.io/metrics-graphite) * Gson (com.google.code.gson:gson:2.13.1 - https://github.com/google/gson) @@ -927,8 +916,6 @@ The license texts of these dependencies can be found in the licenses directory. BSD 3-Clause License - * Commons Compiler (org.codehaus.janino:commons-compiler:2.7.6 - http://docs.codehaus.org/display/JANINO/Home/commons-compiler) - * Janino (org.codehaus.janino:janino:2.7.6 - http://docs.codehaus.org/display/JANINO/Home/janino) * Protocol Buffer Java API (com.google.protobuf:protobuf-java:2.5.0 - http://code.google.com/p/protobuf) BSD License diff --git a/bin/storm.py b/bin/storm.py index 309a426719a..956abde0018 100755 --- a/bin/storm.py +++ b/bin/storm.py @@ -315,7 +315,6 @@ def initialize_main_command(): initialize_localconfvalue_subcommand(subparsers) initialize_remoteconfvalue_subcommand(subparsers) initialize_local_subcommand(subparsers) - initialize_sql_subcommand(subparsers) initialize_kill_subcommand(subparsers) initialize_upload_credentials_subcommand(subparsers) initialize_blobstore_subcommand(subparsers) @@ -563,31 +562,6 @@ def initialize_upload_credentials_subcommand(subparsers): sub_parser.set_defaults(func=upload_credentials) add_common_options(sub_parser) - -def initialize_sql_subcommand(subparsers): - command_help = """Compiles the SQL statements into a Trident topology and submits it to Storm. - If user activates explain mode, SQL Runner analyzes each query statement - and shows query plan instead of submitting topology. - """ - - sub_parser = subparsers.add_parser("sql", help=command_help, formatter_class=SortingHelpFormatter) - - add_client_jar_options(sub_parser) - - sub_parser.add_argument("sql_file", metavar="sql-file") - - group = sub_parser.add_mutually_exclusive_group(required=True) - - group.add_argument( - "topology_name", metavar="topology-name",nargs='?' - ) - - group.add_argument("--explain", action="store_true", help="activate explain mode") - - sub_parser.set_defaults(func=sql) - add_common_options(sub_parser, main_args=False) - - def initialize_blobstore_subcommand(subparsers): sub_parser = subparsers.add_parser("blobstore", formatter_class=SortingHelpFormatter) command_help = """ @@ -1087,45 +1061,6 @@ def local(args): "org.apache.storm.LocalCluster", args, client=False, daemon=False, extrajvmopts=extrajvmopts) - -def sql(args): - local_jars = [arg for arg in args.jars.split(",") if arg] - - artifact_to_file_jars = resolve_dependencies( - args.artifacts, args.artifactRepositories, - args.mavenLocalRepositoryDirectory, args.proxyUrl, - args.proxyUsername, args.proxyPassword - ) - - # include storm-sql-runtime jar(s) to local jar list - # --jars doesn't support wildcard so it should call get_jars_full - sql_runtime_jars = get_jars_full(os.path.join(STORM_TOOLS_LIB_DIR, "sql", "runtime")) - local_jars.extend(sql_runtime_jars) - - extra_jars = [USER_CONF_DIR, STORM_BIN_DIR] - extra_jars.extend(local_jars) - extra_jars.extend(artifact_to_file_jars.values()) - - # include this for running StormSqlRunner, but not for generated topology - sql_core_jars = get_wildcard_dir(os.path.join(STORM_TOOLS_LIB_DIR, "sql", "core")) - extra_jars.extend(sql_core_jars) - - if args.explain: - sql_args = ["--file", args.sql_file, "--explain"] - else: - sql_args = ["--file", args.sql_file, "--topology", args.topology_name] - - exec_storm_class( - "org.apache.storm.sql.StormSqlRunner", storm_config_opts=args.storm_config_opts, - jvmtype="-client", - extrajars=extra_jars, - main_class_args=sql_args, - daemon=False, - jvmopts=["-Dstorm.dependency.jars=" + ",".join(local_jars)] + - ["-Dstorm.dependency.artifacts=" + json.dumps(artifact_to_file_jars)], - overriding_conf_file=args.config) - - def kill(args): exec_storm_class( "org.apache.storm.command.KillTopology", diff --git a/docs/Implementation-docs.md b/docs/Implementation-docs.md index 93459f3dfd6..46fe88cd38a 100644 --- a/docs/Implementation-docs.md +++ b/docs/Implementation-docs.md @@ -11,4 +11,3 @@ This section of the wiki is dedicated to explaining how Storm is implemented. Yo - [Acking framework implementation](Acking-framework-implementation.html) - [Metrics](Metrics.html) - [Nimbus HA](nimbus-ha-design.html) -- [Storm SQL](storm-sql-internal.html) diff --git a/docs/State-checkpointing.md b/docs/State-checkpointing.md index 42af3a7a11f..687ea818f83 100644 --- a/docs/State-checkpointing.md +++ b/docs/State-checkpointing.md @@ -53,7 +53,7 @@ can be changed by setting the storm config `topology.state.checkpoint.interval.m 5. For state persistence, use a state provider that supports persistence by setting the `topology.state.provider` in the storm config. E.g. for using Redis based key-value state implementation set `topology.state.provider: org.apache.storm.redis.state.RedisKeyValueStateProvider` in storm.yaml. The provider implementation jar should be in the class path, which in this case means adding `storm-redis` -to dependency of your topology, or adding `--artifacts "org.apache.storm:storm-sql-redis:"` when submitting your topology with `storm jar`. +to dependency of your topology. 6. The state provider properties can be overridden by setting `topology.state.provider.config`. For Redis state this is a json config with the following properties. diff --git a/docs/index.md b/docs/index.md index 4dd603afbec..e4367e669d0 100644 --- a/docs/index.md +++ b/docs/index.md @@ -38,18 +38,6 @@ We're also notifying it via annotating classes with marker interface `@Interface * [Streams API](Stream-API.html) -#### SQL - -The Storm SQL integration allows users to run SQL queries over streaming data in Storm. - -NOTE: Storm SQL is an `experimental` feature, so the internals of Storm SQL and supported features are subject to change. -But small change will not affect the user experience. We will notify the user when breaking UX change is introduced. - -* [Storm SQL overview](storm-sql.html) -* [Storm SQL example](storm-sql-example.html) -* [Storm SQL reference](storm-sql-reference.html) -* [Storm SQL internal](storm-sql-internal.html) - #### Flux * [Flux Data Driven Topology Builder](flux.html) diff --git a/docs/storm-sql-example.md b/docs/storm-sql-example.md deleted file mode 100644 index 0d68d7f67fb..00000000000 --- a/docs/storm-sql-example.md +++ /dev/null @@ -1,289 +0,0 @@ ---- -title: Storm SQL example -layout: documentation -documentation: true ---- - -This page shows how to use Storm SQL by showing the example of processing Apache logs. -This page is written by "how-to" style so you can follow the step and learn how to utilize Storm SQL step by step. - -## Preparation - -This page assumes that Apache Zookeeper, Apache Storm and Apache Kafka are installed locally and running properly configured. -For convenience, this page assumes that Apache Kafka 0.10.0 is installed via `brew`. - -We'll use below tools to prepare the JSON data which will be fed to the input data source. -Since they're Python projects, this page assumes Python 3.0.x with `pip3`, `virtualenv` is installed locally. - -* https://github.com/kiritbasu/Fake-Apache-Log-Generator -* https://github.com/rory/apache-log-parser - -## Creating Topics - -In this page, we will use four topics, `apache-logs`, `apache-errorlogs`, `apache-slowlogs`. -Please create topics according to your environment. - -For Apache Kafka 0.10.0 with brew installed, - -``` -kafka-topics --create --topic apache-logs --zookeeper localhost:2181 --replication-factor 1 --partitions 5 -kafka-topics --create --topic apache-errorlogs --zookeeper localhost:2181 --replication-factor 1 --partitions 5 -kafka-topics --create --topic apache-slowlogs --zookeeper localhost:2181 --replication-factor 1 --partitions 5 -``` - -## Feeding Data - -Let's feed the data to input topics. In this page we will generate fake Apache logs, and parse to JSON format, and feed JSON to Kafka topic. - -Let's create your working directory, since we will clone the project and also setup virtualenv. - -In your working directory, `virtualenv env` to setup virtualenv to env directory, and activate. - -``` -$ virtualenv env -$ source env/bin/activate -``` - -Feel free to `deactivate` when you're done with example. - -### Install and modify Fake-Apache-Log-Generator - -`Fake-Apache-Log-Generator` is not presented to package, and also we need to modify the script. - -``` -$ git clone https://github.com/kiritbasu/Fake-Apache-Log-Generator.git -$ cd Fake-Apache-Log-Generator -``` - -Open `apache-fake-log-gen.py` and replace `while (flag):` statements to below: - -``` - elapsed_us = random.randint(1 * 1000,1000 * 1000) # 1 ms to 1 sec - seconds=random.randint(30,300) - increment = datetime.timedelta(seconds=seconds) - otime += increment - - ip = faker.ipv4() - dt = otime.strftime('%d/%b/%Y:%H:%M:%S') - tz = datetime.datetime.now(pytz.timezone('US/Pacific')).strftime('%z') - vrb = numpy.random.choice(verb,p=[0.6,0.1,0.1,0.2]) - - uri = random.choice(resources) - if uri.find("apps")>0: - uri += `random.randint(1000,10000)` - - resp = numpy.random.choice(response,p=[0.9,0.04,0.02,0.04]) - byt = int(random.gauss(5000,50)) - referer = faker.uri() - useragent = numpy.random.choice(ualist,p=[0.5,0.3,0.1,0.05,0.05] )() - f.write('%s - - [%s %s] %s "%s %s HTTP/1.0" %s %s "%s" "%s"\n' % (ip,dt,tz,elapsed_us,vrb,uri,resp,byt,referer,useragent)) - - log_lines = log_lines - 1 - flag = False if log_lines == 0 else True -``` - -to make sure fake elapsed_us is included to fake log. - -For convenience, you can skip cloning project and download modified file from here: [apache-fake-log-gen.py (gist)](https://gist.github.com/HeartSaVioR/79fd4e461604fabecf535ffece47e6c2) - -### Install apache-log-parser and write parsing script - -`apache-log-parser` can be installed via `pip`. - -``` -$ pip3 install apache-log-parser -``` - -Since apache-log-parser is a library, in order to parse fake log we need to write small Python script. -Let's create file `parse-fake-log-gen-to-json-with-incrementing-id.py` with below content: - -``` -import sys -import apache_log_parser -import json - -auto_incr_id = 1 -parser_format = '%a - - %t %D "%r" %s %b "%{Referer}i" "%{User-Agent}i"' -line_parser = apache_log_parser.make_parser(parser_format) -while True: - # we'll use pipe - line = sys.stdin.readline() - if not line: - break - parsed_dict = line_parser(line) - parsed_dict['id'] = auto_incr_id - auto_incr_id += 1 - - parsed_dict = {k.upper(): v for k, v in parsed_dict.iteritems() if not k.endswith('datetimeobj')} - print(json.dumps(parsed_dict)) -``` - -### Feed parsed JSON Apache Log to Kafka - -OK! We're prepared to feed the data to Kafka topic. Let's use `kafka-console-producer` to feed parsed JSON. - -``` -$ python3 apache-fake-log-gen.py -n 0 | python3 parse-fake-log-gen-to-json-with-incrementing-id.py | kafka-console-producer --broker-list localhost:9092 --topic apache-logs -``` - -and execute below to another terminal session to confirm data is being fed. - -``` -$ kafka-console-consumer --zookeeper localhost:2181 --topic apache-logs -``` - -If you can see the json like below, it's done: - -``` -{"TIME_US": "757467", "REQUEST_FIRST_LINE": "GET /wp-content HTTP/1.0", "REQUEST_METHOD": "GET", "RESPONSE_BYTES_CLF": "4988", "TIME_RECEIVED_ISOFORMAT": "2021-06-30T22:02:53", "TIME_RECEIVED_TZ_ISOFORMAT": "2021-06-30T22:02:53-07:00", "REQUEST_HTTP_VER": "1.0", "REQUEST_HEADER_USER_AGENT__BROWSER__FAMILY": "Firefox", "REQUEST_HEADER_USER_AGENT__IS_MOBILE": false, "REQUEST_HEADER_USER_AGENT__BROWSER__VERSION_STRING": "3.6.13", "REQUEST_URL_FRAGMENT": "", "REQUEST_HEADER_USER_AGENT": "Mozilla/5.0 (X11; Linux x86_64; rv:1.9.7.20) Gecko/2010-10-13 13:52:34 Firefox/3.6.13", "REQUEST_URL_SCHEME": "", "REQUEST_URL_PATH": "/wp-content", "REQUEST_URL_QUERY_SIMPLE_DICT": {}, "TIME_RECEIVED_UTC_ISOFORMAT": "2021-07-01T05:02:53+00:00", "REQUEST_URL_QUERY_DICT": {}, "STATUS": "200", "REQUEST_URL_NETLOC": "", "REQUEST_URL_QUERY_LIST": [], "REQUEST_URL_QUERY": "", "REQUEST_URL_USERNAME": null, "REQUEST_HEADER_USER_AGENT__OS__VERSION_STRING": "", "REQUEST_URL_HOSTNAME": null, "REQUEST_HEADER_USER_AGENT__OS__FAMILY": "Linux", "REQUEST_URL": "/wp-content", "ID": 904128, "REQUEST_HEADER_REFERER": "http://white.com/terms/", "REQUEST_URL_PORT": null, "REQUEST_URL_PASSWORD": null, "TIME_RECEIVED": "[30/Jun/2021:22:02:53 -0700]", "REMOTE_IP": "88.203.90.62"} -``` - -## Example: filtering error logs - -In this example we'll filter error logs from entire logs and store them to another topics. `project` and `filter` features will be used. - -The content of script file is here: - -``` -CREATE EXTERNAL TABLE APACHE_LOGS (ID INT PRIMARY KEY, REMOTE_IP VARCHAR, REQUEST_URL VARCHAR, REQUEST_METHOD VARCHAR, STATUS VARCHAR, REQUEST_HEADER_USER_AGENT VARCHAR, TIME_RECEIVED_UTC_ISOFORMAT VARCHAR, TIME_US DOUBLE) LOCATION 'kafka://apache-logs?bootstrap-servers=localhost:9092' -CREATE EXTERNAL TABLE APACHE_ERROR_LOGS (ID INT PRIMARY KEY, REMOTE_IP VARCHAR, REQUEST_URL VARCHAR, REQUEST_METHOD VARCHAR, STATUS INT, REQUEST_HEADER_USER_AGENT VARCHAR, TIME_RECEIVED_UTC_ISOFORMAT VARCHAR, TIME_ELAPSED_MS INT) LOCATION 'kafka://apache-error-logs?bootstrap-servers=localhost:9092' TBLPROPERTIES '{"producer":{"acks":"1","key.serializer":"org.apache.storm.kafka.IntSerializer"}}' -INSERT INTO APACHE_ERROR_LOGS SELECT ID, REMOTE_IP, REQUEST_URL, REQUEST_METHOD, CAST(STATUS AS INT) AS STATUS_INT, REQUEST_HEADER_USER_AGENT, TIME_RECEIVED_UTC_ISOFORMAT, (TIME_US / 1000) AS TIME_ELAPSED_MS FROM APACHE_LOGS WHERE (CAST(STATUS AS INT) / 100) >= 4 -``` - -Save this file to `apache_log_error_filtering.sql`. - -Let's take a look at the script. - -The first statement defines the table `APACHE_LOGS` which represents the input stream. The `LOCATION` clause specifies the Kafka host (`localhost:9092`) and the topic (`apache-logs`). -Note that Kafka data source requires primary key to be defined. That's why we put integer id for parsed JSON data. - -Similarly, the second statement specifies the table `APACHE_ERROR_LOGS` which represents the output stream. The `TBLPROPERTIES` clause specifies the configuration of [KafkaProducer](http://kafka.apache.org/documentation.html#producerconfigs) and is required for a Kafka sink table. - -The last statement defines the topology. Storm SQL only define the topology and run topology on DML statement. -DDL statements define input data source, output data source, and user defined function which will be referred by DML statement. - -Let's look at the `where` statement first. Since we want to filter error logs, we divide status by 100 and compare quotient is equal or greater than 4. (easier representation is `>= 400`) -Since status in JSON is string format (hence represented as VARCHAR for APACHE_LOGS table), we apply CAST(STATUS AS INT) to convert to integer type before applying division. -Now we have filtered only error logs. - -Let's transform some columns to match the output stream. In this statement we apply CAST(STATUS AS INT) to convert to integer type, and divide TIME_US by 1000 to convert microsecond to millisecond. - -Last, insert statement stores filtered and transformed rows (tuples) to the output stream. - -To run this example, users need to include the data sources (`storm-sql-kafka` in this case) and its dependency in the -class path. The Storm SQL core dependencies are automatically handled when users run `storm sql`. -Users can include data sources at the submission step like below: - -``` -$ $STORM_DIR/bin/storm sql apache_log_error_filtering.sql apache_log_error_filtering --artifacts "org.apache.storm:storm-sql-kafka:2.0.0-SNAPSHOT,org.apache.storm:storm-kafka-client:2.0.0-SNAPSHOT,org.apache.kafka:kafka-clients:1.1.0^org.slf4j:slf4j-log4j12" -``` - -Above command submits the SQL statements to StormSQL. The command line syntax of Storm SQL is `storm sql [script file] [topology name]`. -Users need to modify each artifacts' version if users are using different version of Storm or Kafka. - -If your statements pass the validation phase, topology will be shown to Storm UI page. - -You can see the output via console: - -``` -$ kafka-console-consumer --zookeeper localhost:2181 --topic apache-error-logs -``` - -and the output will be similar to: - -``` -{"ID":854643,"REMOTE_IP":"4.227.214.159","REQUEST_URL":"/wp-content","REQUEST_METHOD":"GET","STATUS":404,"REQUEST_HEADER_USER_AGENT":"Mozilla/5.0 (Windows 98; Win 9x 4.90; it-IT; rv:1.9.2.20) Gecko/2015-06-03 11:20:16 Firefox/3.6.17","TIME_RECEIVED_UTC_ISOFORMAT":"2021-03-28T19:14:44+00:00","TIME_RECEIVED_TIMESTAMP":1616958884000,"TIME_ELAPSED_MS":274.222} -{"ID":854693,"REMOTE_IP":"223.50.249.7","REQUEST_URL":"/apps/cart.jsp?appID=5578","REQUEST_METHOD":"GET","STATUS":404,"REQUEST_HEADER_USER_AGENT":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_6; rv:1.9.2.20) Gecko/2015-11-06 00:20:43 Firefox/3.8","TIME_RECEIVED_UTC_ISOFORMAT":"2021-03-28T21:41:02+00:00","TIME_RECEIVED_TIMESTAMP":1616967662000,"TIME_ELAPSED_MS":716.851} -... -``` - -You can also run Storm SQL runner to see the logical plan via placing `--explain` to topology name: - -``` -$ $STORM_DIR/bin/storm sql apache_log_error_filtering.sql --explain --artifacts "org.apache.storm:storm-sql-kafka:2.0.0-SNAPSHOT,org.apache.storm:storm-kafka-client:2.0.0-SNAPSHOT,org.apache.kafka:kafka-clients:1.1.0^org.slf4j:slf4j-log4j12" -``` - -and the output will be similar to: - -``` -LogicalTableModify(table=[[APACHE_ERROR_LOGS]], operation=[INSERT], updateColumnList=[[]], flattened=[true]), id = 8 - LogicalProject(ID=[$0], REMOTE_IP=[$1], REQUEST_URL=[$2], REQUEST_METHOD=[$3], STATUS=[CAST($4):INTEGER NOT NULL], REQUEST_HEADER_USER_AGENT=[$5], TIME_RECEIVED_UTC_ISOFORMAT=[$6], TIME_ELAPSED_MS=[/($7, 1000)]), id = 7 - LogicalFilter(condition=[>=(/(CAST($4):INTEGER NOT NULL, 100), 4)]), id = 6 - EnumerableTableScan(table=[[APACHE_LOGS]]), id = 5 -``` - -It might be not same as you are seeing if Storm SQL applies query optimizations. - -We're executing the first Storm SQL topology! Please kill the topology when you see enough output and the logs. - -To be concise, we'll skip explaining the things we've already seen. - -## Example: filtering slow logs - -In this example we'll filter slow logs from entire logs and store them to another topics. `project` and `filter`, and `User Defined Function (UDF)` features will be used. -This is very similar to `filtering error logs` but we'll see how to define `User Defined Function (UDF)`. - -The content of script file is here: - -``` -CREATE EXTERNAL TABLE APACHE_LOGS (ID INT PRIMARY KEY, REMOTE_IP VARCHAR, REQUEST_URL VARCHAR, REQUEST_METHOD VARCHAR, STATUS VARCHAR, REQUEST_HEADER_USER_AGENT VARCHAR, TIME_RECEIVED_UTC_ISOFORMAT VARCHAR, TIME_US DOUBLE) LOCATION 'kafka://apache-logs?bootstrap-servers=localhost:9092' TBLPROPERTIES '{"producer":{"acks":"1","key.serializer":"org.apache.storm.kafka.IntSerializer"}}' -CREATE EXTERNAL TABLE APACHE_SLOW_LOGS (ID INT PRIMARY KEY, REMOTE_IP VARCHAR, REQUEST_URL VARCHAR, REQUEST_METHOD VARCHAR, STATUS INT, REQUEST_HEADER_USER_AGENT VARCHAR, TIME_RECEIVED_UTC_ISOFORMAT VARCHAR, TIME_RECEIVED_TIMESTAMP BIGINT, TIME_ELAPSED_MS INT) LOCATION 'kafka://apache-slow-logs?bootstrap-servers=localhost:9092' TBLPROPERTIES '{"producer":{"acks":"1","key.serializer":"org.apache.storm.kafka.IntSerializer"}}' -CREATE FUNCTION GET_TIME AS 'org.apache.storm.sql.runtime.functions.scalar.datetime.GetTime2' -INSERT INTO APACHE_SLOW_LOGS SELECT ID, REMOTE_IP, REQUEST_URL, REQUEST_METHOD, CAST(STATUS AS INT) AS STATUS_INT, REQUEST_HEADER_USER_AGENT, TIME_RECEIVED_UTC_ISOFORMAT, GET_TIME(TIME_RECEIVED_UTC_ISOFORMAT, 'yyyy-MM-dd''T''HH:mm:ssZZ') AS TIME_RECEIVED_TIMESTAMP, TIME_US / 1000 AS TIME_ELAPSED_MS FROM APACHE_LOGS WHERE (TIME_US / 1000) >= 100 -``` - -Save this file to `apache_log_slow_filtering.sql`. - -We can skip the first 2 statements since it's almost same to the last example. - -The third statement defines the `User defined function`. We're defining `GET_TIME` which uses `org.apache.storm.sql.runtime.functions.scalar.datetime.GetTime2` class. - -The implementation of GetTime2 is here: - -``` -package org.apache.storm.sql.runtime.functions.scalar.datetime; - -public class GetTime2 { - public static Long evaluate(String dateString, String dateFormat) { - try { - DateTimeFormatter df = DateTimeFormat.forPattern(dateFormat).withZoneUTC(); - return df.parseDateTime(dateString).getMillis(); - } catch (Exception ex) { - throw new RuntimeException(ex); - } - } -} -``` - -This class can be used for UDF since it defines static `evaluate` method. The SQL type of parameters and return are determined by Calcite which Storm SQL depends on. - -Note that this class should be in classpath, so in order to define UDF, you need to create jar file which contains UDF classes and run `storm sql` with `--jar` option. -This page assumes that GetTime2 is in classpath, for simplicity. - -The last statement is very similar to filtering error logs. The only new thing is that we call `GET_TIME(TIME_RECEIVED_UTC_ISOFORMAT, 'yyyy-MM-dd''T''HH:mm:ssZZ')` to convert string time to unix timestamp (BIGINT). - -Let's execute it. - -``` -$ $STORM_DIR/bin/storm sql apache_log_slow_filtering.sql apache_log_slow_filtering --artifacts "org.apache.storm:storm-sql-kafka:2.0.0-SNAPSHOT,org.apache.storm:storm-kafka-client:2.0.0-SNAPSHOT,org.apache.kafka:kafka-clients:1.1.0^org.slf4j:slf4j-log4j12" -``` - -You can see the output via console: - -``` -$ kafka-console-consumer --zookeeper localhost:2181 --topic apache-slow-logs -``` - -and the output will be similar to: - -``` -{"ID":890502,"REMOTE_IP":"136.156.159.160","REQUEST_URL":"/list","REQUEST_METHOD":"GET","STATUS":200,"REQUEST_HEADER_USER_AGENT":"Mozilla/5.0 (Windows NT 5.01) AppleWebKit/5311 (KHTML, like Gecko) Chrome/13.0.860.0 Safari/5311","TIME_RECEIVED_UTC_ISOFORMAT":"2021-06-05T03:44:59+00:00","TIME_RECEIVED_TIMESTAMP":1622864699000,"TIME_ELAPSED_MS":638.579} -{"ID":890542,"REMOTE_IP":"105.146.3.190","REQUEST_URL":"/search/tag/list","REQUEST_METHOD":"DELETE","STATUS":200,"REQUEST_HEADER_USER_AGENT":"Mozilla/5.0 (X11; Linux i686) AppleWebKit/5332 (KHTML, like Gecko) Chrome/13.0.891.0 Safari/5332","TIME_RECEIVED_UTC_ISOFORMAT":"2021-06-05T05:54:27+00:00","TIME_RECEIVED_TIMESTAMP":1622872467000,"TIME_ELAPSED_MS":403.957} -... -``` - -That's it! Supposing we have UDF which queries geo location via remote ip, we can filter via geo location, or enrich geo location to transformed result. - -## Summary - -We looked through several simple use cases for Storm SQL to learn Storm SQL features. If you haven't looked at [Storm SQL integration](storm-sql.html) and [Storm SQL language](storm-sql-reference.html), you need to read it to see full supported features. \ No newline at end of file diff --git a/docs/storm-sql-internal.md b/docs/storm-sql-internal.md deleted file mode 100644 index e4972f5d8e3..00000000000 --- a/docs/storm-sql-internal.md +++ /dev/null @@ -1,59 +0,0 @@ ---- -title: The Internals of Storm SQL -layout: documentation -documentation: true ---- - -This page describes the design and the implementation of the Storm SQL integration. - -## Overview - -SQL is a well-adopted yet complicated standard. Several projects including Drill, Hive, Phoenix and Spark have invested significantly in their SQL layers. One of the main design goal of StormSQL is to leverage the existing investments for these projects. StormSQL leverages [Apache Calcite](///calcite.apache.org) to implement the SQL standard. StormSQL focuses on compiling the SQL statements to Storm topologies so that they can be executed in Storm clusters. - -Figure 1 describes the workflow of executing a SQL query in StormSQL. First, users provide a sequence of SQL statements. StormSQL parses the SQL statements and translates them to a Calcite logical plan. A logical plan consists of a sequence of SQL logical operators that describe how the query should be executed irrespective to the underlying execution engines. Some examples of logical operators include `TableScan`, `Filter`, `Projection` and `GroupBy`. - -
- - -

Figure 1: Workflow of StormSQL.

-
- -The next step is to compile the logical execution plan down to a physical execution plan. A physical plan consists of physical operators that describes how to execute the SQL query in *StormSQL*. Physical operators such as `Filter`, `Projection`, and `GroupBy` are directly mapped to operations in Storm topologies. StormSQL also compiles expressions in the SQL statements into Java code blocks and plugs them into the Storm Streams API functions which will be compiled once and executed in runtime. - -Finally, StormSQL submits created Storm topology with empty packaged JAR to the Storm cluster. Storm schedules and executes the Storm topology in the same way it executes other Storm topologies. - -The following code block shows an example query that filters and projects results from a Kafka stream. - -``` -CREATE EXTERNAL TABLE ORDERS (ID INT PRIMARY KEY, UNIT_PRICE INT, QUANTITY INT) LOCATION 'kafka://...' ... - -CREATE EXTERNAL TABLE LARGE_ORDERS (ID INT PRIMARY KEY, TOTAL INT) 'kafka://...' ... - -INSERT INTO LARGE_ORDERS SELECT ID, UNIT_PRICE * QUANTITY AS TOTAL FROM ORDERS WHERE UNIT_PRICE * QUANTITY > 50 -``` - -The first two SQL statements define the inputs and outputs of external data. Figure 2 describes the processes of how StormSQL takes the last `SELECT` query and compiles it down to Storm topology. - -
- - -

Figure 2: Compiling the example query to Storm topology.

-
- - -## Constraints of querying streaming tables - -There are several constraints when querying tables that represent a real-time data stream: - -* The `ORDER BY` clause cannot be applied to a stream. -* There is at least one monotonic field in the `GROUP BY` clauses to allow StormSQL bounds the size of the buffer. - -For more information please refer to http://calcite.apache.org/docs/stream.html. - -## Dependency - -Storm takes care about necessary dependencies of Storm SQL except the data source JAR which is used by `EXTERNAL TABLE`. -You can use `--jars` or `--artifacts` option to `storm sql` so that data source JAR can be included to Storm SQL Runner and also Storm Topology runtime classpath. -(Use `--artifacts` if your data source JARs are available in Maven repository since it handles transitive dependencies.) - -Please refer [Storm SQL integration](storm-sql.html) page to how to do it. \ No newline at end of file diff --git a/docs/storm-sql-reference.md b/docs/storm-sql-reference.md deleted file mode 100644 index d7afa379226..00000000000 --- a/docs/storm-sql-reference.md +++ /dev/null @@ -1,1314 +0,0 @@ ---- -title: Storm SQL language -layout: documentation -documentation: true ---- - -Storm SQL uses Apache Calcite to parse and evaluate the SQL statements. -Storm SQL also adopts Rex compiler from Calcite, so Storm SQL is expected to handle SQL dialect recognized by Calcite's default SQL parser. - -The page is based on Calcite SQL reference on website, and removes the area Storm SQL doesn't support, and also adds the area Storm SQL supports. - -Please read [Storm SQL integration](storm-sql.html) page first to see what features Storm SQL supports. - -## Grammar - -Calcite provides broader SQL Grammar. But Storm SQL is not a database system and handles streaming data, so only subset of grammar is supported. -Storm SQL doesn't redefine SQL Grammar and just utilize the parser Calcite provided, so SQL statements are still parsed based on Calcite's SQL Grammar. - -SQL grammar in [BNF](http://en.wikipedia.org/wiki/Backus%E2%80%93Naur_Form)-like form. - -{% highlight sql %} -statement: - setStatement - | resetStatement - | explain - | describe - | insert - | update - | merge - | delete - | query - -setStatement: - [ ALTER ( SYSTEM | SESSION ) ] SET identifier '=' expression - -resetStatement: - [ ALTER ( SYSTEM | SESSION ) ] RESET identifier - | [ ALTER ( SYSTEM | SESSION ) ] RESET ALL - -explain: - EXPLAIN PLAN - [ WITH TYPE | WITH IMPLEMENTATION | WITHOUT IMPLEMENTATION ] - [ EXCLUDING ATTRIBUTES | INCLUDING [ ALL ] ATTRIBUTES ] - FOR ( query | insert | update | merge | delete ) - -describe: - DESCRIBE DATABASE databaseName - | DESCRIBE CATALOG [ databaseName . ] catalogName - | DESCRIBE SCHEMA [ [ databaseName . ] catalogName ] . schemaName - | DESCRIBE [ TABLE ] [ [ [ databaseName . ] catalogName . ] schemaName . ] tableName [ columnName ] - | DESCRIBE [ STATEMENT ] ( query | insert | update | merge | delete ) - -insert: - ( INSERT | UPSERT ) INTO tablePrimary - [ '(' column [, column ]* ')' ] - query - -update: - UPDATE tablePrimary - SET assign [, assign ]* - [ WHERE booleanExpression ] - -assign: - identifier '=' expression - -merge: - MERGE INTO tablePrimary [ [ AS ] alias ] - USING tablePrimary - ON booleanExpression - [ WHEN MATCHED THEN UPDATE SET assign [, assign ]* ] - [ WHEN NOT MATCHED THEN INSERT VALUES '(' value [ , value ]* ')' ] - -delete: - DELETE FROM tablePrimary [ [ AS ] alias ] - [ WHERE booleanExpression ] - -query: - values - | WITH withItem [ , withItem ]* query - | { - select - | selectWithoutFrom - | query UNION [ ALL ] query - | query EXCEPT query - | query INTERSECT query - } - [ ORDER BY orderItem [, orderItem ]* ] - [ LIMIT { count | ALL } ] - [ OFFSET start { ROW | ROWS } ] - [ FETCH { FIRST | NEXT } [ count ] { ROW | ROWS } ] - -withItem: - name - [ '(' column [, column ]* ')' ] - AS '(' query ')' - -orderItem: - expression [ ASC | DESC ] [ NULLS FIRST | NULLS LAST ] - -select: - SELECT [ STREAM ] [ ALL | DISTINCT ] - { * | projectItem [, projectItem ]* } - FROM tableExpression - [ WHERE booleanExpression ] - [ GROUP BY { groupItem [, groupItem ]* } ] - [ HAVING booleanExpression ] - [ WINDOW windowName AS windowSpec [, windowName AS windowSpec ]* ] - -selectWithoutFrom: - SELECT [ ALL | DISTINCT ] - { * | projectItem [, projectItem ]* } - -projectItem: - expression [ [ AS ] columnAlias ] - | tableAlias . * - -tableExpression: - tableReference [, tableReference ]* - | tableExpression [ NATURAL ] [ LEFT | RIGHT | FULL ] JOIN tableExpression [ joinCondition ] - -joinCondition: - ON booleanExpression - | USING '(' column [, column ]* ')' - -tableReference: - tablePrimary - [ [ AS ] alias [ '(' columnAlias [, columnAlias ]* ')' ] ] - -tablePrimary: - [ [ catalogName . ] schemaName . ] tableName - '(' TABLE [ [ catalogName . ] schemaName . ] tableName ')' - | [ LATERAL ] '(' query ')' - | UNNEST '(' expression ')' [ WITH ORDINALITY ] - | [ LATERAL ] TABLE '(' [ SPECIFIC ] functionName '(' expression [, expression ]* ')' ')' - -values: - VALUES expression [, expression ]* - -groupItem: - expression - | '(' ')' - | '(' expression [, expression ]* ')' - | CUBE '(' expression [, expression ]* ')' - | ROLLUP '(' expression [, expression ]* ')' - | GROUPING SETS '(' groupItem [, groupItem ]* ')' - -windowRef: - windowName - | windowSpec - -windowSpec: - [ windowName ] - '(' - [ ORDER BY orderItem [, orderItem ]* ] - [ PARTITION BY expression [, expression ]* ] - [ - RANGE numericOrIntervalExpression { PRECEDING | FOLLOWING } - | ROWS numericExpression { PRECEDING | FOLLOWING } - ] - ')' -{% endhighlight %} - -In *merge*, at least one of the WHEN MATCHED and WHEN NOT MATCHED clauses must -be present. - -In *orderItem*, if *expression* is a positive integer *n*, it denotes -the nth item in the SELECT clause. - -An aggregate query is a query that contains a GROUP BY or a HAVING -clause, or aggregate functions in the SELECT clause. In the SELECT, -HAVING and ORDER BY clauses of an aggregate query, all expressions -must be constant within the current group (that is, grouping constants -as defined by the GROUP BY clause, or constants), or aggregate -functions, or a combination of constants and aggregate -functions. Aggregate and grouping functions may only appear in an -aggregate query, and only in a SELECT, HAVING or ORDER BY clause. - -A scalar sub-query is a sub-query used as an expression. -If the sub-query returns no rows, the value is NULL; if it -returns more than one row, it is an error. - -IN, EXISTS and scalar sub-queries can occur -in any place where an expression can occur (such as the SELECT clause, -WHERE clause, ON clause of a JOIN, or as an argument to an aggregate -function). - -An IN, EXISTS or scalar sub-query may be correlated; that is, it -may refer to tables in the FROM clause of an enclosing query. - -*selectWithoutFrom* is equivalent to VALUES, -but is not standard SQL and is only allowed in certain -[conformance levels](https://calcite.apache.org/apidocs/org/apache/calcite/sql/validate/SqlConformance.html#isFromRequired--). - -## Keywords - -The following is a list of SQL keywords. This list is also from Calcite SQL reference. -Reserved keywords are **bold**. - -{% comment %} start {% endcomment %} -A, -**ABS**, -ABSOLUTE, -ACTION, -ADA, -ADD, -ADMIN, -AFTER, -**ALL**, -**ALLOCATE**, -**ALLOW**, -**ALTER**, -ALWAYS, -**AND**, -**ANY**, -**ARE**, -**ARRAY**, -**AS**, -ASC, -**ASENSITIVE**, -ASSERTION, -ASSIGNMENT, -**ASYMMETRIC**, -**AT**, -**ATOMIC**, -ATTRIBUTE, -ATTRIBUTES, -**AUTHORIZATION**, -**AVG**, -BEFORE, -**BEGIN**, -BERNOULLI, -**BETWEEN**, -**BIGINT**, -**BINARY**, -**BIT**, -**BLOB**, -**BOOLEAN**, -**BOTH**, -BREADTH, -**BY**, -C, -**CALL**, -**CALLED**, -**CARDINALITY**, -CASCADE, -**CASCADED**, -**CASE**, -**CAST**, -CATALOG, -CATALOG_NAME, -**CEIL**, -**CEILING**, -CENTURY, -CHAIN, -**CHAR**, -**CHARACTER**, -CHARACTERISTICTS, -CHARACTERS, -**CHARACTER_LENGTH**, -CHARACTER_SET_CATALOG, -CHARACTER_SET_NAME, -CHARACTER_SET_SCHEMA, -**CHAR_LENGTH**, -**CHECK**, -CLASS_ORIGIN, -**CLOB**, -**CLOSE**, -**COALESCE**, -COBOL, -**COLLATE**, -COLLATION, -COLLATION_CATALOG, -COLLATION_NAME, -COLLATION_SCHEMA, -**COLLECT**, -**COLUMN**, -COLUMN_NAME, -COMMAND_FUNCTION, -COMMAND_FUNCTION_CODE, -**COMMIT**, -COMMITTED, -**CONDITION**, -CONDITION_NUMBER, -**CONNECT**, -CONNECTION, -CONNECTION_NAME, -**CONSTRAINT**, -CONSTRAINTS, -CONSTRAINT_CATALOG, -CONSTRAINT_NAME, -CONSTRAINT_SCHEMA, -CONSTRUCTOR, -CONTAINS, -CONTINUE, -**CONVERT**, -**CORR**, -**CORRESPONDING**, -**COUNT**, -**COVAR_POP**, -**COVAR_SAMP**, -**CREATE**, -**CROSS**, -**CUBE**, -**CUME_DIST**, -**CURRENT**, -**CURRENT_CATALOG**, -**CURRENT_DATE**, -**CURRENT_DEFAULT_TRANSFORM_GROUP**, -**CURRENT_PATH**, -**CURRENT_ROLE**, -**CURRENT_SCHEMA**, -**CURRENT_TIME**, -**CURRENT_TIMESTAMP**, -**CURRENT_TRANSFORM_GROUP_FOR_TYPE**, -**CURRENT_USER**, -**CURSOR**, -CURSOR_NAME, -**CYCLE**, -DATA, -DATABASE, -**DATE**, -DATETIME_INTERVAL_CODE, -DATETIME_INTERVAL_PRECISION, -**DAY**, -**DEALLOCATE**, -**DEC**, -DECADE, -**DECIMAL**, -**DECLARE**, -**DEFAULT**, -DEFAULTS, -DEFERRABLE, -DEFERRED, -DEFINED, -DEFINER, -DEGREE, -**DELETE**, -**DENSE_RANK**, -DEPTH, -**DEREF**, -DERIVED, -DESC, -**DESCRIBE**, -DESCRIPTION, -DESCRIPTOR, -**DETERMINISTIC**, -DIAGNOSTICS, -**DISALLOW**, -**DISCONNECT**, -DISPATCH, -**DISTINCT**, -DOMAIN, -**DOUBLE**, -DOW, -DOY, -**DROP**, -**DYNAMIC**, -DYNAMIC_FUNCTION, -DYNAMIC_FUNCTION_CODE, -**EACH**, -**ELEMENT**, -**ELSE**, -**END**, -**END-EXEC**, -EPOCH, -EQUALS, -**ESCAPE**, -**EVERY**, -**EXCEPT**, -EXCEPTION, -EXCLUDE, -EXCLUDING, -**EXEC**, -**EXECUTE**, -**EXISTS**, -**EXP**, -**EXPLAIN**, -**EXTEND**, -**EXTERNAL**, -**EXTRACT**, -**FALSE**, -**FETCH**, -**FILTER**, -FINAL, -FIRST, -**FIRST_VALUE**, -**FLOAT**, -**FLOOR**, -FOLLOWING, -**FOR**, -**FOREIGN**, -FORTRAN, -FOUND, -FRAC_SECOND, -**FREE**, -**FROM**, -**FULL**, -**FUNCTION**, -**FUSION**, -G, -GENERAL, -GENERATED, -**GET**, -**GLOBAL**, -GO, -GOTO, -**GRANT**, -GRANTED, -**GROUP**, -**GROUPING**, -**HAVING**, -HIERARCHY, -**HOLD**, -**HOUR**, -**IDENTITY**, -IMMEDIATE, -IMPLEMENTATION, -**IMPORT**, -**IN**, -INCLUDING, -INCREMENT, -**INDICATOR**, -INITIALLY, -**INNER**, -**INOUT**, -INPUT, -**INSENSITIVE**, -**INSERT**, -INSTANCE, -INSTANTIABLE, -**INT**, -**INTEGER**, -**INTERSECT**, -**INTERSECTION**, -**INTERVAL**, -**INTO**, -INVOKER, -**IS**, -ISOLATION, -JAVA, -**JOIN**, -K, -KEY, -KEY_MEMBER, -KEY_TYPE, -LABEL, -**LANGUAGE**, -**LARGE**, -LAST, -**LAST_VALUE**, -**LATERAL**, -**LEADING**, -**LEFT**, -LENGTH, -LEVEL, -LIBRARY, -**LIKE**, -**LIMIT**, -**LN**, -**LOCAL**, -**LOCALTIME**, -**LOCALTIMESTAMP**, -LOCATOR, -**LOWER**, -M, -MAP, -**MATCH**, -MATCHED, -**MAX**, -MAXVALUE, -**MEMBER**, -**MERGE**, -MESSAGE_LENGTH, -MESSAGE_OCTET_LENGTH, -MESSAGE_TEXT, -**METHOD**, -MICROSECOND, -MILLENNIUM, -**MIN**, -**MINUTE**, -MINVALUE, -**MOD**, -**MODIFIES**, -**MODULE**, -**MONTH**, -MORE, -**MULTISET**, -MUMPS, -NAME, -NAMES, -**NATIONAL**, -**NATURAL**, -**NCHAR**, -**NCLOB**, -NESTING, -**NEW**, -**NEXT**, -**NO**, -**NONE**, -**NORMALIZE**, -NORMALIZED, -**NOT**, -**NULL**, -NULLABLE, -**NULLIF**, -NULLS, -NUMBER, -**NUMERIC**, -OBJECT, -OCTETS, -**OCTET_LENGTH**, -**OF**, -**OFFSET**, -**OLD**, -**ON**, -**ONLY**, -**OPEN**, -OPTION, -OPTIONS, -**OR**, -**ORDER**, -ORDERING, -ORDINALITY, -OTHERS, -**OUT**, -**OUTER**, -OUTPUT, -**OVER**, -**OVERLAPS**, -**OVERLAY**, -OVERRIDING, -PAD, -**PARAMETER**, -PARAMETER_MODE, -PARAMETER_NAME, -PARAMETER_ORDINAL_POSITION, -PARAMETER_SPECIFIC_CATALOG, -PARAMETER_SPECIFIC_NAME, -PARAMETER_SPECIFIC_SCHEMA, -PARTIAL, -**PARTITION**, -PASCAL, -PASSTHROUGH, -PATH, -**PERCENTILE_CONT**, -**PERCENTILE_DISC**, -**PERCENT_RANK**, -PLACING, -PLAN, -PLI, -**POSITION**, -**POWER**, -PRECEDING, -**PRECISION**, -**PREPARE**, -PRESERVE, -**PRIMARY**, -PRIOR, -PRIVILEGES, -**PROCEDURE**, -PUBLIC, -QUARTER, -**RANGE**, -**RANK**, -READ, -**READS**, -**REAL**, -**RECURSIVE**, -**REF**, -**REFERENCES**, -**REFERENCING**, -**REGR_AVGX**, -**REGR_AVGY**, -**REGR_COUNT**, -**REGR_INTERCEPT**, -**REGR_R2**, -**REGR_SLOPE**, -**REGR_SXX**, -**REGR_SXY**, -**REGR_SYY**, -RELATIVE, -**RELEASE**, -REPEATABLE, -**RESET**, -RESTART, -RESTRICT, -**RESULT**, -**RETURN**, -RETURNED_CARDINALITY, -RETURNED_LENGTH, -RETURNED_OCTET_LENGTH, -RETURNED_SQLSTATE, -**RETURNS**, -**REVOKE**, -**RIGHT**, -ROLE, -**ROLLBACK**, -**ROLLUP**, -ROUTINE, -ROUTINE_CATALOG, -ROUTINE_NAME, -ROUTINE_SCHEMA, -**ROW**, -**ROWS**, -ROW_COUNT, -**ROW_NUMBER**, -**SAVEPOINT**, -SCALE, -SCHEMA, -SCHEMA_NAME, -**SCOPE**, -SCOPE_CATALOGS, -SCOPE_NAME, -SCOPE_SCHEMA, -**SCROLL**, -**SEARCH**, -**SECOND**, -SECTION, -SECURITY, -**SELECT**, -SELF, -**SENSITIVE**, -SEQUENCE, -SERIALIZABLE, -SERVER, -SERVER_NAME, -SESSION, -**SESSION_USER**, -**SET**, -SETS, -**SIMILAR**, -SIMPLE, -SIZE, -**SMALLINT**, -**SOME**, -SOURCE, -SPACE, -**SPECIFIC**, -**SPECIFICTYPE**, -SPECIFIC_NAME, -**SQL**, -**SQLEXCEPTION**, -**SQLSTATE**, -**SQLWARNING**, -SQL_TSI_DAY, -SQL_TSI_FRAC_SECOND, -SQL_TSI_HOUR, -SQL_TSI_MICROSECOND, -SQL_TSI_MINUTE, -SQL_TSI_MONTH, -SQL_TSI_QUARTER, -SQL_TSI_SECOND, -SQL_TSI_WEEK, -SQL_TSI_YEAR, -**SQRT**, -**START**, -STATE, -STATEMENT, -**STATIC**, -**STDDEV_POP**, -**STDDEV_SAMP**, -**STREAM**, -STRUCTURE, -STYLE, -SUBCLASS_ORIGIN, -**SUBMULTISET**, -SUBSTITUTE, -**SUBSTRING**, -**SUM**, -**SYMMETRIC**, -**SYSTEM**, -**SYSTEM_USER**, -**TABLE**, -**TABLESAMPLE**, -TABLE_NAME, -TEMPORARY, -**THEN**, -TIES, -**TIME**, -**TIMESTAMP**, -TIMESTAMPADD, -TIMESTAMPDIFF, -**TIMEZONE_HOUR**, -**TIMEZONE_MINUTE**, -**TINYINT**, -**TO**, -TOP_LEVEL_COUNT, -**TRAILING**, -TRANSACTION, -TRANSACTIONS_ACTIVE, -TRANSACTIONS_COMMITTED, -TRANSACTIONS_ROLLED_BACK, -TRANSFORM, -TRANSFORMS, -**TRANSLATE**, -**TRANSLATION**, -**TREAT**, -**TRIGGER**, -TRIGGER_CATALOG, -TRIGGER_NAME, -TRIGGER_SCHEMA, -**TRIM**, -**TRUE**, -TYPE, -**UESCAPE**, -UNBOUNDED, -UNCOMMITTED, -UNDER, -**UNION**, -**UNIQUE**, -**UNKNOWN**, -UNNAMED, -**UNNEST**, -**UPDATE**, -**UPPER**, -**UPSERT**, -USAGE, -**USER**, -USER_DEFINED_TYPE_CATALOG, -USER_DEFINED_TYPE_CODE, -USER_DEFINED_TYPE_NAME, -USER_DEFINED_TYPE_SCHEMA, -**USING**, -**VALUE**, -**VALUES**, -**VARBINARY**, -**VARCHAR**, -**VARYING**, -**VAR_POP**, -**VAR_SAMP**, -VERSION, -VIEW, -WEEK, -**WHEN**, -**WHENEVER**, -**WHERE**, -**WIDTH_BUCKET**, -**WINDOW**, -**WITH**, -**WITHIN**, -**WITHOUT**, -WORK, -WRAPPER, -WRITE, -XML, -**YEAR**, -ZONE. -{% comment %} end {% endcomment %} - -## Identifiers - -Identifiers are the names of tables, columns and other metadata -elements used in a SQL query. - -Unquoted identifiers, such as emp, must start with a letter and can -only contain letters, digits, and underscores. They are implicitly -converted to upper case. - -Quoted identifiers, such as `"Employee Name"`, start and end with -double quotes. They may contain virtually any character, including -spaces and other punctuation. If you wish to include a double quote -in an identifier, use another double quote to escape it, like this: -`"An employee called ""Fred""."`. - -In Calcite, matching identifiers to the name of the referenced object is -case-sensitive. But remember that unquoted identifiers are implicitly -converted to upper case before matching, and if the object it refers -to was created using an unquoted identifier for its name, then its -name will have been converted to upper case also. - -## Data types - -### Scalar types - -| Data type | Description | Range and examples | -|:----------- |:------------------------- |:---------------------| -| BOOLEAN | Logical values | Values: TRUE, FALSE, UNKNOWN -| TINYINT | 1 byte signed integer | Range is -255 to 256 -| SMALLINT | 2 byte signed integer | Range is -32768 to 32767 -| INTEGER, INT | 4 byte signed integer | Range is -2147483648 to 2147483647 -| BIGINT | 8 byte signed integer | Range is -9223372036854775808 to 9223372036854775807 -| DECIMAL(p, s) | Fixed point | Example: 123.45 is a DECIMAL(5, 2) value. -| NUMERIC | Fixed point | -| REAL, FLOAT | 4 byte floating point | 6 decimal digits precision -| DOUBLE | 8 byte floating point | 15 decimal digits precision -| CHAR(n), CHARACTER(n) | Fixed-width character string | 'Hello', '' (empty string), _latin1'Hello', n'Hello', _UTF16'Hello', 'Hello' 'there' (literal split into multiple parts) -| VARCHAR(n), CHARACTER VARYING(n) | Variable-length character string | As CHAR(n) -| BINARY(n) | Fixed-width binary string | x'45F0AB', x'' (empty binary string), x'AB' 'CD' (multi-part binary string literal) -| VARBINARY(n), BINARY VARYING(n) | Variable-length binary string | As BINARY(n) -| DATE | Date | Example: DATE '1969-07-20' -| TIME | Time of day | Example: TIME '20:17:40' -| TIMESTAMP [ WITHOUT TIME ZONE ] | Date and time | Example: TIMESTAMP '1969-07-20 20:17:40' -| TIMESTAMP WITH TIME ZONE | Date and time with time zone | Example: TIMESTAMP '1969-07-20 20:17:40 America/Los Angeles' -| INTERVAL timeUnit [ TO timeUnit ] | Date time interval | Examples: INTERVAL '1:5' YEAR TO MONTH, INTERVAL '45' DAY -| Anchored interval | Date time interval | Example: (DATE '1969-07-20', DATE '1972-08-29') - -Where: - -{% highlight sql %} -timeUnit: - MILLENNIUM | CENTURY | DECADE | YEAR | QUARTER | MONTH | WEEK | DOY | DOW | DAY | HOUR | MINUTE | SECOND | EPOCH -{% endhighlight %} - -Note: - -* DATE, TIME and TIMESTAMP have no time zone. There is not even an implicit - time zone, such as UTC (as in Java) or the local time zone. It is left to - the user or application to supply a time zone. - -### Non-scalar types - -| Type | Description -|:-------- |:----------------------------------------------------------- -| ANY | A value of an unknown type -| ROW | Row with 1 or more columns -| MAP | Collection of keys mapped to values -| MULTISET | Unordered collection that may contain duplicates -| ARRAY | Ordered, contiguous collection that may contain duplicates -| CURSOR | Cursor over the result of executing a query - -## Operators and functions - -### Operator precedence - -The operator precedence and associativity, highest to lowest. - -| Operator | Associativity -|:------------------------------------------------- |:------------- -| . | left -| \[ \] (array element) | left -| + - (unary plus, minus) | right -| * / | left -| + - | left -| BETWEEN, IN, LIKE, SIMILAR | - -| < > = <= >= <> != | left -| IS NULL, IS FALSE, IS NOT TRUE etc. | - -| NOT | right -| AND | left -| OR | left - -### Comparison operators - -| Operator syntax | Description -|:------------------------------------------------- |:----------- -| value1 = value2 | Equals -| value1 <> value2 | Not equal -| value1 != value2 | Not equal (only available at some conformance levels) -| value1 > value2 | Greater than -| value1 >= value2 | Greater than or equal -| value1 < value2 | Less than -| value1 <= value2 | Less than or equal -| value IS NULL | Whether *value* is null -| value IS NOT NULL | Whether *value* is not null -| value1 IS DISTINCT FROM value2 | Whether two values are not equal, treating null values as the same -| value1 IS NOT DISTINCT FROM value2 | Whether two values are equal, treating null values as the same -| value1 BETWEEN value2 AND value3 | Whether *value1* is greater than or equal to *value2* and less than or equal to *value3* -| value1 NOT BETWEEN value2 AND value3 | Whether *value1* is less than *value2* or greater than *value3* -| string1 LIKE string2 [ ESCAPE string3 ] | Whether *string1* matches pattern *string2* -| string1 NOT LIKE string2 [ ESCAPE string3 ] | Whether *string1* does not match pattern *string2* -| string1 SIMILAR TO string2 [ ESCAPE string3 ] | Whether *string1* matches regular expression *string2* -| string1 NOT SIMILAR TO string2 [ ESCAPE string3 ] | Whether *string1* does not match regular expression *string2* -| value IN (value [, value]* ) | Whether *value* is equal to a value in a list -| value NOT IN (value [, value]* ) | Whether *value* is not equal to every value in a list - -Not supported yet on Storm SQL: - -| Operator syntax | Description -|:------------------------------------------------- |:----------- -| value IN (sub-query) | Whether *value* is equal to a row returned by *sub-query* -| value NOT IN (sub-query) | Whether *value* is not equal to every row returned by *sub-query* -| EXISTS (sub-query) | Whether *sub-query* returns at least one row - -Storm SQL doesn't support sub-query yet, so above operators don't work properly. This will be addressed in near future. - -### Logical operators - -| Operator syntax | Description -|:---------------------- |:----------- -| boolean1 OR boolean2 | Whether *boolean1* is TRUE or *boolean2* is TRUE -| boolean1 AND boolean2 | Whether *boolean1* and *boolean2* are both TRUE -| NOT boolean | Whether *boolean* is not TRUE; returns UNKNOWN if *boolean* is UNKNOWN -| boolean IS FALSE | Whether *boolean* is FALSE; returns FALSE if *boolean* is UNKNOWN -| boolean IS NOT FALSE | Whether *boolean* is not FALSE; returns TRUE if *boolean* is UNKNOWN -| boolean IS TRUE | Whether *boolean* is TRUE; returns FALSE if *boolean* is UNKNOWN -| boolean IS NOT TRUE | Whether *boolean* is not TRUE; returns TRUE if *boolean* is UNKNOWN -| boolean IS UNKNOWN | Whether *boolean* is UNKNOWN -| boolean IS NOT UNKNOWN | Whether *boolean* is not UNKNOWN - -### Arithmetic operators and functions - -| Operator syntax | Description -|:------------------------- |:----------- -| + numeric | Returns *numeric* -|:- numeric | Returns negative *numeric* -| numeric1 + numeric2 | Returns *numeric1* plus *numeric2* -| numeric1 - numeric2 | Returns *numeric1* minus *numeric2* -| numeric1 * numeric2 | Returns *numeric1* multiplied by *numeric2* -| numeric1 / numeric2 | Returns *numeric1* divided by *numeric2* -| POWER(numeric1, numeric2) | Returns *numeric1* raised to the power of *numeric2* -| ABS(numeric) | Returns the absolute value of *numeric* -| MOD(numeric, numeric) | Returns the remainder (modulus) of *numeric1* divided by *numeric2*. The result is negative only if *numeric1* is negative -| SQRT(numeric) | Returns the square root of *numeric* -| LN(numeric) | Returns the natural logarithm (base *e*) of *numeric* -| LOG10(numeric) | Returns the base 10 logarithm of *numeric* -| EXP(numeric) | Returns *e* raised to the power of *numeric* -| CEIL(numeric) | Rounds *numeric* up, and returns the smallest number that is greater than or equal to *numeric* -| FLOOR(numeric) | Rounds *numeric* down, and returns the largest number that is less than or equal to *numeric* - -### Character string operators and functions - -| Operator syntax | Description -|:-------------------------- |:----------- -| string || string | Concatenates two character strings. -| CHAR_LENGTH(string) | Returns the number of characters in a character string -| CHARACTER_LENGTH(string) | As CHAR_LENGTH(*string*) -| UPPER(string) | Returns a character string converted to upper case -| LOWER(string) | Returns a character string converted to lower case -| POSITION(string1 IN string2) | Returns the position of the first occurrence of *string1* in *string2* -| TRIM( { BOTH | LEADING | TRAILING } string1 FROM string2) | Removes the longest string containing only the characters in *string1* from the start/end/both ends of *string1* -| OVERLAY(string1 PLACING string2 FROM integer [ FOR integer2 ]) | Replaces a substring of *string1* with *string2* -| SUBSTRING(string FROM integer) | Returns a substring of a character string starting at a given point. -| SUBSTRING(string FROM integer FOR integer) | Returns a substring of a character string starting at a given point with a given length. -| INITCAP(string) | Returns *string* with the first letter of each word converter to upper case and the rest to lower case. Words are sequences of alphanumeric characters separated by non-alphanumeric characters. - -Not implemented: - -* SUBSTRING(string FROM regexp FOR regexp) - -### Binary string operators and functions - -| Operator syntax | Description -|:--------------- |:----------- -| binary || binary | Concatenates two binary strings. -| POSITION(binary1 IN binary2) | Returns the position of the first occurrence of *binary1* in *binary2* -| OVERLAY(binary1 PLACING binary2 FROM integer [ FOR integer2 ]) | Replaces a substring of *binary1* with *binary2* - -Known bugs: - -| Operator syntax | Description -|:--------------- |:----------- -| SUBSTRING(binary FROM integer) | Returns a substring of *binary* starting at a given point -| SUBSTRING(binary FROM integer FOR integer) | Returns a substring of *binary* starting at a given point with a given length - -Calcite 1.9.0 has bugs on binary SUBSTRING functions which throws exception while compiling SQL statements. This can be fixed to higher version of Calcite. - -### Date/time functions - -| Operator syntax | Description -|:------------------------- |:----------- -| EXTRACT(timeUnit FROM datetime) | Extracts and returns the value of a specified datetime field from a datetime value expression -| FLOOR(datetime TO timeUnit) | Rounds *datetime* down to *timeUnit* -| CEIL(datetime TO timeUnit) | Rounds *datetime* up to *timeUnit* - -Not implemented: - -* EXTRACT(timeUnit FROM interval) -* CEIL(interval) -* FLOOR(interval) -* datetime - datetime timeUnit [ TO timeUnit ] -* interval OVERLAPS interval -* \+ interval -* \- interval -* interval + interval -* interval - interval -* interval / interval -* datetime + interval -* datetime - interval - -Note on Storm SQL: - -| Operator syntax | Description -|:------------------------- |:----------- -| LOCALTIME | Returns the current date and time in the session time zone in a value of datatype TIME -| LOCALTIME(precision) | Returns the current date and time in the session time zone in a value of datatype TIME, with *precision* digits of precision -| LOCALTIMESTAMP | Returns the current date and time in the session time zone in a value of datatype TIMESTAMP -| LOCALTIMESTAMP(precision) | Returns the current date and time in the session time zone in a value of datatype TIMESTAMP, with *precision* digits of precision -| CURRENT_TIME | Returns the current time in the session time zone, in a value of datatype TIMESTAMP WITH TIME ZONE -| CURRENT_DATE | Returns the current date in the session time zone, in a value of datatype DATE -| CURRENT_TIMESTAMP | Returns the current date and time in the session time zone, in a value of datatype TIMESTAMP WITH TIME ZONE - -SQL standard states that above operators should return the same value while evaluating query. -Storm SQL converts each query to Trident topology and run, so technically current date / time should be fixed while evaluating SQL statement. -Because of this limitation, current date / time will be fixed while creating Trident topology, and these operators should return the same value in the lifecycle of topology. - -### System functions - -Not supported yet on Storm SQL: - -| Operator syntax | Description -|:--------------- |:----------- -| USER | Equivalent to CURRENT_USER -| CURRENT_USER | User name of current execution context -| SESSION_USER | Session user name -| SYSTEM_USER | Returns the name of the current data store user as identified by the operating system -| CURRENT_PATH | Returns a character string representing the current lookup scope for references to user-defined routines and types -| CURRENT_ROLE | Returns the current active role - -These operators are not making sense of Storm SQL's runtime, so it may be never supported unless we find out proper semantics. - -### Conditional functions and operators - -| Operator syntax | Description -|:--------------- |:----------- -| CASE value
WHEN value1 [, value11 ]* THEN result1
[ WHEN valueN [, valueN1 ]* THEN resultN ]*
[ ELSE resultZ ]
END | Simple case -| CASE
WHEN condition1 THEN result1
[ WHEN conditionN THEN resultN ]*
[ ELSE resultZ ]
END | Searched case -| NULLIF(value, value) | Returns NULL if the values are the same.

For example, NULLIF(5, 5) returns NULL; NULLIF(5, 0) returns 5. -| COALESCE(value, value [, value ]* ) | Provides a value if the first value is null.

For example, COALESCE(NULL, 5) returns 5. - -### Type conversion - -| Operator syntax | Description -|:--------------- | :---------- -| CAST(value AS type) | Converts a value to a given type. - -### Value constructors - -| Operator syntax | Description -|:--------------- |:----------- -| ROW (value [, value]* ) | Creates a row from a list of values. -| (value [, value]* ) | Creates a row from a list of values. -| map '[' key ']' | Returns the element of a map with a particular key. -| array '[' index ']' | Returns the element at a particular location in an array. -| ARRAY '[' value [, value ]* ']' | Creates an array from a list of values. -| MAP '[' key, value [, key, value ]* ']' | Creates a map from a list of key-value pairs. - -### Collection functions - -| Operator syntax | Description -|:--------------- |:----------- -| ELEMENT(value) | Returns the sole element of a array or multiset; null if the collection is empty; throws if it has more than one element. -| CARDINALITY(value) | Returns the number of elements in an array or multiset. - -See also: UNNEST relational operator converts a collection to a relation. - -### JDBC function escape - -#### Numeric - -| Operator syntax | Description -|:------------------------------ |:----------- -| {fn ABS(numeric)} | Returns the absolute value of *numeric* -| {fn EXP(numeric)} | Returns *e* raised to the power of *numeric* -| {fn LOG(numeric)} | Returns the natural logarithm (base *e*) of *numeric* -| {fn LOG10(numeric)} | Returns the base-10 logarithm of *numeric* -| {fn MOD(numeric1, numeric2)} | Returns the remainder (modulus) of *numeric1* divided by *numeric2*. The result is negative only if *numeric1* is negative -| {fn POWER(numeric1, numeric2)} | Returns *numeric1* raised to the power of *numeric2* - -Not implemented: - -* {fn ACOS(numeric)} - Returns the arc cosine of *numeric* -* {fn ASIN(numeric)} - Returns the arc sine of *numeric* -* {fn ATAN(numeric)} - Returns the arc tangent of *numeric* -* {fn ATAN2(numeric, numeric)} -* {fn CEILING(numeric)} - Rounds *numeric* up, and returns the smallest number that is greater than or equal to *numeric* -* {fn COS(numeric)} - Returns the cosine of *numeric* -* {fn COT(numeric)} -* {fn DEGREES(numeric)} - Converts *numeric* from radians to degrees -* {fn FLOOR(numeric)} - Rounds *numeric* down, and returns the largest number that is less than or equal to *numeric* -* {fn PI()} - Returns a value that is closer than any other value to *pi* -* {fn RADIANS(numeric)} - Converts *numeric* from degrees to radians -* {fn RAND(numeric)} -* {fn ROUND(numeric, numeric)} -* {fn SIGN(numeric)} -* {fn SIN(numeric)} - Returns the sine of *numeric* -* {fn SQRT(numeric)} - Returns the square root of *numeric* -* {fn TAN(numeric)} - Returns the tangent of *numeric* -* {fn TRUNCATE(numeric, numeric)} - -#### String - -| Operator syntax | Description -|:--------------- |:----------- -| {fn CONCAT(character, character)} | Returns the concatenation of character strings -| {fn LOCATE(string1, string2)} | Returns the position in *string2* of the first occurrence of *string1*. Searches from the beginning of the second CharacterExpression, unless the startIndex parameter is specified. -| {fn INSERT(string1, start, length, string2)} | Inserts *string2* into a slot in *string1* -| {fn LCASE(string)} | Returns a string in which all alphabetic characters in *string* have been converted to lower case -| {fn LENGTH(string)} | Returns the number of characters in a string -| {fn SUBSTRING(string, offset, length)} | Returns a character string that consists of *length* characters from *string* starting at the *offset* position -| {fn UCASE(string)} | Returns a string in which all alphabetic characters in *string* have been converted to upper case - -Known bugs: - -| Operator syntax | Description -|:--------------- |:----------- -| {fn LOCATE(string1, string2 [, integer])} | Returns the position in *string2* of the first occurrence of *string1*. Searches from the beginning of *string2*, unless *integer* is specified. -| {fn LTRIM(string)} | Returns *string* with leading space characters removed -| {fn RTRIM(string)} | Returns *string* with trailing space characters removed - -Calcite 1.9.0 throws exception on {fn LOCATE} with position parameter, {fn LTRIM} and {fn RTRIM} while compiling SQL statement. -This can be fixed to higher version of Calcite. - -Not implemented: - -* {fn ASCII(string)} - Convert a single-character string to the corresponding ASCII code, an integer between 0 and 255 -* {fn CHAR(string)} -* {fn DIFFERENCE(string, string)} -* {fn LEFT(string, integer)} -* {fn REPEAT(string, integer)} -* {fn REPLACE(string, string, string)} -* {fn RIGHT(string, integer)} -* {fn SOUNDEX(string)} -* {fn SPACE(integer)} - -#### Date/time - -| Operator syntax | Description -|:--------------- |:----------- -| {fn CURDATE()} | Equivalent to `CURRENT_DATE` -| {fn CURTIME()} | Equivalent to `LOCALTIME` -| {fn NOW()} | Equivalent to `LOCALTIMESTAMP` -| {fn QUARTER(date)} | Equivalent to `EXTRACT(QUARTER FROM date)`. Returns an integer between 1 and 4. -| {fn TIMESTAMPADD(timeUnit, count, timestamp)} | Adds an interval of *count* *timeUnit*s to a timestamp -| {fn TIMESTAMPDIFF(timeUnit, timestamp1, timestamp2)} | Subtracts *timestamp1* from *timestamp2* and returns the result in *timeUnit*s - -Not implemented: - -* {fn DAYNAME(date)} -* {fn DAYOFMONTH(date)} -* {fn DAYOFWEEK(date)} -* {fn DAYOFYEAR(date)} -* {fn HOUR(time)} -* {fn MINUTE(time)} -* {fn MONTH(date)} -* {fn MONTHNAME(date)} -* {fn SECOND(time)} -* {fn WEEK(date)} -* {fn YEAR(date)} - -#### System - -Not implemented: - -* {fn DATABASE()} -* {fn IFNULL(value, value)} -* {fn USER(value, value)} -* {fn CONVERT(value, type)} - -### Aggregate functions - -Storm SQL doesn't support aggregation yet. - -### Window functions - -Storm SQL doesn't support windowing yet. - -### Grouping functions - -Storm SQL doesn't support grouping functions. - -### User-defined functions - -Users can define user defined function (scalar) using `CREATE FUNCTION` statement. -For example, the following statement defines `MYPLUS` function which uses `org.apache.storm.sql.TestUtils$MyPlus` class. - -``` -CREATE FUNCTION MYPLUS AS 'org.apache.storm.sql.TestUtils$MyPlus' -``` - -Storm SQL determines whether the function as scalar or aggregate by checking which methods are defined. -If the class defines `evaluate` method, Storm SQL treats the function as `scalar`. - -Example of class for scalar function is here: - -``` - public class MyPlus { - public static Integer evaluate(Integer x, Integer y) { - return x + y; - } - } - -``` - -Please note that users should use `--jars` or `--artifacts` while running Storm SQL runner to make sure UDFs are available in classpath. - -## External Data Sources - -### Specifying External Data Sources - -In StormSQL data is represented by external tables. Users can specify data sources using the `CREATE EXTERNAL TABLE` statement. The syntax of `CREATE EXTERNAL TABLE` closely follows the one defined in [Hive Data Definition Language](https://cwiki.apache.org/confluence/display/Hive/LanguageManual+DDL): - -``` -CREATE EXTERNAL TABLE table_name field_list - [ STORED AS - INPUTFORMAT input_format_classname - OUTPUTFORMAT output_format_classname - ] - LOCATION location - [ TBLPROPERTIES tbl_properties ] - [ AS select_stmt ] -``` - -Default input format and output format are JSON. We will introduce `supported formats` from further section. - -For example, the following statement specifies a Kafka spout and sink: - -``` -CREATE EXTERNAL TABLE FOO (ID INT PRIMARY KEY) LOCATION 'kafka://test?bootstrap-hosts=localhost:9092' TBLPROPERTIES '{"producer":{"acks":"1","key.serializer":"org.apache.storm.kafka.IntSerializer"}}' -``` - -Please note that users should use `--jars` or `--artifacts` while running Storm SQL runner to make sure UDFs are available in classpath. - -### Plugging in External Data Sources - -Users plug in external data sources through implementing the `ISqlTridentDataSource` interface and registers them using the mechanisms of Java's service loader. The external data source will be chosen based on the scheme of the URI of the tables. Please refer to the implementation of `storm-sql-kafka` for more details. - -### Supported Formats - -| Format | Input format class | Output format class | Requires properties -|:--------------- |:------------------ |:------------------- |:-------------------- -| JSON | org.apache.storm.sql.runtime.serde.json.JsonScheme | org.apache.storm.sql.runtime.serde.json.JsonSerializer | No -| Avro | org.apache.storm.sql.runtime.serde.avro.AvroScheme | org.apache.storm.sql.runtime.serde.avro.AvroSerializer | Yes -| CSV | org.apache.storm.sql.runtime.serde.csv.CsvScheme | org.apache.storm.sql.runtime.serde.csv.CsvSerializer | No -| TSV | org.apache.storm.sql.runtime.serde.tsv.TsvScheme | org.apache.storm.sql.runtime.serde.tsv.TsvSerializer | No - -#### Avro - -Avro requires users to describe the schema of record (both input and output). Schema should be described on `TBLPROPERTIES`. -Input format needs to be described to `input.avro.schema`, and output format needs to be described to `output.avro.schema`. -Schema string should be an escaped JSON so that `TBLPROPERTIES` is valid JSON. - -Example Schema description: - -`"input.avro.schema": "{\"type\": \"record\", \"name\": \"large_orders\", \"fields\" : [ {\"name\": \"ID\", \"type\": \"int\"}, {\"name\": \"TOTAL\", \"type\": \"int\"} ]}"` - -`"output.avro.schema": "{\"type\": \"record\", \"name\": \"large_orders\", \"fields\" : [ {\"name\": \"ID\", \"type\": \"int\"}, {\"name\": \"TOTAL\", \"type\": \"int\"} ]}"` - -#### CSV - -It uses [Standard RFC4180](https://tools.ietf.org/html/rfc4180) CSV Parser and doesn't need any other properties. - -#### TSV - -By default TSV uses `\t` as delimiter, but users can set another delimiter by setting `input.tsv.delimiter` and/or `output.tsv.delimiter`. -Please note that it supports only one letter for delimiter. - -### Supported Data Sources - -| Data Source | Artifact Name | Location prefix | Support Input data source | Support Output data source | Requires properties -|:--------------- |:------------------ |:------------------- |:------------------------- |:-------------------------- |:------------------- -| Socket | | `socket://host:port` | Yes | Yes | No -| Kafka | org.apache.storm:storm-sql-kafka | `kafka://topic?bootstrap-servers=host1:port1,host2:port2` | Yes | Yes | Yes -| Redis | org.apache.storm:storm-sql-redis | `redis://:[password]@host:port/[dbIdx]` | No | Yes | Yes -| MongoDB | org.apache.stormg:storm-sql-mongodb | `mongodb://[username:password@]host1[:port1][,host2[:port2],...[,hostN[:portN]]][/[database][?options]]` | No | Yes | Yes -| HDFS | org.apache.storm:storm-sql-hdfs | `hdfs://host:port/path-to-file` | No | Yes | Yes - -#### Socket - -Socket data source is a built-in feature so users don't need to add any artifacts to `--artifacts` options. - -Please note that Socket data source is only for testing: it doesn't ensure any delivery guarantee. - -TIP: `netcat` is a convenient tool for Socket: users can use netcat to connect Socket data source for either or both input and output purposes. - -#### Kafka - -Kafka data source requires below properties only when its used for output data source: - -* `producer`: Specify Kafka Producer configuration - Please refer [Kafka producer configs](http://kafka.apache.org/documentation.html#producerconfigs) for details. - * Do not set `bootstrap.servers`. It is extracted from the URI you provide for the data source (e.g. `kafka://topic?bootstrap-servers=localhost:9092,localhost:9093` would extract to `localhost:9092,localhost:9093`). - * Do not set `value.serializer`. It is hardcoded to use `ByteBufferSerializer`. Instead use the `STORED AS INPUTFORMAT ... OUTPUTFORMAT ...` syntax to specify the output serializer Storm will use to create the ByteBuffer from input tuples. - -Please note that `storm-sql-kafka` requires users to provide `storm-kafka-client`, and `storm-kafka-client` requires users to provide `kafka-clients`. -You can use below as working reference for `--artifacts` option, and change dependencies version, and see it works: - -`org.apache.storm:storm-sql-kafka:2.0.0-SNAPSHOT,org.apache.storm:storm-kafka-client:2.0.0-SNAPSHOT,org.apache.kafka:kafka-clients:1.1.0^org.slf4j:slf4j-log4j12` - -#### Redis - -Redis data source requires below properties to be set: - -* `data.type`: data type to be used for storing - only `"STRING"` and `"HASH"` are supported -* `data.additional.key`: key if data type needs both key and field (field will be used as field) -* `redis.timeout`: timeout in milliseconds (ex. `"3000"`) -* `use.redis.cluster`: `"true"` if data source is Redis Cluster env., `"false"` otherwise. - -Please note that `storm-sql-redis` requires users to provide `storm-redis`. -You can use below as working reference for `--artifacts` option, and change dependencies version if really needed: - -`org.apache.storm:storm-sql-redis:2.0.0-SNAPSHOT,org.apache.storm:storm-redis:2.0.0-SNAPSHOT` - -#### MongoDB - -MongoDB data source requires below properties to be set: - -`{"collection.name": "storm_sql_mongo", "ser.field": "serfield"}` - -* `ser.field`: field to store - record will be serialized and stored as BSON in this field -* `collection.name`: Collection name - -Please note that `storm-sql-mongodb` requires users to provide `storm-mongodb`. -You can use below as working reference for `--artifacts` option, and change dependencies version if really needed: - -`org.apache.storm:storm-sql-mongodb:2.0.0-SNAPSHOT,org.apache.storm:storm-mongodb:2.0.0-SNAPSHOT` - -Storing record with preserving fields are not supported for now. - -#### HDFS - -HDFS data source requires below properties to be set: - -* `hdfs.file.path`: HDFS file path -* `hdfs.file.name`: HDFS file name - please refer to [SimpleFileNameFormat]({{page.git-blob-base}}/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/format/SimpleFileNameFormat.java) -* `hdfs.rotation.size.kb`: HDFS FileSizeRotationPolicy in KB -* `hdfs.rotation.time.seconds`: HDFS TimedRotationPolicy in seconds - -Please note that `hdfs.rotation.size.kb` and `hdfs.rotation.time.seconds` only one can be used for hdfs rotation. - -And note that `storm-sql-hdfs` requires users to provide `storm-hdfs`. -You can use below as working reference for `--artifacts` option, and change dependencies version if really needed: - -`org.apache.storm:storm-sql-hdfs:2.0.0-SNAPSHOT,org.apache.storm:storm-hdfs:2.0.0-SNAPSHOT` - -Also, hdfs configuration files should be provided. -You can put the `core-site.xml` and `hdfs-site.xml` into the `conf` directory which is in Storm installation directory. diff --git a/docs/storm-sql.md b/docs/storm-sql.md deleted file mode 100644 index facb68bc93c..00000000000 --- a/docs/storm-sql.md +++ /dev/null @@ -1,198 +0,0 @@ ---- -title: Storm SQL integration -layout: documentation -documentation: true ---- - -The Storm SQL integration allows users to run SQL queries over streaming data in Storm. Not only the SQL interface allows faster development cycles on streaming analytics, but also opens up the opportunities to unify batch data processing like [Apache Hive](///hive.apache.org) and real-time streaming data analytics. - -At a very high level StormSQL compiles the SQL queries to Storm topologies leveraging Streams API and executes them in Storm clusters. This document provides information of how to use StormSQL as end users. For people that are interested in more details in the design and the implementation of StormSQL please refer to [this](storm-sql-internal.html) page. - -Storm SQL integration is an `experimental` feature, so the internal of Storm SQL and supported features are subject to change. -But small change will not affect the user experience. We will notice/announce the user when breaking UX change is introduced. - -## Usage - -Run the ``storm sql`` command to compile SQL statements into Storm topology, and submit it to the Storm cluster - -``` -$ bin/storm sql -``` - -In which `sql-file` contains a list of SQL statements to be executed, and `topo-name` is the name of the topology. - -StormSQL activates `explain mode` and shows query plan instead of submitting topology when user specifies `topo-name` as `--explain`. -Detailed explanation is available from `Showing Query Plan (explain mode)` section. - -## Supported Features - -The following features are supported in the current repository: - -* Streaming from and to external data sources -* Filtering tuples -* Projections -* User defined function (scalar) - -Aggregations and Join are not supported for now. When Storm SQL will support native `Streaming SQL`, these features will be introduced. - -Please be aware that as Storm uses [Apache Calcite](///calcite.apache.org) to parse the supplied SQL, you will likely benefit from skimming the Calcite documentation, in particular the section on [identifiers](https://calcite.apache.org/docs/reference.html#identifiers). - -## Specifying External Data Sources - -In StormSQL data is represented by external tables. Users can specify data sources using the `CREATE EXTERNAL TABLE` statement. The syntax of `CREATE EXTERNAL TABLE` closely follows the one defined in [Hive Data Definition Language](https://cwiki.apache.org/confluence/display/Hive/LanguageManual+DDL): - -``` -CREATE EXTERNAL TABLE table_name field_list - [ STORED AS - INPUTFORMAT input_format_classname - OUTPUTFORMAT output_format_classname - ] - LOCATION location - [ PARALLELISM parallelism ] - [ TBLPROPERTIES tbl_properties ] - [ AS select_stmt ] -``` - -You can find detailed explanations of the properties in [Hive Data Definition Language](https://cwiki.apache.org/confluence/display/Hive/LanguageManual+DDL). - -`PARALLELISM` is StormSQL's own keyword which describes parallelism hint for input data source. This is same as providing parallelism hint to Spout. -Downstream operators are executed with same parallelism before repartition (Aggregation triggers repartition). - -Default value is 1, and this option is no effect on output data source. (We might change if needed. Normally repartition is the thing to avoid.) - -For example, the following statement specifies a Kafka spout and sink: - -``` -CREATE EXTERNAL TABLE FOO (ID INT PRIMARY KEY) LOCATION 'kafka://test?bootstrap-servers=localhost:9092' TBLPROPERTIES '{"producer":{"acks":"1","key.serializer":"org.apache.storm.kafka.IntSerializer"}}' -``` - -## Plugging in External Data Sources - -Users plug in external data sources through implementing the `ISqlStreamsDataSource` interface and registers them using the mechanisms of Java's service loader. The external data source will be chosen based on the scheme of the URI of the tables. Please refer to the implementation of `storm-sql-kafka` for more details. - -## Specifying User Defined Function (UDF) - -Users can define user defined function (scalar) using `CREATE FUNCTION` statement. -For example, the following statement defines `MYPLUS` function which uses `org.apache.storm.sql.TestUtils$MyPlus` class. - -``` -CREATE FUNCTION MYPLUS AS 'org.apache.storm.sql.TestUtils$MyPlus' -``` - -Storm SQL determines whether the function as scalar or aggregate by checking which methods are defined. -If the class defines `evaluate` method, Storm SQL treats the function as `scalar`. - -Example of class for scalar function is here: - -``` - public class MyPlus { - public static Integer evaluate(Integer x, Integer y) { - return x + y; - } - } - -``` - -## Example: Filtering Kafka Stream - -Let's say there is a Kafka stream that represents the transactions of orders. Each message in the stream contains the id of the order, the unit price of the product and the quantity of the orders. The goal is to filter orders where the transactions are significant and to insert these orders into another Kafka stream for further analysis. - -The user can specify the following SQL statements in the SQL file: - -``` -CREATE EXTERNAL TABLE ORDERS (ID INT PRIMARY KEY, UNIT_PRICE INT, QUANTITY INT) LOCATION 'kafka://orders?bootstrap-servers=localhost:9092,localhost:9093' -CREATE EXTERNAL TABLE LARGE_ORDERS (ID INT PRIMARY KEY, TOTAL INT) LOCATION 'kafka://large_orders?bootstrap-servers=localhost:9092,localhost:9093' TBLPROPERTIES '{"producer":{"acks":"1","key.serializer":"org.apache.storm.kafka.IntSerializer"}}' -INSERT INTO LARGE_ORDERS SELECT ID, UNIT_PRICE * QUANTITY AS TOTAL FROM ORDERS WHERE UNIT_PRICE * QUANTITY > 50 -``` - -The first statement defines the table `ORDER` which represents the input stream. The `LOCATION` clause specifies the Kafka bootstrap servers (`localhost:9092,localhost:9093`) and the topic (`orders`). - -Similarly, the second statement specifies the table `LARGE_ORDERS` which represents the output stream. The `TBLPROPERTIES` clause specifies the configuration of [KafkaProducer](http://kafka.apache.org/documentation.html#producerconfigs) and is required for a Kafka sink table. - -The third statement is a `SELECT` statement which defines the topology: it instructs StormSQL to filter all orders in the external table `ORDERS`, calculates the total price and inserts matching records into the Kafka stream specified by `LARGE_ORDER`. - -To run this example, users need to include the data sources (`storm-sql-kafka` in this case) and its dependency in the -class path. The Storm SQL core dependencies are automatically handled when users run `storm sql`. Users can include data sources at the submission step like below: - -``` -$ bin/storm sql order_filtering.sql order_filtering --artifacts "org.apache.storm:storm-sql-kafka:2.0.0-SNAPSHOT,org.apache.storm:storm-kafka-client:2.0.0-SNAPSHOT,org.apache.kafka:kafka-clients:1.1.0^org.slf4j:slf4j-log4j12" -``` - -Above command submits the SQL statements to StormSQL. Users need to modify each artifacts' version if users are using different version of Storm or Kafka. - -Having run the above command, you should now be able to see the `order_filtering` topology in Storm UI. - -## Showing Query Plan (explain mode) - -Like `explain` on SQL statement, StormSQL provides `explain mode` when running Storm SQL Runner. In explain mode, StormSQL analyzes each query statement (only DML) and show plan instead of submitting topology. - -In order to run `explain mode`, you need to provide topology name as `--explain` and run `storm sql` as same as submitting. - -For example, when you run the example seen above with explain mode: - -``` -$ bin/storm sql order_filtering.sql --explain --artifacts "org.apache.storm:storm-sql-kafka:2.0.0-SNAPSHOT,org.apache.storm:storm-kafka-client:2.0.0-SNAPSHOT,org.apache.kafka:kafka-clients:1.1.0^org.slf4j:slf4j-log4j12" -``` - -StormSQL prints out like below: - -``` - -=========================================================== -query> -CREATE EXTERNAL TABLE ORDERS (ID INT PRIMARY KEY, UNIT_PRICE INT, QUANTITY INT) LOCATION 'kafka://orders?bootstrap-servers=localhost:9092,localhost:9093' ------------------------------------------------------------ -17:03:06.040 [main] INFO o.a.s.s.r.DataSourcesRegistry - Registering scheme socket with org.apache.storm.sql.runtime.datasource.socket.SocketDataSourcesProvider@d62fe5b -17:03:06.090 [main] INFO o.a.s.s.r.DataSourcesRegistry - Registering scheme kafka with org.apache.storm.sql.kafka.KafkaDataSourcesProvider@34158c08 -17:03:06.290 [main] INFO o.a.s.k.s.KafkaSpoutConfig - Setting Kafka consumer property 'auto.offset.reset' to 'earliest' to ensure at-least-once processing -17:03:06.290 [main] INFO o.a.s.k.s.KafkaSpoutConfig - Setting Kafka consumer property 'enable.auto.commit' to 'false', because the spout does not support auto-commit -No plan presented on DDL -=========================================================== -=========================================================== -query> -CREATE EXTERNAL TABLE LARGE_ORDERS (ID INT PRIMARY KEY, TOTAL INT) LOCATION 'kafka://large_orders?bootstrap-servers=localhost:9092,localhost:9093' TBLPROPERTIES '{"producer":{"acks":"1","key.serializer":"org.apache.storm.kafka.IntSerializer"}}' ------------------------------------------------------------ -17:03:06.504 [main] INFO o.a.s.k.s.KafkaSpoutConfig - Setting Kafka consumer property 'auto.offset.reset' to 'earliest' to ensure at-least-once processing -17:03:06.504 [main] INFO o.a.s.k.s.KafkaSpoutConfig - Setting Kafka consumer property 'enable.auto.commit' to 'false', because the spout does not support auto-commit -No plan presented on DDL -=========================================================== -=========================================================== -query> -INSERT INTO LARGE_ORDERS SELECT ID, UNIT_PRICE * QUANTITY AS TOTAL FROM ORDERS WHERE UNIT_PRICE * QUANTITY > 50 ------------------------------------------------------------ -plan> -LogicalTableModify(table=[[LARGE_ORDERS]], operation=[INSERT], flattened=[true]) - LogicalProject(ID=[$0], TOTAL=[*($1, $2)]) - LogicalFilter(condition=[>(*($1, $2), 50)]) - EnumerableTableScan(table=[[ORDERS]]) - -=========================================================== - -``` - -## Debugging expressions -When a Storm SQL topology is submitted, the code generated based on the SQL expressions will be logged. For example, the WHERE clause in the order filtering example will produce the following log during submit: - -``` -17:37:55.344 [main] INFO o.a.s.s.r.s.f.EvaluationCalc - Expression code for filter: -public class Generated_STREAMSCALCREL_33_0 implements org.apache.storm.sql.runtime.calcite.ExecutableExpression { - public void execute(org.apache.calcite.interpreter.Context context, Object[] outputValues) { - final Object[] current = context.values; - outputValues[0] = org.apache.calcite.runtime.SqlFunctions.toInt(current[1]) * org.apache.calcite.runtime.SqlFunctions.toInt(current[2]) > 50; - } - - public Object execute(org.apache.calcite.interpreter.Context context) { - final Object[] values = new Object[1]; - this.execute(context, values); - return values[0]; - } - -} -``` - -This can be helpful in case you need to debug the topology. - -## Current Limitations - -- Windowing is yet to be implemented. -- Aggregation and join will be introduced after supporting windowing. diff --git a/pom.xml b/pom.xml index b6371800485..15177e9048c 100644 --- a/pom.xml +++ b/pom.xml @@ -128,7 +128,6 @@ 4.3.0 2.2.2 3.0 - 1.16.0 5.2.0 6.1.7 @@ -505,7 +504,6 @@ flux - sql @@ -934,21 +932,6 @@ pom import - - org.apache.calcite - calcite-core - ${calcite.version} - - - ch.qos.reload4j - reload4j - - - org.slf4j - slf4j-reload4j - - - org.apache.kafka kafka-clients diff --git a/sql/README.md b/sql/README.md deleted file mode 100644 index 297c9deec2d..00000000000 --- a/sql/README.md +++ /dev/null @@ -1,28 +0,0 @@ -# Storm SQL - -Compile SQL queries to Storm topologies and run. - -Please see [here](../docs/storm-sql.md) for details on how to use it. - -## License - -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, -software distributed under the License is distributed on an -"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -KIND, either express or implied. See the License for the -specific language governing permissions and limitations -under the License. - -## Committer Sponsors - * Sriharsha Chintalapani ([sriharsha@apache.org](mailto:sriharsha@apache.org)) - * P. Taylor Goetz ([ptgoetz@apache.org](mailto:ptgoetz@apache.org)) diff --git a/sql/pom.xml b/sql/pom.xml deleted file mode 100644 index 70a5ed6be3d..00000000000 --- a/sql/pom.xml +++ /dev/null @@ -1,59 +0,0 @@ - - - - 4.0.0 - - - storm - org.apache.storm - 2.8.3-SNAPSHOT - ../pom.xml - - - sql - pom - - - - haohui - Haohui Mai - ricetons@gmail.com - - - - - storm-sql-core - storm-sql-runtime - storm-sql-external/storm-sql-kafka - storm-sql-external/storm-sql-redis - storm-sql-external/storm-sql-hdfs - - - - - - org.apache.maven.plugins - maven-checkstyle-plugin - - - org.apache.maven.plugins - maven-pmd-plugin - - - - diff --git a/sql/storm-sql-core/pom.xml b/sql/storm-sql-core/pom.xml deleted file mode 100644 index a92b722d4f2..00000000000 --- a/sql/storm-sql-core/pom.xml +++ /dev/null @@ -1,331 +0,0 @@ - - - - 4.0.0 - - - storm - org.apache.storm - 2.8.3-SNAPSHOT - ../../pom.xml - - - storm-sql-core - - - - --add-opens java.base/java.nio=ALL-UNNAMED - - - - haohui - Haohui Mai - ricetons@gmail.com - - - - - - org.apache.storm - storm-client - ${project.version} - ${provided.scope} - - - org.apache.storm - storm-server - ${project.version} - test - - - org.apache.storm - storm-sql-runtime - ${project.version} - ${provided.scope} - - - org.apache.storm - storm-sql-runtime - ${project.version} - test - test-jar - - - org.apache.calcite - calcite-core - - - org.apache.calcite.avatica - avatica-metrics - - - com.fasterxml.jackson.core - jackson-core - - - com.fasterxml.jackson.core - jackson-annotations - - - com.fasterxml.jackson.core - jackson-databind - - - org.apache.httpcomponents - httpclient - - - org.apache.httpcomponents - httpcore - - - commons-dbcp - commons-dbcp - - - ch.qos.reload4j - reload4j - - - org.slf4j - slf4j-reload4j - - - - - com.fasterxml.jackson.core - jackson-annotations - - - commons-cli - commons-cli - - - commons-lang - commons-lang - runtime - - - org.mockito - mockito-core - - - org.hamcrest - hamcrest - - - org.apache.commons - commons-collections4 - 4.5.0 - test - - - - src/jvm - src/test - - - org.apache.maven.plugins - maven-resources-plugin - - - copy-fmpp-resources - initialize - - copy-resources - - - ${project.build.directory}/codegen - - - src/codegen - false - - - - - - copy-java-sources - process-sources - - copy-resources - - - ${basedir}/target/classes/ - - - src/jvm - true - - - src/test - true - - - target/generated-sources - - true - - - - - - - - org.apache.maven.plugins - maven-jar-plugin - 3.4.2 - - - - test-jar - - - - - - org.apache.maven.plugins - maven-surefire-plugin - - ${storm-sql-core.test.introspection.argLine} -Xmx3g -XX:+HeapDumpOnOutOfMemoryError - true - IntegrationTest | ${java.unit.test.exclude.groups} - false - 1.0C - true - - ${project.basedir}/target/testhome - - - - - org.apache.maven.plugins - maven-checkstyle-plugin - - - - org.apache.maven.plugins - maven-pmd-plugin - - - - org.apache.maven.plugins - maven-dependency-plugin - - - unpack-parser-template - initialize - - unpack - - - - - org.apache.calcite - calcite-core - ${calcite.version} - jar - true - ${project.build.directory}/ - **/Parser.jj - - - - - - - - com.googlecode.fmpp-maven-plugin - fmpp-maven-plugin - 1.0 - - - org.freemarker - freemarker - 2.3.34 - - - - - generate-fmpp-sources - generate-sources - - generate - - - ${project.build.directory}/codegen/config.fmpp - target/generated-sources - ${project.build.directory}/codegen/templates - - - - - - org.codehaus.mojo - build-helper-maven-plugin - 3.6.1 - - - add-generated-sources - process-sources - - add-source - - - - ${project.build.directory}/generated-sources - - - - - - - org.codehaus.mojo - javacc-maven-plugin - 3.1.1 - - - generate-sources - javacc - - javacc - - - ${project.build.directory}/generated-sources/ - - **/*.jj - - 2 - false - ${project.build.directory}/generated-sources/ - - - - - - - diff --git a/sql/storm-sql-core/src/codegen/config.fmpp b/sql/storm-sql-core/src/codegen/config.fmpp deleted file mode 100644 index be5a792ca0a..00000000000 --- a/sql/storm-sql-core/src/codegen/config.fmpp +++ /dev/null @@ -1,23 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http:# www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -data: { - parser: tdd(../data/Parser.tdd) -} - -freemarkerLinks: { - includes: includes/ -} diff --git a/sql/storm-sql-core/src/codegen/data/Parser.tdd b/sql/storm-sql-core/src/codegen/data/Parser.tdd deleted file mode 100644 index b0dccb6fb35..00000000000 --- a/sql/storm-sql-core/src/codegen/data/Parser.tdd +++ /dev/null @@ -1,80 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http:# www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -{ - # Generated parser implementation class package and name - package: "org.apache.storm.sql.parser.impl", - class: "StormParserImpl", - - # List of import statements. - imports: [ - "org.apache.calcite.sql.validate.*", - "org.apache.calcite.util.*", - "org.apache.storm.sql.parser.*", - "java.util.*" - ] - - # List of keywords. - keywords: [ - "LOCATION", - "INPUTFORMAT", - "OUTPUTFORMAT", - "PARALLELISM", - "STORED", - "TBLPROPERTIES", - "JAR" - ] - - # List of methods for parsing custom SQL statements. - statementParserMethods: [ - "SqlCreateTable()", - "SqlCreateFunction()" - ] - - # List of methods for parsing custom literals. - # Example: ParseJsonLiteral(). - literalParserMethods: [ - ] - - # List of methods for parsing custom data types. - dataTypeParserMethods: [ - ] - - nonReservedKeywords: [ - ] - - createStatementParserMethods: [ - ] - - alterStatementParserMethods: [ - ] - - dropStatementParserMethods: [ - ] - - # List of files in @includes directory that have parser method - # implementations for custom SQL statements, literals or types - # given as part of "statementParserMethods", "literalParserMethods" or - # "dataTypeParserMethods". - implementationFiles: [ - "parserImpls.ftl" - ] - - includeCompoundIdentifier: true, - includeBraces: true, - includeAdditionalDeclarations: false, - allowBangEqual: false -} diff --git a/sql/storm-sql-core/src/codegen/includes/license.ftl b/sql/storm-sql-core/src/codegen/includes/license.ftl deleted file mode 100644 index 7e66353b7db..00000000000 --- a/sql/storm-sql-core/src/codegen/includes/license.ftl +++ /dev/null @@ -1,17 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ diff --git a/sql/storm-sql-core/src/codegen/includes/parserImpls.ftl b/sql/storm-sql-core/src/codegen/includes/parserImpls.ftl deleted file mode 100644 index 41438400a05..00000000000 --- a/sql/storm-sql-core/src/codegen/includes/parserImpls.ftl +++ /dev/null @@ -1,113 +0,0 @@ -<#-- Licensed to the Apache Software Foundation (ASF) under one or more contributor - license agreements. See the NOTICE file distributed with this work for additional - information regarding copyright ownership. The ASF licenses this file to - You under the Apache License, Version 2.0 (the "License"); you may not use - this file except in compliance with the License. You may obtain a copy of - the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required - by applicable law or agreed to in writing, software distributed under the - License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS - OF ANY KIND, either express or implied. See the License for the specific - language governing permissions and limitations under the License. --> - - -private void ColumnDef(List list) : -{ - SqlParserPos pos; - SqlIdentifier name; - SqlDataTypeSpec type; - ColumnConstraint constraint = null; - SqlMonotonicity monotonicity = SqlMonotonicity.NOT_MONOTONIC; -} -{ - name = SimpleIdentifier() { pos = getPos(); } - type = DataType() - [ - - [ { monotonicity = SqlMonotonicity.INCREASING; } - | { monotonicity = SqlMonotonicity.DECREASING; } - ] - { constraint = new ColumnConstraint.PrimaryKey(monotonicity, getPos()); } - ] - { - list.add(new ColumnDefinition(name, type, constraint, pos)); - } -} - -SqlNodeList ColumnDefinitionList() : -{ - SqlParserPos pos; - List list = Lists.newArrayList(); -} -{ - { pos = getPos(); } - ColumnDef(list) - ( ColumnDef(list) )* - { - return new SqlNodeList(list, pos.plus(getPos())); - } -} - -/** - * CREATE EXTERNAL TABLE ( IF NOT EXISTS )? - * ( database_name '.' )? table_name ( '(' column_def ( ',' column_def )* ')' - * ( STORED AS INPUTFORMAT input_format_classname OUTPUTFORMAT output_format_classname )? - * LOCATION location_uri - * ( TBLPROPERTIES tbl_properties )? - * ( AS select_stmt ) - */ -SqlNode SqlCreateTable() : -{ - SqlParserPos pos; - SqlIdentifier tblName; - SqlNodeList fieldList; - SqlNode location; - SqlNode parallelism = null; - SqlNode input_format_class_name = null, output_format_class_name = null; - SqlNode tbl_properties = null; - SqlNode select = null; -} -{ - { pos = getPos(); } - - tblName = CompoundIdentifier() - fieldList = ColumnDefinitionList() - [ - - input_format_class_name = StringLiteral() - output_format_class_name = StringLiteral() - ] - - location = StringLiteral() - [ parallelism = UnsignedNumericLiteral() ] - [ tbl_properties = StringLiteral() ] - [ select = OrderedQueryOrExpr(ExprContext.ACCEPT_QUERY) ] { - return new SqlCreateTable(pos, tblName, fieldList, - input_format_class_name, output_format_class_name, location, - parallelism, tbl_properties, select); - } -} - -/** - * CREATE FUNCTION functionname AS 'classname' - */ -SqlNode SqlCreateFunction() : -{ - SqlParserPos pos; - SqlIdentifier functionName; - SqlNode className; - SqlNode jarName = null; -} -{ - { pos = getPos(); } - - functionName = CompoundIdentifier() - - className = StringLiteral() - [ - - jarName = StringLiteral() - ] - { - return new SqlCreateFunction(pos, functionName, className, jarName); - } -} \ No newline at end of file diff --git a/sql/storm-sql-core/src/jvm/org/apache/storm/sql/AbstractStreamsProcessor.java b/sql/storm-sql-core/src/jvm/org/apache/storm/sql/AbstractStreamsProcessor.java deleted file mode 100644 index e0610456cf4..00000000000 --- a/sql/storm-sql-core/src/jvm/org/apache/storm/sql/AbstractStreamsProcessor.java +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.apache.storm.sql; - -import java.util.List; - -import org.apache.calcite.DataContext; -import org.apache.storm.generated.StormTopology; -import org.apache.storm.sql.javac.CompilingClassLoader; -import org.apache.storm.streams.Stream; -import org.apache.storm.tuple.Values; - -public abstract class AbstractStreamsProcessor { - protected Stream outputStream; - protected DataContext dataContext; - protected List classLoaders; - - /** - * Return final output stream of SQL topology structure. - * - * @return the output stream of the SQL - */ - public Stream outputStream() { - return outputStream; - } - - /** - * Construct the Storm topology based on the SQL. - */ - public abstract StormTopology build(); - - /** - * Return DataContext instance which is used with execution of query. - * - * @return DataContext instance which is used with execution of query - */ - public DataContext getDataContext() { - return dataContext; - } - - /** - * Return the list of Classloaders which need to be compiled and included to the jar. - * They're all chaining so the last classloader can access all classes. - * - * @return Classloaders to compile. - */ - public List getClassLoaders() { - return classLoaders; - } -} \ No newline at end of file diff --git a/sql/storm-sql-core/src/jvm/org/apache/storm/sql/StormSql.java b/sql/storm-sql-core/src/jvm/org/apache/storm/sql/StormSql.java deleted file mode 100644 index e4715f88869..00000000000 --- a/sql/storm-sql-core/src/jvm/org/apache/storm/sql/StormSql.java +++ /dev/null @@ -1,45 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions - * and limitations under the License. - */ - -package org.apache.storm.sql; - -import java.util.Map; -import org.apache.storm.StormSubmitter; -import org.apache.storm.generated.SubmitOptions; - -/** - * The StormSql class provides standalone, interactive interfaces to execute - * SQL statements over streaming data. - * - *

The StormSql class is stateless. The user needs to submit the data - * definition language (DDL) statements and the query statements in the same - * batch. - */ -public abstract class StormSql { - public static StormSql construct() { - return new StormSqlImpl(); - } - - /** - * Submit the SQL statements to Nimbus and run it as a topology. - */ - public abstract void submit( - String name, Iterable statements, Map topoConf, SubmitOptions opts, - StormSubmitter.ProgressListener progressListener, String asUser) - throws Exception; - - /** - * Print out query plan for each query. - */ - public abstract void explain(Iterable statements) throws Exception; -} - diff --git a/sql/storm-sql-core/src/jvm/org/apache/storm/sql/StormSqlContext.java b/sql/storm-sql-core/src/jvm/org/apache/storm/sql/StormSqlContext.java deleted file mode 100644 index e5413c34bf8..00000000000 --- a/sql/storm-sql-core/src/jvm/org/apache/storm/sql/StormSqlContext.java +++ /dev/null @@ -1,169 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.apache.storm.sql; - -import java.lang.reflect.Method; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Properties; - -import org.apache.calcite.adapter.java.JavaTypeFactory; -import org.apache.calcite.config.CalciteConnectionConfigImpl; -import org.apache.calcite.jdbc.CalciteSchema; -import org.apache.calcite.prepare.CalciteCatalogReader; -import org.apache.calcite.rel.RelNode; -import org.apache.calcite.rel.type.RelDataType; -import org.apache.calcite.rel.type.RelDataTypeSystem; -import org.apache.calcite.schema.Function; -import org.apache.calcite.schema.SchemaPlus; -import org.apache.calcite.schema.Table; -import org.apache.calcite.schema.impl.AggregateFunctionImpl; -import org.apache.calcite.schema.impl.ScalarFunctionImpl; -import org.apache.calcite.sql.SqlExplainLevel; -import org.apache.calcite.sql.SqlNode; -import org.apache.calcite.sql.SqlOperatorTable; -import org.apache.calcite.sql.fun.SqlStdOperatorTable; -import org.apache.calcite.sql.parser.SqlParseException; -import org.apache.calcite.sql.util.ChainedSqlOperatorTable; -import org.apache.calcite.tools.FrameworkConfig; -import org.apache.calcite.tools.Frameworks; -import org.apache.calcite.tools.Planner; -import org.apache.calcite.tools.RelConversionException; -import org.apache.calcite.tools.ValidationException; -import org.apache.storm.sql.compiler.CompilerUtil; -import org.apache.storm.sql.compiler.StormSqlTypeFactoryImpl; -import org.apache.storm.sql.parser.ColumnConstraint; -import org.apache.storm.sql.parser.ColumnDefinition; -import org.apache.storm.sql.parser.SqlCreateFunction; -import org.apache.storm.sql.parser.SqlCreateTable; -import org.apache.storm.sql.planner.StormRelUtils; -import org.apache.storm.sql.planner.streams.QueryPlanner; -import org.apache.storm.sql.runtime.DataSourcesRegistry; -import org.apache.storm.sql.runtime.FieldInfo; -import org.apache.storm.sql.runtime.ISqlStreamsDataSource; - -public class StormSqlContext { - private final JavaTypeFactory typeFactory = new StormSqlTypeFactoryImpl( - RelDataTypeSystem.DEFAULT); - private final SchemaPlus schema = Frameworks.createRootSchema(true); - private boolean hasUdf = false; - private Map dataSources = new HashMap<>(); - - public void interpretCreateTable(SqlCreateTable n) { - CompilerUtil.TableBuilderInfo builder = new CompilerUtil.TableBuilderInfo(typeFactory); - List fields = new ArrayList<>(); - for (ColumnDefinition col : n.fieldList()) { - builder.field(col.name(), col.type(), col.constraint()); - RelDataType dataType = col.type().deriveType(typeFactory); - Class javaType = (Class) typeFactory.getJavaClass(dataType); - ColumnConstraint constraint = col.constraint(); - boolean isPrimary = constraint != null && constraint instanceof ColumnConstraint.PrimaryKey; - fields.add(new FieldInfo(col.name(), javaType, isPrimary)); - } - - if (n.parallelism() != null) { - builder.parallelismHint(n.parallelism()); - } - Table table = builder.build(); - schema.add(n.tableName(), table); - - ISqlStreamsDataSource ds = DataSourcesRegistry.constructStreamsDataSource(n.location(), n - .inputFormatClass(), n.outputFormatClass(), n.properties(), fields); - if (ds == null) { - throw new RuntimeException("Failed to find data source for " + n - .tableName() + " URI: " + n.location()); - } else if (dataSources.containsKey(n.tableName())) { - throw new RuntimeException("Duplicated definition for table " + n - .tableName()); - } - dataSources.put(n.tableName(), ds); - } - - public void interpretCreateFunction(SqlCreateFunction sqlCreateFunction) throws ClassNotFoundException { - if (sqlCreateFunction.jarName() != null) { - throw new UnsupportedOperationException("UDF 'USING JAR' not implemented"); - } - Method method; - Function function; - if ((method = findMethod(sqlCreateFunction.className(), "evaluate")) != null) { - function = ScalarFunctionImpl.create(method); - } else if (findMethod(sqlCreateFunction.className(), "add") != null) { - function = AggregateFunctionImpl.create(Class.forName(sqlCreateFunction.className())); - } else { - throw new RuntimeException("Invalid scalar or aggregate function"); - } - schema.add(sqlCreateFunction.functionName().toUpperCase(), function); - hasUdf = true; - } - - public AbstractStreamsProcessor compileSql(String query) throws Exception { - QueryPlanner planner = new QueryPlanner(schema); - return planner.compile(dataSources, query); - } - - public String explain(String query) throws SqlParseException, ValidationException, RelConversionException { - FrameworkConfig config = buildFrameWorkConfig(); - Planner planner = Frameworks.getPlanner(config); - SqlNode parse = planner.parse(query); - SqlNode validate = planner.validate(parse); - RelNode tree = planner.convert(validate); - - return StormRelUtils.explain(tree, SqlExplainLevel.ALL_ATTRIBUTES); - } - - public FrameworkConfig buildFrameWorkConfig() { - if (hasUdf) { - List sqlOperatorTables = new ArrayList<>(); - sqlOperatorTables.add(SqlStdOperatorTable.instance()); - sqlOperatorTables.add(new CalciteCatalogReader(CalciteSchema.from(schema), - Collections.emptyList(), typeFactory, new CalciteConnectionConfigImpl(new Properties()))); - return Frameworks.newConfigBuilder().defaultSchema(schema) - .operatorTable(new ChainedSqlOperatorTable(sqlOperatorTables)).build(); - } else { - return Frameworks.newConfigBuilder().defaultSchema(schema).build(); - } - } - - public JavaTypeFactory getTypeFactory() { - return typeFactory; - } - - public SchemaPlus getSchema() { - return schema; - } - - public Map getDataSources() { - return dataSources; - } - - private Method findMethod(String clazzName, String methodName) throws ClassNotFoundException { - Class clazz = Class.forName(clazzName); - for (Method method : clazz.getMethods()) { - if (method.getName().equals(methodName)) { - return method; - } - } - return null; - } - -} diff --git a/sql/storm-sql-core/src/jvm/org/apache/storm/sql/StormSqlImpl.java b/sql/storm-sql-core/src/jvm/org/apache/storm/sql/StormSqlImpl.java deleted file mode 100644 index a91dcae22fa..00000000000 --- a/sql/storm-sql-core/src/jvm/org/apache/storm/sql/StormSqlImpl.java +++ /dev/null @@ -1,123 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions - * and limitations under the License. - */ - -package org.apache.storm.sql; - -import java.io.BufferedOutputStream; -import java.io.ByteArrayOutputStream; -import java.io.FileOutputStream; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.List; -import java.util.Map; -import java.util.jar.Attributes; -import java.util.jar.JarOutputStream; -import java.util.jar.Manifest; -import java.util.zip.ZipEntry; -import org.apache.calcite.sql.SqlNode; -import org.apache.storm.StormSubmitter; -import org.apache.storm.generated.StormTopology; -import org.apache.storm.generated.SubmitOptions; -import org.apache.storm.sql.javac.CompilingClassLoader; -import org.apache.storm.sql.parser.SqlCreateFunction; -import org.apache.storm.sql.parser.SqlCreateTable; -import org.apache.storm.sql.parser.StormParser; - -class StormSqlImpl extends StormSql { - private final StormSqlContext sqlContext; - - StormSqlImpl() { - sqlContext = new StormSqlContext(); - } - - @Override - public void submit( - String name, Iterable statements, Map topoConf, SubmitOptions opts, - StormSubmitter.ProgressListener progressListener, String asUser) - throws Exception { - for (String sql : statements) { - StormParser parser = new StormParser(sql); - SqlNode node = parser.impl().parseSqlStmtEof(); - if (node instanceof SqlCreateTable) { - sqlContext.interpretCreateTable((SqlCreateTable) node); - } else if (node instanceof SqlCreateFunction) { - sqlContext.interpretCreateFunction((SqlCreateFunction) node); - } else { - AbstractStreamsProcessor processor = sqlContext.compileSql(sql); - StormTopology topo = processor.build(); - - Path jarPath = null; - try { - // QueryPlanner on Streams mode configures the topology with compiled classes, - // so we need to add new classes into topology jar - // Topology will be serialized and sent to Nimbus, and deserialized and executed in workers. - - jarPath = Files.createTempFile("storm-sql", ".jar"); - System.setProperty("storm.jar", jarPath.toString()); - packageTopology(jarPath, processor); - StormSubmitter.submitTopologyAs(name, topoConf, topo, opts, progressListener, asUser); - } finally { - if (jarPath != null) { - Files.delete(jarPath); - } - } - } - } - } - - @Override - public void explain(Iterable statements) throws Exception { - for (String sql : statements) { - System.out.println("==========================================================="); - System.out.println("query>"); - System.out.println(sql); - System.out.println("-----------------------------------------------------------"); - - StormParser parser = new StormParser(sql); - SqlNode node = parser.impl().parseSqlStmtEof(); - if (node instanceof SqlCreateTable) { - sqlContext.interpretCreateTable((SqlCreateTable) node); - System.out.println("No plan presented on DDL"); - } else if (node instanceof SqlCreateFunction) { - sqlContext.interpretCreateFunction((SqlCreateFunction) node); - System.out.println("No plan presented on DDL"); - } else { - String plan = sqlContext.explain(sql); - System.out.println("plan>"); - System.out.println(plan); - } - - System.out.println("==========================================================="); - } - } - - private void packageTopology(Path jar, AbstractStreamsProcessor processor) throws IOException { - Manifest manifest = new Manifest(); - Attributes attr = manifest.getMainAttributes(); - attr.put(Attributes.Name.MANIFEST_VERSION, "1.0"); - attr.put(Attributes.Name.MAIN_CLASS, processor.getClass().getCanonicalName()); - try (JarOutputStream out = new JarOutputStream( - new BufferedOutputStream(new FileOutputStream(jar.toFile())), manifest)) { - List classLoaders = processor.getClassLoaders(); - if (classLoaders != null && !classLoaders.isEmpty()) { - for (CompilingClassLoader classLoader : classLoaders) { - for (Map.Entry e : classLoader.getClasses().entrySet()) { - out.putNextEntry(new ZipEntry(e.getKey().replace(".", "/") + ".class")); - out.write(e.getValue().toByteArray()); - out.closeEntry(); - } - } - } - } - } -} diff --git a/sql/storm-sql-core/src/jvm/org/apache/storm/sql/StormSqlRunner.java b/sql/storm-sql-core/src/jvm/org/apache/storm/sql/StormSqlRunner.java deleted file mode 100644 index edcc46d2d18..00000000000 --- a/sql/storm-sql-core/src/jvm/org/apache/storm/sql/StormSqlRunner.java +++ /dev/null @@ -1,80 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions - * and limitations under the License. - */ - -package org.apache.storm.sql; - -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.Paths; -import java.util.List; -import java.util.Map; -import org.apache.commons.cli.CommandLine; -import org.apache.commons.cli.CommandLineParser; -import org.apache.commons.cli.DefaultParser; -import org.apache.commons.cli.HelpFormatter; -import org.apache.commons.cli.Options; -import org.apache.storm.generated.SubmitOptions; -import org.apache.storm.generated.TopologyInitialStatus; -import org.apache.storm.utils.Utils; - -public class StormSqlRunner { - private static final String OPTION_SQL_FILE_SHORT = "f"; - private static final String OPTION_SQL_FILE_LONG = "file"; - private static final String OPTION_SQL_TOPOLOGY_NAME_SHORT = "t"; - private static final String OPTION_SQL_TOPOLOGY_NAME_LONG = "topology"; - private static final String OPTION_SQL_EXPLAIN_SHORT = "e"; - private static final String OPTION_SQL_EXPLAIN_LONG = "explain"; - - public static void main(String[] args) throws Exception { - Options options = buildOptions(); - CommandLineParser parser = new DefaultParser(); - CommandLine commandLine = parser.parse(options, args); - - if (!commandLine.hasOption(OPTION_SQL_FILE_LONG)) { - printUsageAndExit(options, OPTION_SQL_FILE_LONG + " is required"); - } - - String filePath = commandLine.getOptionValue(OPTION_SQL_FILE_LONG); - List stmts = Files.readAllLines(Paths.get(filePath), StandardCharsets.UTF_8); - StormSql sql = StormSql.construct(); - @SuppressWarnings("unchecked") - Map conf = Utils.readStormConfig(); - - if (commandLine.hasOption(OPTION_SQL_EXPLAIN_LONG)) { - sql.explain(stmts); - } else if (commandLine.hasOption(OPTION_SQL_TOPOLOGY_NAME_LONG)) { - String topoName = commandLine.getOptionValue(OPTION_SQL_TOPOLOGY_NAME_LONG); - SubmitOptions submitOptions = new SubmitOptions(TopologyInitialStatus.ACTIVE); - sql.submit(topoName, stmts, conf, submitOptions, null, null); - } else { - printUsageAndExit(options, "Either " + OPTION_SQL_TOPOLOGY_NAME_LONG - + " or " + OPTION_SQL_EXPLAIN_LONG + " must be presented"); - } - } - - private static void printUsageAndExit(Options options, String message) { - System.out.println(message); - HelpFormatter formatter = new HelpFormatter(); - formatter.printHelp("storm-sql-runner ", options); - System.exit(1); - } - - private static Options buildOptions() { - Options options = new Options(); - options.addOption(OPTION_SQL_FILE_SHORT, OPTION_SQL_FILE_LONG, true, "REQUIRED SQL file which has sql statements"); - options.addOption(OPTION_SQL_TOPOLOGY_NAME_SHORT, OPTION_SQL_TOPOLOGY_NAME_LONG, true, "Topology name to submit"); - options - .addOption(OPTION_SQL_EXPLAIN_SHORT, OPTION_SQL_EXPLAIN_LONG, false, "Activate explain mode (topology name will be ignored)"); - return options; - } - -} diff --git a/sql/storm-sql-core/src/jvm/org/apache/storm/sql/calcite/ParallelStreamableTable.java b/sql/storm-sql-core/src/jvm/org/apache/storm/sql/calcite/ParallelStreamableTable.java deleted file mode 100644 index 3a6a9164861..00000000000 --- a/sql/storm-sql-core/src/jvm/org/apache/storm/sql/calcite/ParallelStreamableTable.java +++ /dev/null @@ -1,29 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions - * and limitations under the License. - */ - -package org.apache.storm.sql.calcite; - -import org.apache.calcite.rel.stream.Delta; -import org.apache.calcite.schema.StreamableTable; - -/** - * Table that can be converted to a stream. This table also has its parallelism information. - * - * @see Delta - */ -public interface ParallelStreamableTable extends StormStreamableTable { - - /** - * Returns parallelism hint of this table. Returns null if don't know. - */ - Integer parallelismHint(); -} diff --git a/sql/storm-sql-core/src/jvm/org/apache/storm/sql/calcite/ParallelTable.java b/sql/storm-sql-core/src/jvm/org/apache/storm/sql/calcite/ParallelTable.java deleted file mode 100644 index 025077b939d..00000000000 --- a/sql/storm-sql-core/src/jvm/org/apache/storm/sql/calcite/ParallelTable.java +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.apache.storm.sql.calcite; - -public interface ParallelTable extends StormTable { - - /** - * Returns parallelism hint of this table. Returns null if not known. - */ - Integer parallelismHint(); -} \ No newline at end of file diff --git a/sql/storm-sql-core/src/jvm/org/apache/storm/sql/calcite/StormStreamableTable.java b/sql/storm-sql-core/src/jvm/org/apache/storm/sql/calcite/StormStreamableTable.java deleted file mode 100644 index 5833b1ca9e7..00000000000 --- a/sql/storm-sql-core/src/jvm/org/apache/storm/sql/calcite/StormStreamableTable.java +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.apache.storm.sql.calcite; - -import org.apache.calcite.schema.StreamableTable; - -public interface StormStreamableTable extends StreamableTable { - int primaryKey(); -} \ No newline at end of file diff --git a/sql/storm-sql-core/src/jvm/org/apache/storm/sql/calcite/StormTable.java b/sql/storm-sql-core/src/jvm/org/apache/storm/sql/calcite/StormTable.java deleted file mode 100644 index 26bb05f99a8..00000000000 --- a/sql/storm-sql-core/src/jvm/org/apache/storm/sql/calcite/StormTable.java +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.apache.storm.sql.calcite; - -import org.apache.calcite.schema.Table; - -public interface StormTable extends Table { - int primaryKey(); -} \ No newline at end of file diff --git a/sql/storm-sql-core/src/jvm/org/apache/storm/sql/compiler/CompilerUtil.java b/sql/storm-sql-core/src/jvm/org/apache/storm/sql/compiler/CompilerUtil.java deleted file mode 100644 index a91199c4fa6..00000000000 --- a/sql/storm-sql-core/src/jvm/org/apache/storm/sql/compiler/CompilerUtil.java +++ /dev/null @@ -1,227 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions - * and limitations under the License. - */ - -package org.apache.storm.sql.compiler; - -import static org.apache.calcite.rel.RelFieldCollation.Direction; -import static org.apache.calcite.rel.RelFieldCollation.Direction.ASCENDING; -import static org.apache.calcite.rel.RelFieldCollation.Direction.DESCENDING; -import static org.apache.calcite.rel.RelFieldCollation.NullDirection; -import static org.apache.calcite.sql.validate.SqlMonotonicity.INCREASING; - -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Preconditions; -import com.google.common.collect.ImmutableList; - -import java.util.ArrayList; - -import org.apache.calcite.config.CalciteConnectionConfig; -import org.apache.calcite.rel.RelCollations; -import org.apache.calcite.rel.RelFieldCollation; -import org.apache.calcite.rel.type.RelDataType; -import org.apache.calcite.rel.type.RelDataTypeFactory; -import org.apache.calcite.schema.Schema; -import org.apache.calcite.schema.Statistic; -import org.apache.calcite.schema.Statistics; -import org.apache.calcite.schema.StreamableTable; -import org.apache.calcite.schema.Table; -import org.apache.calcite.sql.SqlCall; -import org.apache.calcite.sql.SqlDataTypeSpec; -import org.apache.calcite.sql.SqlNode; -import org.apache.calcite.sql.type.SqlTypeName; -import org.apache.calcite.sql.validate.SqlMonotonicity; -import org.apache.calcite.util.ImmutableBitSet; - -import org.apache.storm.sql.calcite.ParallelStreamableTable; -import org.apache.storm.sql.calcite.ParallelTable; -import org.apache.storm.sql.parser.ColumnConstraint; - -public class CompilerUtil { - - public static class TableBuilderInfo { - private final RelDataTypeFactory typeFactory; - private final ArrayList fields = new ArrayList<>(); - private final ArrayList rows = new ArrayList<>(); - private int primaryKey = -1; - private Integer parallelismHint; - private SqlMonotonicity primaryKeyMonotonicity; - private Statistic stats; - - public TableBuilderInfo(RelDataTypeFactory typeFactory) { - this.typeFactory = typeFactory; - } - - public TableBuilderInfo field(String name, SqlTypeName type) { - return field(name, typeFactory.createSqlType(type)); - } - - public TableBuilderInfo field(String name, RelDataType type) { - fields.add(new FieldType(name, type)); - return this; - } - - public TableBuilderInfo field(String name, SqlTypeName type, ColumnConstraint constraint) { - interpretConstraint(constraint, fields.size()); - return field(name, typeFactory.createSqlType(type)); - } - - public TableBuilderInfo field(String name, RelDataType type, ColumnConstraint constraint) { - interpretConstraint(constraint, fields.size()); - fields.add(new FieldType(name, type)); - return this; - } - - public TableBuilderInfo field(String name, SqlDataTypeSpec type, ColumnConstraint constraint) { - RelDataType dataType = type.deriveType(typeFactory); - interpretConstraint(constraint, fields.size()); - fields.add(new FieldType(name, dataType)); - return this; - } - - private void interpretConstraint(ColumnConstraint constraint, int fieldIdx) { - if (constraint instanceof ColumnConstraint.PrimaryKey) { - ColumnConstraint.PrimaryKey pk = (ColumnConstraint.PrimaryKey) constraint; - Preconditions.checkState(primaryKey == -1, "There are more than one primary key in the table"); - primaryKey = fieldIdx; - primaryKeyMonotonicity = pk.monotonicity(); - } - } - - public TableBuilderInfo statistics(Statistic stats) { - this.stats = stats; - return this; - } - - @VisibleForTesting - public TableBuilderInfo rows(Object[] data) { - rows.add(data); - return this; - } - - public TableBuilderInfo parallelismHint(int parallelismHint) { - this.parallelismHint = parallelismHint; - return this; - } - - public StreamableTable build() { - final Statistic stat = buildStatistic(); - - final Table tbl = new ParallelTable() { - @Override - public Integer parallelismHint() { - return parallelismHint; - } - - @Override - public int primaryKey() { - return primaryKey; - } - - @Override - public RelDataType getRowType( - RelDataTypeFactory relDataTypeFactory) { - RelDataTypeFactory.FieldInfoBuilder b = relDataTypeFactory.builder(); - for (FieldType f : fields) { - b.add(f.name, f.relDataType); - } - return b.build(); - } - - @Override - public Statistic getStatistic() { - return stat != null ? stat : Statistics.of(rows.size(), - ImmutableList.of()); - } - - @Override - public Schema.TableType getJdbcTableType() { - return Schema.TableType.STREAM; - } - - @Override - public boolean isRolledUp(String column) { - return false; - } - - @Override - public boolean rolledUpColumnValidInsideAgg(String column, SqlCall call, SqlNode parent, - CalciteConnectionConfig config) { - return false; - } - }; - - return new ParallelStreamableTable() { - @Override - public int primaryKey() { - return primaryKey; - } - - @Override - public Integer parallelismHint() { - return parallelismHint; - } - - @Override - public Table stream() { - return tbl; - } - - @Override - public RelDataType getRowType(RelDataTypeFactory relDataTypeFactory) { - return tbl.getRowType(relDataTypeFactory); - } - - @Override - public Statistic getStatistic() { - return tbl.getStatistic(); - } - - @Override - public Schema.TableType getJdbcTableType() { - return Schema.TableType.STREAM; - } - - @Override - public boolean isRolledUp(String column) { - return false; - } - - @Override - public boolean rolledUpColumnValidInsideAgg(String column, SqlCall call, SqlNode parent, - CalciteConnectionConfig config) { - return false; - } - }; - } - - private Statistic buildStatistic() { - if (stats != null || primaryKey == -1) { - return stats; - } - Direction dir = primaryKeyMonotonicity == INCREASING ? ASCENDING : DESCENDING; - RelFieldCollation collation = new RelFieldCollation(primaryKey, dir, NullDirection.UNSPECIFIED); - return Statistics.of(fields.size(), ImmutableList.of(ImmutableBitSet.of(primaryKey)), - ImmutableList.of(RelCollations.of(collation))); - } - - private static class FieldType { - private final String name; - private final RelDataType relDataType; - - private FieldType(String name, RelDataType relDataType) { - this.name = name; - this.relDataType = relDataType; - } - - } - } -} diff --git a/sql/storm-sql-core/src/jvm/org/apache/storm/sql/compiler/RexNodeToJavaCodeCompiler.java b/sql/storm-sql-core/src/jvm/org/apache/storm/sql/compiler/RexNodeToJavaCodeCompiler.java deleted file mode 100644 index ed6227169c2..00000000000 --- a/sql/storm-sql-core/src/jvm/org/apache/storm/sql/compiler/RexNodeToJavaCodeCompiler.java +++ /dev/null @@ -1,234 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to you under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.storm.sql.compiler; - -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.Lists; -import java.lang.reflect.Constructor; -import java.lang.reflect.Field; -import java.lang.reflect.Method; -import java.lang.reflect.Modifier; -import java.lang.reflect.Type; -import java.util.List; -import org.apache.calcite.adapter.enumerable.JavaRowFormat; -import org.apache.calcite.adapter.enumerable.PhysType; -import org.apache.calcite.adapter.enumerable.PhysTypeImpl; -import org.apache.calcite.adapter.enumerable.RexToLixTranslator; -import org.apache.calcite.interpreter.Context; -import org.apache.calcite.jdbc.JavaTypeFactoryImpl; -import org.apache.calcite.linq4j.function.Function1; -import org.apache.calcite.linq4j.tree.BlockBuilder; -import org.apache.calcite.linq4j.tree.BlockStatement; -import org.apache.calcite.linq4j.tree.ClassDeclaration; -import org.apache.calcite.linq4j.tree.Expression; -import org.apache.calcite.linq4j.tree.Expressions; -import org.apache.calcite.linq4j.tree.MemberDeclaration; -import org.apache.calcite.linq4j.tree.ParameterExpression; -import org.apache.calcite.linq4j.tree.Types; -import org.apache.calcite.rel.type.RelDataType; -import org.apache.calcite.rex.RexBuilder; -import org.apache.calcite.rex.RexNode; -import org.apache.calcite.rex.RexProgram; -import org.apache.calcite.rex.RexProgramBuilder; -import org.apache.calcite.util.BuiltInMethod; -import org.apache.calcite.util.Pair; -import org.apache.storm.sql.runtime.calcite.ExecutableExpression; - -/** - * Compiles a scalar expression ({@link org.apache.calcite.rex.RexNode}) to Java source code String. - * - *

This code is inspired by JaninoRexCompiler in Calcite, but while it is returning - * {@link org.apache.calcite.interpreter.Scalar} which is executable, we need to pass the source code to compile and - * serialize instance so that it can be executed on worker efficiently. - */ -public class RexNodeToJavaCodeCompiler { - private final RexBuilder rexBuilder; - - public RexNodeToJavaCodeCompiler(RexBuilder rexBuilder) { - this.rexBuilder = rexBuilder; - } - - /** - * Given a method that implements {@link ExecutableExpression#execute(Context, Object[])}, adds a bridge method that - * implements {@link ExecutableExpression#execute(Context)}, and compiles. - */ - static String baz(ParameterExpression context, - ParameterExpression outputValues, BlockStatement block, String className) { - final List declarations = Lists.newArrayList(); - - // public void execute(Context, Object[] outputValues) - declarations.add( - Expressions.methodDecl(Modifier.PUBLIC, void.class, - StormBuiltInMethod.EXPR_EXECUTE2.method.getName(), - ImmutableList.of(context, outputValues), block)); - - // public Object execute(Context) - final BlockBuilder builder = new BlockBuilder(); - final Expression values_ = builder.append("values", - Expressions.newArrayBounds(Object.class, 1, - Expressions.constant(1))); - builder.add( - Expressions.statement( - Expressions.call( - Expressions.parameter(ExecutableExpression.class, "this"), - StormBuiltInMethod.EXPR_EXECUTE2.method, context, values_))); - builder.add( - Expressions.return_(null, - Expressions.arrayIndex(values_, Expressions.constant(0)))); - declarations.add( - Expressions.methodDecl(Modifier.PUBLIC, Object.class, - StormBuiltInMethod.EXPR_EXECUTE1.method.getName(), - ImmutableList.of(context), builder.toBlock())); - - final ClassDeclaration classDeclaration = - Expressions.classDecl(Modifier.PUBLIC, className, null, - ImmutableList.of(ExecutableExpression.class), declarations); - - return Expressions.toString(Lists.newArrayList(classDeclaration), "\n", false); - } - - public BlockStatement compileToBlock(List nodes, RelDataType inputRowType) { - final RexProgramBuilder programBuilder = - new RexProgramBuilder(inputRowType, rexBuilder); - for (RexNode node : nodes) { - programBuilder.addProject(node, null); - } - - return compileToBlock(programBuilder.getProgram()); - } - - public BlockStatement compileToBlock(final RexProgram program) { - final ParameterExpression context_ = - Expressions.parameter(Context.class, "context"); - final ParameterExpression outputValues_ = - Expressions.parameter(Object[].class, "outputValues"); - - return compileToBlock(program, context_, outputValues_).toBlock(); - } - - private BlockBuilder compileToBlock(final RexProgram program, ParameterExpression context, - ParameterExpression outputValues) { - RelDataType inputRowType = program.getInputRowType(); - final BlockBuilder builder = new BlockBuilder(); - final JavaTypeFactoryImpl javaTypeFactory = - new JavaTypeFactoryImpl(rexBuilder.getTypeFactory().getTypeSystem()); - - final RexToLixTranslator.InputGetter inputGetter = - new RexToLixTranslator.InputGetterImpl( - ImmutableList.of( - Pair.of( - Expressions.field(context, - BuiltInMethod.CONTEXT_VALUES.field), - PhysTypeImpl.of(javaTypeFactory, inputRowType, - JavaRowFormat.ARRAY, false)))); - final Function1 correlates = - new Function1() { - @Override - public RexToLixTranslator.InputGetter apply(String a0) { - throw new UnsupportedOperationException(); - } - }; - final Expression root = - Expressions.field(context, BuiltInMethod.CONTEXT_ROOT.field); - final List list = - RexToLixTranslator.translateProjects(program, javaTypeFactory, builder, - null, root, inputGetter, correlates); - for (int i = 0; i < list.size(); i++) { - builder.add( - Expressions.statement( - Expressions.assign( - Expressions.arrayIndex(outputValues, - Expressions.constant(i)), - list.get(i)))); - } - - return builder; - } - - public String compile(List nodes, RelDataType inputRowType, String className) { - final RexProgramBuilder programBuilder = - new RexProgramBuilder(inputRowType, rexBuilder); - for (RexNode node : nodes) { - programBuilder.addProject(node, null); - } - - return compile(programBuilder.getProgram(), className); - } - - public String compile(final RexProgram program, String className) { - final ParameterExpression context_ = - Expressions.parameter(Context.class, "context"); - final ParameterExpression outputValues_ = - Expressions.parameter(Object[].class, "outputValues"); - - BlockBuilder builder = compileToBlock(program, context_, outputValues_); - return baz(context_, outputValues_, builder.toBlock(), className); - } - - enum StormBuiltInMethod { - EXPR_EXECUTE1(ExecutableExpression.class, "execute", Context.class), - EXPR_EXECUTE2(ExecutableExpression.class, "execute", Context.class, Object[].class); - - public static final ImmutableMap MAP; - - static { - final ImmutableMap.Builder builder = - ImmutableMap.builder(); - for (BuiltInMethod value : BuiltInMethod.values()) { - if (value.method != null) { - builder.put(value.method, value); - } - } - MAP = builder.build(); - } - - public final Method method; - public final Constructor constructor; - public final Field field; - - StormBuiltInMethod(Method method, Constructor constructor, Field field) { - this.method = method; - this.constructor = constructor; - this.field = field; - } - - /** - * Defines a method. - */ - StormBuiltInMethod(Class clazz, String methodName, Class... argumentTypes) { - this(Types.lookupMethod(clazz, methodName, argumentTypes), null, null); - } - - /** - * Defines a constructor. - */ - StormBuiltInMethod(Class clazz, Class... argumentTypes) { - this(null, Types.lookupConstructor(clazz, argumentTypes), null); - } - - /** - * Defines a field. - */ - StormBuiltInMethod(Class clazz, String fieldName, boolean dummy) { - this(null, null, Types.lookupField(clazz, fieldName)); - assert dummy : "dummy value for method overloading must be true"; - } - } - -} diff --git a/sql/storm-sql-core/src/jvm/org/apache/storm/sql/compiler/StormSqlTypeFactoryImpl.java b/sql/storm-sql-core/src/jvm/org/apache/storm/sql/compiler/StormSqlTypeFactoryImpl.java deleted file mode 100644 index 621ba492f8a..00000000000 --- a/sql/storm-sql-core/src/jvm/org/apache/storm/sql/compiler/StormSqlTypeFactoryImpl.java +++ /dev/null @@ -1,42 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions - * and limitations under the License. - */ - -package org.apache.storm.sql.compiler; - -import org.apache.calcite.jdbc.JavaTypeFactoryImpl; -import org.apache.calcite.rel.type.RelDataType; -import org.apache.calcite.rel.type.RelDataTypeSystem; -import org.apache.calcite.sql.type.JavaToSqlTypeConversionRules; -import org.apache.calcite.sql.type.SqlTypeName; - -public class StormSqlTypeFactoryImpl extends JavaTypeFactoryImpl { - - public StormSqlTypeFactoryImpl() { - } - - public StormSqlTypeFactoryImpl(RelDataTypeSystem typeSystem) { - super(typeSystem); - } - - @Override - public RelDataType toSql(RelDataType type) { - if (type instanceof JavaType) { - JavaType javaType = (JavaType) type; - SqlTypeName sqlTypeName = JavaToSqlTypeConversionRules.instance().lookup(javaType.getJavaClass()); - if (sqlTypeName == null) { - sqlTypeName = SqlTypeName.ANY; - } - return createTypeWithNullability(createSqlType(sqlTypeName), type.isNullable()); - } - return super.toSql(type); - } -} diff --git a/sql/storm-sql-core/src/jvm/org/apache/storm/sql/javac/CompilingClassLoader.java b/sql/storm-sql-core/src/jvm/org/apache/storm/sql/javac/CompilingClassLoader.java deleted file mode 100644 index aaf4bf77413..00000000000 --- a/sql/storm-sql-core/src/jvm/org/apache/storm/sql/javac/CompilingClassLoader.java +++ /dev/null @@ -1,227 +0,0 @@ -/* - * Copyright (C) 2010 Google, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.storm.sql.javac; - -import static java.util.Collections.singleton; - -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.OutputStream; -import java.net.URI; -import java.net.URISyntaxException; -import java.util.HashMap; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import javax.tools.DiagnosticListener; -import javax.tools.FileObject; -import javax.tools.ForwardingJavaFileManager; -import javax.tools.JavaCompiler; -import javax.tools.JavaFileManager; -import javax.tools.JavaFileObject; -import javax.tools.SimpleJavaFileObject; -import javax.tools.ToolProvider; - -/** - * This is a Java ClassLoader that will attempt to load a class from a string of source code. - *

- *

Example

- *

- * 

- * String className = "com.foo.MyClass";
- * String classSource =
- *   "package com.foo;\n" +
- *   "public class MyClass implements Runnable {\n" +
- *   "  @Override public void run() {\n" +
- *   "   log(\"Hello world\");\n" +
- *   "  }\n" +
- *   "}";
- *
- * // Load class from source.
- * ClassLoader classLoader = new CompilingClassLoader(
- *     parentClassLoader, className, classSource);
- * Class myClass = classLoader.loadClass(className);
- *
- * // Use it.
- * Runnable instance = (Runnable)myClass.newInstance();
- * instance.run();
- *
- *

- * Only one chunk of source can be compiled per instance of CompilingClassLoader. If you need to compile more, create multiple - * CompilingClassLoader instances. - *

- * Uses Java 1.6's in built compiler API. - *

- * If the class cannot be compiled, loadClass() will throw a ClassNotFoundException and log the compile errors to System.err. If you don't - * want the messages logged, or want to explicitly handle the messages you can provide your own {@link javax.tools.DiagnosticListener} - * through {#setDiagnosticListener()}. - *

- * @see java.lang.ClassLoader - * @see javax.tools.JavaCompiler - */ -public class CompilingClassLoader extends ClassLoader { - - private static final URI EMPTY_URI; - - static { - try { - // Needed to keep SimpleFileObject constructor happy. - EMPTY_URI = new URI(""); - } catch (URISyntaxException e) { - throw new Error(e); - } - } - - private final Map byteCodeForClasses = new HashMap<>(); - - /** - * Constructor. - * - * @param parent Parent classloader to resolve dependencies from. - * @param className Name of class to compile. eg. "com.foo.MyClass". - * @param sourceCode Java source for class. e.g. "package com.foo; class MyClass { ... }". - * @param diagnosticListener Notified of compiler errors (may be null). - */ - public CompilingClassLoader( - ClassLoader parent, - String className, - String sourceCode, - DiagnosticListener diagnosticListener) - throws CompilerException { - super(parent); - if (!compileSourceCodeToByteCode(className, sourceCode, diagnosticListener)) { - throw new CompilerException("Could not compile " + className); - } - } - - public Map getClasses() { - return byteCodeForClasses; - } - - /** - * Override ClassLoader's class resolving method. Don't call this directly, instead use {@link ClassLoader#loadClass(String)}. - */ - @Override - public Class findClass(String name) throws ClassNotFoundException { - ByteArrayOutputStream byteCode = byteCodeForClasses.get(name); - if (byteCode == null) { - throw new ClassNotFoundException(name); - } - return defineClass(name, byteCode.toByteArray(), 0, byteCode.size()); - } - - /** - * Compiles source code to byte code. - * @return indicates whether compilation was successful - */ - private boolean compileSourceCodeToByteCode( - String className, String sourceCode, DiagnosticListener diagnosticListener) { - JavaCompiler javaCompiler = ToolProvider.getSystemJavaCompiler(); - - // Set up the in-memory filesystem. - InMemoryFileManager fileManager = - new InMemoryFileManager(javaCompiler.getStandardFileManager(null, null, null)); - JavaFileObject javaFile = new InMemoryJavaFile(className, sourceCode); - - // Javac option: remove these when the javac zip impl is fixed - // (http://b/issue?id=1822932) - System.setProperty("useJavaUtilZip", "true"); // setting value to any non-null string - List options = new LinkedList<>(); - // this is ignored by javac currently but useJavaUtilZip should be - // a valid javac XD option, which is another bug - options.add("-XDuseJavaUtilZip"); - - // Now compile! - JavaCompiler.CompilationTask compilationTask = - javaCompiler.getTask( - null, // Null: log any unhandled errors to stderr. - fileManager, - diagnosticListener, - options, - null, - singleton(javaFile)); - return compilationTask.call(); - } - - /** - * Thrown when code cannot be compiled. - */ - public static class CompilerException extends Exception { - private static final long serialVersionUID = -2936958840023603270L; - - CompilerException(String message) { - super(message); - } - } - - private static class InMemoryJavaFile extends SimpleJavaFileObject { - private final String sourceCode; - - /** - * Constructor. - */ - InMemoryJavaFile(String className, String sourceCode) { - super(makeUri(className), Kind.SOURCE); - this.sourceCode = sourceCode; - } - - private static URI makeUri(String className) { - try { - return new URI(className.replaceAll("\\.", "/") + Kind.SOURCE.extension); - } catch (URISyntaxException e) { - throw new RuntimeException(e); // Not sure what could cause this. - } - } - - @Override - public CharSequence getCharContent(boolean ignoreEncodingErrors) throws IOException { - return sourceCode; - } - } - - /** - * Provides an in-memory representation of JavaFileManager abstraction, so we do not need to write any files to disk. - * - *

When files are written to, rather than putting the bytes on disk, they are appended to buffers in byteCodeForClasses. - * - * @see javax.tools.JavaFileManager - */ - private class InMemoryFileManager extends ForwardingJavaFileManager { - InMemoryFileManager(JavaFileManager fileManager) { - super(fileManager); - } - - @Override - public JavaFileObject getJavaFileForOutput( - Location location, final String className, JavaFileObject.Kind kind, FileObject sibling) - throws IOException { - return new SimpleJavaFileObject(EMPTY_URI, kind) { - @Override - public OutputStream openOutputStream() throws IOException { - ByteArrayOutputStream outputStream = byteCodeForClasses.get(className); - if (outputStream != null) { - throw new IllegalStateException("Cannot write more than once"); - } - // Reasonable size for a simple .class. - outputStream = new ByteArrayOutputStream(256); - byteCodeForClasses.put(className, outputStream); - return outputStream; - } - }; - } - } -} diff --git a/sql/storm-sql-core/src/jvm/org/apache/storm/sql/parser/ColumnConstraint.java b/sql/storm-sql-core/src/jvm/org/apache/storm/sql/parser/ColumnConstraint.java deleted file mode 100644 index 3be6bc988fd..00000000000 --- a/sql/storm-sql-core/src/jvm/org/apache/storm/sql/parser/ColumnConstraint.java +++ /dev/null @@ -1,38 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions - * and limitations under the License. - */ - -package org.apache.storm.sql.parser; - -import org.apache.calcite.sql.SqlLiteral; -import org.apache.calcite.sql.parser.SqlParserPos; -import org.apache.calcite.sql.type.SqlTypeName; -import org.apache.calcite.sql.validate.SqlMonotonicity; - -public class ColumnConstraint extends SqlLiteral { - private ColumnConstraint( - Object value, SqlTypeName typeName, SqlParserPos pos) { - super(value, typeName, pos); - } - - public static class PrimaryKey extends ColumnConstraint { - private final SqlMonotonicity monotonicity; - - public PrimaryKey(SqlMonotonicity monotonicity, SqlParserPos pos) { - super(SqlDdlKeywords.PRIMARY, SqlTypeName.SYMBOL, pos); - this.monotonicity = monotonicity; - } - - public SqlMonotonicity monotonicity() { - return monotonicity; - } - } -} diff --git a/sql/storm-sql-core/src/jvm/org/apache/storm/sql/parser/ColumnDefinition.java b/sql/storm-sql-core/src/jvm/org/apache/storm/sql/parser/ColumnDefinition.java deleted file mode 100644 index 95e335366b8..00000000000 --- a/sql/storm-sql-core/src/jvm/org/apache/storm/sql/parser/ColumnDefinition.java +++ /dev/null @@ -1,38 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions - * and limitations under the License. - */ - -package org.apache.storm.sql.parser; - -import java.util.Arrays; -import org.apache.calcite.sql.SqlDataTypeSpec; -import org.apache.calcite.sql.SqlIdentifier; -import org.apache.calcite.sql.SqlNodeList; -import org.apache.calcite.sql.parser.SqlParserPos; - -public class ColumnDefinition extends SqlNodeList { - public ColumnDefinition( - SqlIdentifier name, SqlDataTypeSpec type, ColumnConstraint constraint, SqlParserPos pos) { - super(Arrays.asList(name, type, constraint), pos); - } - - public String name() { - return get(0).toString(); - } - - public SqlDataTypeSpec type() { - return (SqlDataTypeSpec) get(1); - } - - public ColumnConstraint constraint() { - return (ColumnConstraint) get(2); - } -} diff --git a/sql/storm-sql-core/src/jvm/org/apache/storm/sql/parser/SqlCreateFunction.java b/sql/storm-sql-core/src/jvm/org/apache/storm/sql/parser/SqlCreateFunction.java deleted file mode 100644 index 157ac718b36..00000000000 --- a/sql/storm-sql-core/src/jvm/org/apache/storm/sql/parser/SqlCreateFunction.java +++ /dev/null @@ -1,88 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions - * and limitations under the License. - */ - -package org.apache.storm.sql.parser; - -import java.util.List; -import org.apache.calcite.sql.SqlCall; -import org.apache.calcite.sql.SqlIdentifier; -import org.apache.calcite.sql.SqlKind; -import org.apache.calcite.sql.SqlLiteral; -import org.apache.calcite.sql.SqlNode; -import org.apache.calcite.sql.SqlOperator; -import org.apache.calcite.sql.SqlSpecialOperator; -import org.apache.calcite.sql.SqlWriter; -import org.apache.calcite.sql.parser.SqlParserPos; -import org.apache.calcite.util.ImmutableNullableList; -import org.apache.calcite.util.NlsString; - -public class SqlCreateFunction extends SqlCall { - public static final SqlSpecialOperator OPERATOR = new SqlSpecialOperator( - "CREATE_FUNCTION", SqlKind.OTHER) { - @Override - public SqlCall createCall( - SqlLiteral functionQualifier, SqlParserPos pos, SqlNode... o) { - assert functionQualifier == null; - return new SqlCreateFunction(pos, (SqlIdentifier) o[0], o[1], o[2]); - } - - @Override - public void unparse( - SqlWriter writer, SqlCall call, int leftPrec, int rightPrec) { - SqlCreateFunction t = (SqlCreateFunction) call; - UnparseUtil u = new UnparseUtil(writer, leftPrec, rightPrec); - u.keyword("CREATE", "FUNCTION").node(t.functionName).keyword("AS").node(t.className); - if (t.jarName != null) { - u.keyword("USING", "JAR").node(t.jarName); - } - } - }; - - private final SqlIdentifier functionName; - private final SqlNode className; - private final SqlNode jarName; - - public SqlCreateFunction(SqlParserPos pos, SqlIdentifier functionName, SqlNode className, SqlNode jarName) { - super(pos); - this.functionName = functionName; - this.className = className; - this.jarName = jarName; - } - - @Override - public SqlOperator getOperator() { - return OPERATOR; - } - - @Override - public List getOperandList() { - return ImmutableNullableList.of(functionName, className); - } - - - @Override - public void unparse(SqlWriter writer, int leftPrec, int rightPrec) { - getOperator().unparse(writer, this, leftPrec, rightPrec); - } - - public String functionName() { - return functionName.toString(); - } - - public String className() { - return ((NlsString) SqlLiteral.value(className)).getValue(); - } - - public String jarName() { - return jarName == null ? null : ((NlsString) SqlLiteral.value(jarName)).getValue(); - } -} diff --git a/sql/storm-sql-core/src/jvm/org/apache/storm/sql/parser/SqlCreateTable.java b/sql/storm-sql-core/src/jvm/org/apache/storm/sql/parser/SqlCreateTable.java deleted file mode 100644 index bd58dd67ecd..00000000000 --- a/sql/storm-sql-core/src/jvm/org/apache/storm/sql/parser/SqlCreateTable.java +++ /dev/null @@ -1,164 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions - * and limitations under the License. - */ - -package org.apache.storm.sql.parser; - -import com.fasterxml.jackson.databind.ObjectMapper; -import java.io.IOException; -import java.math.BigDecimal; -import java.net.URI; -import java.util.HashMap; -import java.util.List; -import java.util.Properties; -import org.apache.calcite.sql.SqlCall; -import org.apache.calcite.sql.SqlIdentifier; -import org.apache.calcite.sql.SqlKind; -import org.apache.calcite.sql.SqlLiteral; -import org.apache.calcite.sql.SqlNode; -import org.apache.calcite.sql.SqlNodeList; -import org.apache.calcite.sql.SqlOperator; -import org.apache.calcite.sql.SqlSpecialOperator; -import org.apache.calcite.sql.SqlWriter; -import org.apache.calcite.sql.parser.SqlParserPos; -import org.apache.calcite.util.ImmutableNullableList; -import org.apache.calcite.util.NlsString; - -public class SqlCreateTable extends SqlCall { - public static final SqlSpecialOperator OPERATOR = new SqlSpecialOperator( - "CREATE_TABLE", SqlKind.OTHER) { - @Override - public SqlCall createCall( - SqlLiteral functionQualifier, SqlParserPos pos, SqlNode... o) { - assert functionQualifier == null; - return new SqlCreateTable(pos, (SqlIdentifier) o[0], (SqlNodeList) o[1], - o[2], o[3], o[4], o[5], o[6], o[7]); - } - - @Override - public void unparse( - SqlWriter writer, SqlCall call, int leftPrec, int rightPrec) { - SqlCreateTable t = (SqlCreateTable) call; - UnparseUtil u = new UnparseUtil(writer, leftPrec, rightPrec); - u.keyword("CREATE", "EXTERNAL", "TABLE").node(t.tblName).nodeList( - t.fieldList); - if (t.inputFormatClass != null && t.outputFormatClass != null) { - u.keyword("STORED", "AS", "INPUTFORMAT").node( - t.inputFormatClass).keyword("OUTPUTFORMAT").node( - t.outputFormatClass); - } - u.keyword("LOCATION").node(t.location); - if (t.parallelism != null) { - u.keyword("PARALLELISM").node(t.parallelism); - } - if (t.properties != null) { - u.keyword("TBLPROPERTIES").node(t.properties); - } - if (t.query != null) { - u.keyword("AS").node(t.query); - } - } - }; - private static final int DEFAULT_PARALLELISM = 1; - private final SqlIdentifier tblName; - private final SqlNodeList fieldList; - private final SqlNode inputFormatClass; - private final SqlNode outputFormatClass; - private final SqlNode location; - private final SqlNode parallelism; - private final SqlNode properties; - private final SqlNode query; - - public SqlCreateTable( - SqlParserPos pos, SqlIdentifier tblName, SqlNodeList fieldList, - SqlNode inputFormatClass, SqlNode outputFormatClass, SqlNode location, - SqlNode parallelism, SqlNode properties, SqlNode query) { - super(pos); - this.tblName = tblName; - this.fieldList = fieldList; - this.inputFormatClass = inputFormatClass; - this.outputFormatClass = outputFormatClass; - this.location = location; - this.parallelism = parallelism; - this.properties = properties; - this.query = query; - } - - @Override - public SqlOperator getOperator() { - return OPERATOR; - } - - @Override - public void unparse(SqlWriter writer, int leftPrec, int rightPrec) { - getOperator().unparse(writer, this, leftPrec, rightPrec); - } - - @Override - public List getOperandList() { - return ImmutableNullableList.of(tblName, fieldList, inputFormatClass, - outputFormatClass, location, properties, - query); - } - - public String tableName() { - return tblName.toString(); - } - - public URI location() { - return URI.create(getString(location)); - } - - public Integer parallelism() { - Integer para = getInteger(parallelism); - if (para != null) { - return para; - } else { - return DEFAULT_PARALLELISM; - } - } - - public String inputFormatClass() { - return getString(inputFormatClass); - } - - public String outputFormatClass() { - return getString(outputFormatClass); - } - - public Properties properties() { - Properties props = new Properties(); - if (properties != null) { - try { - ObjectMapper mapper = new ObjectMapper(); - HashMap map = mapper.readValue(getString(properties), HashMap.class); - props.putAll(map); - } catch (IOException e) { - throw new RuntimeException(e); - } - } - return props; - } - - private Integer getInteger(SqlNode n) { - return n == null ? null : ((BigDecimal) SqlLiteral.value(n)).intValueExact(); - } - - private String getString(SqlNode n) { - return n == null ? null : ((NlsString) SqlLiteral.value(n)).getValue(); - } - - @SuppressWarnings("unchecked") - public List fieldList() { - return (List) ((List) fieldList.getList()); - } - -} diff --git a/sql/storm-sql-core/src/jvm/org/apache/storm/sql/parser/SqlDdlKeywords.java b/sql/storm-sql-core/src/jvm/org/apache/storm/sql/parser/SqlDdlKeywords.java deleted file mode 100644 index 327c55c9cfe..00000000000 --- a/sql/storm-sql-core/src/jvm/org/apache/storm/sql/parser/SqlDdlKeywords.java +++ /dev/null @@ -1,22 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions - * and limitations under the License. - */ - -package org.apache.storm.sql.parser; - -import org.apache.calcite.sql.SqlLiteral; - -/** - * Define the keywords that can occur in a CREATE TABLE statement. - */ -public enum SqlDdlKeywords implements SqlLiteral.SqlSymbol { - PRIMARY -} diff --git a/sql/storm-sql-core/src/jvm/org/apache/storm/sql/parser/StormParser.java b/sql/storm-sql-core/src/jvm/org/apache/storm/sql/parser/StormParser.java deleted file mode 100644 index e1d3a60fa1f..00000000000 --- a/sql/storm-sql-core/src/jvm/org/apache/storm/sql/parser/StormParser.java +++ /dev/null @@ -1,44 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions - * and limitations under the License. - */ - -package org.apache.storm.sql.parser; - -import com.google.common.annotations.VisibleForTesting; -import java.io.StringReader; -import org.apache.calcite.config.Lex; -import org.apache.storm.sql.parser.impl.StormParserImpl; - -public class StormParser { - public static final int DEFAULT_IDENTIFIER_MAX_LENGTH = 128; - private final StormParserImpl impl; - - /** - * Constructor. - */ - public StormParser(String s) { - this.impl = new StormParserImpl(new StringReader(s)); - this.impl.setTabSize(1); - this.impl.setQuotedCasing(Lex.ORACLE.quotedCasing); - this.impl.setUnquotedCasing(Lex.ORACLE.unquotedCasing); - this.impl.setIdentifierMaxLength(DEFAULT_IDENTIFIER_MAX_LENGTH); - /* - * By default parser uses [ ] for quoting identifiers. Switching to DQID (double quoted identifiers) - * is needed for array and map access (m['x'] = 1 or arr[2] = 10 etc) to work. - */ - this.impl.switchTo("DQID"); - } - - @VisibleForTesting - public StormParserImpl impl() { - return impl; - } -} diff --git a/sql/storm-sql-core/src/jvm/org/apache/storm/sql/parser/UnparseUtil.java b/sql/storm-sql-core/src/jvm/org/apache/storm/sql/parser/UnparseUtil.java deleted file mode 100644 index a868f3b82cb..00000000000 --- a/sql/storm-sql-core/src/jvm/org/apache/storm/sql/parser/UnparseUtil.java +++ /dev/null @@ -1,54 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions - * and limitations under the License. - */ - -package org.apache.storm.sql.parser; - -import org.apache.calcite.sql.SqlNode; -import org.apache.calcite.sql.SqlNodeList; -import org.apache.calcite.sql.SqlWriter; - -class UnparseUtil { - private final SqlWriter writer; - private final int leftPrec; - private final int rightPrec; - - UnparseUtil(SqlWriter writer, int leftPrec, int rightPrec) { - this.writer = writer; - this.leftPrec = leftPrec; - this.rightPrec = rightPrec; - } - - UnparseUtil keyword(String... keywords) { - for (String k : keywords) { - writer.keyword(k); - } - return this; - } - - UnparseUtil node(SqlNode n) { - n.unparse(writer, leftPrec, rightPrec); - return this; - } - - UnparseUtil nodeList(SqlNodeList l) { - writer.keyword("("); - if (l.size() > 0) { - l.get(0).unparse(writer, leftPrec, rightPrec); - for (int i = 1; i < l.size(); ++i) { - writer.keyword(","); - l.get(i).unparse(writer, leftPrec, rightPrec); - } - } - writer.keyword(")"); - return this; - } -} diff --git a/sql/storm-sql-core/src/jvm/org/apache/storm/sql/planner/StormRelDataTypeSystem.java b/sql/storm-sql-core/src/jvm/org/apache/storm/sql/planner/StormRelDataTypeSystem.java deleted file mode 100644 index 12a63b5979e..00000000000 --- a/sql/storm-sql-core/src/jvm/org/apache/storm/sql/planner/StormRelDataTypeSystem.java +++ /dev/null @@ -1,31 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions - * and limitations under the License. - */ - -package org.apache.storm.sql.planner; - -import org.apache.calcite.rel.type.RelDataTypeSystem; -import org.apache.calcite.rel.type.RelDataTypeSystemImpl; - -public class StormRelDataTypeSystem extends RelDataTypeSystemImpl { - public static final RelDataTypeSystem STORM_REL_DATATYPE_SYSTEM = new StormRelDataTypeSystem(); - - @Override - public int getMaxNumericScale() { - return 38; - } - - @Override - public int getMaxNumericPrecision() { - return 38; - } - -} diff --git a/sql/storm-sql-core/src/jvm/org/apache/storm/sql/planner/StormRelUtils.java b/sql/storm-sql-core/src/jvm/org/apache/storm/sql/planner/StormRelUtils.java deleted file mode 100644 index f3cc2b80e00..00000000000 --- a/sql/storm-sql-core/src/jvm/org/apache/storm/sql/planner/StormRelUtils.java +++ /dev/null @@ -1,62 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions - * and limitations under the License. - */ - -package org.apache.storm.sql.planner; - -import java.util.concurrent.atomic.AtomicInteger; -import org.apache.calcite.plan.RelOptUtil; -import org.apache.calcite.plan.volcano.RelSubset; -import org.apache.calcite.rel.RelNode; -import org.apache.calcite.sql.SqlExplainLevel; -import org.apache.storm.sql.planner.streams.rel.StreamsRel; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class StormRelUtils { - private static final Logger LOG = LoggerFactory.getLogger(StormRelUtils.class); - - private static final AtomicInteger sequence = new AtomicInteger(0); - private static final AtomicInteger classSequence = new AtomicInteger(0); - - public static String getClassName(StreamsRel relNode) { - return "Generated_" - + relNode.getClass().getSimpleName().toUpperCase() - + "_" - + relNode.getId() - + "_" - + classSequence.getAndIncrement(); - } - - public static StreamsRel getStormRelInput(RelNode input) { - if (input instanceof RelSubset) { - // go with known best input - input = ((RelSubset) input).getBest(); - } - return (StreamsRel) input; - } - - public static String explain(final RelNode rel) { - return explain(rel, SqlExplainLevel.EXPPLAN_ATTRIBUTES); - } - - public static String explain(final RelNode rel, SqlExplainLevel detailLevel) { - String explain = ""; - try { - explain = RelOptUtil.toString(rel); - } catch (StackOverflowError e) { - LOG.error("StackOverflowError occurred while extracting plan. Please report it to the dev@ mailing list."); - LOG.error("RelNode " + rel + " ExplainLevel " + detailLevel, e); - LOG.error("Forcing plan to empty string and continue... SQL Runner may not working properly after."); - } - return explain; - } -} diff --git a/sql/storm-sql-core/src/jvm/org/apache/storm/sql/planner/UnsupportedOperatorsVisitor.java b/sql/storm-sql-core/src/jvm/org/apache/storm/sql/planner/UnsupportedOperatorsVisitor.java deleted file mode 100644 index 39193df0a17..00000000000 --- a/sql/storm-sql-core/src/jvm/org/apache/storm/sql/planner/UnsupportedOperatorsVisitor.java +++ /dev/null @@ -1,19 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions - * and limitations under the License. - */ - -package org.apache.storm.sql.planner; - -import org.apache.calcite.sql.util.SqlShuttle; - -public class UnsupportedOperatorsVisitor extends SqlShuttle { - -} diff --git a/sql/storm-sql-core/src/jvm/org/apache/storm/sql/planner/rel/StormCalcRelBase.java b/sql/storm-sql-core/src/jvm/org/apache/storm/sql/planner/rel/StormCalcRelBase.java deleted file mode 100644 index 10072d01607..00000000000 --- a/sql/storm-sql-core/src/jvm/org/apache/storm/sql/planner/rel/StormCalcRelBase.java +++ /dev/null @@ -1,25 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions - * and limitations under the License. - */ - -package org.apache.storm.sql.planner.rel; - -import org.apache.calcite.plan.RelOptCluster; -import org.apache.calcite.plan.RelTraitSet; -import org.apache.calcite.rel.RelNode; -import org.apache.calcite.rel.core.Calc; -import org.apache.calcite.rex.RexProgram; - -public abstract class StormCalcRelBase extends Calc implements StormRelNode { - protected StormCalcRelBase(RelOptCluster cluster, RelTraitSet traits, RelNode child, RexProgram program) { - super(cluster, traits, child, program); - } -} diff --git a/sql/storm-sql-core/src/jvm/org/apache/storm/sql/planner/rel/StormFilterRelBase.java b/sql/storm-sql-core/src/jvm/org/apache/storm/sql/planner/rel/StormFilterRelBase.java deleted file mode 100644 index f040cbb60e8..00000000000 --- a/sql/storm-sql-core/src/jvm/org/apache/storm/sql/planner/rel/StormFilterRelBase.java +++ /dev/null @@ -1,25 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions - * and limitations under the License. - */ - -package org.apache.storm.sql.planner.rel; - -import org.apache.calcite.plan.RelOptCluster; -import org.apache.calcite.plan.RelTraitSet; -import org.apache.calcite.rel.RelNode; -import org.apache.calcite.rel.core.Filter; -import org.apache.calcite.rex.RexNode; - -public abstract class StormFilterRelBase extends Filter implements StormRelNode { - protected StormFilterRelBase(RelOptCluster cluster, RelTraitSet traits, RelNode child, RexNode condition) { - super(cluster, traits, child, condition); - } -} diff --git a/sql/storm-sql-core/src/jvm/org/apache/storm/sql/planner/rel/StormJoinRelBase.java b/sql/storm-sql-core/src/jvm/org/apache/storm/sql/planner/rel/StormJoinRelBase.java deleted file mode 100644 index 13dc010f634..00000000000 --- a/sql/storm-sql-core/src/jvm/org/apache/storm/sql/planner/rel/StormJoinRelBase.java +++ /dev/null @@ -1,29 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions - * and limitations under the License. - */ - -package org.apache.storm.sql.planner.rel; - -import java.util.Set; -import org.apache.calcite.plan.RelOptCluster; -import org.apache.calcite.plan.RelTraitSet; -import org.apache.calcite.rel.RelNode; -import org.apache.calcite.rel.core.CorrelationId; -import org.apache.calcite.rel.core.Join; -import org.apache.calcite.rel.core.JoinRelType; -import org.apache.calcite.rex.RexNode; - -public abstract class StormJoinRelBase extends Join implements StormRelNode { - protected StormJoinRelBase(RelOptCluster cluster, RelTraitSet traitSet, RelNode left, RelNode right, RexNode condition, - Set variablesSet, JoinRelType joinType) { - super(cluster, traitSet, left, right, condition, variablesSet, joinType); - } -} diff --git a/sql/storm-sql-core/src/jvm/org/apache/storm/sql/planner/rel/StormProjectRelBase.java b/sql/storm-sql-core/src/jvm/org/apache/storm/sql/planner/rel/StormProjectRelBase.java deleted file mode 100644 index e30a52d7d4b..00000000000 --- a/sql/storm-sql-core/src/jvm/org/apache/storm/sql/planner/rel/StormProjectRelBase.java +++ /dev/null @@ -1,28 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions - * and limitations under the License. - */ - -package org.apache.storm.sql.planner.rel; - -import java.util.List; -import org.apache.calcite.plan.RelOptCluster; -import org.apache.calcite.plan.RelTraitSet; -import org.apache.calcite.rel.RelNode; -import org.apache.calcite.rel.core.Project; -import org.apache.calcite.rel.type.RelDataType; -import org.apache.calcite.rex.RexNode; - -public abstract class StormProjectRelBase extends Project implements StormRelNode { - protected StormProjectRelBase(RelOptCluster cluster, RelTraitSet traits, RelNode input, List projects, - RelDataType rowType) { - super(cluster, traits, input, projects, rowType); - } -} diff --git a/sql/storm-sql-core/src/jvm/org/apache/storm/sql/planner/rel/StormRelNode.java b/sql/storm-sql-core/src/jvm/org/apache/storm/sql/planner/rel/StormRelNode.java deleted file mode 100644 index cdea2e1bfd4..00000000000 --- a/sql/storm-sql-core/src/jvm/org/apache/storm/sql/planner/rel/StormRelNode.java +++ /dev/null @@ -1,18 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions - * and limitations under the License. - */ - -package org.apache.storm.sql.planner.rel; - -import org.apache.calcite.rel.RelNode; - -public interface StormRelNode extends RelNode { -} diff --git a/sql/storm-sql-core/src/jvm/org/apache/storm/sql/planner/rel/StormStreamInsertRelBase.java b/sql/storm-sql-core/src/jvm/org/apache/storm/sql/planner/rel/StormStreamInsertRelBase.java deleted file mode 100644 index 424822e7fe5..00000000000 --- a/sql/storm-sql-core/src/jvm/org/apache/storm/sql/planner/rel/StormStreamInsertRelBase.java +++ /dev/null @@ -1,30 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions - * and limitations under the License. - */ - -package org.apache.storm.sql.planner.rel; - -import java.util.List; -import org.apache.calcite.plan.RelOptCluster; -import org.apache.calcite.plan.RelOptTable; -import org.apache.calcite.plan.RelTraitSet; -import org.apache.calcite.prepare.Prepare; -import org.apache.calcite.rel.RelNode; -import org.apache.calcite.rel.core.TableModify; -import org.apache.calcite.rex.RexNode; - -public abstract class StormStreamInsertRelBase extends TableModify implements StormRelNode { - protected StormStreamInsertRelBase(RelOptCluster cluster, RelTraitSet traits, RelOptTable table, Prepare.CatalogReader catalogReader, - RelNode child, Operation operation, List updateColumnList, - List sourceExpressionList, boolean flattened) { - super(cluster, traits, table, catalogReader, child, operation, updateColumnList, sourceExpressionList, flattened); - } -} diff --git a/sql/storm-sql-core/src/jvm/org/apache/storm/sql/planner/rel/StormStreamScanRelBase.java b/sql/storm-sql-core/src/jvm/org/apache/storm/sql/planner/rel/StormStreamScanRelBase.java deleted file mode 100644 index 756ed07be05..00000000000 --- a/sql/storm-sql-core/src/jvm/org/apache/storm/sql/planner/rel/StormStreamScanRelBase.java +++ /dev/null @@ -1,25 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions - * and limitations under the License. - */ - -package org.apache.storm.sql.planner.rel; - -import org.apache.calcite.plan.RelOptCluster; -import org.apache.calcite.plan.RelOptTable; -import org.apache.calcite.plan.RelTraitSet; -import org.apache.calcite.rel.core.TableScan; - -public abstract class StormStreamScanRelBase extends TableScan implements StormRelNode { - - protected StormStreamScanRelBase(RelOptCluster cluster, RelTraitSet traitSet, RelOptTable table) { - super(cluster, traitSet, table); - } -} diff --git a/sql/storm-sql-core/src/jvm/org/apache/storm/sql/planner/streams/QueryPlanner.java b/sql/storm-sql-core/src/jvm/org/apache/storm/sql/planner/streams/QueryPlanner.java deleted file mode 100644 index 180232e2857..00000000000 --- a/sql/storm-sql-core/src/jvm/org/apache/storm/sql/planner/streams/QueryPlanner.java +++ /dev/null @@ -1,158 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.apache.storm.sql.planner.streams; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.Map; -import java.util.Properties; - -import org.apache.calcite.DataContext; -import org.apache.calcite.adapter.java.JavaTypeFactory; -import org.apache.calcite.config.CalciteConnectionConfigImpl; -import org.apache.calcite.jdbc.CalciteSchema; -import org.apache.calcite.jdbc.JavaTypeFactoryImpl; -import org.apache.calcite.plan.Contexts; -import org.apache.calcite.plan.ConventionTraitDef; -import org.apache.calcite.plan.RelTraitDef; -import org.apache.calcite.plan.RelTraitSet; -import org.apache.calcite.prepare.CalciteCatalogReader; -import org.apache.calcite.rel.RelCollationTraitDef; -import org.apache.calcite.rel.RelNode; -import org.apache.calcite.rel.type.RelDataTypeSystem; -import org.apache.calcite.rex.RexBuilder; -import org.apache.calcite.schema.SchemaPlus; -import org.apache.calcite.sql.SqlNode; -import org.apache.calcite.sql.SqlOperatorTable; -import org.apache.calcite.sql.fun.SqlStdOperatorTable; -import org.apache.calcite.sql.parser.SqlParseException; -import org.apache.calcite.sql.util.ChainedSqlOperatorTable; -import org.apache.calcite.tools.FrameworkConfig; -import org.apache.calcite.tools.Frameworks; -import org.apache.calcite.tools.Planner; -import org.apache.calcite.tools.RelConversionException; -import org.apache.calcite.tools.ValidationException; -import org.apache.storm.generated.StormTopology; -import org.apache.storm.sql.AbstractStreamsProcessor; -import org.apache.storm.sql.javac.CompilingClassLoader; -import org.apache.storm.sql.planner.StormRelDataTypeSystem; -import org.apache.storm.sql.planner.UnsupportedOperatorsVisitor; -import org.apache.storm.sql.planner.streams.rel.StreamsLogicalConvention; -import org.apache.storm.sql.planner.streams.rel.StreamsRel; -import org.apache.storm.sql.runtime.ISqlStreamsDataSource; -import org.apache.storm.streams.Stream; -import org.apache.storm.streams.StreamBuilder; -import org.apache.storm.tuple.Values; - -public class QueryPlanner { - - public static final int STORM_REL_CONVERSION_RULES = 1; - - private final Planner planner; - - private final JavaTypeFactory typeFactory = new JavaTypeFactoryImpl( - RelDataTypeSystem.DEFAULT); - - public QueryPlanner(SchemaPlus schema) { - final List traitDefs = new ArrayList(); - - traitDefs.add(ConventionTraitDef.INSTANCE); - traitDefs.add(RelCollationTraitDef.INSTANCE); - - List sqlOperatorTables = new ArrayList<>(); - sqlOperatorTables.add(SqlStdOperatorTable.instance()); - sqlOperatorTables.add(new CalciteCatalogReader(CalciteSchema.from(schema), - Collections.emptyList(), typeFactory, new CalciteConnectionConfigImpl(new Properties()))); - - FrameworkConfig config = Frameworks.newConfigBuilder() - .defaultSchema(schema) - .operatorTable(new ChainedSqlOperatorTable(sqlOperatorTables)) - .traitDefs(traitDefs) - .context(Contexts.EMPTY_CONTEXT) - .ruleSets(StreamsStormRuleSets.getRuleSets()) - .costFactory(null) - .typeSystem(StormRelDataTypeSystem.STORM_REL_DATATYPE_SYSTEM) - .build(); - this.planner = Frameworks.getPlanner(config); - } - - public AbstractStreamsProcessor compile(Map sources, String query) throws Exception { - StreamsRel relNode = getPlan(query); - - StreamsPlanCreator streamsPlanCreator = new StreamsPlanCreator(sources, new RexBuilder(typeFactory)); - relNode.streamsPlan(streamsPlanCreator); - - final StreamBuilder streamBuilder = streamsPlanCreator.getStreamBuilder(); - final Stream lastStream = streamsPlanCreator.pop(); - final DataContext dc = streamsPlanCreator.getDataContext(); - final List cls = streamsPlanCreator.getClassLoaders(); - - return new AbstractStreamsProcessor() { - @Override - public StormTopology build() { - return streamBuilder.build(); - } - - @Override - public Stream outputStream() { - return lastStream; - } - - @Override - public DataContext getDataContext() { - return dc; - } - - @Override - public List getClassLoaders() { - return cls; - } - }; - } - - public StreamsRel getPlan(String query) throws ValidationException, RelConversionException, SqlParseException { - return (StreamsRel) validateAndConvert(planner.parse(query)); - } - - private RelNode validateAndConvert(SqlNode sqlNode) throws ValidationException, RelConversionException { - SqlNode validated = validateNode(sqlNode); - RelNode relNode = convertToRelNode(validated); - return convertToStormRel(relNode); - } - - private RelNode convertToStormRel(RelNode relNode) throws RelConversionException { - RelTraitSet traitSet = relNode.getTraitSet(); - traitSet = traitSet.simplify(); - - // PlannerImpl.transform() optimizes RelNode with ruleset - return planner.transform(STORM_REL_CONVERSION_RULES, traitSet.plus(StreamsLogicalConvention.INSTANCE), relNode); - } - - private RelNode convertToRelNode(SqlNode sqlNode) throws RelConversionException { - return planner.rel(sqlNode).rel; - } - - private SqlNode validateNode(SqlNode sqlNode) throws ValidationException { - SqlNode validatedSqlNode = planner.validate(sqlNode); - validatedSqlNode.accept(new UnsupportedOperatorsVisitor()); - return validatedSqlNode; - } -} diff --git a/sql/storm-sql-core/src/jvm/org/apache/storm/sql/planner/streams/StreamsPlanCreator.java b/sql/storm-sql-core/src/jvm/org/apache/storm/sql/planner/streams/StreamsPlanCreator.java deleted file mode 100644 index 977c1ee229e..00000000000 --- a/sql/storm-sql-core/src/jvm/org/apache/storm/sql/planner/streams/StreamsPlanCreator.java +++ /dev/null @@ -1,126 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.apache.storm.sql.planner.streams; - -import java.util.ArrayDeque; -import java.util.ArrayList; -import java.util.Deque; -import java.util.List; -import java.util.Map; - -import org.apache.calcite.DataContext; -import org.apache.calcite.adapter.java.JavaTypeFactory; -import org.apache.calcite.rel.type.RelDataType; -import org.apache.calcite.rex.RexBuilder; -import org.apache.calcite.rex.RexNode; -import org.apache.calcite.rex.RexProgram; -import org.apache.storm.sql.compiler.RexNodeToJavaCodeCompiler; -import org.apache.storm.sql.javac.CompilingClassLoader; -import org.apache.storm.sql.runtime.ISqlStreamsDataSource; -import org.apache.storm.sql.runtime.calcite.DebuggableExecutableExpression; -import org.apache.storm.sql.runtime.calcite.ExecutableExpression; -import org.apache.storm.sql.runtime.calcite.StormDataContext; -import org.apache.storm.streams.Stream; -import org.apache.storm.streams.StreamBuilder; -import org.apache.storm.tuple.Values; - -public class StreamsPlanCreator { - private final Map sources; - private final JavaTypeFactory typeFactory; - private final RexNodeToJavaCodeCompiler rexCompiler; - private final StreamBuilder streamBuilder; - private final DataContext dataContext; - - private final Deque> streamStack = new ArrayDeque<>(); - private final List classLoaders = new ArrayList<>(); - - public StreamsPlanCreator(Map sources, RexBuilder rexBuilder) { - this.sources = sources; - this.rexCompiler = new RexNodeToJavaCodeCompiler(rexBuilder); - this.typeFactory = (JavaTypeFactory) rexBuilder.getTypeFactory(); - - this.streamBuilder = new StreamBuilder(); - this.dataContext = new StormDataContext(); - } - - public void addStream(Stream stream) throws Exception { - push(stream); - } - - public Stream pop() { - return streamStack.pop(); - } - - public Map getSources() { - return sources; - } - - public DataContext getDataContext() { - return dataContext; - } - - public JavaTypeFactory getTypeFactory() { - return typeFactory; - } - - public StreamBuilder getStreamBuilder() { - return streamBuilder; - } - - public ExecutableExpression createScalarInstance(List nodes, RelDataType inputRowType, String className) - throws CompilingClassLoader.CompilerException, ClassNotFoundException, IllegalAccessException, InstantiationException { - String expr = rexCompiler.compile(nodes, inputRowType, className); - CompilingClassLoader classLoader = new CompilingClassLoader( - getLastClassLoader(), className, expr, null); - ExecutableExpression instance = (ExecutableExpression) classLoader.loadClass(className).newInstance(); - addClassLoader(classLoader); - return new DebuggableExecutableExpression(instance, expr); - } - - public ExecutableExpression createScalarInstance(RexProgram program, String className) - throws CompilingClassLoader.CompilerException, ClassNotFoundException, IllegalAccessException, InstantiationException { - String expr = rexCompiler.compile(program, className); - CompilingClassLoader classLoader = new CompilingClassLoader( - getLastClassLoader(), className, expr, null); - ExecutableExpression instance = (ExecutableExpression) classLoader.loadClass(className).newInstance(); - addClassLoader(classLoader); - return new DebuggableExecutableExpression(instance, expr); - } - - private void push(Stream stream) { - streamStack.push(stream); - } - - public void addClassLoader(CompilingClassLoader compilingClassLoader) { - this.classLoaders.add(compilingClassLoader); - } - - public ClassLoader getLastClassLoader() { - if (this.classLoaders.size() > 0) { - return this.classLoaders.get(this.classLoaders.size() - 1); - } else { - return this.getClass().getClassLoader(); - } - } - - public List getClassLoaders() { - return classLoaders; - } -} diff --git a/sql/storm-sql-core/src/jvm/org/apache/storm/sql/planner/streams/StreamsStormRuleSets.java b/sql/storm-sql-core/src/jvm/org/apache/storm/sql/planner/streams/StreamsStormRuleSets.java deleted file mode 100644 index 53a17b841f0..00000000000 --- a/sql/storm-sql-core/src/jvm/org/apache/storm/sql/planner/streams/StreamsStormRuleSets.java +++ /dev/null @@ -1,109 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.apache.storm.sql.planner.streams; - -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableSet; -import java.util.Iterator; -import org.apache.calcite.plan.RelOptRule; -import org.apache.calcite.rel.rules.CalcMergeRule; -import org.apache.calcite.rel.rules.FilterCalcMergeRule; -import org.apache.calcite.rel.rules.FilterProjectTransposeRule; -import org.apache.calcite.rel.rules.FilterToCalcRule; -import org.apache.calcite.rel.rules.ProjectCalcMergeRule; -import org.apache.calcite.rel.rules.ProjectFilterTransposeRule; -import org.apache.calcite.rel.rules.ProjectRemoveRule; -import org.apache.calcite.rel.rules.ProjectToCalcRule; -import org.apache.calcite.rel.rules.PruneEmptyRules; -import org.apache.calcite.rel.rules.ReduceExpressionsRule; -import org.apache.calcite.rel.rules.SortRemoveRule; -import org.apache.calcite.rel.rules.UnionEliminatorRule; -import org.apache.calcite.rel.stream.StreamRules; -import org.apache.calcite.tools.RuleSet; -import org.apache.storm.sql.planner.streams.rules.StreamsAggregateRule; -import org.apache.storm.sql.planner.streams.rules.StreamsCalcRule; -import org.apache.storm.sql.planner.streams.rules.StreamsFilterRule; -import org.apache.storm.sql.planner.streams.rules.StreamsJoinRule; -import org.apache.storm.sql.planner.streams.rules.StreamsModifyRule; -import org.apache.storm.sql.planner.streams.rules.StreamsProjectRule; -import org.apache.storm.sql.planner.streams.rules.StreamsScanRule; - -public class StreamsStormRuleSets { - private static final ImmutableSet calciteToStormConversionRules = - ImmutableSet.builder().add( - SortRemoveRule.INSTANCE, - - FilterToCalcRule.INSTANCE, - ProjectToCalcRule.INSTANCE, - FilterCalcMergeRule.INSTANCE, - ProjectCalcMergeRule.INSTANCE, - CalcMergeRule.INSTANCE, - - PruneEmptyRules.FILTER_INSTANCE, - PruneEmptyRules.PROJECT_INSTANCE, - PruneEmptyRules.UNION_INSTANCE, - - ProjectFilterTransposeRule.INSTANCE, - FilterProjectTransposeRule.INSTANCE, - ProjectRemoveRule.INSTANCE, - - ReduceExpressionsRule.FILTER_INSTANCE, - ReduceExpressionsRule.PROJECT_INSTANCE, - ReduceExpressionsRule.CALC_INSTANCE, - - // merge and push unions rules - UnionEliminatorRule.INSTANCE, - - StreamsScanRule.INSTANCE, - StreamsFilterRule.INSTANCE, - StreamsProjectRule.INSTANCE, - StreamsAggregateRule.INSTANCE, - StreamsJoinRule.INSTANCE, - StreamsModifyRule.INSTANCE, - StreamsCalcRule.INSTANCE - ).build(); - - public static RuleSet[] getRuleSets() { - return new RuleSet[]{ - new StormRuleSet(StreamRules.RULES), - new StormRuleSet(ImmutableSet.builder().addAll(StreamRules.RULES).addAll(calciteToStormConversionRules).build()) - }; - } - - private static class StormRuleSet implements RuleSet { - final ImmutableSet rules; - - StormRuleSet(ImmutableSet rules) { - this.rules = rules; - } - - StormRuleSet(ImmutableList rules) { - this.rules = ImmutableSet.builder() - .addAll(rules) - .build(); - } - - @Override - public Iterator iterator() { - return rules.iterator(); - } - } - -} diff --git a/sql/storm-sql-core/src/jvm/org/apache/storm/sql/planner/streams/rel/StreamsCalcRel.java b/sql/storm-sql-core/src/jvm/org/apache/storm/sql/planner/streams/rel/StreamsCalcRel.java deleted file mode 100644 index 9cdda935243..00000000000 --- a/sql/storm-sql-core/src/jvm/org/apache/storm/sql/planner/streams/rel/StreamsCalcRel.java +++ /dev/null @@ -1,94 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.apache.storm.sql.planner.streams.rel; - -import com.google.common.collect.Lists; -import java.util.ArrayList; -import java.util.List; -import org.apache.calcite.plan.RelOptCluster; -import org.apache.calcite.plan.RelTraitSet; -import org.apache.calcite.rel.RelNode; -import org.apache.calcite.rel.core.Calc; -import org.apache.calcite.rel.type.RelDataType; -import org.apache.calcite.rex.RexLocalRef; -import org.apache.calcite.rex.RexNode; -import org.apache.calcite.rex.RexProgram; -import org.apache.storm.sql.planner.StormRelUtils; -import org.apache.storm.sql.planner.rel.StormCalcRelBase; -import org.apache.storm.sql.planner.streams.StreamsPlanCreator; -import org.apache.storm.sql.runtime.calcite.ExecutableExpression; -import org.apache.storm.sql.runtime.streams.functions.EvaluationCalc; -import org.apache.storm.streams.Stream; -import org.apache.storm.tuple.Values; - -public class StreamsCalcRel extends StormCalcRelBase implements StreamsRel { - public StreamsCalcRel(RelOptCluster cluster, RelTraitSet traits, RelNode child, RexProgram program) { - super(cluster, traits, child, program); - } - - @Override - public Calc copy(RelTraitSet traitSet, RelNode child, RexProgram program) { - return new StreamsCalcRel(getCluster(), traitSet, child, program); - } - - @Override - public void streamsPlan(StreamsPlanCreator planCreator) throws Exception { - // SingleRel - RelNode input = getInput(); - StormRelUtils.getStormRelInput(input).streamsPlan(planCreator); - - RelDataType inputRowType = getInput(0).getRowType(); - - // filter - ExecutableExpression filterInstance = null; - RexLocalRef condition = program.getCondition(); - if (condition != null) { - RexNode conditionNode = program.expandLocalRef(condition); - filterInstance = planCreator.createScalarInstance(Lists.newArrayList(conditionNode), inputRowType, - StormRelUtils.getClassName(this)); - } - - // projection - ExecutableExpression projectionInstance = null; - List projectList = program.getProjectList(); - if (projectList != null && !projectList.isEmpty()) { - List expandedNodes = new ArrayList<>(); - for (RexLocalRef project : projectList) { - expandedNodes.add(program.expandLocalRef(project)); - } - - projectionInstance = planCreator.createScalarInstance(expandedNodes, inputRowType, - StormRelUtils.getClassName(this)); - } - - if (projectionInstance == null && filterInstance == null) { - // it shouldn't be happen - throw new IllegalStateException("Either projection or condition, or both should be provided."); - } - - List outputFieldNames = getRowType().getFieldNames(); - int outputCount = outputFieldNames.size(); - EvaluationCalc evalCalc = new EvaluationCalc(filterInstance, projectionInstance, outputCount, planCreator.getDataContext()); - final Stream inputStream = planCreator.pop(); - final Stream finalStream = inputStream.flatMap(evalCalc); - - planCreator.addStream(finalStream); - } -} diff --git a/sql/storm-sql-core/src/jvm/org/apache/storm/sql/planner/streams/rel/StreamsFilterRel.java b/sql/storm-sql-core/src/jvm/org/apache/storm/sql/planner/streams/rel/StreamsFilterRel.java deleted file mode 100644 index ce17bf6d3e9..00000000000 --- a/sql/storm-sql-core/src/jvm/org/apache/storm/sql/planner/streams/rel/StreamsFilterRel.java +++ /dev/null @@ -1,66 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.apache.storm.sql.planner.streams.rel; - -import java.util.List; - -import org.apache.calcite.plan.RelOptCluster; -import org.apache.calcite.plan.RelTraitSet; -import org.apache.calcite.rel.RelNode; -import org.apache.calcite.rel.core.Filter; -import org.apache.calcite.rel.type.RelDataType; -import org.apache.calcite.rex.RexNode; -import org.apache.storm.sql.planner.StormRelUtils; -import org.apache.storm.sql.planner.rel.StormFilterRelBase; -import org.apache.storm.sql.planner.streams.StreamsPlanCreator; -import org.apache.storm.sql.runtime.calcite.ExecutableExpression; -import org.apache.storm.sql.runtime.streams.functions.EvaluationFilter; -import org.apache.storm.streams.Stream; -import org.apache.storm.tuple.Values; - -public class StreamsFilterRel extends StormFilterRelBase implements StreamsRel { - public StreamsFilterRel(RelOptCluster cluster, RelTraitSet traits, RelNode child, RexNode condition) { - super(cluster, traits, child, condition); - } - - @Override - public Filter copy(RelTraitSet traitSet, RelNode input, RexNode condition) { - return new StreamsFilterRel(getCluster(), traitSet, input, condition); - } - - @Override - public void streamsPlan(StreamsPlanCreator planCreator) throws Exception { - // SingleRel - RelNode input = getInput(); - StormRelUtils.getStormRelInput(input).streamsPlan(planCreator); - Stream inputStream = planCreator.pop(); - - List childExps = getChildExps(); - RelDataType inputRowType = getInput(0).getRowType(); - - String filterClassName = StormRelUtils.getClassName(this); - ExecutableExpression filterInstance = planCreator.createScalarInstance(childExps, inputRowType, filterClassName); - - EvaluationFilter evalFilter = new EvaluationFilter(filterInstance, planCreator.getDataContext()); - final Stream finalStream = inputStream.filter(evalFilter); - - planCreator.addStream(finalStream); - } -} diff --git a/sql/storm-sql-core/src/jvm/org/apache/storm/sql/planner/streams/rel/StreamsLogicalConvention.java b/sql/storm-sql-core/src/jvm/org/apache/storm/sql/planner/streams/rel/StreamsLogicalConvention.java deleted file mode 100644 index 48a7540260c..00000000000 --- a/sql/storm-sql-core/src/jvm/org/apache/storm/sql/planner/streams/rel/StreamsLogicalConvention.java +++ /dev/null @@ -1,69 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.apache.storm.sql.planner.streams.rel; - -import org.apache.calcite.plan.Convention; -import org.apache.calcite.plan.ConventionTraitDef; -import org.apache.calcite.plan.RelOptPlanner; -import org.apache.calcite.plan.RelTrait; -import org.apache.calcite.plan.RelTraitDef; -import org.apache.calcite.plan.RelTraitSet; - -public enum StreamsLogicalConvention implements Convention { - INSTANCE; - - @Override - public Class getInterface() { - return StreamsRel.class; - } - - @Override - public String getName() { - return "STORM_LOGICAL"; - } - - @Override - public RelTraitDef getTraitDef() { - return ConventionTraitDef.INSTANCE; - } - - @Override - public boolean satisfies(RelTrait trait) { - return this == trait; - } - - @Override - public void register(RelOptPlanner planner) {} - - @Override - public String toString() { - return getName(); - } - - @Override - public boolean canConvertConvention(Convention toConvention) { - return false; - } - - @Override - public boolean useAbstractConvertersForConversion(RelTraitSet fromTraits, RelTraitSet toTraits) { - return false; - } -} diff --git a/sql/storm-sql-core/src/jvm/org/apache/storm/sql/planner/streams/rel/StreamsProjectRel.java b/sql/storm-sql-core/src/jvm/org/apache/storm/sql/planner/streams/rel/StreamsProjectRel.java deleted file mode 100644 index 4ad70e15687..00000000000 --- a/sql/storm-sql-core/src/jvm/org/apache/storm/sql/planner/streams/rel/StreamsProjectRel.java +++ /dev/null @@ -1,69 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.apache.storm.sql.planner.streams.rel; - -import java.util.List; - -import org.apache.calcite.plan.RelOptCluster; -import org.apache.calcite.plan.RelTraitSet; -import org.apache.calcite.rel.RelNode; -import org.apache.calcite.rel.core.Project; -import org.apache.calcite.rel.type.RelDataType; -import org.apache.calcite.rex.RexNode; -import org.apache.storm.sql.planner.StormRelUtils; -import org.apache.storm.sql.planner.rel.StormProjectRelBase; -import org.apache.storm.sql.planner.streams.StreamsPlanCreator; -import org.apache.storm.sql.runtime.calcite.ExecutableExpression; -import org.apache.storm.sql.runtime.streams.functions.EvaluationFunction; -import org.apache.storm.streams.Stream; -import org.apache.storm.tuple.Values; - -public class StreamsProjectRel extends StormProjectRelBase implements StreamsRel { - public StreamsProjectRel(RelOptCluster cluster, RelTraitSet traits, RelNode input, List projects, - RelDataType rowType) { - super(cluster, traits, input, projects, rowType); - } - - @Override - public Project copy(RelTraitSet traitSet, RelNode input, List projects, RelDataType rowType) { - return new StreamsProjectRel(getCluster(), traitSet, input, projects, rowType); - } - - @Override - public void streamsPlan(StreamsPlanCreator planCreator) throws Exception { - // SingleRel - RelNode input = getInput(); - StormRelUtils.getStormRelInput(input).streamsPlan(planCreator); - Stream inputStream = planCreator.pop(); - - String projectionClassName = StormRelUtils.getClassName(this); - - List outputFieldNames = getRowType().getFieldNames(); - int outputCount = outputFieldNames.size(); - - List childExps = getChildExps(); - RelDataType inputRowType = getInput(0).getRowType(); - - ExecutableExpression projectionInstance = planCreator.createScalarInstance(childExps, inputRowType, projectionClassName); - EvaluationFunction evalFunc = new EvaluationFunction(projectionInstance, outputCount, planCreator.getDataContext()); - final Stream finalStream = inputStream.map(evalFunc); - planCreator.addStream(finalStream); - } -} diff --git a/sql/storm-sql-core/src/jvm/org/apache/storm/sql/planner/streams/rel/StreamsRel.java b/sql/storm-sql-core/src/jvm/org/apache/storm/sql/planner/streams/rel/StreamsRel.java deleted file mode 100644 index 70d02ec43b8..00000000000 --- a/sql/storm-sql-core/src/jvm/org/apache/storm/sql/planner/streams/rel/StreamsRel.java +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.apache.storm.sql.planner.streams.rel; - -import org.apache.storm.sql.planner.rel.StormRelNode; -import org.apache.storm.sql.planner.streams.StreamsPlanCreator; - -public interface StreamsRel extends StormRelNode { - void streamsPlan(StreamsPlanCreator planCreator) throws Exception; -} diff --git a/sql/storm-sql-core/src/jvm/org/apache/storm/sql/planner/streams/rel/StreamsStreamInsertRel.java b/sql/storm-sql-core/src/jvm/org/apache/storm/sql/planner/streams/rel/StreamsStreamInsertRel.java deleted file mode 100644 index 50c488df0ba..00000000000 --- a/sql/storm-sql-core/src/jvm/org/apache/storm/sql/planner/streams/rel/StreamsStreamInsertRel.java +++ /dev/null @@ -1,82 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.apache.storm.sql.planner.streams.rel; - -import com.google.common.base.Joiner; -import com.google.common.base.Preconditions; - -import java.util.List; - -import org.apache.calcite.plan.RelOptCluster; -import org.apache.calcite.plan.RelOptTable; -import org.apache.calcite.plan.RelTraitSet; -import org.apache.calcite.prepare.Prepare; -import org.apache.calcite.rel.RelNode; -import org.apache.calcite.rex.RexNode; -import org.apache.storm.sql.planner.StormRelUtils; -import org.apache.storm.sql.planner.rel.StormStreamInsertRelBase; -import org.apache.storm.sql.planner.streams.StreamsPlanCreator; -import org.apache.storm.sql.runtime.streams.functions.StreamInsertMapToPairFunction; -import org.apache.storm.streams.Stream; -import org.apache.storm.topology.IRichBolt; -import org.apache.storm.tuple.Values; - -public class StreamsStreamInsertRel extends StormStreamInsertRelBase implements StreamsRel { - private final int primaryKeyIndex; - - public StreamsStreamInsertRel(RelOptCluster cluster, RelTraitSet traits, RelOptTable table, Prepare.CatalogReader catalogReader, - RelNode child, Operation operation, List updateColumnList, List sourceExpressionList, - boolean flattened, int primaryKeyIndex) { - super(cluster, traits, table, catalogReader, child, operation, updateColumnList, sourceExpressionList, flattened); - this.primaryKeyIndex = primaryKeyIndex; - } - - @Override - public RelNode copy(RelTraitSet traitSet, List inputs) { - return new StreamsStreamInsertRel(getCluster(), traitSet, getTable(), getCatalogReader(), - sole(inputs), getOperation(), getUpdateColumnList(), getSourceExpressionList(), isFlattened(), primaryKeyIndex); - } - - @Override - public void streamsPlan(StreamsPlanCreator planCreator) throws Exception { - // SingleRel - RelNode input = getInput(); - StormRelUtils.getStormRelInput(input).streamsPlan(planCreator); - Stream inputStream = planCreator.pop(); - - Preconditions.checkArgument(isInsert(), "Only INSERT statement is supported."); - - // Calcite ensures that the value is structurized to the table definition - // hence we can use PK index directly - // To elaborate, if table BAR is defined as ID INTEGER PK, NAME VARCHAR, DEPTID INTEGER - // and query like INSERT INTO BAR SELECT NAME, ID FROM FOO is executed, - // Calcite makes the projection ($1 <- ID, $0 <- NAME, null) to the value before INSERT. - - String tableName = Joiner.on('.').join(getTable().getQualifiedName()); - IRichBolt consumer = planCreator.getSources().get(tableName).getConsumer(); - - // To make logic simple, it assumes that all the tables have one PK (which it should be extended to support composed key), - // and provides PairStream(KeyedStream) to consumer bolt. - inputStream.mapToPair(new StreamInsertMapToPairFunction(primaryKeyIndex)).to(consumer); - - planCreator.addStream(inputStream); - } - -} diff --git a/sql/storm-sql-core/src/jvm/org/apache/storm/sql/planner/streams/rel/StreamsStreamScanRel.java b/sql/storm-sql-core/src/jvm/org/apache/storm/sql/planner/streams/rel/StreamsStreamScanRel.java deleted file mode 100644 index 6ecf9b844d4..00000000000 --- a/sql/storm-sql-core/src/jvm/org/apache/storm/sql/planner/streams/rel/StreamsStreamScanRel.java +++ /dev/null @@ -1,57 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.apache.storm.sql.planner.streams.rel; - -import com.google.common.base.Joiner; -import java.util.List; -import java.util.Map; -import org.apache.calcite.plan.RelOptCluster; -import org.apache.calcite.plan.RelOptTable; -import org.apache.calcite.plan.RelTraitSet; -import org.apache.storm.sql.planner.rel.StormStreamScanRelBase; -import org.apache.storm.sql.planner.streams.StreamsPlanCreator; -import org.apache.storm.sql.runtime.ISqlStreamsDataSource; -import org.apache.storm.sql.runtime.streams.functions.StreamsScanTupleValueMapper; -import org.apache.storm.streams.Stream; -import org.apache.storm.tuple.Values; - -public class StreamsStreamScanRel extends StormStreamScanRelBase implements StreamsRel { - private final int parallelismHint; - - public StreamsStreamScanRel(RelOptCluster cluster, RelTraitSet traitSet, RelOptTable table, int parallelismHint) { - super(cluster, traitSet, table); - this.parallelismHint = parallelismHint; - } - - @Override - public void streamsPlan(StreamsPlanCreator planCreator) throws Exception { - String sourceName = Joiner.on('.').join(getTable().getQualifiedName()); - - Map sources = planCreator.getSources(); - if (!sources.containsKey(sourceName)) { - throw new RuntimeException("Cannot find table " + sourceName); - } - - List fieldNames = getRowType().getFieldNames(); - final Stream finalStream = planCreator.getStreamBuilder() - .newStream(sources.get(sourceName).getProducer(), new StreamsScanTupleValueMapper(fieldNames), parallelismHint); - planCreator.addStream(finalStream); - } -} diff --git a/sql/storm-sql-core/src/jvm/org/apache/storm/sql/planner/streams/rules/StreamsAggregateRule.java b/sql/storm-sql-core/src/jvm/org/apache/storm/sql/planner/streams/rules/StreamsAggregateRule.java deleted file mode 100644 index 3d86f26ec11..00000000000 --- a/sql/storm-sql-core/src/jvm/org/apache/storm/sql/planner/streams/rules/StreamsAggregateRule.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.apache.storm.sql.planner.streams.rules; - -import org.apache.calcite.plan.Convention; -import org.apache.calcite.plan.RelOptRule; -import org.apache.calcite.rel.RelNode; -import org.apache.calcite.rel.convert.ConverterRule; -import org.apache.calcite.rel.logical.LogicalAggregate; -import org.apache.storm.sql.planner.streams.rel.StreamsLogicalConvention; - -public class StreamsAggregateRule extends ConverterRule { - public static final RelOptRule INSTANCE = new StreamsAggregateRule(); - - private StreamsAggregateRule() { - super(LogicalAggregate.class, Convention.NONE, StreamsLogicalConvention.INSTANCE, "StreamsAggregateRule"); - } - - @Override - public RelNode convert(RelNode rel) { - throw new UnsupportedOperationException("Aggregate operation is not supported."); - } -} diff --git a/sql/storm-sql-core/src/jvm/org/apache/storm/sql/planner/streams/rules/StreamsCalcRule.java b/sql/storm-sql-core/src/jvm/org/apache/storm/sql/planner/streams/rules/StreamsCalcRule.java deleted file mode 100644 index 328d4f58de0..00000000000 --- a/sql/storm-sql-core/src/jvm/org/apache/storm/sql/planner/streams/rules/StreamsCalcRule.java +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.apache.storm.sql.planner.streams.rules; - -import org.apache.calcite.plan.Convention; -import org.apache.calcite.rel.RelNode; -import org.apache.calcite.rel.convert.ConverterRule; -import org.apache.calcite.rel.core.Calc; -import org.apache.calcite.rel.logical.LogicalCalc; -import org.apache.storm.sql.planner.streams.rel.StreamsCalcRel; -import org.apache.storm.sql.planner.streams.rel.StreamsLogicalConvention; - -public class StreamsCalcRule extends ConverterRule { - public static final StreamsCalcRule INSTANCE = new StreamsCalcRule(); - - private StreamsCalcRule() { - super(LogicalCalc.class, Convention.NONE, StreamsLogicalConvention.INSTANCE, "StreamsCalcRule"); - } - - @Override - public RelNode convert(RelNode rel) { - final Calc calc = (Calc) rel; - final RelNode input = calc.getInput(); - - return new StreamsCalcRel(calc.getCluster(), calc.getTraitSet().replace(StreamsLogicalConvention.INSTANCE), - convert(input, input.getTraitSet().replace(StreamsLogicalConvention.INSTANCE)), - calc.getProgram()); - } -} diff --git a/sql/storm-sql-core/src/jvm/org/apache/storm/sql/planner/streams/rules/StreamsFilterRule.java b/sql/storm-sql-core/src/jvm/org/apache/storm/sql/planner/streams/rules/StreamsFilterRule.java deleted file mode 100644 index 1e5ab199007..00000000000 --- a/sql/storm-sql-core/src/jvm/org/apache/storm/sql/planner/streams/rules/StreamsFilterRule.java +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.apache.storm.sql.planner.streams.rules; - -import org.apache.calcite.plan.Convention; -import org.apache.calcite.rel.RelNode; -import org.apache.calcite.rel.convert.ConverterRule; -import org.apache.calcite.rel.core.Filter; -import org.apache.calcite.rel.logical.LogicalFilter; -import org.apache.storm.sql.planner.streams.rel.StreamsFilterRel; -import org.apache.storm.sql.planner.streams.rel.StreamsLogicalConvention; - -public class StreamsFilterRule extends ConverterRule { - public static StreamsFilterRule INSTANCE = new StreamsFilterRule(); - - private StreamsFilterRule() { - super(LogicalFilter.class, Convention.NONE, StreamsLogicalConvention.INSTANCE, "StreamsFilterRule"); - } - - @Override - public RelNode convert(RelNode rel) { - final Filter filter = (Filter) rel; - final RelNode input = filter.getInput(); - - return new StreamsFilterRel(filter.getCluster(), - filter.getTraitSet().replace(StreamsLogicalConvention.INSTANCE), - convert(input, input.getTraitSet().replace(StreamsLogicalConvention.INSTANCE)), - filter.getCondition()); - } -} diff --git a/sql/storm-sql-core/src/jvm/org/apache/storm/sql/planner/streams/rules/StreamsJoinRule.java b/sql/storm-sql-core/src/jvm/org/apache/storm/sql/planner/streams/rules/StreamsJoinRule.java deleted file mode 100644 index e3d5a7ddc0a..00000000000 --- a/sql/storm-sql-core/src/jvm/org/apache/storm/sql/planner/streams/rules/StreamsJoinRule.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.apache.storm.sql.planner.streams.rules; - -import org.apache.calcite.plan.Convention; -import org.apache.calcite.rel.RelNode; -import org.apache.calcite.rel.convert.ConverterRule; -import org.apache.calcite.rel.logical.LogicalJoin; -import org.apache.storm.sql.planner.streams.rel.StreamsLogicalConvention; - -public class StreamsJoinRule extends ConverterRule { - public static final StreamsJoinRule INSTANCE = new StreamsJoinRule(); - - private StreamsJoinRule() { - super(LogicalJoin.class, Convention.NONE, StreamsLogicalConvention.INSTANCE, "StreamsJoinRule"); - } - - @Override - public RelNode convert(RelNode rel) { - throw new UnsupportedOperationException("Join operation is not supported."); - } -} diff --git a/sql/storm-sql-core/src/jvm/org/apache/storm/sql/planner/streams/rules/StreamsModifyRule.java b/sql/storm-sql-core/src/jvm/org/apache/storm/sql/planner/streams/rules/StreamsModifyRule.java deleted file mode 100644 index f4776072772..00000000000 --- a/sql/storm-sql-core/src/jvm/org/apache/storm/sql/planner/streams/rules/StreamsModifyRule.java +++ /dev/null @@ -1,89 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.apache.storm.sql.planner.streams.rules; - -import java.util.List; - -import org.apache.calcite.plan.Convention; -import org.apache.calcite.plan.RelOptCluster; -import org.apache.calcite.plan.RelOptTable; -import org.apache.calcite.plan.RelTraitSet; -import org.apache.calcite.prepare.Prepare; -import org.apache.calcite.rel.RelNode; -import org.apache.calcite.rel.convert.ConverterRule; -import org.apache.calcite.rel.core.TableModify; -import org.apache.calcite.rel.logical.LogicalTableModify; -import org.apache.calcite.rex.RexNode; -import org.apache.calcite.schema.Table; -import org.apache.storm.sql.calcite.StormStreamableTable; -import org.apache.storm.sql.calcite.StormTable; -import org.apache.storm.sql.planner.streams.rel.StreamsLogicalConvention; -import org.apache.storm.sql.planner.streams.rel.StreamsStreamInsertRel; - -public class StreamsModifyRule extends ConverterRule { - public static final StreamsModifyRule INSTANCE = new StreamsModifyRule(); - - private StreamsModifyRule() { - super(LogicalTableModify.class, Convention.NONE, StreamsLogicalConvention.INSTANCE, "StreamsModifyRule"); - } - - @Override - public RelNode convert(RelNode rel) { - final TableModify tableModify = (TableModify) rel; - final RelNode input = tableModify.getInput(); - - final RelOptCluster cluster = tableModify.getCluster(); - final RelTraitSet traitSet = tableModify.getTraitSet().replace(StreamsLogicalConvention.INSTANCE); - final RelOptTable relOptTable = tableModify.getTable(); - final Prepare.CatalogReader catalogReader = tableModify.getCatalogReader(); - final RelNode convertedInput = convert(input, input.getTraitSet().replace(StreamsLogicalConvention.INSTANCE)); - final TableModify.Operation operation = tableModify.getOperation(); - final List updateColumnList = tableModify.getUpdateColumnList(); - final List sourceExpressionList = tableModify.getSourceExpressionList(); - final boolean flattened = tableModify.isFlattened(); - - int primaryKey; - - StormTable stormTable = tableModify.getTable().unwrap(StormTable.class); - if (stormTable != null) { - primaryKey = stormTable.primaryKey(); - } else { - StormStreamableTable streamableTable = tableModify.getTable().unwrap(StormStreamableTable.class); - if (streamableTable != null) { - primaryKey = streamableTable.primaryKey(); - } else { - throw new IllegalStateException("Table must be able to unwrap with StormTable or StormStreamableTable."); - } - } - - final Table table = tableModify.getTable().unwrap(Table.class); - - switch (table.getJdbcTableType()) { - case STREAM: - if (operation != TableModify.Operation.INSERT) { - throw new UnsupportedOperationException(String.format("Stream doesn't support %s modify operation", operation)); - } - return new StreamsStreamInsertRel(cluster, traitSet, relOptTable, catalogReader, convertedInput, operation, - updateColumnList, sourceExpressionList, flattened, primaryKey); - default: - throw new IllegalArgumentException(String.format("Unsupported table type: %s", table.getJdbcTableType())); - } - } -} diff --git a/sql/storm-sql-core/src/jvm/org/apache/storm/sql/planner/streams/rules/StreamsProjectRule.java b/sql/storm-sql-core/src/jvm/org/apache/storm/sql/planner/streams/rules/StreamsProjectRule.java deleted file mode 100644 index 7508046c6fe..00000000000 --- a/sql/storm-sql-core/src/jvm/org/apache/storm/sql/planner/streams/rules/StreamsProjectRule.java +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.apache.storm.sql.planner.streams.rules; - -import org.apache.calcite.plan.Convention; -import org.apache.calcite.rel.RelNode; -import org.apache.calcite.rel.convert.ConverterRule; -import org.apache.calcite.rel.core.Project; -import org.apache.calcite.rel.logical.LogicalProject; -import org.apache.storm.sql.planner.streams.rel.StreamsLogicalConvention; -import org.apache.storm.sql.planner.streams.rel.StreamsProjectRel; - -public class StreamsProjectRule extends ConverterRule { - public static final StreamsProjectRule INSTANCE = new StreamsProjectRule(); - - private StreamsProjectRule() { - super(LogicalProject.class, Convention.NONE, StreamsLogicalConvention.INSTANCE, - "StreamsProjectRule"); - } - - @Override - public RelNode convert(RelNode rel) { - final Project project = (Project) rel; - final RelNode input = project.getInput(); - - return new StreamsProjectRel(project.getCluster(), - project.getTraitSet().replace(StreamsLogicalConvention.INSTANCE), - convert(input, input.getTraitSet().replace(StreamsLogicalConvention.INSTANCE)), project.getProjects(), - project.getRowType()); - } -} diff --git a/sql/storm-sql-core/src/jvm/org/apache/storm/sql/planner/streams/rules/StreamsScanRule.java b/sql/storm-sql-core/src/jvm/org/apache/storm/sql/planner/streams/rules/StreamsScanRule.java deleted file mode 100644 index 536fcfe2ecd..00000000000 --- a/sql/storm-sql-core/src/jvm/org/apache/storm/sql/planner/streams/rules/StreamsScanRule.java +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.apache.storm.sql.planner.streams.rules; - -import org.apache.calcite.adapter.enumerable.EnumerableConvention; -import org.apache.calcite.adapter.enumerable.EnumerableTableScan; -import org.apache.calcite.rel.RelNode; -import org.apache.calcite.rel.convert.ConverterRule; -import org.apache.calcite.rel.core.TableScan; -import org.apache.calcite.schema.Table; -import org.apache.storm.sql.calcite.ParallelStreamableTable; -import org.apache.storm.sql.calcite.ParallelTable; -import org.apache.storm.sql.planner.streams.rel.StreamsLogicalConvention; -import org.apache.storm.sql.planner.streams.rel.StreamsStreamScanRel; - -public class StreamsScanRule extends ConverterRule { - public static final StreamsScanRule INSTANCE = new StreamsScanRule(); - public static final int DEFAULT_PARALLELISM_HINT = 1; - - private StreamsScanRule() { - super(EnumerableTableScan.class, EnumerableConvention.INSTANCE, StreamsLogicalConvention.INSTANCE, "StreamsScanRule"); - } - - @Override - public RelNode convert(RelNode rel) { - final TableScan scan = (TableScan) rel; - int parallelismHint = DEFAULT_PARALLELISM_HINT; - - final ParallelTable parallelTable = scan.getTable().unwrap(ParallelTable.class); - if (parallelTable != null && parallelTable.parallelismHint() != null) { - parallelismHint = parallelTable.parallelismHint(); - } - - final Table table = scan.getTable().unwrap(Table.class); - switch (table.getJdbcTableType()) { - case STREAM: - return new StreamsStreamScanRel(scan.getCluster(), - scan.getTraitSet().replace(StreamsLogicalConvention.INSTANCE), - scan.getTable(), parallelismHint); - default: - throw new IllegalArgumentException(String.format("Unsupported table type: %s", table.getJdbcTableType())); - } - } -} diff --git a/sql/storm-sql-core/src/test/org/apache/storm/sql/SqlTestUtil.java b/sql/storm-sql-core/src/test/org/apache/storm/sql/SqlTestUtil.java deleted file mode 100644 index 7a6d97936a3..00000000000 --- a/sql/storm-sql-core/src/test/org/apache/storm/sql/SqlTestUtil.java +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.apache.storm.sql; - -import java.util.List; -import java.util.concurrent.Callable; -import org.apache.storm.Config; -import org.apache.storm.LocalCluster; -import org.apache.storm.generated.StormTopology; -import org.apache.storm.sql.javac.CompilingClassLoader; -import org.apache.storm.utils.Utils; - -public final class SqlTestUtil { - - public static void runStormTopology(LocalCluster cluster, final List watchedList, final int expectedValueSize, - AbstractStreamsProcessor proc, StormTopology topo) throws Exception { - final Config conf = new Config(); - conf.setMaxSpoutPending(20); - conf.setDebug(true); - - if (proc.getClassLoaders() != null && proc.getClassLoaders().size() > 0) { - CompilingClassLoader lastClassloader = proc.getClassLoaders().get(proc.getClassLoaders().size() - 1); - Utils.setClassLoaderForJavaDeSerialize(lastClassloader); - } - - try (LocalCluster.LocalTopology stormTopo = cluster.submitTopology("storm-sql", conf, topo)) { - waitForCompletion(1000 * 1000, () -> watchedList.size() < expectedValueSize); - } finally { - while (cluster.getTopologySummaries().size() > 0) { - Thread.sleep(10); - } - Utils.resetClassLoaderForJavaDeSerialize(); - } - } - - private static void waitForCompletion(long timeout, Callable cond) throws Exception { - long start = TestUtils.monotonicNow(); - while (TestUtils.monotonicNow() - start < timeout && cond.call()) { - Thread.sleep(100); - } - } -} diff --git a/sql/storm-sql-core/src/test/org/apache/storm/sql/StormSqlLocalClusterImpl.java b/sql/storm-sql-core/src/test/org/apache/storm/sql/StormSqlLocalClusterImpl.java deleted file mode 100644 index f654e63eb19..00000000000 --- a/sql/storm-sql-core/src/test/org/apache/storm/sql/StormSqlLocalClusterImpl.java +++ /dev/null @@ -1,79 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.apache.storm.sql; - -import java.util.function.Predicate; -import org.apache.calcite.sql.SqlNode; -import org.apache.storm.Config; -import org.apache.storm.LocalCluster; -import org.apache.storm.generated.StormTopology; -import org.apache.storm.sql.javac.CompilingClassLoader; -import org.apache.storm.sql.parser.SqlCreateFunction; -import org.apache.storm.sql.parser.SqlCreateTable; -import org.apache.storm.sql.parser.StormParser; -import org.apache.storm.utils.Utils; - -public class StormSqlLocalClusterImpl { - private final StormSqlContext sqlContext; - - public StormSqlLocalClusterImpl() { - sqlContext = new StormSqlContext(); - } - - private static void waitForCompletion(long timeout, Predicate cond) throws Exception { - long start = TestUtils.monotonicNow(); - while (TestUtils.monotonicNow() - start < timeout && !cond.test(null)) { - Thread.sleep(100); - } - } - - public void runLocal(LocalCluster localCluster, Iterable statements, - Predicate waitCondition, long waitTimeoutMs) throws Exception { - final Config conf = new Config(); - conf.setMaxSpoutPending(20); - - for (String sql : statements) { - StormParser parser = new StormParser(sql); - SqlNode node = parser.impl().parseSqlStmtEof(); - if (node instanceof SqlCreateTable) { - sqlContext.interpretCreateTable((SqlCreateTable) node); - } else if (node instanceof SqlCreateFunction) { - sqlContext.interpretCreateFunction((SqlCreateFunction) node); - } else { - AbstractStreamsProcessor processor = sqlContext.compileSql(sql); - StormTopology topo = processor.build(); - - if (processor.getClassLoaders() != null && processor.getClassLoaders().size() > 0) { - CompilingClassLoader lastClassloader = processor.getClassLoaders().get(processor.getClassLoaders().size() - 1); - Utils.setClassLoaderForJavaDeSerialize(lastClassloader); - } - - try (LocalCluster.LocalTopology stormTopo = localCluster.submitTopology("storm-sql", conf, topo)) { - waitForCompletion(waitTimeoutMs, waitCondition); - } finally { - while (localCluster.getTopologySummaries().size() > 0) { - Thread.sleep(10); - } - Utils.resetClassLoaderForJavaDeSerialize(); - } - } - } - } -} diff --git a/sql/storm-sql-core/src/test/org/apache/storm/sql/TestStormSql.java b/sql/storm-sql-core/src/test/org/apache/storm/sql/TestStormSql.java deleted file mode 100644 index 511a7aaa851..00000000000 --- a/sql/storm-sql-core/src/test/org/apache/storm/sql/TestStormSql.java +++ /dev/null @@ -1,297 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions - * and limitations under the License. - */ - -package org.apache.storm.sql; - -import java.net.URI; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.Map; -import java.util.Properties; - -import com.google.common.collect.ImmutableMap; -import org.apache.calcite.tools.ValidationException; -import org.apache.storm.LocalCluster; -import org.apache.storm.sql.javac.CompilingClassLoader; -import org.apache.storm.sql.runtime.DataSourcesProvider; -import org.apache.storm.sql.runtime.DataSourcesRegistry; -import org.apache.storm.sql.runtime.FieldInfo; -import org.apache.storm.sql.runtime.ISqlStreamsDataSource; -import org.apache.storm.streams.Pair; -import org.apache.storm.tuple.Values; -import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertThrows; - -@ExtendWith(TestUtils.MockBoltExtension.class) -@ExtendWith(TestUtils.MockInsertBoltExtension.class) -public class TestStormSql { - - public static final int WAIT_TIMEOUT_MS = 1000 * 1000; - public static final int WAIT_TIMEOUT_MS_NO_RECORDS_EXPECTED = 1000 * 10; - public static final int WAIT_TIMEOUT_MS_ERROR_EXPECTED = 1000; - - private static LocalCluster cluster; - - @BeforeAll - public static void staticSetup() throws Exception { - DataSourcesRegistry.providerMap().put("mock", new MockDataSourceProvider()); - DataSourcesRegistry.providerMap().put("mocknested", new MockNestedDataSourceProvider()); - DataSourcesRegistry.providerMap().put("mockgroup", new MockGroupDataSourceProvider()); - DataSourcesRegistry.providerMap().put("mockemp", new MockEmpDataSourceProvider()); - DataSourcesRegistry.providerMap().put("mockdept", new MockDeptDataSourceProvider()); - - cluster = new LocalCluster(); - } - - @AfterAll - public static void staticCleanup() { - DataSourcesRegistry.providerMap().remove("mock"); - DataSourcesRegistry.providerMap().remove("mocknested"); - DataSourcesRegistry.providerMap().remove("mockgroup"); - DataSourcesRegistry.providerMap().remove("mockemp"); - DataSourcesRegistry.providerMap().remove("mockdept"); - - if (cluster != null) { - cluster.shutdown(); - cluster = null; - } - } - - @Test - public void testExternalDataSource() throws Exception { - List stmt = new ArrayList<>(); - stmt.add("CREATE EXTERNAL TABLE FOO (ID INT PRIMARY KEY) LOCATION 'mock:///foo'"); - stmt.add("CREATE EXTERNAL TABLE BAR (ID INT PRIMARY KEY) LOCATION 'mock:///foo'"); - stmt.add("INSERT INTO BAR SELECT STREAM ID + 1 FROM FOO WHERE ID > 2"); - StormSqlLocalClusterImpl impl = new StormSqlLocalClusterImpl(); - - List> values = TestUtils.MockInsertBolt.getCollectedValues(); - impl.runLocal(cluster, stmt, (__) -> values.size() >= 2, WAIT_TIMEOUT_MS); - - assertEquals(2, values.size()); - assertEquals(4, values.get(0).getFirst()); - assertEquals(5, values.get(1).getFirst()); - } - - @Test - public void testExternalDataSourceNested() throws Exception { - List stmt = new ArrayList<>(); - stmt.add("CREATE EXTERNAL TABLE FOO (ID INT PRIMARY KEY, MAPFIELD ANY, NESTEDMAPFIELD ANY, ARRAYFIELD ANY) LOCATION 'mocknested:///foo'"); - stmt.add("CREATE EXTERNAL TABLE BAR (ID INT PRIMARY KEY, MAPFIELD ANY, NESTEDMAPFIELD ANY, ARRAYFIELD ANY) LOCATION 'mocknested:///foo'"); - stmt.add("INSERT INTO BAR SELECT STREAM ID, MAPFIELD['c'], NESTEDMAPFIELD, ARRAYFIELD " + - "FROM FOO " + - "WHERE CAST(MAPFIELD['b'] AS INTEGER) = 2 AND CAST(ARRAYFIELD[2] AS INTEGER) = 200"); - StormSqlLocalClusterImpl impl = new StormSqlLocalClusterImpl(); - - List> values = TestUtils.MockInsertBolt.getCollectedValues();; - impl.runLocal(cluster, stmt, (__) -> values.size() >= 1, WAIT_TIMEOUT_MS); - - Map map = ImmutableMap.of("b", 2, "c", 4); - Map> nestedMap = ImmutableMap.of("a", map); - assertEquals(2, values.get(0).getFirst()); - assertEquals(new Values(2, 4, nestedMap, Arrays.asList(100, 200, 300)), values.get(0).getSecond()); - } - - @Test - public void testExternalNestedNonExistKeyAccess() throws Exception { - List stmt = new ArrayList<>(); - // this triggers java.lang.RuntimeException: Cannot convert null to int - stmt.add("CREATE EXTERNAL TABLE FOO (ID INT, MAPFIELD ANY, NESTEDMAPFIELD ANY, ARRAYFIELD ANY) LOCATION 'mocknested:///foo'"); - stmt.add("CREATE EXTERNAL TABLE BAR (ID INT, MAPFIELD ANY, NESTEDMAPFIELD ANY, ARRAYFIELD ANY) LOCATION 'mocknested:///foo'"); - stmt.add("INSERT INTO BAR SELECT STREAM ID, MAPFIELD, NESTEDMAPFIELD, ARRAYFIELD " + - "FROM FOO " + - "WHERE CAST(MAPFIELD['a'] AS INTEGER) = 2"); - StormSqlLocalClusterImpl impl = new StormSqlLocalClusterImpl(); - - List> values = TestUtils.MockInsertBolt.getCollectedValues(); - impl.runLocal(cluster, stmt, (__) -> true, WAIT_TIMEOUT_MS_NO_RECORDS_EXPECTED); - - assertEquals(0, values.size()); - } - - @Test - public void testExternalNestedNonExistKeyAccess2() throws Exception { - List stmt = new ArrayList<>(); - // this triggers java.lang.RuntimeException: Cannot convert null to int - stmt.add("CREATE EXTERNAL TABLE FOO (ID INT, MAPFIELD ANY, NESTEDMAPFIELD ANY, ARRAYFIELD ANY) LOCATION 'mocknested:///foo'"); - stmt.add("CREATE EXTERNAL TABLE BAR (ID INT, MAPFIELD ANY, NESTEDMAPFIELD ANY, ARRAYFIELD ANY) LOCATION 'mocknested:///foo'"); - stmt.add("INSERT INTO BAR SELECT STREAM ID, MAPFIELD, NESTEDMAPFIELD, ARRAYFIELD " + - "FROM FOO " + - "WHERE CAST(NESTEDMAPFIELD['b']['c'] AS INTEGER) = 4"); - StormSqlLocalClusterImpl impl = new StormSqlLocalClusterImpl(); - - List> values = TestUtils.MockInsertBolt.getCollectedValues(); - impl.runLocal(cluster, stmt, (__) -> true, WAIT_TIMEOUT_MS_NO_RECORDS_EXPECTED); - - assertEquals(0, values.size()); - } - - @Test - public void testExternalNestedInvalidAccessStringIndexOnArray() throws Exception { - List stmt = new ArrayList<>(); - stmt.add("CREATE EXTERNAL TABLE FOO (ID INT, MAPFIELD ANY, NESTEDMAPFIELD ANY, ARRAYFIELD ANY) LOCATION 'mocknested:///foo'"); - stmt.add("CREATE EXTERNAL TABLE BAR (ID INT, MAPFIELD ANY, NESTEDMAPFIELD ANY, ARRAYFIELD ANY) LOCATION 'mocknested:///foo'"); - stmt.add("INSERT INTO BAR SELECT STREAM ID, MAPFIELD, NESTEDMAPFIELD, ARRAYFIELD " + - "FROM FOO " + - "WHERE CAST(ARRAYFIELD['a'] AS INTEGER) = 200"); - StormSqlLocalClusterImpl impl = new StormSqlLocalClusterImpl(); - - List> values = TestUtils.MockInsertBolt.getCollectedValues(); - impl.runLocal(cluster, stmt, (__) -> true, WAIT_TIMEOUT_MS_NO_RECORDS_EXPECTED); - - assertEquals(0, values.size()); - } - - @Test - public void testExternalNestedArrayOutOfBoundAccess() throws Exception { - List stmt = new ArrayList<>(); - stmt.add("CREATE EXTERNAL TABLE FOO (ID INT, MAPFIELD ANY, NESTEDMAPFIELD ANY, ARRAYFIELD ANY) LOCATION 'mocknested:///foo'"); - stmt.add("CREATE EXTERNAL TABLE BAR (ID INT, MAPFIELD ANY, NESTEDMAPFIELD ANY, ARRAYFIELD ANY) LOCATION 'mocknested:///foo'"); - stmt.add("INSERT INTO BAR SELECT STREAM ID, MAPFIELD, NESTEDMAPFIELD, ARRAYFIELD " + - "FROM FOO " + - "WHERE CAST(ARRAYFIELD[10] AS INTEGER) = 200"); - StormSqlLocalClusterImpl impl = new StormSqlLocalClusterImpl(); - - List> values = TestUtils.MockInsertBolt.getCollectedValues(); - impl.runLocal(cluster, stmt, (__) -> true, WAIT_TIMEOUT_MS_NO_RECORDS_EXPECTED); - - assertEquals(0, values.size()); - } - - @Test - public void testExternalUdfType() throws Exception { - List stmt = new ArrayList<>(); - stmt.add("CREATE EXTERNAL TABLE FOO (ID INT, NAME VARCHAR) LOCATION 'mock:///foo'"); - stmt.add("CREATE EXTERNAL TABLE BAR (ID INT) LOCATION 'mock:///foo'"); - stmt.add("CREATE FUNCTION MYPLUS AS 'org.apache.storm.sql.TestUtils$MyPlus'"); - stmt.add("INSERT INTO BAR SELECT STREAM MYPLUS(NAME, 1) FROM FOO WHERE ID = 0"); - StormSqlLocalClusterImpl impl = new StormSqlLocalClusterImpl(); - - assertThrows(ValidationException.class, - () -> impl.runLocal(cluster, stmt, (__) -> true, WAIT_TIMEOUT_MS_ERROR_EXPECTED)); - } - - @Test - public void testExternalUdfType2() throws Exception { - List stmt = new ArrayList<>(); - // generated code will be not compilable since return type of MYPLUS and type of 'x' are different - stmt.add("CREATE EXTERNAL TABLE FOO (ID INT, NAME VARCHAR) LOCATION 'mock:///foo'"); - stmt.add("CREATE EXTERNAL TABLE BAR (ID INT) LOCATION 'mock:///foo'"); - stmt.add("CREATE FUNCTION MYPLUS AS 'org.apache.storm.sql.TestUtils$MyPlus'"); - stmt.add("INSERT INTO BAR SELECT STREAM ID FROM FOO WHERE MYPLUS(ID, 1) = 'x'"); - StormSqlLocalClusterImpl impl = new StormSqlLocalClusterImpl(); - - assertThrows(CompilingClassLoader.CompilerException.class, - () -> impl.runLocal(cluster, stmt, (__) -> true, WAIT_TIMEOUT_MS_ERROR_EXPECTED)); - } - - @Test - public void testExternalUdf() throws Exception { - List stmt = new ArrayList<>(); - stmt.add("CREATE EXTERNAL TABLE FOO (ID INT PRIMARY KEY) LOCATION 'mock:///foo'"); - stmt.add("CREATE EXTERNAL TABLE BAR (ID INT PRIMARY KEY) LOCATION 'mock:///foo'"); - stmt.add("CREATE FUNCTION MYPLUS AS 'org.apache.storm.sql.TestUtils$MyPlus'"); - stmt.add("INSERT INTO BAR SELECT STREAM MYPLUS(ID, 1) FROM FOO WHERE ID > 2"); - StormSqlLocalClusterImpl impl = new StormSqlLocalClusterImpl(); - - List> values = TestUtils.MockInsertBolt.getCollectedValues(); - impl.runLocal(cluster, stmt, (__) -> values.size() >= 2, WAIT_TIMEOUT_MS); - - assertEquals(2, values.size()); - assertEquals(4, values.get(0).getFirst()); - assertEquals(5, values.get(1).getFirst()); - } - - @Test - public void testExternalUdfUsingJar() throws Exception { - List stmt = new ArrayList<>(); - stmt.add("CREATE EXTERNAL TABLE FOO (ID INT PRIMARY KEY) LOCATION 'mock:///foo'"); - stmt.add("CREATE EXTERNAL TABLE BAR (ID INT PRIMARY KEY) LOCATION 'mock:///foo'"); - stmt.add("CREATE FUNCTION MYPLUS AS 'org.apache.storm.sql.TestUtils$MyPlus' USING JAR 'foo'"); - stmt.add("INSERT INTO BAR SELECT STREAM MYPLUS(ID, 1) FROM FOO WHERE ID > 2"); - StormSqlLocalClusterImpl impl = new StormSqlLocalClusterImpl(); - - assertThrows(UnsupportedOperationException.class, - () -> impl.runLocal(cluster, stmt, (__) -> true, WAIT_TIMEOUT_MS_ERROR_EXPECTED)); - } - - private static class MockDataSourceProvider implements DataSourcesProvider { - @Override - public String scheme() { - return "mock"; - } - - @Override - public ISqlStreamsDataSource constructStreams(URI uri, String inputFormatClass, String outputFormatClass, - Properties properties, List fields) { - return new TestUtils.MockSqlStreamsInsertDataSource(); - } - } - - private static class MockNestedDataSourceProvider implements DataSourcesProvider { - @Override - public String scheme() { - return "mocknested"; - } - - @Override - public ISqlStreamsDataSource constructStreams(URI uri, String inputFormatClass, String outputFormatClass, - Properties properties, List fields) { - return new TestUtils.MockSqlStreamsInsertNestedDataSource(); - } - } - - private static class MockGroupDataSourceProvider implements DataSourcesProvider { - @Override - public String scheme() { - return "mockgroup"; - } - - @Override - public ISqlStreamsDataSource constructStreams(URI uri, String inputFormatClass, String outputFormatClass, - Properties properties, List fields) { - return new TestUtils.MockSqlStreamsInsertGroupedDataSource(); - } - } - - private static class MockEmpDataSourceProvider implements DataSourcesProvider { - @Override - public String scheme() { - return "mockemp"; - } - - @Override - public ISqlStreamsDataSource constructStreams(URI uri, String inputFormatClass, String outputFormatClass, - Properties properties, List fields) { - return new TestUtils.MockSqlStreamsInsertJoinDataSourceEmp(); - } - } - - private static class MockDeptDataSourceProvider implements DataSourcesProvider { - @Override - public String scheme() { - return "mockdept"; - } - - @Override - public ISqlStreamsDataSource constructStreams(URI uri, String inputFormatClass, String outputFormatClass, - Properties properties, List fields) { - return new TestUtils.MockSqlStreamsInsertJoinDataSourceDept(); - } - } -} diff --git a/sql/storm-sql-core/src/test/org/apache/storm/sql/compiler/backends/streams/TestCompilerUtils.java b/sql/storm-sql-core/src/test/org/apache/storm/sql/compiler/backends/streams/TestCompilerUtils.java deleted file mode 100644 index b7fb5b35db9..00000000000 --- a/sql/storm-sql-core/src/test/org/apache/storm/sql/compiler/backends/streams/TestCompilerUtils.java +++ /dev/null @@ -1,217 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.apache.storm.sql.compiler.backends.streams; - -import org.apache.calcite.adapter.java.JavaTypeFactory; -import org.apache.calcite.jdbc.JavaTypeFactoryImpl; -import org.apache.calcite.rel.RelNode; -import org.apache.calcite.rel.type.RelDataTypeSystem; -import org.apache.calcite.schema.SchemaPlus; -import org.apache.calcite.schema.StreamableTable; -import org.apache.calcite.schema.Table; -import org.apache.calcite.schema.impl.AggregateFunctionImpl; -import org.apache.calcite.schema.impl.ScalarFunctionImpl; -import org.apache.calcite.sql.SqlExplainLevel; -import org.apache.calcite.sql.parser.SqlParseException; -import org.apache.calcite.sql.parser.SqlParserPos; -import org.apache.calcite.sql.type.SqlTypeName; -import org.apache.calcite.sql.validate.SqlMonotonicity; -import org.apache.calcite.tools.Frameworks; -import org.apache.calcite.tools.RelConversionException; -import org.apache.calcite.tools.ValidationException; -import org.apache.storm.sql.compiler.CompilerUtil; -import org.apache.storm.sql.parser.ColumnConstraint; -import org.apache.storm.sql.planner.StormRelUtils; -import org.apache.storm.sql.planner.streams.QueryPlanner; -import org.apache.storm.sql.planner.streams.rel.StreamsRel; - -public class TestCompilerUtils { - - public static CalciteState sqlOverDummyTable(String sql) - throws RelConversionException, ValidationException, SqlParseException { - SchemaPlus schema = Frameworks.createRootSchema(true); - JavaTypeFactory typeFactory = new JavaTypeFactoryImpl - (RelDataTypeSystem.DEFAULT); - StreamableTable streamableTable = new CompilerUtil.TableBuilderInfo(typeFactory) - .field("ID", SqlTypeName.INTEGER, new ColumnConstraint.PrimaryKey(SqlMonotonicity.MONOTONIC, SqlParserPos.ZERO)) - .field("NAME", typeFactory.createType(String.class)) - .field("ADDR", typeFactory.createType(String.class)) - .build(); - Table table = streamableTable.stream(); - schema.add("FOO", table); - schema.add("BAR", table); - schema.add("MYPLUS", ScalarFunctionImpl.create(MyPlus.class, "eval")); - - QueryPlanner queryPlanner = new QueryPlanner(schema); - StreamsRel tree = queryPlanner.getPlan(sql); - System.out.println(StormRelUtils.explain(tree, SqlExplainLevel.ALL_ATTRIBUTES)); - return new CalciteState(schema, tree); - } - - public static CalciteState sqlOverDummyGroupByTable(String sql) - throws RelConversionException, ValidationException, SqlParseException { - SchemaPlus schema = Frameworks.createRootSchema(true); - JavaTypeFactory typeFactory = new JavaTypeFactoryImpl - (RelDataTypeSystem.DEFAULT); - StreamableTable streamableTable = new CompilerUtil.TableBuilderInfo(typeFactory) - .field("ID", SqlTypeName.INTEGER, new ColumnConstraint.PrimaryKey(SqlMonotonicity.MONOTONIC, SqlParserPos.ZERO)) - .field("GRPID", SqlTypeName.INTEGER) - .field("NAME", typeFactory.createType(String.class)) - .field("ADDR", typeFactory.createType(String.class)) - .field("AGE", SqlTypeName.INTEGER) - .field("SCORE", SqlTypeName.INTEGER) - .build(); - Table table = streamableTable.stream(); - schema.add("FOO", table); - schema.add("BAR", table); - schema.add("MYSTATICSUM", AggregateFunctionImpl.create(MyStaticSumFunction.class)); - schema.add("MYSUM", AggregateFunctionImpl.create(MySumFunction.class)); - - QueryPlanner queryPlanner = new QueryPlanner(schema); - StreamsRel tree = queryPlanner.getPlan(sql); - System.out.println(StormRelUtils.explain(tree, SqlExplainLevel.ALL_ATTRIBUTES)); - return new CalciteState(schema, tree); - } - - public static CalciteState sqlOverNestedTable(String sql) - throws RelConversionException, ValidationException, SqlParseException { - SchemaPlus schema = Frameworks.createRootSchema(true); - JavaTypeFactory typeFactory = new JavaTypeFactoryImpl - (RelDataTypeSystem.DEFAULT); - - StreamableTable streamableTable = new CompilerUtil.TableBuilderInfo(typeFactory) - .field("ID", SqlTypeName.INTEGER, new ColumnConstraint.PrimaryKey(SqlMonotonicity.MONOTONIC, SqlParserPos.ZERO)) - .field("MAPFIELD", - typeFactory.createTypeWithNullability( - typeFactory.createMapType( - typeFactory.createTypeWithNullability( - typeFactory.createSqlType(SqlTypeName.VARCHAR), true), - typeFactory.createTypeWithNullability( - typeFactory.createSqlType(SqlTypeName.INTEGER), true)) - , true)) - .field("NESTEDMAPFIELD", - typeFactory.createTypeWithNullability( - typeFactory.createMapType( - typeFactory.createTypeWithNullability( - typeFactory.createSqlType(SqlTypeName.VARCHAR), true), - typeFactory.createTypeWithNullability( - typeFactory.createMapType( - typeFactory.createTypeWithNullability( - typeFactory.createSqlType(SqlTypeName.VARCHAR), true), - typeFactory.createTypeWithNullability( - typeFactory.createSqlType(SqlTypeName.INTEGER), true)) - , true)) - , true)) - .field("ARRAYFIELD", typeFactory.createTypeWithNullability( - typeFactory.createArrayType( - typeFactory.createTypeWithNullability( - typeFactory.createSqlType(SqlTypeName.INTEGER), true), -1L) - , true)) - .build(); - Table table = streamableTable.stream(); - schema.add("FOO", table); - schema.add("BAR", table); - schema.add("MYPLUS", ScalarFunctionImpl.create(MyPlus.class, "eval")); - - QueryPlanner queryPlanner = new QueryPlanner(schema); - StreamsRel tree = queryPlanner.getPlan(sql); - System.out.println(StormRelUtils.explain(tree, SqlExplainLevel.ALL_ATTRIBUTES)); - return new CalciteState(schema, tree); - } - - public static CalciteState sqlOverSimpleEquiJoinTables(String sql) - throws RelConversionException, ValidationException, SqlParseException { - SchemaPlus schema = Frameworks.createRootSchema(true); - JavaTypeFactory typeFactory = new JavaTypeFactoryImpl - (RelDataTypeSystem.DEFAULT); - - StreamableTable streamableTable = new CompilerUtil.TableBuilderInfo(typeFactory) - .field("EMPID", SqlTypeName.INTEGER, new ColumnConstraint.PrimaryKey(SqlMonotonicity.MONOTONIC, SqlParserPos.ZERO)) - .field("EMPNAME", SqlTypeName.VARCHAR) - .field("DEPTID", SqlTypeName.INTEGER) - .build(); - - StreamableTable streamableTable2 = new CompilerUtil.TableBuilderInfo(typeFactory) - .field("DEPTID", SqlTypeName.INTEGER, new ColumnConstraint.PrimaryKey(SqlMonotonicity.MONOTONIC, SqlParserPos.ZERO)) - .field("DEPTNAME", SqlTypeName.VARCHAR) - .build(); - - Table table = streamableTable.stream(); - Table table2 = streamableTable2.stream(); - schema.add("EMP", table); - schema.add("DEPT", table2); - - QueryPlanner queryPlanner = new QueryPlanner(schema); - StreamsRel tree = queryPlanner.getPlan(sql); - System.out.println(StormRelUtils.explain(tree, SqlExplainLevel.ALL_ATTRIBUTES)); - return new CalciteState(schema, tree); - } - - public static class MyPlus { - public static Integer eval(Integer x, Integer y) { - return x + y; - } - } - - public static class MyStaticSumFunction { - public static long init() { - return 0L; - } - - public static long add(long accumulator, int v) { - return accumulator + v; - } - } - - public static class MySumFunction { - public MySumFunction() { - } - - public long init() { - return 0L; - } - - public long add(long accumulator, int v) { - return accumulator + v; - } - - public long result(long accumulator) { - return accumulator; - } - } - - public static class CalciteState { - final SchemaPlus schema; - final RelNode tree; - - private CalciteState(SchemaPlus schema, RelNode tree) { - this.schema = schema; - this.tree = tree; - } - - public SchemaPlus schema() { - return schema; - } - - public RelNode tree() { - return tree; - } - } -} diff --git a/sql/storm-sql-core/src/test/org/apache/storm/sql/compiler/backends/streams/TestPlanCompiler.java b/sql/storm-sql-core/src/test/org/apache/storm/sql/compiler/backends/streams/TestPlanCompiler.java deleted file mode 100644 index 9b126f1151f..00000000000 --- a/sql/storm-sql-core/src/test/org/apache/storm/sql/compiler/backends/streams/TestPlanCompiler.java +++ /dev/null @@ -1,231 +0,0 @@ -/* - * * - * * Licensed to the Apache Software Foundation (ASF) under one - * * or more contributor license agreements. See the NOTICE file - * * distributed with this work for additional information - * * regarding copyright ownership. The ASF licenses this file - * * to you under the Apache License, Version 2.0 (the - * * "License"); you may not use this file except in compliance - * * with the License. You may obtain a copy of the License at - * *

- * * http://www.apache.org/licenses/LICENSE-2.0 - * *

- * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, - * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * * See the License for the specific language governing permissions and - * * limitations under the License. - * - */ -package org.apache.storm.sql.compiler.backends.streams; - -import static org.junit.jupiter.api.Assertions.assertArrayEquals; -import static org.junit.jupiter.api.Assertions.assertEquals; - -import com.google.common.collect.ImmutableMap; -import org.apache.calcite.DataContext; -import org.apache.calcite.avatica.util.DateTimeUtils; -import org.apache.storm.LocalCluster; -import org.apache.storm.sql.SqlTestUtil; -import org.apache.storm.sql.TestUtils; -import org.apache.storm.generated.StormTopology; -import org.apache.storm.sql.AbstractStreamsProcessor; -import org.apache.storm.sql.planner.streams.QueryPlanner; -import org.apache.storm.sql.runtime.ISqlStreamsDataSource; -import org.apache.storm.streams.Pair; -import org.apache.storm.tuple.Values; - -import java.time.ZoneOffset; -import java.util.Arrays; -import java.util.HashMap; -import java.util.Map; -import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; - -@ExtendWith(TestUtils.MockBoltExtension.class) -@ExtendWith(TestUtils.MockInsertBoltExtension.class) -public class TestPlanCompiler { - - private static LocalCluster cluster; - - @BeforeAll - public static void staticSetup() throws Exception { - cluster = new LocalCluster(); - } - - @AfterAll - public static void staticCleanup() { - if (cluster != null) { - cluster.shutdown(); - cluster = null; - } - } - - @Test - public void testCompile() throws Exception { - final int EXPECTED_VALUE_SIZE = 2; - String sql = "SELECT ID FROM FOO WHERE ID > 2"; - TestCompilerUtils.CalciteState state = TestCompilerUtils.sqlOverDummyTable(sql); - final Map data = new HashMap<>(); - data.put("FOO", new TestUtils.MockSqlStreamsDataSource()); - - QueryPlanner planner = new QueryPlanner(state.schema()); - AbstractStreamsProcessor proc = planner.compile(data, sql); - // inject output bolt - proc.outputStream().to(new TestUtils.MockBolt()); - final StormTopology topo = proc.build(); - - SqlTestUtil.runStormTopology(cluster, TestUtils.MockBolt.getCollectedValues(), EXPECTED_VALUE_SIZE, proc, topo); - assertArrayEquals(new Values[]{new Values(3), new Values(4)}, TestUtils.MockBolt.getCollectedValues().toArray()); - } - - @Test - public void testInsert() throws Exception { - final int EXPECTED_VALUE_SIZE = 1; - String sql = "INSERT INTO BAR SELECT ID, NAME, ADDR FROM FOO WHERE ID > 3"; - TestCompilerUtils.CalciteState state = TestCompilerUtils.sqlOverDummyTable(sql); - final Map data = new HashMap<>(); - data.put("FOO", new TestUtils.MockSqlStreamsDataSource()); - data.put("BAR", new TestUtils.MockSqlStreamsOutputDataSource()); - - QueryPlanner planner = new QueryPlanner(state.schema()); - AbstractStreamsProcessor proc = planner.compile(data, sql); - final StormTopology topo = proc.build(); - - SqlTestUtil.runStormTopology(cluster, TestUtils.MockInsertBolt.getCollectedValues(), EXPECTED_VALUE_SIZE, proc, topo); - assertArrayEquals(new Pair[]{Pair.of(4, new Values(4, "abcde", "y"))}, TestUtils.MockInsertBolt.getCollectedValues().toArray()); - } - - @Test - public void testUdf() throws Exception { - int EXPECTED_VALUE_SIZE = 1; - String sql = "SELECT MYPLUS(ID, 3)" - + "FROM FOO " - + "WHERE ID = 2"; - TestCompilerUtils.CalciteState state = TestCompilerUtils.sqlOverDummyTable(sql); - Map data = new HashMap<>(); - data.put("FOO", new TestUtils.MockSqlStreamsDataSource()); - - QueryPlanner planner = new QueryPlanner(state.schema()); - AbstractStreamsProcessor proc = planner.compile(data, sql); - // inject output bolt - proc.outputStream().to(new TestUtils.MockBolt()); - final StormTopology topo = proc.build(); - - SqlTestUtil.runStormTopology(cluster, TestUtils.MockBolt.getCollectedValues(), EXPECTED_VALUE_SIZE, proc, topo); - assertArrayEquals(new Values[]{new Values(5)}, TestUtils.MockBolt.getCollectedValues().toArray()); - } - - @Test - public void testNested() throws Exception { - int EXPECTED_VALUE_SIZE = 1; - String sql = "SELECT ID, MAPFIELD['c'], NESTEDMAPFIELD, ARRAYFIELD " - + "FROM FOO " - + "WHERE NESTEDMAPFIELD['a']['b'] = 2 AND ARRAYFIELD[2] = 200"; - TestCompilerUtils.CalciteState state = TestCompilerUtils.sqlOverNestedTable(sql); - - final Map data = new HashMap<>(); - data.put("FOO", new TestUtils.MockSqlStreamsNestedDataSource()); - - QueryPlanner planner = new QueryPlanner(state.schema()); - AbstractStreamsProcessor proc = planner.compile(data, sql); - // inject output bolt - proc.outputStream().to(new TestUtils.MockBolt()); - final StormTopology topo = proc.build(); - - SqlTestUtil.runStormTopology(cluster, TestUtils.MockBolt.getCollectedValues(), EXPECTED_VALUE_SIZE, proc, topo); - - Map map = ImmutableMap.of("b", 2, "c", 4); - Map> nestedMap = ImmutableMap.of("a", map); - assertArrayEquals(new Values[]{new Values(2, 4, nestedMap, Arrays.asList(100, 200, 300))}, - TestUtils.MockBolt.getCollectedValues().toArray()); - } - - /** - * All the binary literal tests are done here, because Avatica converts the result to byte[] whereas Stream provides the result to - * ByteString which makes different semantic from Stream implementation. - */ - @Test - public void testBinaryStringFunctions() throws Exception { - int EXPECTED_VALUE_SIZE = 1; - String sql = "SELECT x'45F0AB' || x'45F0AB', " - + "POSITION(x'F0' IN x'453423F0ABBC'), " - + "OVERLAY(x'453423F0ABBC45' PLACING x'4534' FROM 3), " - + "SUBSTRING(x'453423F0ABBC' FROM 3), " - + "SUBSTRING(x'453423F0ABBC453423F0ABBC' FROM 3 FOR 4) " - + "FROM FOO " - + "WHERE ID > 0 AND ID < 2"; - - TestCompilerUtils.CalciteState state = TestCompilerUtils.sqlOverDummyTable(sql); - final Map data = new HashMap<>(); - data.put("FOO", new TestUtils.MockSqlStreamsDataSource()); - - QueryPlanner planner = new QueryPlanner(state.schema()); - AbstractStreamsProcessor proc = planner.compile(data, sql); - // inject output bolt - proc.outputStream().to(new TestUtils.MockBolt()); - final StormTopology topo = proc.build(); - - SqlTestUtil.runStormTopology(cluster, TestUtils.MockBolt.getCollectedValues(), EXPECTED_VALUE_SIZE, proc, topo); - - Values v = TestUtils.MockBolt.getCollectedValues().get(0); - - assertEquals("45f0ab45f0ab", v.get(0).toString()); - assertEquals(4, v.get(1)); - assertEquals("45344534abbc45", v.get(2).toString()); - assertEquals("23f0abbc", v.get(3).toString()); - assertEquals("23f0abbc", v.get(4).toString()); - } - - /** - * All the date/time/timestamp related tests are done here, because Avatica converts the result of date functions to java.sql classes - * whereas Stream provides long type which makes different semantic from Stream implementation. - */ - @Test - public void testDateKeywordsAndFunctions() throws Exception { - int EXPECTED_VALUE_SIZE = 1; - String sql = "SELECT " - + "LOCALTIME, CURRENT_TIME, LOCALTIMESTAMP, CURRENT_TIMESTAMP, CURRENT_DATE, " - + "DATE '1970-05-15' AS datefield, TIME '00:00:00' AS timefield, TIMESTAMP '2016-01-01 00:00:00' as timestampfield, " - + "EXTRACT(MONTH FROM TIMESTAMP '2010-01-23 12:34:56')," - + "FLOOR(DATE '2016-01-23' TO MONTH)," - + "CEIL(TIME '12:34:56' TO MINUTE)," - + "{fn CURDATE()} = CURRENT_DATE, {fn CURTIME()} = LOCALTIME, {fn NOW()} = LOCALTIMESTAMP," - + "{fn QUARTER(DATE '2016-10-07')}, {fn TIMESTAMPADD(MINUTE, 15, TIMESTAMP '2016-10-07 00:00:00')}," - + "{fn TIMESTAMPDIFF(SECOND, TIMESTAMP '2016-10-06 00:00:00', TIMESTAMP '2016-10-07 00:00:00')}," - + "INTERVAL '1-5' YEAR TO MONTH AS intervalfield, " - + "(DATE '1970-01-01', DATE '1970-01-15') AS anchoredinterval_field " - + "FROM FOO " - + "WHERE ID > 0 AND ID < 2"; - TestCompilerUtils.CalciteState state = TestCompilerUtils.sqlOverDummyTable(sql); - - final Map data = new HashMap<>(); - data.put("FOO", new TestUtils.MockSqlStreamsDataSource()); - - QueryPlanner planner = new QueryPlanner(state.schema()); - AbstractStreamsProcessor proc = planner.compile(data, sql); - // inject output bolt - proc.outputStream().to(new TestUtils.MockBolt()); - final DataContext dataContext = proc.getDataContext(); - final StormTopology topo = proc.build(); - - SqlTestUtil.runStormTopology(cluster, TestUtils.MockBolt.getCollectedValues(), EXPECTED_VALUE_SIZE, proc, topo); - - long utcTimestamp = (long) dataContext.get(DataContext.Variable.UTC_TIMESTAMP.camelName); - long currentTimestamp = (long) dataContext.get(DataContext.Variable.CURRENT_TIMESTAMP.camelName); - long localTimestamp = (long) dataContext.get(DataContext.Variable.LOCAL_TIMESTAMP.camelName); - - System.out.println(TestUtils.MockBolt.getCollectedValues()); - - java.sql.Timestamp timestamp = new java.sql.Timestamp(utcTimestamp); - int dateInt = (int) timestamp.toLocalDateTime().atOffset(ZoneOffset.UTC).toLocalDate().toEpochDay(); - int localTimeInt = (int) (localTimestamp % DateTimeUtils.MILLIS_PER_DAY); - int currentTimeInt = (int) (currentTimestamp % DateTimeUtils.MILLIS_PER_DAY); - - assertArrayEquals(new Values[]{new Values(localTimeInt, currentTimeInt, localTimestamp, currentTimestamp, dateInt, - 134, 0, 1451606400000L, 1L, 0L, 45300000, true, true, true, 4L, 1475799300000L, 86400, 17, 0, 14)}, - TestUtils.MockBolt.getCollectedValues().toArray()); - } -} diff --git a/sql/storm-sql-core/src/test/org/apache/storm/sql/parser/TestSqlParser.java b/sql/storm-sql-core/src/test/org/apache/storm/sql/parser/TestSqlParser.java deleted file mode 100644 index c81b3faf1ec..00000000000 --- a/sql/storm-sql-core/src/test/org/apache/storm/sql/parser/TestSqlParser.java +++ /dev/null @@ -1,50 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version - * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions - * and limitations under the License. - */ - -package org.apache.storm.sql.parser; - -import org.apache.calcite.sql.SqlNode; -import org.apache.storm.sql.parser.impl.ParseException; -import org.junit.jupiter.api.Test; - -import static org.junit.jupiter.api.Assertions.assertThrows; - -public class TestSqlParser { - private static SqlNode parse(String sql) throws Exception { - StormParser parser = new StormParser(sql); - return parser.impl().parseSqlStmtEof(); - } - - @Test - public void testCreateTable() throws Exception { - String sql = "CREATE EXTERNAL TABLE foo (bar INT) LOCATION 'kafka:///foo'"; - parse(sql); - } - - @Test - public void testCreateTableWithPrimaryKey() throws Exception { - String sql = "CREATE EXTERNAL TABLE foo (bar INT PRIMARY KEY ASC) LOCATION 'kafka:///foo'"; - parse(sql); - } - - @Test - public void testCreateTableWithoutLocation() throws Exception { - String sql = "CREATE EXTERNAL TABLE foo (bar INT)"; - assertThrows(ParseException.class, () -> parse(sql)); - } - - @Test - public void testCreateFunction() throws Exception { - String sql = "CREATE FUNCTION foo AS 'org.apache.storm.sql.MyUDF'"; - parse(sql); - } -} diff --git a/sql/storm-sql-external/storm-sql-hdfs/pom.xml b/sql/storm-sql-external/storm-sql-hdfs/pom.xml deleted file mode 100644 index 517fd13d757..00000000000 --- a/sql/storm-sql-external/storm-sql-hdfs/pom.xml +++ /dev/null @@ -1,152 +0,0 @@ - - - - 4.0.0 - - - storm - org.apache.storm - 2.8.3-SNAPSHOT - ../../../pom.xml - - - storm-sql-hdfs - - - - --add-opens java.security.jgss/sun.security.krb5=ALL-UNNAMED - - 33.4.8-jre - - - - - vesense - Xin Wang - xinwang@apache.org - - - - - - org.apache.storm - storm-client - ${project.version} - ${provided.scope} - - - - org.slf4j - log4j-over-slf4j - - - - - org.apache.storm - storm-sql-runtime - ${project.version} - ${provided.scope} - - - org.apache.storm - storm-sql-runtime - ${project.version} - test - test-jar - - - org.apache.storm - storm-hdfs - ${project.version} - ${provided.scope} - - - org.apache.hadoop - hadoop-client-minicluster - ${hadoop.version} - test - - - org.eclipse.jetty - jetty-servlet - test - - - org.eclipse.jetty - jetty-webapp - test - - - org.mockito - mockito-core - - - org.hamcrest - hamcrest - - - com.google.guava - guava - ${guava.version.downgrade} - - - - src/jvm - src/test - - - ${basedir}/src/resources - - - - - org.apache.maven.plugins - maven-checkstyle-plugin - - - - org.apache.maven.plugins - maven-pmd-plugin - - - org.apache.maven.plugins - maven-surefire-plugin - - ${storm-sql-hdfs.test.introspection.argLine} -Xmx3g -XX:+HeapDumpOnOutOfMemoryError - true - IntegrationTest | ${java.unit.test.exclude.groups} - false - 1.0C - true - - ${project.basedir}/target/testhome - - - - - - diff --git a/sql/storm-sql-external/storm-sql-hdfs/src/jvm/org/apache/storm/sql/hdfs/HdfsDataSourcesProvider.java b/sql/storm-sql-external/storm-sql-hdfs/src/jvm/org/apache/storm/sql/hdfs/HdfsDataSourcesProvider.java deleted file mode 100644 index 0845808e5ae..00000000000 --- a/sql/storm-sql-external/storm-sql-hdfs/src/jvm/org/apache/storm/sql/hdfs/HdfsDataSourcesProvider.java +++ /dev/null @@ -1,131 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.storm.sql.hdfs; - -import com.google.common.base.Preconditions; - -import java.net.URI; -import java.util.List; -import java.util.Properties; - -import org.apache.storm.hdfs.bolt.HdfsBolt; -import org.apache.storm.hdfs.bolt.format.FileNameFormat; -import org.apache.storm.hdfs.bolt.format.RecordFormat; -import org.apache.storm.hdfs.bolt.format.SimpleFileNameFormat; -import org.apache.storm.hdfs.bolt.rotation.FileRotationPolicy; -import org.apache.storm.hdfs.bolt.rotation.FileSizeRotationPolicy; -import org.apache.storm.hdfs.bolt.rotation.TimedRotationPolicy; -import org.apache.storm.sql.runtime.DataSourcesProvider; -import org.apache.storm.sql.runtime.FieldInfo; -import org.apache.storm.sql.runtime.IOutputSerializer; -import org.apache.storm.sql.runtime.ISqlStreamsDataSource; -import org.apache.storm.sql.runtime.utils.FieldInfoUtils; -import org.apache.storm.sql.runtime.utils.SerdeUtils; -import org.apache.storm.topology.IRichBolt; -import org.apache.storm.topology.IRichSpout; -import org.apache.storm.tuple.Tuple; -import org.apache.storm.tuple.Values; - -/** - * Create a HDFS sink based on the URI and properties. The URI has the format of hdfs://host:port/path-to-file - * The properties are in JSON format which specifies the name / path of the hdfs file and etc. - */ -public class HdfsDataSourcesProvider implements DataSourcesProvider { - - private static final String PROPERTY_HDFS_FILE_PATH = "hdfs.file.path"; - private static final String PROPERTY_HDFS_FILE_NAME = "hdfs.file.name"; - private static final String DEFAULT_VALUE_HDFS_FILE_PATH = "/storm"; - private static final String DEFAULT_VALUE_HDF_FILE_NAME = "$TIME.$NUM.txt"; - private static final String PROPERTY_HDFS_ROTATION_SIZE_KB = "hdfs.rotation.size.kb"; - private static final String PROPERTY_HDFS_ROTATION_TIME_SECONDS = "hdfs.rotation.time.seconds"; - private static final String SCHEME_NAME = "hdfs"; - - private static class HdfsStreamsDataSource implements ISqlStreamsDataSource { - private final String url; - private final Properties props; - private final IOutputSerializer serializer; - - private HdfsStreamsDataSource(String url, Properties props, IOutputSerializer serializer) { - this.url = url; - this.props = props; - this.serializer = serializer; - } - - @Override - public IRichSpout getProducer() { - throw new UnsupportedOperationException(this.getClass().getName() + " doesn't provide Producer"); - } - - @Override - public IRichBolt getConsumer() { - FileNameFormat fileNameFormat = new SimpleFileNameFormat() - .withPath(props.getProperty(PROPERTY_HDFS_FILE_PATH, DEFAULT_VALUE_HDFS_FILE_PATH)) - .withName(props.getProperty(PROPERTY_HDFS_FILE_NAME, DEFAULT_VALUE_HDF_FILE_NAME)); - - RecordFormat recordFormat = new StreamsRecordFormat(serializer); - - FileRotationPolicy rotationPolicy; - String size = props.getProperty(PROPERTY_HDFS_ROTATION_SIZE_KB); - String interval = props.getProperty(PROPERTY_HDFS_ROTATION_TIME_SECONDS); - Preconditions.checkArgument(size != null || interval != null, "Hdfs data source must contain file rotation config"); - - if (size != null) { - rotationPolicy = new FileSizeRotationPolicy(Float.parseFloat(size), FileSizeRotationPolicy.Units.KB); - } else { - rotationPolicy = new TimedRotationPolicy(Float.parseFloat(interval), TimedRotationPolicy.TimeUnit.SECONDS); - } - - return new HdfsBolt() - .withFileNameFormat(fileNameFormat) - .withRecordFormat(recordFormat) - .withRotationPolicy(rotationPolicy) - .withFsUrl(url); - } - } - - private static class StreamsRecordFormat implements RecordFormat { - private final IOutputSerializer serializer; - - private StreamsRecordFormat(IOutputSerializer serializer) { - this.serializer = serializer; - } - - @Override - public byte[] format(Tuple tuple) { - //TODO we should handle '\n'. ref DelimitedRecordFormat - Values values = (Values) tuple.getValue(1); - return serializer.write(values, null).array(); - } - - } - - @Override - public String scheme() { - return SCHEME_NAME; - } - - @Override - public ISqlStreamsDataSource constructStreams(URI uri, String inputFormatClass, String outputFormatClass, - Properties properties, List fields) { - List fieldNames = FieldInfoUtils.getFieldNames(fields); - IOutputSerializer serializer = SerdeUtils.getSerializer(outputFormatClass, properties, fieldNames); - return new HdfsStreamsDataSource(uri.toString(), properties, serializer); - } - -} diff --git a/sql/storm-sql-external/storm-sql-hdfs/src/resources/META-INF/services/org.apache.storm.sql.runtime.DataSourcesProvider b/sql/storm-sql-external/storm-sql-hdfs/src/resources/META-INF/services/org.apache.storm.sql.runtime.DataSourcesProvider deleted file mode 100644 index 5fac84fc8f2..00000000000 --- a/sql/storm-sql-external/storm-sql-hdfs/src/resources/META-INF/services/org.apache.storm.sql.runtime.DataSourcesProvider +++ /dev/null @@ -1,16 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -org.apache.storm.sql.hdfs.HdfsDataSourcesProvider \ No newline at end of file diff --git a/sql/storm-sql-external/storm-sql-hdfs/src/test/org/apache/storm/sql/hdfs/TestHdfsDataSourcesProvider.java b/sql/storm-sql-external/storm-sql-hdfs/src/test/org/apache/storm/sql/hdfs/TestHdfsDataSourcesProvider.java deleted file mode 100644 index 823f71f3919..00000000000 --- a/sql/storm-sql-external/storm-sql-hdfs/src/test/org/apache/storm/sql/hdfs/TestHdfsDataSourcesProvider.java +++ /dev/null @@ -1,88 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.storm.sql.hdfs; - -import com.google.common.collect.ImmutableList; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileUtil; -import org.apache.hadoop.hdfs.MiniDFSCluster; -import org.apache.storm.hdfs.bolt.HdfsBolt; -import org.apache.storm.sql.runtime.DataSourcesRegistry; -import org.apache.storm.sql.runtime.FieldInfo; -import org.apache.storm.sql.runtime.ISqlStreamsDataSource; -import org.apache.storm.topology.IRichBolt; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - -import java.io.File; -import java.io.IOException; -import java.net.URI; -import java.util.List; -import java.util.Properties; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNotNull; - -public class TestHdfsDataSourcesProvider { - - private static final List FIELDS = ImmutableList.of( - new FieldInfo("ID", int.class, true), - new FieldInfo("val", String.class, false)); - private static final Properties TBL_PROPERTIES = new Properties(); - - private static String hdfsURI; - private static MiniDFSCluster hdfsCluster; - - static { - TBL_PROPERTIES.put("hdfs.file.path", "/unittest"); - TBL_PROPERTIES.put("hdfs.file.name", "test1.txt"); - TBL_PROPERTIES.put("hdfs.rotation.time.seconds", "120"); - } - - @BeforeEach - public void setup() throws Exception { - Configuration conf = new Configuration(); - conf.set("fs.trash.interval", "10"); - conf.setBoolean("dfs.permissions", true); - File baseDir = new File("./target/hdfs/").getAbsoluteFile(); - FileUtil.fullyDelete(baseDir); - conf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, baseDir.getAbsolutePath()); - - MiniDFSCluster.Builder builder = new MiniDFSCluster.Builder(conf); - hdfsCluster = builder.build(); - hdfsURI = "hdfs://localhost:" + hdfsCluster.getNameNodePort() + "/"; - } - - @AfterEach - public void shutDown() throws IOException { - hdfsCluster.shutdown(); - } - - @SuppressWarnings("unchecked") - @Test - public void testHdfsSink() throws Exception { - ISqlStreamsDataSource ds = DataSourcesRegistry.constructStreamsDataSource( - URI.create(hdfsURI), null, null, TBL_PROPERTIES, FIELDS); - assertNotNull(ds); - - IRichBolt consumer = ds.getConsumer(); - - assertEquals(HdfsBolt.class, consumer.getClass()); - } -} diff --git a/sql/storm-sql-external/storm-sql-kafka/pom.xml b/sql/storm-sql-external/storm-sql-kafka/pom.xml deleted file mode 100644 index ccfc289d761..00000000000 --- a/sql/storm-sql-external/storm-sql-kafka/pom.xml +++ /dev/null @@ -1,107 +0,0 @@ - - - - 4.0.0 - - - storm - org.apache.storm - 2.8.3-SNAPSHOT - ../../../pom.xml - - - storm-sql-kafka - - - - 33.4.8-jre - - - - - haohui - Haohui Mai - ricetons@gmail.com - - - - - - org.apache.storm - storm-client - ${project.version} - ${provided.scope} - - - org.apache.storm - storm-sql-runtime - ${project.version} - ${provided.scope} - - - org.apache.storm - storm-sql-runtime - ${project.version} - test - test-jar - - - org.apache.storm - storm-kafka-client - ${project.version} - ${provided.scope} - - - org.apache.kafka - kafka-clients - - - org.mockito - mockito-core - - - org.hamcrest - hamcrest - - - com.google.guava - guava - ${guava.version} - - - - src/jvm - src/test - - - ${basedir}/src/resources - - - - - org.apache.maven.plugins - maven-checkstyle-plugin - - - - org.apache.maven.plugins - maven-pmd-plugin - - - - diff --git a/sql/storm-sql-external/storm-sql-kafka/src/jvm/org/apache/storm/sql/kafka/KafkaDataSourcesProvider.java b/sql/storm-sql-external/storm-sql-kafka/src/jvm/org/apache/storm/sql/kafka/KafkaDataSourcesProvider.java deleted file mode 100644 index 2af340d0e5b..00000000000 --- a/sql/storm-sql-external/storm-sql-kafka/src/jvm/org/apache/storm/sql/kafka/KafkaDataSourcesProvider.java +++ /dev/null @@ -1,175 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.storm.sql.kafka; - -import com.google.common.base.Preconditions; - -import java.net.URI; -import java.nio.ByteBuffer; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Properties; -import java.util.UUID; -import org.apache.kafka.clients.consumer.ConsumerConfig; -import org.apache.kafka.clients.producer.ProducerConfig; -import org.apache.kafka.common.serialization.ByteBufferDeserializer; -import org.apache.kafka.common.serialization.ByteBufferSerializer; -import org.apache.storm.kafka.bolt.KafkaBolt; -import org.apache.storm.kafka.bolt.mapper.TupleToKafkaMapper; -import org.apache.storm.kafka.bolt.selector.DefaultTopicSelector; -import org.apache.storm.kafka.spout.KafkaSpout; -import org.apache.storm.kafka.spout.KafkaSpoutConfig; - -import org.apache.storm.spout.Scheme; -import org.apache.storm.sql.runtime.DataSourcesProvider; -import org.apache.storm.sql.runtime.FieldInfo; -import org.apache.storm.sql.runtime.IOutputSerializer; -import org.apache.storm.sql.runtime.ISqlStreamsDataSource; -import org.apache.storm.sql.runtime.utils.SerdeUtils; -import org.apache.storm.topology.IRichBolt; -import org.apache.storm.topology.IRichSpout; -import org.apache.storm.tuple.Tuple; -import org.apache.storm.tuple.Values; - -/** - * Create a Kafka spout/sink based on the URI and properties. The URI has the format of - * kafka://topic?bootstrap-servers=ip:port[,ip:port]. The properties are in JSON format which specifies the producer config - * of the Kafka broker. - */ -public class KafkaDataSourcesProvider implements DataSourcesProvider { - private static final String CONFIG_KEY_PRODUCER = "producer"; - private static final String URI_PARAMS_BOOTSTRAP_SERVERS = "bootstrap-servers"; - - private static class SqlKafkaMapper implements TupleToKafkaMapper { - private final IOutputSerializer serializer; - - private SqlKafkaMapper(IOutputSerializer serializer) { - this.serializer = serializer; - } - - @Override - public Object getKeyFromTuple(Tuple tuple) { - return tuple.getValue(0); - } - - @Override - public ByteBuffer getMessageFromTuple(Tuple tuple) { - return serializer.write((Values) tuple.getValue(1), null); - } - } - - private static class KafkaStreamsDataSource implements ISqlStreamsDataSource { - private final KafkaSpoutConfig kafkaSpoutConfig; - private final String bootstrapServers; - private final String topic; - private final Properties props; - private final IOutputSerializer serializer; - - KafkaStreamsDataSource(KafkaSpoutConfig kafkaSpoutConfig, String bootstrapServers, - String topic, Properties props, IOutputSerializer serializer) { - this.kafkaSpoutConfig = kafkaSpoutConfig; - this.bootstrapServers = bootstrapServers; - this.topic = topic; - this.props = props; - this.serializer = serializer; - } - - @Override - public IRichSpout getProducer() { - return new KafkaSpout<>(kafkaSpoutConfig); - } - - @Override - public IRichBolt getConsumer() { - Preconditions.checkArgument(!props.isEmpty(), - "Writable Kafka table " + topic + " must contain producer config"); - HashMap producerConfig = (HashMap) props.get(CONFIG_KEY_PRODUCER); - props.putAll(producerConfig); - Preconditions.checkState(!props.containsKey(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG), - "Writable Kafka table " + topic + " must not contain \"bootstrap.servers\" config, set it in the kafka URL instead"); - Preconditions.checkState(!props.containsKey(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG), - "Writable Kafka table " + topic + "must not contain " + ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG - + ", it will be hardcoded to be " + ByteBufferSerializer.class); - props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, ByteBufferSerializer.class); - props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers); - - TupleToKafkaMapper mapper = new SqlKafkaMapper(serializer); - - return new KafkaBolt() - .withTopicSelector(new DefaultTopicSelector(topic)) - .withProducerProperties(props) - .withTupleToKafkaMapper(mapper); - } - } - - @Override - public String scheme() { - return "kafka"; - } - - @Override - public ISqlStreamsDataSource constructStreams(URI uri, String inputFormatClass, String outputFormatClass, - Properties properties, List fields) { - List fieldNames = new ArrayList<>(); - int primaryIndex = -1; - for (int i = 0; i < fields.size(); ++i) { - FieldInfo f = fields.get(i); - fieldNames.add(f.name()); - if (f.isPrimary()) { - primaryIndex = i; - } - } - Preconditions.checkState(primaryIndex != -1, "Kafka stream table must have a primary key"); - Scheme scheme = SerdeUtils.getScheme(inputFormatClass, properties, fieldNames); - - Map values = parseUriParams(uri.getQuery()); - String bootstrapServers = values.get(URI_PARAMS_BOOTSTRAP_SERVERS); - Preconditions.checkNotNull(bootstrapServers, "bootstrap-servers must be specified"); - String topic = uri.getHost(); - KafkaSpoutConfig kafkaSpoutConfig = - new KafkaSpoutConfig.Builder(bootstrapServers, topic) - .setProp(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, ByteBufferDeserializer.class) - .setProp(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, ByteBufferDeserializer.class) - .setProp(ConsumerConfig.GROUP_ID_CONFIG, "storm-sql-kafka-" + UUID.randomUUID().toString()) - .setRecordTranslator(new RecordTranslatorSchemeAdapter(scheme)) - .build(); - - IOutputSerializer serializer = SerdeUtils.getSerializer(outputFormatClass, properties, fieldNames); - - return new KafkaStreamsDataSource(kafkaSpoutConfig, bootstrapServers, topic, properties, serializer); - } - - private static Map parseUriParams(String query) { - HashMap res = new HashMap<>(); - if (query == null) { - return res; - } - - String[] params = query.split("&"); - for (String p : params) { - String[] v = p.split("=", 2); - if (v.length > 1) { - res.put(v[0], v[1]); - } - } - return res; - } -} diff --git a/sql/storm-sql-external/storm-sql-kafka/src/jvm/org/apache/storm/sql/kafka/RecordTranslatorSchemeAdapter.java b/sql/storm-sql-external/storm-sql-kafka/src/jvm/org/apache/storm/sql/kafka/RecordTranslatorSchemeAdapter.java deleted file mode 100644 index 093bfcb7e9a..00000000000 --- a/sql/storm-sql-external/storm-sql-kafka/src/jvm/org/apache/storm/sql/kafka/RecordTranslatorSchemeAdapter.java +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Copyright 2018 The Apache Software Foundation. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.storm.sql.kafka; - -import java.nio.ByteBuffer; -import java.util.List; -import org.apache.kafka.clients.consumer.ConsumerRecord; -import org.apache.storm.kafka.spout.RecordTranslator; -import org.apache.storm.spout.Scheme; -import org.apache.storm.tuple.Fields; - -/** - * RecordTranslator that delegates to a Scheme. - */ -public class RecordTranslatorSchemeAdapter implements RecordTranslator { - - private static final long serialVersionUID = 1L; - private final Scheme delegate; - - public RecordTranslatorSchemeAdapter(Scheme delegate) { - this.delegate = delegate; - } - - @Override - public List apply(ConsumerRecord record) { - return this.delegate.deserialize(record.value()); - } - - @Override - public Fields getFieldsFor(String stream) { - //We ensure there is only the default stream when configuring the spout, so it should be safe to ignore the parameter here. - return this.delegate.getOutputFields(); - } -} diff --git a/sql/storm-sql-external/storm-sql-kafka/src/resources/META-INF/services/org.apache.storm.sql.runtime.DataSourcesProvider b/sql/storm-sql-external/storm-sql-kafka/src/resources/META-INF/services/org.apache.storm.sql.runtime.DataSourcesProvider deleted file mode 100644 index 7f687cc84b9..00000000000 --- a/sql/storm-sql-external/storm-sql-kafka/src/resources/META-INF/services/org.apache.storm.sql.runtime.DataSourcesProvider +++ /dev/null @@ -1,16 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -org.apache.storm.sql.kafka.KafkaDataSourcesProvider \ No newline at end of file diff --git a/sql/storm-sql-external/storm-sql-kafka/src/test/org/apache/storm/sql/kafka/TestKafkaDataSourcesProvider.java b/sql/storm-sql-external/storm-sql-kafka/src/test/org/apache/storm/sql/kafka/TestKafkaDataSourcesProvider.java deleted file mode 100644 index 5caf8e3122d..00000000000 --- a/sql/storm-sql-external/storm-sql-kafka/src/test/org/apache/storm/sql/kafka/TestKafkaDataSourcesProvider.java +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.storm.sql.kafka; - -import com.google.common.collect.ImmutableList; -import org.apache.storm.kafka.bolt.KafkaBolt; -import org.apache.storm.sql.runtime.DataSourcesRegistry; -import org.apache.storm.sql.runtime.FieldInfo; -import org.apache.storm.sql.runtime.ISqlStreamsDataSource; -import org.apache.storm.topology.IRichBolt; -import org.junit.jupiter.api.Test; - -import java.net.URI; -import java.util.*; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNotNull; - -public class TestKafkaDataSourcesProvider { - - private static final List FIELDS = ImmutableList.of( - new FieldInfo("ID", int.class, true), - new FieldInfo("val", String.class, false)); - private static final Properties TBL_PROPERTIES = new Properties(); - - static { - Map map = new HashMap<>(); - map.put("acks", "1"); - map.put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer"); - TBL_PROPERTIES.put("producer", map); - } - - @SuppressWarnings("unchecked") - @Test - public void testKafkaSink() throws Exception { - ISqlStreamsDataSource ds = DataSourcesRegistry.constructStreamsDataSource( - URI.create("kafka://foo?bootstrap-servers=foo"), null, null, TBL_PROPERTIES, FIELDS); - assertNotNull(ds); - - IRichBolt consumer = ds.getConsumer(); - - assertEquals(KafkaBolt.class, consumer.getClass()); - } -} diff --git a/sql/storm-sql-external/storm-sql-redis/pom.xml b/sql/storm-sql-external/storm-sql-redis/pom.xml deleted file mode 100644 index c28fad6e382..00000000000 --- a/sql/storm-sql-external/storm-sql-redis/pom.xml +++ /dev/null @@ -1,85 +0,0 @@ - - - - 4.0.0 - - - storm - org.apache.storm - 2.8.3-SNAPSHOT - ../../../pom.xml - - - storm-sql-redis - - - - org.apache.storm - storm-client - ${project.version} - ${provided.scope} - - - org.apache.storm - storm-sql-runtime - ${project.version} - ${provided.scope} - - - org.apache.storm - storm-sql-runtime - ${project.version} - test - test-jar - - - org.apache.storm - storm-redis - ${project.version} - ${provided.scope} - - - org.mockito - mockito-core - - - org.hamcrest - hamcrest - - - - src/jvm - src/test - - - ${basedir}/src/resources - - - - - org.apache.maven.plugins - maven-checkstyle-plugin - - - - org.apache.maven.plugins - maven-pmd-plugin - - - - diff --git a/sql/storm-sql-external/storm-sql-redis/src/jvm/org/apache/storm/sql/redis/RedisDataSourcesProvider.java b/sql/storm-sql-external/storm-sql-redis/src/jvm/org/apache/storm/sql/redis/RedisDataSourcesProvider.java deleted file mode 100644 index b14b7f31c11..00000000000 --- a/sql/storm-sql-external/storm-sql-redis/src/jvm/org/apache/storm/sql/redis/RedisDataSourcesProvider.java +++ /dev/null @@ -1,213 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.storm.sql.redis; - -import com.google.common.base.Preconditions; - -import java.io.Serializable; -import java.net.InetSocketAddress; -import java.net.URI; -import java.util.Collections; -import java.util.List; -import java.util.Properties; - -import org.apache.storm.redis.bolt.RedisStoreBolt; -import org.apache.storm.redis.common.config.JedisClusterConfig; -import org.apache.storm.redis.common.config.JedisPoolConfig; -import org.apache.storm.redis.common.mapper.RedisDataTypeDescription; -import org.apache.storm.redis.common.mapper.RedisStoreMapper; -import org.apache.storm.sql.runtime.DataSourcesProvider; -import org.apache.storm.sql.runtime.FieldInfo; -import org.apache.storm.sql.runtime.IOutputSerializer; -import org.apache.storm.sql.runtime.ISqlStreamsDataSource; -import org.apache.storm.sql.runtime.utils.FieldInfoUtils; -import org.apache.storm.sql.runtime.utils.SerdeUtils; -import org.apache.storm.topology.IRichBolt; -import org.apache.storm.topology.IRichSpout; -import org.apache.storm.tuple.ITuple; -import org.apache.storm.tuple.Values; -import redis.clients.jedis.util.JedisURIHelper; - -/** - * Create a Redis sink based on the URI and properties. The URI has the format of - * redis://:[password]@[host]:[port]/[dbIdx]. Only host is mandatory and others can be set to default. - *

- * The properties are in JSON format which specifies the config of the Redis data type and etc. - * Please note that when "use.redis.cluster" is "true", cluster discovery is only done from given URI. - */ -public class RedisDataSourcesProvider implements DataSourcesProvider { - private static final int DEFAULT_REDIS_PORT = 6379; - private static final int DEFAULT_TIMEOUT = 2000; - private static final String PROPERTY_DATA_TYPE = "data.type"; - private static final String PROPERTY_DATA_ADDITIONAL_KEY = "data.additional.key"; - private static final String PROPERTY_REDIS_TIMEOUT = "redis.timeout"; - private static final String PROPERTY_USE_REDIS_CLUSTER = "use.redis.cluster"; - private static final String DEFAULT_USE_REDIS_CLUSTER = "false"; - - private abstract static class AbstractRedisStreamsDataSource implements ISqlStreamsDataSource, Serializable { - protected abstract IRichBolt newRedisBolt(RedisStoreMapper storeMapper); - - private final Properties props; - private final List fields; - private final IOutputSerializer serializer; - - AbstractRedisStreamsDataSource(Properties props, List fields, IOutputSerializer serializer) { - this.props = props; - this.fields = fields; - this.serializer = serializer; - } - - @Override - public IRichSpout getProducer() { - throw new UnsupportedOperationException(this.getClass().getName() + " doesn't provide Producer"); - } - - @Override - public IRichBolt getConsumer() { - RedisDataTypeDescription dataTypeDescription = getDataTypeDesc(props); - - RedisStoreMapper storeMapper = new SqlRedisStoreMapper(dataTypeDescription, fields, serializer); - return newRedisBolt(storeMapper); - } - - private RedisDataTypeDescription getDataTypeDesc(Properties props) { - Preconditions.checkArgument(props.containsKey(PROPERTY_DATA_TYPE), - "Redis data source must contain " + PROPERTY_DATA_TYPE + " config"); - - RedisDataTypeDescription.RedisDataType dataType = RedisDataTypeDescription.RedisDataType.valueOf( - props.getProperty(PROPERTY_DATA_TYPE).toUpperCase()); - String additionalKey = props.getProperty(PROPERTY_DATA_ADDITIONAL_KEY); - - return new RedisDataTypeDescription(dataType, additionalKey); - } - } - - private static class RedisClusterStreamsDataSource extends AbstractRedisStreamsDataSource { - private final JedisClusterConfig config; - - RedisClusterStreamsDataSource(JedisClusterConfig config, Properties props, List fields, IOutputSerializer serializer) { - super(props, fields, serializer); - this.config = config; - } - - @Override - protected IRichBolt newRedisBolt(RedisStoreMapper storeMapper) { - return new RedisStoreBolt(config, storeMapper); - } - } - - private static class RedisStreamsDataSource extends AbstractRedisStreamsDataSource { - private final JedisPoolConfig config; - - RedisStreamsDataSource(JedisPoolConfig config, Properties props, List fields, IOutputSerializer serializer) { - super(props, fields, serializer); - this.config = config; - } - - @Override - protected IRichBolt newRedisBolt(RedisStoreMapper storeMapper) { - return new RedisStoreBolt(config, storeMapper); - } - } - - @Override - public String scheme() { - return "redis"; - } - - @Override - public ISqlStreamsDataSource constructStreams( - URI uri, String inputFormatClass, String outputFormatClass, Properties props, List fields) { - Preconditions.checkArgument(JedisURIHelper.isValid(uri), "URI is not valid for Redis: " + uri); - - String host = uri.getHost(); - int port = uri.getPort() != -1 ? uri.getPort() : DEFAULT_REDIS_PORT; - int dbIdx = JedisURIHelper.getDBIndex(uri); - String password = JedisURIHelper.getPassword(uri); - - int timeout = Integer.parseInt(props.getProperty(PROPERTY_REDIS_TIMEOUT, String.valueOf(DEFAULT_TIMEOUT))); - - boolean clusterMode = Boolean.valueOf(props.getProperty(PROPERTY_USE_REDIS_CLUSTER, "false")); - - List fieldNames = FieldInfoUtils.getFieldNames(fields); - IOutputSerializer serializer = SerdeUtils.getSerializer(outputFormatClass, props, fieldNames); - if (clusterMode) { - JedisClusterConfig config = new JedisClusterConfig.Builder() - .setNodes(Collections.singleton(new InetSocketAddress(host, port))) - .setTimeout(timeout) - .build(); - return new RedisClusterStreamsDataSource(config, props, fields, serializer); - } else { - JedisPoolConfig config = new JedisPoolConfig(host, port, timeout, password, dbIdx); - return new RedisStreamsDataSource(config, props, fields, serializer); - } - } - - private static class SqlRedisStoreMapper implements RedisStoreMapper { - private final RedisDataTypeDescription dataTypeDescription; - private final FieldInfo primaryKeyField; - private final IOutputSerializer outputSerializer; - - private SqlRedisStoreMapper(RedisDataTypeDescription dataTypeDescription, List fields, - IOutputSerializer outputSerializer) { - this.dataTypeDescription = dataTypeDescription; - this.outputSerializer = outputSerializer; - - // find primary key from constructor - FieldInfo pkField = findPrimaryKeyField(fields); - Preconditions.checkArgument(pkField != null, "Primary key must be presented to field list"); - - this.primaryKeyField = pkField; - } - - private FieldInfo findPrimaryKeyField(List fields) { - FieldInfo pkField = null; - for (FieldInfo field : fields) { - if (field.isPrimary()) { - // TODO: this assumes key is only from the one field - // if not we need to have order of fields in PK - pkField = field; - break; - } - } - return pkField; - } - - @Override - public RedisDataTypeDescription getDataTypeDescription() { - return dataTypeDescription; - } - - @Override - public String getKeyFromTuple(ITuple tuple) { - Object key = tuple.getValue(0); - if (key == null) { - throw new NullPointerException("key field is null"); - } - return String.valueOf(key); - } - - @Override - public String getValueFromTuple(ITuple tuple) { - Values values = (Values) tuple.getValue(1); - byte[] array = outputSerializer.write(values, null).array(); - return new String(array); - } - } -} diff --git a/sql/storm-sql-external/storm-sql-redis/src/resources/META-INF/services/org.apache.storm.sql.runtime.DataSourcesProvider b/sql/storm-sql-external/storm-sql-redis/src/resources/META-INF/services/org.apache.storm.sql.runtime.DataSourcesProvider deleted file mode 100644 index 23b044404fa..00000000000 --- a/sql/storm-sql-external/storm-sql-redis/src/resources/META-INF/services/org.apache.storm.sql.runtime.DataSourcesProvider +++ /dev/null @@ -1,16 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -org.apache.storm.sql.redis.RedisDataSourcesProvider \ No newline at end of file diff --git a/sql/storm-sql-external/storm-sql-redis/src/test/org/apache/storm/sql/redis/TestRedisDataSourcesProvider.java b/sql/storm-sql-external/storm-sql-redis/src/test/org/apache/storm/sql/redis/TestRedisDataSourcesProvider.java deleted file mode 100644 index 6bcc30c547e..00000000000 --- a/sql/storm-sql-external/storm-sql-redis/src/test/org/apache/storm/sql/redis/TestRedisDataSourcesProvider.java +++ /dev/null @@ -1,78 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.storm.sql.redis; - -import com.google.common.collect.ImmutableList; -import org.apache.storm.redis.bolt.RedisStoreBolt; -import org.apache.storm.sql.runtime.DataSourcesRegistry; -import org.apache.storm.sql.runtime.FieldInfo; -import org.apache.storm.sql.runtime.ISqlStreamsDataSource; -import org.apache.storm.sql.runtime.serde.json.JsonSerializer; -import org.apache.storm.topology.IRichBolt; -import org.junit.jupiter.api.Test; - -import java.net.URI; -import java.util.List; -import java.util.Properties; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNotNull; - -public class TestRedisDataSourcesProvider { - - private static final List FIELDS = ImmutableList.of( - new FieldInfo("ID", int.class, true), - new FieldInfo("val", String.class, false)); - private static final List FIELD_NAMES = ImmutableList.of("ID", "val"); - private static final String ADDITIONAL_KEY = "hello"; - private static final JsonSerializer SERIALIZER = new JsonSerializer(FIELD_NAMES); - private static final Properties TBL_PROPERTIES = new Properties(); - private static final Properties CLUSTER_TBL_PROPERTIES = new Properties(); - - static { - TBL_PROPERTIES.put("data.type", "HASH"); - TBL_PROPERTIES.put("data.additional.key", ADDITIONAL_KEY); - CLUSTER_TBL_PROPERTIES.put("data.type", "HASH"); - CLUSTER_TBL_PROPERTIES.put("data.additional.key", ADDITIONAL_KEY); - CLUSTER_TBL_PROPERTIES.put("use.redis.cluster", "true"); - } - - @SuppressWarnings("unchecked") - @Test - public void testRedisSink() throws Exception { - ISqlStreamsDataSource ds = DataSourcesRegistry.constructStreamsDataSource( - URI.create("redis://:foobared@localhost:6380/2"), null, null, TBL_PROPERTIES, FIELDS); - assertNotNull(ds); - - IRichBolt consumer = ds.getConsumer(); - - assertEquals(RedisStoreBolt.class, consumer.getClass()); - } - - @SuppressWarnings("unchecked") - @Test - public void testRedisClusterSink() throws Exception { - ISqlStreamsDataSource ds = DataSourcesRegistry.constructStreamsDataSource( - URI.create("redis://localhost:6380"), null, null, CLUSTER_TBL_PROPERTIES, FIELDS); - assertNotNull(ds); - - IRichBolt consumer = ds.getConsumer(); - - assertEquals(RedisStoreBolt.class, consumer.getClass()); - } -} diff --git a/sql/storm-sql-runtime/pom.xml b/sql/storm-sql-runtime/pom.xml deleted file mode 100644 index 1982f00337f..00000000000 --- a/sql/storm-sql-runtime/pom.xml +++ /dev/null @@ -1,169 +0,0 @@ - - - - 4.0.0 - - - storm - org.apache.storm - 2.8.3-SNAPSHOT - ../../pom.xml - - - storm-sql-runtime - - - - 33.4.8-jre - - - - - haohui - Haohui Mai - ricetons@gmail.com - - - - - - org.apache.storm - storm-client - ${project.version} - ${provided.scope} - - - org.apache.calcite - calcite-core - ${calcite.version} - - - org.apache.calcite.avatica - avatica-metrics - - - com.fasterxml.jackson.core - jackson-core - - - com.fasterxml.jackson.core - jackson-annotations - - - com.fasterxml.jackson.core - jackson-databind - - - org.apache.httpcomponents - httpclient - - - org.apache.httpcomponents - httpcore - - - com.google.protobuf - protobuf-java - - - commons-dbcp - commons-dbcp - - - com.google.code.findbugs - jsr305 - - - org.pentaho - pentaho-aggdesigner-algorithm - - - ch.qos.reload4j - reload4j - - - org.slf4j - slf4j-reload4j - - - - - com.fasterxml.jackson.core - jackson-databind - ${jackson.databind.version} - - - org.apache.avro - avro - 1.12.0 - - - commons-lang - commons-lang - - - org.apache.commons - commons-csv - 1.14.0 - - - org.mockito - mockito-core - - - org.hamcrest - hamcrest - - - com.google.guava - guava - ${guava.version} - - - - src/jvm - src/test - - - ${basedir}/src/resources - - - - - org.apache.maven.plugins - maven-jar-plugin - - - - test-jar - - - - - - org.apache.maven.plugins - maven-checkstyle-plugin - - - - org.apache.maven.plugins - maven-pmd-plugin - - - - diff --git a/sql/storm-sql-runtime/src/jvm/org/apache/calcite/interpreter/StormContext.java b/sql/storm-sql-runtime/src/jvm/org/apache/calcite/interpreter/StormContext.java deleted file mode 100644 index 700284c12e9..00000000000 --- a/sql/storm-sql-runtime/src/jvm/org/apache/calcite/interpreter/StormContext.java +++ /dev/null @@ -1,32 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.calcite.interpreter; - -import java.io.Serializable; - -import org.apache.calcite.DataContext; - -/** - * This is a hack to use Calcite Context. - */ -public class StormContext extends Context implements Serializable { - public StormContext(DataContext root) { - super(root); - } -} diff --git a/sql/storm-sql-runtime/src/jvm/org/apache/storm/sql/runtime/DataSourcesProvider.java b/sql/storm-sql-runtime/src/jvm/org/apache/storm/sql/runtime/DataSourcesProvider.java deleted file mode 100644 index f8966109c32..00000000000 --- a/sql/storm-sql-runtime/src/jvm/org/apache/storm/sql/runtime/DataSourcesProvider.java +++ /dev/null @@ -1,46 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.storm.sql.runtime; - -import java.net.URI; -import java.util.List; -import java.util.Properties; - -public interface DataSourcesProvider { - /** - * Get the scheme of the data source. - * @return the scheme of the data source - */ - String scheme(); - - /** - * Construct a new data source for streams mode. - * - * @param uri The URI that specifies the data source. The format of the URI - * is fully customizable. - * @param inputFormatClass the name of the class that deserializes data. - * It is null when unspecified. - * @param outputFormatClass the name of the class that serializes data. It - * is null when unspecified. - * @param fields The name of the fields and the schema of the table. - */ - ISqlStreamsDataSource constructStreams( - URI uri, String inputFormatClass, String outputFormatClass, - Properties properties, List fields); -} diff --git a/sql/storm-sql-runtime/src/jvm/org/apache/storm/sql/runtime/DataSourcesRegistry.java b/sql/storm-sql-runtime/src/jvm/org/apache/storm/sql/runtime/DataSourcesRegistry.java deleted file mode 100644 index 9fe963e72fd..00000000000 --- a/sql/storm-sql-runtime/src/jvm/org/apache/storm/sql/runtime/DataSourcesRegistry.java +++ /dev/null @@ -1,75 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.storm.sql.runtime; - -import java.net.URI; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Properties; -import java.util.ServiceLoader; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class DataSourcesRegistry { - private static final Logger LOG = LoggerFactory.getLogger( - DataSourcesRegistry.class); - private static final Map providers; - - static { - providers = new HashMap<>(); - ServiceLoader loader = ServiceLoader.load( - DataSourcesProvider.class); - for (DataSourcesProvider p : loader) { - LOG.info("Registering scheme {} with {}", p.scheme(), p); - providers.put(p.scheme(), p); - } - } - - private DataSourcesRegistry() { - } - - /** - * Construct a streams data source. - * @param uri data source uri - * @param inputFormatClass input format class - * @param outputFormatClass output format class - * @param properties Properties - * @param fields fields info list - * @return StreamsDataSource object - */ - public static ISqlStreamsDataSource constructStreamsDataSource( - URI uri, String inputFormatClass, String outputFormatClass, - Properties properties, List fields) { - DataSourcesProvider provider = providers.get(uri.getScheme()); - if (provider == null) { - return null; - } - - return provider.constructStreams(uri, inputFormatClass, outputFormatClass, properties, fields); - } - - /** - * Allow unit tests to inject data sources. - */ - public static Map providerMap() { - return providers; - } -} diff --git a/sql/storm-sql-runtime/src/jvm/org/apache/storm/sql/runtime/FieldInfo.java b/sql/storm-sql-runtime/src/jvm/org/apache/storm/sql/runtime/FieldInfo.java deleted file mode 100644 index c5478519d77..00000000000 --- a/sql/storm-sql-runtime/src/jvm/org/apache/storm/sql/runtime/FieldInfo.java +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.storm.sql.runtime; - -import java.io.Serializable; - -/** - * Describe each column of the field. - */ -public class FieldInfo implements Serializable { - private final String name; - private final Class type; - private final boolean isPrimary; - - /** - * FieldInfo Constructor. - * @param name field name - * @param type filed type - * @param isPrimary primary or not - */ - public FieldInfo(String name, Class type, boolean isPrimary) { - this.name = name; - this.type = type; - this.isPrimary = isPrimary; - } - - public String name() { - return name; - } - - public Class type() { - return type; - } - - public boolean isPrimary() { - return isPrimary; - } -} diff --git a/sql/storm-sql-runtime/src/jvm/org/apache/storm/sql/runtime/IOutputSerializer.java b/sql/storm-sql-runtime/src/jvm/org/apache/storm/sql/runtime/IOutputSerializer.java deleted file mode 100644 index 46d56b747b5..00000000000 --- a/sql/storm-sql-runtime/src/jvm/org/apache/storm/sql/runtime/IOutputSerializer.java +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.storm.sql.runtime; - -import java.nio.ByteBuffer; -import java.util.List; - -public interface IOutputSerializer { - /** - * Serialize the data to a ByteBuffer. The caller can pass in a ByteBuffer so that the serializer can reuse the - * memory. - * - * @return A ByteBuffer contains the serialized result. - */ - ByteBuffer write(List data, ByteBuffer buffer); -} diff --git a/sql/storm-sql-runtime/src/jvm/org/apache/storm/sql/runtime/ISqlStreamsDataSource.java b/sql/storm-sql-runtime/src/jvm/org/apache/storm/sql/runtime/ISqlStreamsDataSource.java deleted file mode 100644 index da569adce5f..00000000000 --- a/sql/storm-sql-runtime/src/jvm/org/apache/storm/sql/runtime/ISqlStreamsDataSource.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.apache.storm.sql.runtime; - -import org.apache.storm.topology.IRichBolt; -import org.apache.storm.topology.IRichSpout; - -/** - * An ISqlStreamsDataSource specifies how an external data source produces and consumes data. - */ -public interface ISqlStreamsDataSource { - /** - * Provides instance of IRichSpout which can be used as producer in topology. - * - * @see org.apache.storm.topology.IRichSpout - */ - IRichSpout getProducer(); - - /** - * Provides instance of IRichBolt which can be used as consumer in topology. - */ - IRichBolt getConsumer(); -} \ No newline at end of file diff --git a/sql/storm-sql-runtime/src/jvm/org/apache/storm/sql/runtime/calcite/DebuggableExecutableExpression.java b/sql/storm-sql-runtime/src/jvm/org/apache/storm/sql/runtime/calcite/DebuggableExecutableExpression.java deleted file mode 100644 index e78f354549f..00000000000 --- a/sql/storm-sql-runtime/src/jvm/org/apache/storm/sql/runtime/calcite/DebuggableExecutableExpression.java +++ /dev/null @@ -1,45 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.storm.sql.runtime.calcite; - -import org.apache.calcite.interpreter.Context; - -public class DebuggableExecutableExpression implements ExecutableExpression { - private ExecutableExpression delegate; - private String delegateCode; - - public DebuggableExecutableExpression(ExecutableExpression delegate, String delegateCode) { - this.delegate = delegate; - this.delegateCode = delegateCode; - } - - @Override - public Object execute(Context context) { - return delegate.execute(context); - } - - @Override - public void execute(Context context, Object[] results) { - delegate.execute(context, results); - } - - public String getDelegateCode() { - return delegateCode; - } -} diff --git a/sql/storm-sql-runtime/src/jvm/org/apache/storm/sql/runtime/calcite/ExecutableExpression.java b/sql/storm-sql-runtime/src/jvm/org/apache/storm/sql/runtime/calcite/ExecutableExpression.java deleted file mode 100644 index 699a0560489..00000000000 --- a/sql/storm-sql-runtime/src/jvm/org/apache/storm/sql/runtime/calcite/ExecutableExpression.java +++ /dev/null @@ -1,32 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.storm.sql.runtime.calcite; - -import java.io.Serializable; - -import org.apache.calcite.interpreter.Context; - -/** - * Compiled executable expression. - */ -public interface ExecutableExpression extends Serializable { - Object execute(Context context); - - void execute(Context context, Object[] results); -} diff --git a/sql/storm-sql-runtime/src/jvm/org/apache/storm/sql/runtime/calcite/StormDataContext.java b/sql/storm-sql-runtime/src/jvm/org/apache/storm/sql/runtime/calcite/StormDataContext.java deleted file mode 100644 index 6f506cc860f..00000000000 --- a/sql/storm-sql-runtime/src/jvm/org/apache/storm/sql/runtime/calcite/StormDataContext.java +++ /dev/null @@ -1,84 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.storm.sql.runtime.calcite; - -import com.google.common.collect.ImmutableMap; - -import java.io.Serializable; -import java.util.Calendar; -import java.util.TimeZone; - -import org.apache.calcite.DataContext; -import org.apache.calcite.adapter.java.JavaTypeFactory; -import org.apache.calcite.linq4j.QueryProvider; -import org.apache.calcite.runtime.Hook; -import org.apache.calcite.schema.SchemaPlus; -import org.apache.calcite.util.Holder; - -/** - * This is based on SlimDataContext in Calcite, and borrow some from DataContextImpl in Calcite. - */ -public class StormDataContext implements DataContext, Serializable { - private final ImmutableMap map; - - /** - * StormDataContext Constructor. - */ - public StormDataContext() { - // Store the time at which the query started executing. The SQL - // standard says that functions such as CURRENT_TIMESTAMP return the - // same value throughout the query. - - final Holder timeHolder = Holder.of(System.currentTimeMillis()); - - // Give a hook chance to alter the clock. - Hook.CURRENT_TIME.run(timeHolder); - final long time = timeHolder.get(); - final TimeZone timeZone = Calendar.getInstance().getTimeZone(); - final long localOffset = timeZone.getOffset(time); - final long currentOffset = localOffset; - - ImmutableMap.Builder builder = ImmutableMap.builder(); - builder.put(Variable.UTC_TIMESTAMP.camelName, time) - .put(Variable.CURRENT_TIMESTAMP.camelName, time + currentOffset) - .put(Variable.LOCAL_TIMESTAMP.camelName, time + localOffset) - .put(Variable.TIME_ZONE.camelName, timeZone); - map = builder.build(); - } - - @Override - public SchemaPlus getRootSchema() { - return null; - } - - @Override - public JavaTypeFactory getTypeFactory() { - return null; - } - - @Override - public QueryProvider getQueryProvider() { - return null; - } - - @Override - public Object get(String name) { - return map.get(name); - } -} diff --git a/sql/storm-sql-runtime/src/jvm/org/apache/storm/sql/runtime/datasource/socket/SocketDataSourcesProvider.java b/sql/storm-sql-runtime/src/jvm/org/apache/storm/sql/runtime/datasource/socket/SocketDataSourcesProvider.java deleted file mode 100644 index 48f3273ba0e..00000000000 --- a/sql/storm-sql-runtime/src/jvm/org/apache/storm/sql/runtime/datasource/socket/SocketDataSourcesProvider.java +++ /dev/null @@ -1,90 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.storm.sql.runtime.datasource.socket; - -import java.net.URI; -import java.util.List; -import java.util.Properties; - -import org.apache.storm.spout.Scheme; -import org.apache.storm.sql.runtime.DataSourcesProvider; -import org.apache.storm.sql.runtime.FieldInfo; -import org.apache.storm.sql.runtime.IOutputSerializer; -import org.apache.storm.sql.runtime.ISqlStreamsDataSource; -import org.apache.storm.sql.runtime.datasource.socket.bolt.SocketBolt; -import org.apache.storm.sql.runtime.datasource.socket.spout.SocketSpout; -import org.apache.storm.sql.runtime.utils.FieldInfoUtils; -import org.apache.storm.sql.runtime.utils.SerdeUtils; -import org.apache.storm.topology.IRichBolt; -import org.apache.storm.topology.IRichSpout; - -/** - * Create a Socket data source based on the URI and properties. The URI has the format of - * socket://[host]:[port]. Both of host and port are mandatory. - *

- * Note that it connects to given host and port, and receive the message if it's used for input source, - * and send the message if it's used for output data source. - */ -public class SocketDataSourcesProvider implements DataSourcesProvider { - @Override - public String scheme() { - return "socket"; - } - - private static class SocketStreamsDataSource implements ISqlStreamsDataSource { - - private final String host; - private final int port; - private final Scheme scheme; - private final IOutputSerializer serializer; - - SocketStreamsDataSource(String host, int port, Scheme scheme, IOutputSerializer serializer) { - this.host = host; - this.port = port; - this.scheme = scheme; - this.serializer = serializer; - } - - @Override - public IRichSpout getProducer() { - return new SocketSpout(scheme, host, port); - } - - @Override - public IRichBolt getConsumer() { - return new SocketBolt(serializer, host, port); - } - } - - @Override - public ISqlStreamsDataSource constructStreams(URI uri, String inputFormatClass, String outputFormatClass, - Properties properties, List fields) { - String host = uri.getHost(); - int port = uri.getPort(); - if (port == -1) { - throw new RuntimeException("Port information is not available. URI: " + uri); - } - - List fieldNames = FieldInfoUtils.getFieldNames(fields); - Scheme scheme = SerdeUtils.getScheme(inputFormatClass, properties, fieldNames); - IOutputSerializer serializer = SerdeUtils.getSerializer(outputFormatClass, properties, fieldNames); - - return new SocketDataSourcesProvider.SocketStreamsDataSource(host, port, scheme, serializer); - } -} diff --git a/sql/storm-sql-runtime/src/jvm/org/apache/storm/sql/runtime/datasource/socket/bolt/SocketBolt.java b/sql/storm-sql-runtime/src/jvm/org/apache/storm/sql/runtime/datasource/socket/bolt/SocketBolt.java deleted file mode 100644 index 78adef33c73..00000000000 --- a/sql/storm-sql-runtime/src/jvm/org/apache/storm/sql/runtime/datasource/socket/bolt/SocketBolt.java +++ /dev/null @@ -1,105 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.apache.storm.sql.runtime.datasource.socket.bolt; - -import java.io.BufferedWriter; -import java.io.IOException; -import java.io.OutputStreamWriter; -import java.net.Socket; -import java.util.Map; - -import org.apache.storm.shade.org.apache.commons.io.IOUtils; -import org.apache.storm.sql.runtime.IOutputSerializer; -import org.apache.storm.task.OutputCollector; -import org.apache.storm.task.TopologyContext; -import org.apache.storm.topology.IRichBolt; -import org.apache.storm.topology.OutputFieldsDeclarer; -import org.apache.storm.topology.base.BaseRichBolt; -import org.apache.storm.tuple.Tuple; -import org.apache.storm.tuple.Values; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * The Bolt implementation for Socket. Only available for Storm SQL. - * The class doesn't handle reconnection so you may not want to use this for production. - */ -public class SocketBolt extends BaseRichBolt { - private static final Logger LOG = LoggerFactory.getLogger(SocketBolt.class); - - private final IOutputSerializer serializer; - private final String host; - private final int port; - - private transient OutputCollector collector; - private transient BufferedWriter writer; - private transient Socket socket; - - /** - * Constructor. - */ - public SocketBolt(IOutputSerializer serializer, String host, int port) { - this.serializer = serializer; - this.host = host; - this.port = port; - } - - @Override - public void prepare(Map topoConf, TopologyContext context, OutputCollector collector) { - this.collector = collector; - - try { - this.socket = new Socket(host, port); - this.writer = new BufferedWriter(new OutputStreamWriter(socket.getOutputStream())); - } catch (IOException e) { - throw new RuntimeException("Exception while initializing socket for State. host " - + host + " port " + port, e); - } - } - - @Override - public void execute(Tuple input) { - Values values = (Values) input.getValue(0); - byte[] array = serializer.write(values, null).array(); - String data = new String(array); - - try { - writer.write(data + "\n"); - writer.flush(); - collector.ack(input); - } catch (IOException e) { - LOG.error("Error while writing data to socket.", e); - collector.reportError(e); - collector.fail(input); - } - } - - @Override - public void cleanup() { - IOUtils.closeQuietly(writer); - IOUtils.closeQuietly(socket); - } - - @Override - public void declareOutputFields(OutputFieldsDeclarer declarer) { - - } - -} \ No newline at end of file diff --git a/sql/storm-sql-runtime/src/jvm/org/apache/storm/sql/runtime/datasource/socket/spout/SocketSpout.java b/sql/storm-sql-runtime/src/jvm/org/apache/storm/sql/runtime/datasource/socket/spout/SocketSpout.java deleted file mode 100644 index 38cb8e7610d..00000000000 --- a/sql/storm-sql-runtime/src/jvm/org/apache/storm/sql/runtime/datasource/socket/spout/SocketSpout.java +++ /dev/null @@ -1,195 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.apache.storm.sql.runtime.datasource.socket.spout; - -import com.fasterxml.jackson.databind.ObjectMapper; - -import java.io.BufferedReader; -import java.io.Closeable; -import java.io.IOException; -import java.io.InputStreamReader; -import java.net.Socket; -import java.nio.ByteBuffer; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.UUID; -import java.util.concurrent.BlockingDeque; -import java.util.concurrent.LinkedBlockingDeque; - -import org.apache.storm.Config; -import org.apache.storm.spout.Scheme; -import org.apache.storm.spout.SpoutOutputCollector; -import org.apache.storm.task.TopologyContext; -import org.apache.storm.topology.IRichSpout; -import org.apache.storm.topology.OutputFieldsDeclarer; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * Spout for Socket data. Only available for Storm SQL. - * The class doesn't handle reconnection, so you may not want to use this for production. - */ -public class SocketSpout implements IRichSpout { - private static final Logger LOG = LoggerFactory.getLogger(SocketSpout.class); - - private final String host; - private final int port; - private final Scheme scheme; - - private volatile boolean running; - - private BlockingDeque> queue; - private Socket socket; - private Thread readerThread; - private BufferedReader in; - private ObjectMapper objectMapper; - - private SpoutOutputCollector collector; - private Map> emitted; - - /** - * SocketSpout Constructor. - * @param scheme Scheme - * @param host socket host - * @param port socket port - */ - public SocketSpout(Scheme scheme, String host, int port) { - this.scheme = scheme; - this.host = host; - this.port = port; - } - - @Override - public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) { - this.collector = collector; - this.queue = new LinkedBlockingDeque<>(); - this.emitted = new HashMap<>(); - this.objectMapper = new ObjectMapper(); - - try { - socket = new Socket(host, port); - in = new BufferedReader(new InputStreamReader(socket.getInputStream())); - } catch (IOException e) { - throw new RuntimeException("Error opening socket: host " + host + " port " + port); - } - - readerThread = new Thread(new SocketSpout.SocketReaderRunnable()); - readerThread.start(); - } - - @Override - public void close() { - running = false; - readerThread.interrupt(); - queue.clear(); - - closeQuietly(in); - closeQuietly(socket); - } - - @Override - public void activate() { - running = true; - } - - @Override - public void deactivate() { - running = false; - } - - @Override - public void nextTuple() { - if (queue.peek() != null) { - List values = queue.poll(); - if (values != null) { - String id = UUID.randomUUID().toString(); - emitted.put(id, values); - collector.emit(values, id); - } - } - } - - private List convertLineToTuple(String line) { - return scheme.deserialize(ByteBuffer.wrap(line.getBytes())); - } - - @Override - public void ack(Object msgId) { - emitted.remove(msgId); - } - - @Override - public void fail(Object msgId) { - List emittedValues = emitted.remove(msgId); - if (emittedValues != null) { - queue.addLast(emittedValues); - } - } - - @Override - public void declareOutputFields(OutputFieldsDeclarer declarer) { - declarer.declare(scheme.getOutputFields()); - } - - @Override - public Map getComponentConfiguration() { - Config conf = new Config(); - conf.setMaxTaskParallelism(1); - return conf; - } - - private class SocketReaderRunnable implements Runnable { - @Override - public void run() { - while (running) { - try { - String line = in.readLine(); - if (line == null) { - throw new RuntimeException("EOF reached from the socket. We can't read the data any more."); - } - - List values = convertLineToTuple(line.trim()); - queue.push(values); - } catch (Throwable t) { - // This spout is added to test purpose, so just failing fast doesn't hurt much - die(t); - } - } - } - } - - private void die(Throwable t) { - LOG.error("Halting process: TridentSocketSpout died.", t); - if (running || (t instanceof Error)) { //don't exit if not running, unless it is an Error - System.exit(11); - } - } - - private void closeQuietly(final Closeable closeable) { - try { - if (closeable != null) { - closeable.close(); - } - } catch (final IOException ioe) { - // ignore - } - } -} \ No newline at end of file diff --git a/sql/storm-sql-runtime/src/jvm/org/apache/storm/sql/runtime/serde/avro/AvroScheme.java b/sql/storm-sql-runtime/src/jvm/org/apache/storm/sql/runtime/serde/avro/AvroScheme.java deleted file mode 100644 index 147afe67283..00000000000 --- a/sql/storm-sql-runtime/src/jvm/org/apache/storm/sql/runtime/serde/avro/AvroScheme.java +++ /dev/null @@ -1,80 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.storm.sql.runtime.serde.avro; - -import java.io.IOException; -import java.nio.ByteBuffer; -import java.util.ArrayList; -import java.util.List; - -import org.apache.avro.Schema; -import org.apache.avro.generic.GenericDatumReader; -import org.apache.avro.generic.GenericRecord; -import org.apache.avro.io.BinaryDecoder; -import org.apache.avro.io.DatumReader; -import org.apache.avro.io.DecoderFactory; -import org.apache.storm.spout.Scheme; -import org.apache.storm.sql.runtime.utils.SerdeUtils; -import org.apache.storm.tuple.Fields; -import org.apache.storm.utils.Utils; - -/** - * AvroScheme uses generic(without code generation) instead of specific(with code generation) readers. - */ -public class AvroScheme implements Scheme { - private final String schemaString; - private final List fieldNames; - private final CachedSchemas schemas; - - /** - * AvroScheme Constructor. - * @param schemaString schema string - * @param fieldNames field names - */ - public AvroScheme(String schemaString, List fieldNames) { - this.schemaString = schemaString; - this.fieldNames = fieldNames; - this.schemas = new CachedSchemas(); - } - - @Override - public List deserialize(ByteBuffer ser) { - try { - Schema schema = schemas.getSchema(schemaString); - DatumReader reader = new GenericDatumReader<>(schema); - BinaryDecoder decoder = DecoderFactory.get().binaryDecoder(Utils.toByteArray(ser), null); - GenericRecord record = reader.read(null, decoder); - - ArrayList list = new ArrayList<>(fieldNames.size()); - for (String field : fieldNames) { - Object value = record.get(field); - // Avro strings are stored using a special Avro Utf8 type instead of using Java primitives - list.add(SerdeUtils.convertAvroUtf8(value)); - } - return list; - } catch (IOException e) { - throw new RuntimeException(e); - } - } - - @Override - public Fields getOutputFields() { - return new Fields(fieldNames); - } -} diff --git a/sql/storm-sql-runtime/src/jvm/org/apache/storm/sql/runtime/serde/avro/AvroSerializer.java b/sql/storm-sql-runtime/src/jvm/org/apache/storm/sql/runtime/serde/avro/AvroSerializer.java deleted file mode 100644 index 6e4204e041e..00000000000 --- a/sql/storm-sql-runtime/src/jvm/org/apache/storm/sql/runtime/serde/avro/AvroSerializer.java +++ /dev/null @@ -1,79 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.storm.sql.runtime.serde.avro; - -import com.google.common.base.Preconditions; - -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.Serializable; -import java.nio.ByteBuffer; -import java.util.List; - -import org.apache.avro.Schema; -import org.apache.avro.generic.GenericData; -import org.apache.avro.generic.GenericDatumWriter; -import org.apache.avro.generic.GenericRecord; -import org.apache.avro.io.DatumWriter; -import org.apache.avro.io.Encoder; -import org.apache.avro.io.EncoderFactory; -import org.apache.storm.sql.runtime.IOutputSerializer; - -/** - * AvroSerializer uses generic(without code generation) instead of specific(with code generation) writers. - */ -public class AvroSerializer implements IOutputSerializer, Serializable { - private final String schemaString; - private final List fieldNames; - private final CachedSchemas schemas; - - /** - * AvroSerializer Constructor. - * @param schemaString schema string - * @param fieldNames field names - */ - public AvroSerializer(String schemaString, List fieldNames) { - this.schemaString = schemaString; - this.fieldNames = fieldNames; - this.schemas = new CachedSchemas(); - } - - @Override - public ByteBuffer write(List data, ByteBuffer buffer) { - Preconditions.checkArgument(data != null && data.size() == fieldNames.size(), "Invalid schemas"); - try { - Schema schema = schemas.getSchema(schemaString); - GenericRecord record = new GenericData.Record(schema); - for (int i = 0; i < fieldNames.size(); i++) { - record.put(fieldNames.get(i), data.get(i)); - } - - ByteArrayOutputStream out = new ByteArrayOutputStream(); - DatumWriter writer = new GenericDatumWriter<>(record.getSchema()); - Encoder encoder = EncoderFactory.get().directBinaryEncoder(out, null); - writer.write(record, encoder); - encoder.flush(); - byte[] bytes = out.toByteArray(); - out.close(); - return ByteBuffer.wrap(bytes); - } catch (IOException e) { - throw new RuntimeException(e); - } - } -} diff --git a/sql/storm-sql-runtime/src/jvm/org/apache/storm/sql/runtime/serde/avro/CachedSchemas.java b/sql/storm-sql-runtime/src/jvm/org/apache/storm/sql/runtime/serde/avro/CachedSchemas.java deleted file mode 100644 index 28f6e86312a..00000000000 --- a/sql/storm-sql-runtime/src/jvm/org/apache/storm/sql/runtime/serde/avro/CachedSchemas.java +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.storm.sql.runtime.serde.avro; - -import java.io.Serializable; -import java.util.HashMap; -import java.util.Map; - -import org.apache.avro.Schema; - -// TODO this class is reserved for supporting messages with different schemas. -// current only one schema in the cache -public class CachedSchemas implements Serializable { - - private final Map cache = new HashMap<>(); - - /** - * Get a schema based on schema string. - * @param schemaString schema string - * @return Schema object - */ - public Schema getSchema(String schemaString) { - Schema schema = cache.get(schemaString); - if (schema == null) { - schema = new Schema.Parser().parse(schemaString); - cache.put(schemaString, schema); - } - return schema; - } - -} diff --git a/sql/storm-sql-runtime/src/jvm/org/apache/storm/sql/runtime/serde/csv/CsvScheme.java b/sql/storm-sql-runtime/src/jvm/org/apache/storm/sql/runtime/serde/csv/CsvScheme.java deleted file mode 100644 index 1bf3d4ddd87..00000000000 --- a/sql/storm-sql-runtime/src/jvm/org/apache/storm/sql/runtime/serde/csv/CsvScheme.java +++ /dev/null @@ -1,72 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.storm.sql.runtime.serde.csv; - -import com.google.common.base.Preconditions; - -import java.io.IOException; -import java.nio.ByteBuffer; -import java.nio.charset.StandardCharsets; -import java.util.ArrayList; -import java.util.List; - -import org.apache.commons.csv.CSVFormat; -import org.apache.commons.csv.CSVParser; -import org.apache.commons.csv.CSVRecord; -import org.apache.storm.spout.Scheme; -import org.apache.storm.tuple.Fields; -import org.apache.storm.utils.Utils; - -/** - * CsvScheme uses the standard RFC4180 CSV Parser - * One of the difference from Tsv format is that fields with embedded commas will be quoted. - * eg: a,"b,c",d is allowed. - * - * @see RFC4180 - */ -public class CsvScheme implements Scheme { - private final List fieldNames; - - public CsvScheme(List fieldNames) { - this.fieldNames = fieldNames; - } - - @Override - public List deserialize(ByteBuffer ser) { - try { - String data = new String(Utils.toByteArray(ser), StandardCharsets.UTF_8); - CSVParser parser = CSVParser.parse(data, CSVFormat.RFC4180); - CSVRecord record = parser.getRecords().get(0); - Preconditions.checkArgument(record.size() == fieldNames.size(), "Invalid schema"); - - ArrayList list = new ArrayList<>(fieldNames.size()); - for (int i = 0; i < record.size(); i++) { - list.add(record.get(i)); - } - return list; - } catch (IOException e) { - throw new RuntimeException(e); - } - } - - @Override - public Fields getOutputFields() { - return new Fields(fieldNames); - } -} diff --git a/sql/storm-sql-runtime/src/jvm/org/apache/storm/sql/runtime/serde/csv/CsvSerializer.java b/sql/storm-sql-runtime/src/jvm/org/apache/storm/sql/runtime/serde/csv/CsvSerializer.java deleted file mode 100644 index cbcdb3c98f5..00000000000 --- a/sql/storm-sql-runtime/src/jvm/org/apache/storm/sql/runtime/serde/csv/CsvSerializer.java +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.storm.sql.runtime.serde.csv; - -import java.io.IOException; -import java.io.Serializable; -import java.io.StringWriter; -import java.nio.ByteBuffer; -import java.nio.charset.StandardCharsets; -import java.util.List; - -import org.apache.commons.csv.CSVFormat; -import org.apache.commons.csv.CSVPrinter; -import org.apache.storm.sql.runtime.IOutputSerializer; - -/** - * CsvSerializer uses the standard RFC4180 CSV Parser - * One of the difference from Tsv format is that fields with embedded commas will be quoted. - * eg: a,"b,c",d is allowed. - * - * @see RFC4180 - */ -public class CsvSerializer implements IOutputSerializer, Serializable { - private final List fields; //reserved for future - - public CsvSerializer(List fields) { - this.fields = fields; - } - - @Override - public ByteBuffer write(List data, ByteBuffer buffer) { - try { - StringWriter writer = new StringWriter(); - CSVPrinter printer = new CSVPrinter(writer, CSVFormat.RFC4180); - for (Object o : data) { - printer.print(o); - } - //since using StringWriter, we do not need to close it. - return ByteBuffer.wrap(writer.getBuffer().toString().getBytes(StandardCharsets.UTF_8)); - } catch (IOException e) { - throw new RuntimeException(e); - } - } -} diff --git a/sql/storm-sql-runtime/src/jvm/org/apache/storm/sql/runtime/serde/json/JsonScheme.java b/sql/storm-sql-runtime/src/jvm/org/apache/storm/sql/runtime/serde/json/JsonScheme.java deleted file mode 100644 index db202fce979..00000000000 --- a/sql/storm-sql-runtime/src/jvm/org/apache/storm/sql/runtime/serde/json/JsonScheme.java +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.storm.sql.runtime.serde.json; - -import com.fasterxml.jackson.databind.ObjectMapper; - -import java.io.IOException; -import java.nio.ByteBuffer; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; - -import org.apache.storm.spout.Scheme; -import org.apache.storm.tuple.Fields; -import org.apache.storm.utils.Utils; - -public class JsonScheme implements Scheme { - private final List fields; - - public JsonScheme(List fields) { - this.fields = fields; - } - - @Override - public List deserialize(ByteBuffer ser) { - ObjectMapper mapper = new ObjectMapper(); - try { - @SuppressWarnings("unchecked") - HashMap map = mapper.readValue(Utils.toByteArray(ser), HashMap.class); - ArrayList list = new ArrayList<>(fields.size()); - for (String f : fields) { - list.add(map.get(f)); - } - return list; - } catch (IOException e) { - throw new RuntimeException(e); - } - } - - @Override - public Fields getOutputFields() { - return new Fields(fields); - } -} diff --git a/sql/storm-sql-runtime/src/jvm/org/apache/storm/sql/runtime/serde/json/JsonSerializer.java b/sql/storm-sql-runtime/src/jvm/org/apache/storm/sql/runtime/serde/json/JsonSerializer.java deleted file mode 100644 index 0f40fdf8370..00000000000 --- a/sql/storm-sql-runtime/src/jvm/org/apache/storm/sql/runtime/serde/json/JsonSerializer.java +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.storm.sql.runtime.serde.json; - -import com.fasterxml.jackson.core.JsonFactory; -import com.fasterxml.jackson.core.JsonGenerator; -import com.google.common.base.Preconditions; - -import java.io.IOException; -import java.io.Serializable; -import java.io.StringWriter; -import java.nio.ByteBuffer; -import java.nio.charset.StandardCharsets; -import java.util.List; - -import org.apache.storm.sql.runtime.IOutputSerializer; - -public class JsonSerializer implements IOutputSerializer, Serializable { - private final List fieldNames; - private final JsonFactory jsonFactory; - - public JsonSerializer(List fieldNames) { - this.fieldNames = fieldNames; - jsonFactory = new JsonFactory(); - } - - @Override - public ByteBuffer write(List data, ByteBuffer buffer) { - Preconditions.checkArgument(data != null && data.size() == fieldNames.size(), "Invalid schema"); - StringWriter sw = new StringWriter(); - try (JsonGenerator jg = jsonFactory.createGenerator(sw)) { - jg.writeStartObject(); - for (int i = 0; i < fieldNames.size(); ++i) { - jg.writeFieldName(fieldNames.get(i)); - jg.writeObject(data.get(i)); - } - jg.writeEndObject(); - } catch (IOException e) { - throw new RuntimeException(e); - } - return ByteBuffer.wrap(sw.toString().getBytes(StandardCharsets.UTF_8)); - } -} diff --git a/sql/storm-sql-runtime/src/jvm/org/apache/storm/sql/runtime/serde/tsv/TsvScheme.java b/sql/storm-sql-runtime/src/jvm/org/apache/storm/sql/runtime/serde/tsv/TsvScheme.java deleted file mode 100644 index 7d9a896fb13..00000000000 --- a/sql/storm-sql-runtime/src/jvm/org/apache/storm/sql/runtime/serde/tsv/TsvScheme.java +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.storm.sql.runtime.serde.tsv; - -import com.google.common.base.Preconditions; - -import java.nio.ByteBuffer; -import java.nio.charset.StandardCharsets; -import java.util.ArrayList; -import java.util.List; - -import org.apache.storm.spout.Scheme; -import org.apache.storm.tuple.Fields; -import org.apache.storm.utils.Utils; - -/** - * TsvScheme uses a simple delimited format implemention by splitting string, - * and it supports user defined delimiter. - */ -public class TsvScheme implements Scheme { - private final List fieldNames; - private final char delimiter; - - public TsvScheme(List fieldNames, char delimiter) { - this.fieldNames = fieldNames; - this.delimiter = delimiter; - } - - @Override - public List deserialize(ByteBuffer ser) { - String data = new String(Utils.toByteArray(ser), StandardCharsets.UTF_8); - List parts = org.apache.storm.sql.runtime.utils.Utils.split(data, delimiter); - Preconditions.checkArgument(parts.size() == fieldNames.size(), "Invalid schema"); - - ArrayList list = new ArrayList<>(fieldNames.size()); - list.addAll(parts); - return list; - } - - @Override - public Fields getOutputFields() { - return new Fields(fieldNames); - } -} diff --git a/sql/storm-sql-runtime/src/jvm/org/apache/storm/sql/runtime/serde/tsv/TsvSerializer.java b/sql/storm-sql-runtime/src/jvm/org/apache/storm/sql/runtime/serde/tsv/TsvSerializer.java deleted file mode 100644 index d5b41acd017..00000000000 --- a/sql/storm-sql-runtime/src/jvm/org/apache/storm/sql/runtime/serde/tsv/TsvSerializer.java +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.storm.sql.runtime.serde.tsv; - -import java.io.Serializable; -import java.nio.ByteBuffer; -import java.nio.charset.StandardCharsets; -import java.util.List; - -import org.apache.storm.sql.runtime.IOutputSerializer; - -/** - * TsvSerializer uses a simple delimited format implemention by splitting string, - * and it supports user defined delimiter. - */ -public class TsvSerializer implements IOutputSerializer, Serializable { - private final List fields; //reserved for future - private final char delimiter; - - public TsvSerializer(List fields, char delimiter) { - this.fields = fields; - this.delimiter = delimiter; - } - - @Override - public ByteBuffer write(List data, ByteBuffer buffer) { - StringBuilder sb = new StringBuilder(512); // 512: for most scenes to avoid inner array resizing - for (int i = 0; i < data.size(); i++) { - Object o = data.get(i); - if (i == 0) { - sb.append(o); - } else { - sb.append(delimiter); - sb.append(o); - } - } - return ByteBuffer.wrap(sb.toString().getBytes(StandardCharsets.UTF_8)); - } -} diff --git a/sql/storm-sql-runtime/src/jvm/org/apache/storm/sql/runtime/streams/functions/EvaluationCalc.java b/sql/storm-sql-runtime/src/jvm/org/apache/storm/sql/runtime/streams/functions/EvaluationCalc.java deleted file mode 100644 index 215e47fcf07..00000000000 --- a/sql/storm-sql-runtime/src/jvm/org/apache/storm/sql/runtime/streams/functions/EvaluationCalc.java +++ /dev/null @@ -1,85 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.apache.storm.sql.runtime.streams.functions; - -import java.util.Collections; -import java.util.Map; - -import org.apache.calcite.DataContext; -import org.apache.calcite.interpreter.Context; -import org.apache.calcite.interpreter.StormContext; -import org.apache.storm.sql.runtime.calcite.DebuggableExecutableExpression; -import org.apache.storm.sql.runtime.calcite.ExecutableExpression; -import org.apache.storm.streams.operations.FlatMapFunction; -import org.apache.storm.tuple.Values; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class EvaluationCalc implements FlatMapFunction { - private static final Logger LOG = LoggerFactory.getLogger(EvaluationCalc.class); - - private final ExecutableExpression filterInstance; - private final ExecutableExpression projectionInstance; - private final Object[] outputValues; - private final DataContext dataContext; - - /** - * EvaluationCalc Constructor. - * @param filterInstance ExecutableExpression - * @param projectionInstance ExecutableExpression - * @param outputCount output count - * @param dataContext DataContext - */ - public EvaluationCalc(ExecutableExpression filterInstance, ExecutableExpression projectionInstance, - int outputCount, DataContext dataContext) { - this.filterInstance = filterInstance; - this.projectionInstance = projectionInstance; - this.outputValues = new Object[outputCount]; - this.dataContext = dataContext; - - if (projectionInstance != null && projectionInstance instanceof DebuggableExecutableExpression) { - LOG.info("Expression code for projection: \n{}", ((DebuggableExecutableExpression) projectionInstance).getDelegateCode()); - } - if (filterInstance != null && filterInstance instanceof DebuggableExecutableExpression) { - LOG.info("Expression code for filter: \n{}", ((DebuggableExecutableExpression) filterInstance).getDelegateCode()); - } - } - - @Override - public Iterable apply(Values input) { - Context calciteContext = new StormContext(dataContext); - calciteContext.values = input.toArray(); - - if (filterInstance != null) { - filterInstance.execute(calciteContext, outputValues); - // filtered out - if (outputValues[0] == null || !((Boolean) outputValues[0])) { - return Collections.emptyList(); - } - } - - if (projectionInstance != null) { - projectionInstance.execute(calciteContext, outputValues); - return Collections.singletonList(new Values(outputValues)); - } else { - return Collections.singletonList(new Values(input.toArray())); - } - } -} diff --git a/sql/storm-sql-runtime/src/jvm/org/apache/storm/sql/runtime/streams/functions/EvaluationFilter.java b/sql/storm-sql-runtime/src/jvm/org/apache/storm/sql/runtime/streams/functions/EvaluationFilter.java deleted file mode 100644 index c3237fe6911..00000000000 --- a/sql/storm-sql-runtime/src/jvm/org/apache/storm/sql/runtime/streams/functions/EvaluationFilter.java +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.apache.storm.sql.runtime.streams.functions; - -import org.apache.calcite.DataContext; -import org.apache.calcite.interpreter.Context; -import org.apache.calcite.interpreter.StormContext; -import org.apache.storm.sql.runtime.calcite.DebuggableExecutableExpression; -import org.apache.storm.sql.runtime.calcite.ExecutableExpression; -import org.apache.storm.streams.operations.Predicate; -import org.apache.storm.tuple.Values; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class EvaluationFilter implements Predicate { - private static final Logger LOG = LoggerFactory.getLogger(EvaluationFilter.class); - - private final ExecutableExpression filterInstance; - private final DataContext dataContext; - private final Object[] outputValues; - - /** - * EvaluationFilter Constructor. - * @param filterInstance ExecutableExpression - * @param dataContext DataContext - */ - public EvaluationFilter(ExecutableExpression filterInstance, DataContext dataContext) { - this.filterInstance = filterInstance; - this.dataContext = dataContext; - this.outputValues = new Object[1]; - - if (filterInstance != null && filterInstance instanceof DebuggableExecutableExpression) { - LOG.info("Expression code for filter: \n{}", ((DebuggableExecutableExpression) filterInstance).getDelegateCode()); - } - } - - @Override - public boolean test(Values input) { - Context calciteContext = new StormContext(dataContext); - calciteContext.values = input.toArray(); - filterInstance.execute(calciteContext, outputValues); - return (outputValues[0] != null && (boolean) outputValues[0]); - } -} diff --git a/sql/storm-sql-runtime/src/jvm/org/apache/storm/sql/runtime/streams/functions/EvaluationFunction.java b/sql/storm-sql-runtime/src/jvm/org/apache/storm/sql/runtime/streams/functions/EvaluationFunction.java deleted file mode 100644 index b10b4e85c02..00000000000 --- a/sql/storm-sql-runtime/src/jvm/org/apache/storm/sql/runtime/streams/functions/EvaluationFunction.java +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.apache.storm.sql.runtime.streams.functions; - -import java.util.Map; - -import org.apache.calcite.DataContext; -import org.apache.calcite.interpreter.Context; -import org.apache.calcite.interpreter.StormContext; -import org.apache.storm.sql.runtime.calcite.DebuggableExecutableExpression; -import org.apache.storm.sql.runtime.calcite.ExecutableExpression; -import org.apache.storm.streams.operations.Function; -import org.apache.storm.tuple.Values; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class EvaluationFunction implements Function { - private static final Logger LOG = LoggerFactory.getLogger(EvaluationFunction.class); - - private final ExecutableExpression projectionInstance; - private final Object[] outputValues; - private final DataContext dataContext; - - /** - * EvaluationFunction Constructor. - * @param projectionInstance ExecutableExpression - * @param outputCount output count - * @param dataContext DataContext - */ - public EvaluationFunction(ExecutableExpression projectionInstance, int outputCount, DataContext dataContext) { - this.projectionInstance = projectionInstance; - this.outputValues = new Object[outputCount]; - this.dataContext = dataContext; - - if (projectionInstance instanceof DebuggableExecutableExpression) { - LOG.info("Expression code: {}", ((DebuggableExecutableExpression) projectionInstance).getDelegateCode()); - } - } - - @Override - public Values apply(Values input) { - Context calciteContext = new StormContext(dataContext); - calciteContext.values = input.toArray(); - projectionInstance.execute(calciteContext, outputValues); - return new Values(outputValues); - } -} diff --git a/sql/storm-sql-runtime/src/jvm/org/apache/storm/sql/runtime/streams/functions/StreamInsertMapToPairFunction.java b/sql/storm-sql-runtime/src/jvm/org/apache/storm/sql/runtime/streams/functions/StreamInsertMapToPairFunction.java deleted file mode 100644 index fd5035af644..00000000000 --- a/sql/storm-sql-runtime/src/jvm/org/apache/storm/sql/runtime/streams/functions/StreamInsertMapToPairFunction.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.apache.storm.sql.runtime.streams.functions; - -import org.apache.storm.streams.Pair; -import org.apache.storm.streams.operations.PairFunction; -import org.apache.storm.tuple.Values; - -public class StreamInsertMapToPairFunction implements PairFunction { - - private final int primaryKeyIndex; - - public StreamInsertMapToPairFunction(int primaryKeyIndex) { - this.primaryKeyIndex = primaryKeyIndex; - } - - @Override - public Pair apply(Values input) { - return Pair.of(input.get(primaryKeyIndex), input); - } -} \ No newline at end of file diff --git a/sql/storm-sql-runtime/src/jvm/org/apache/storm/sql/runtime/streams/functions/StreamsScanTupleValueMapper.java b/sql/storm-sql-runtime/src/jvm/org/apache/storm/sql/runtime/streams/functions/StreamsScanTupleValueMapper.java deleted file mode 100644 index db5c5dafd5e..00000000000 --- a/sql/storm-sql-runtime/src/jvm/org/apache/storm/sql/runtime/streams/functions/StreamsScanTupleValueMapper.java +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.apache.storm.sql.runtime.streams.functions; - -import java.util.ArrayList; -import java.util.List; - -import org.apache.storm.streams.operations.mappers.TupleValueMapper; -import org.apache.storm.tuple.Tuple; -import org.apache.storm.tuple.Values; - -public class StreamsScanTupleValueMapper implements TupleValueMapper { - private final List fieldNames; - - /** - * Constructor. - * - * @param fieldNames fields to be added to. - */ - public StreamsScanTupleValueMapper(List fieldNames) { - // prevent issue when the implementation of fieldNames is not serializable - // getRowType().getFieldNames() returns Calcite Pair$ which is NOT serializable - this.fieldNames = new ArrayList<>(fieldNames); - } - - @Override - public Values apply(Tuple input) { - Values values = new Values(); - for (String field : fieldNames) { - values.add(input.getValueByField(field)); - } - return values; - } -} \ No newline at end of file diff --git a/sql/storm-sql-runtime/src/jvm/org/apache/storm/sql/runtime/utils/FieldInfoUtils.java b/sql/storm-sql-runtime/src/jvm/org/apache/storm/sql/runtime/utils/FieldInfoUtils.java deleted file mode 100644 index 26017dd7475..00000000000 --- a/sql/storm-sql-runtime/src/jvm/org/apache/storm/sql/runtime/utils/FieldInfoUtils.java +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.storm.sql.runtime.utils; - -import com.google.common.base.Function; -import com.google.common.collect.Lists; - -import java.io.Serializable; -import java.util.List; - -import org.apache.storm.sql.runtime.FieldInfo; - -public final class FieldInfoUtils { - - public static List getFieldNames(List fields) { - return Lists.transform(fields, new FieldNameExtractor()); - } - - private static class FieldNameExtractor implements Function, Serializable { - @Override - public String apply(FieldInfo fieldInfo) { - return fieldInfo.name(); - } - } -} diff --git a/sql/storm-sql-runtime/src/jvm/org/apache/storm/sql/runtime/utils/SerdeUtils.java b/sql/storm-sql-runtime/src/jvm/org/apache/storm/sql/runtime/utils/SerdeUtils.java deleted file mode 100644 index 36e69eb425e..00000000000 --- a/sql/storm-sql-runtime/src/jvm/org/apache/storm/sql/runtime/utils/SerdeUtils.java +++ /dev/null @@ -1,144 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.storm.sql.runtime.utils; - -import static org.apache.commons.lang.StringUtils.isNotEmpty; - -import com.google.common.base.Preconditions; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Properties; - -import org.apache.avro.generic.GenericData; -import org.apache.avro.util.Utf8; -import org.apache.storm.spout.Scheme; -import org.apache.storm.sql.runtime.IOutputSerializer; -import org.apache.storm.sql.runtime.serde.avro.AvroScheme; -import org.apache.storm.sql.runtime.serde.avro.AvroSerializer; -import org.apache.storm.sql.runtime.serde.csv.CsvScheme; -import org.apache.storm.sql.runtime.serde.csv.CsvSerializer; -import org.apache.storm.sql.runtime.serde.json.JsonScheme; -import org.apache.storm.sql.runtime.serde.json.JsonSerializer; -import org.apache.storm.sql.runtime.serde.tsv.TsvScheme; -import org.apache.storm.sql.runtime.serde.tsv.TsvSerializer; -import org.apache.storm.utils.ReflectionUtils; - -public final class SerdeUtils { - /** - * Get a Scheme instance based on specific configurations. - * @param inputFormatClass input format class - * @param properties Properties - * @param fieldNames field names - * @return the Scheme instance - */ - public static Scheme getScheme(String inputFormatClass, Properties properties, List fieldNames) { - Scheme scheme; - if (isNotEmpty(inputFormatClass)) { - if (JsonScheme.class.getName().equals(inputFormatClass)) { - scheme = new JsonScheme(fieldNames); - } else if (TsvScheme.class.getName().equals(inputFormatClass)) { - String delimiter = properties.getProperty("input.tsv.delimiter", "\t"); - scheme = new TsvScheme(fieldNames, delimiter.charAt(0)); - } else if (CsvScheme.class.getName().equals(inputFormatClass)) { - scheme = new CsvScheme(fieldNames); - } else if (AvroScheme.class.getName().equals(inputFormatClass)) { - String schemaString = properties.getProperty("input.avro.schema"); - Preconditions.checkArgument(isNotEmpty(schemaString), "input.avro.schema can not be empty"); - scheme = new AvroScheme(schemaString, fieldNames); - } else { - scheme = ReflectionUtils.newInstance(inputFormatClass); - } - } else { - //use JsonScheme as the default scheme - scheme = new JsonScheme(fieldNames); - } - return scheme; - } - - /** - * Get a OutputSerializer instance based on specific configurations. - * @param outputFormatClass output format class - * @param properties Properties - * @param fieldNames field names - * @return the OutputSerializer instance - */ - public static IOutputSerializer getSerializer(String outputFormatClass, Properties properties, List fieldNames) { - IOutputSerializer serializer; - if (isNotEmpty(outputFormatClass)) { - if (JsonSerializer.class.getName().equals(outputFormatClass)) { - serializer = new JsonSerializer(fieldNames); - } else if (TsvSerializer.class.getName().equals(outputFormatClass)) { - String delimiter = properties.getProperty("output.tsv.delimiter", "\t"); - serializer = new TsvSerializer(fieldNames, delimiter.charAt(0)); - } else if (CsvSerializer.class.getName().equals(outputFormatClass)) { - serializer = new CsvSerializer(fieldNames); - } else if (AvroSerializer.class.getName().equals(outputFormatClass)) { - String schemaString = properties.getProperty("output.avro.schema"); - Preconditions.checkArgument(isNotEmpty(schemaString), "output.avro.schema can not be empty"); - serializer = new AvroSerializer(schemaString, fieldNames); - } else { - serializer = ReflectionUtils.newInstance(outputFormatClass); - } - } else { - //use JsonSerializer as the default serializer - serializer = new JsonSerializer(fieldNames); - } - return serializer; - } - - /** - * Convert a Avro object to a Java object, changing the Avro Utf8 type to Java String. - * @param value Avro object - * @return Java object - */ - public static Object convertAvroUtf8(Object value) { - Object ret; - if (value instanceof Utf8) { - ret = value.toString(); - } else if (value instanceof Map) { - ret = convertAvroUtf8Map((Map) value); - } else if (value instanceof GenericData.Array) { - ret = convertAvroUtf8Array((GenericData.Array) value); - } else { - ret = value; - } - return ret; - } - - private static Object convertAvroUtf8Map(Map value) { - Map map = new HashMap<>(value.size()); - for (Map.Entry entry : value.entrySet()) { - Object k = convertAvroUtf8(entry.getKey()); - Object v = convertAvroUtf8(entry.getValue()); - map.put(k, v); - } - return map; - } - - private static Object convertAvroUtf8Array(GenericData.Array value) { - List ls = new ArrayList<>(value.size()); - for (Object o : value) { - ls.add(convertAvroUtf8(o)); - } - return ls; - } -} diff --git a/sql/storm-sql-runtime/src/jvm/org/apache/storm/sql/runtime/utils/Utils.java b/sql/storm-sql-runtime/src/jvm/org/apache/storm/sql/runtime/utils/Utils.java deleted file mode 100644 index 59de0c0d893..00000000000 --- a/sql/storm-sql-runtime/src/jvm/org/apache/storm/sql/runtime/utils/Utils.java +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.storm.sql.runtime.utils; - -import java.util.LinkedList; -import java.util.List; - -public final class Utils { - - /** - * This method for splitting string into parts by a delimiter. - * It has higher performance than String.split(String regex). - * - * @param data need split string - * @param delimiter the delimiter - * @return string list - */ - public static List split(String data, char delimiter) { - List list = new LinkedList<>(); - //do not use .toCharArray avoid system copy - StringBuilder sb = new StringBuilder(512); - int len = data.length(); - for (int i = 0; i < len; i++) { - char ch = data.charAt(i); - if (ch == delimiter) { - list.add(sb.toString()); - sb.setLength(0); - if (i == len - 1) { - list.add(""); - } - } else { - sb.append(ch); - } - } - if (sb.length() > 0) { - list.add(sb.toString()); - } - return list; - } -} diff --git a/sql/storm-sql-runtime/src/resources/META-INF/services/org.apache.storm.sql.runtime.DataSourcesProvider b/sql/storm-sql-runtime/src/resources/META-INF/services/org.apache.storm.sql.runtime.DataSourcesProvider deleted file mode 100644 index 9a945f7c62a..00000000000 --- a/sql/storm-sql-runtime/src/resources/META-INF/services/org.apache.storm.sql.runtime.DataSourcesProvider +++ /dev/null @@ -1,16 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -org.apache.storm.sql.runtime.datasource.socket.SocketDataSourcesProvider \ No newline at end of file diff --git a/sql/storm-sql-runtime/src/test/org/apache/storm/sql/TestAvroSerializer.java b/sql/storm-sql-runtime/src/test/org/apache/storm/sql/TestAvroSerializer.java deleted file mode 100644 index debb264ebc1..00000000000 --- a/sql/storm-sql-runtime/src/test/org/apache/storm/sql/TestAvroSerializer.java +++ /dev/null @@ -1,72 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.storm.sql; - -import com.google.common.collect.Lists; -import com.google.common.collect.Maps; -import org.apache.storm.sql.runtime.serde.avro.AvroScheme; -import org.apache.storm.sql.runtime.serde.avro.AvroSerializer; -import org.junit.jupiter.api.Test; - -import java.nio.ByteBuffer; -import java.util.List; -import java.util.Map; - -import static org.junit.jupiter.api.Assertions.assertArrayEquals; - -public class TestAvroSerializer { - private static final String schemaString = "{\"type\":\"record\"," + - "\"name\":\"avrotest\"," + - "\"fields\":[{\"name\":\"ID\",\"type\":\"int\"}," + - "{ \"name\":\"val\", \"type\":\"string\" }]}"; - - private static final String schemaString1 = "{\"type\":\"record\"," + - "\"name\":\"avrotest\"," + - "\"fields\":[{\"name\":\"ID\",\"type\":\"int\"}," + - "{ \"type\":{\"type\":\"map\",\"values\": \"long\"}, \"name\":\"val1\" }," + - "{ \"type\":{\"type\":\"array\",\"items\": \"string\"}, \"name\":\"val2\" }]}"; - - @Test - public void testAvroSchemeAndSerializer() { - List fields = Lists.newArrayList("ID", "val"); - List o = Lists.newArrayList(1, "2"); - - AvroSerializer serializer = new AvroSerializer(schemaString, fields); - ByteBuffer byteBuffer = serializer.write(o, null); - - AvroScheme scheme = new AvroScheme(schemaString, fields); - assertArrayEquals(o.toArray(), scheme.deserialize(byteBuffer).toArray()); - } - - @Test - public void testAvroComplexSchemeAndSerializer() { - List fields = Lists.newArrayList("ID", "val1", "val2"); - - Map mp = Maps.newHashMap();; - mp.put("l1",1234L); - mp.put("l2",56789L); - List ls = Lists.newArrayList("s1", "s2"); - List o = Lists.newArrayList(1, mp, ls); - - AvroSerializer serializer = new AvroSerializer(schemaString1, fields); - ByteBuffer byteBuffer = serializer.write(o, null); - - AvroScheme scheme = new AvroScheme(schemaString1, fields); - assertArrayEquals(o.toArray(), scheme.deserialize(byteBuffer).toArray()); - } -} diff --git a/sql/storm-sql-runtime/src/test/org/apache/storm/sql/TestCsvSerializer.java b/sql/storm-sql-runtime/src/test/org/apache/storm/sql/TestCsvSerializer.java deleted file mode 100644 index db8c0272409..00000000000 --- a/sql/storm-sql-runtime/src/test/org/apache/storm/sql/TestCsvSerializer.java +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.storm.sql; - -import com.google.common.collect.Lists; -import org.apache.storm.sql.runtime.serde.csv.CsvScheme; -import org.apache.storm.sql.runtime.serde.csv.CsvSerializer; -import org.junit.jupiter.api.Test; - -import java.nio.ByteBuffer; -import java.util.List; - -import static org.junit.jupiter.api.Assertions.assertArrayEquals; - -public class TestCsvSerializer { - - @Test - public void testCsvSchemeAndSerializer() { - List fields = Lists.newArrayList("ID", "val"); - List o = Lists.newArrayList("1", "2"); - - CsvSerializer serializer = new CsvSerializer(fields); - ByteBuffer byteBuffer = serializer.write(o, null); - - CsvScheme scheme = new CsvScheme(fields); - assertArrayEquals(o.toArray(), scheme.deserialize(byteBuffer).toArray()); - - // Fields with embedded commas or double-quote characters - fields = Lists.newArrayList("ID", "val", "v"); - o = Lists.newArrayList("1,9", "2,\"3\",5", "\"8\""); - - serializer = new CsvSerializer(fields); - byteBuffer = serializer.write(o, null); - - scheme = new CsvScheme(fields); - assertArrayEquals(o.toArray(), scheme.deserialize(byteBuffer).toArray()); - } - -} diff --git a/sql/storm-sql-runtime/src/test/org/apache/storm/sql/TestJsonRepresentation.java b/sql/storm-sql-runtime/src/test/org/apache/storm/sql/TestJsonRepresentation.java deleted file mode 100644 index 980cd9b463a..00000000000 --- a/sql/storm-sql-runtime/src/test/org/apache/storm/sql/TestJsonRepresentation.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.storm.sql; - -import org.apache.storm.sql.runtime.serde.json.JsonScheme; -import org.apache.storm.sql.runtime.serde.json.JsonSerializer; -import org.apache.storm.utils.Utils; -import com.google.common.collect.Lists; -import org.junit.jupiter.api.Test; - -import java.nio.ByteBuffer; -import java.nio.charset.Charset; -import java.util.List; - -import static org.junit.jupiter.api.Assertions.assertArrayEquals; -import static org.junit.jupiter.api.Assertions.assertEquals; - -public class TestJsonRepresentation { - @Test - public void testJsonScheme() { - final List fields = Lists.newArrayList("ID", "val"); - final String s = "{\"ID\": 1, \"val\": \"2\"}"; - JsonScheme scheme = new JsonScheme(fields); - List o = scheme.deserialize(ByteBuffer.wrap(s.getBytes(Charset.defaultCharset()))); - assertArrayEquals(new Object[] {1, "2"}, o.toArray()); - } - - @Test - public void testJsonSerializer() { - final List fields = Lists.newArrayList("ID", "val"); - List o = Lists. newArrayList(1, "2"); - JsonSerializer s = new JsonSerializer(fields); - ByteBuffer buf = s.write(o, null); - byte[] b = Utils.toByteArray(buf); - assertEquals("{\"ID\":1,\"val\":\"2\"}", new String(b)); - } -} diff --git a/sql/storm-sql-runtime/src/test/org/apache/storm/sql/TestTsvSerializer.java b/sql/storm-sql-runtime/src/test/org/apache/storm/sql/TestTsvSerializer.java deleted file mode 100644 index 885c9233c74..00000000000 --- a/sql/storm-sql-runtime/src/test/org/apache/storm/sql/TestTsvSerializer.java +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.storm.sql; - -import com.google.common.collect.Lists; -import org.apache.storm.sql.runtime.serde.tsv.TsvScheme; -import org.apache.storm.sql.runtime.serde.tsv.TsvSerializer; -import org.junit.jupiter.api.Test; - -import java.nio.ByteBuffer; -import java.util.List; - -import static org.junit.jupiter.api.Assertions.assertArrayEquals; - -public class TestTsvSerializer { - - @Test - public void testTsvSchemeAndSerializer() { - final char delimiter = '\t'; - - List fields = Lists.newArrayList("ID", "val"); - List o = Lists.newArrayList("1", "2"); - - TsvSerializer serializer = new TsvSerializer(fields, delimiter); - ByteBuffer byteBuffer = serializer.write(o, null); - - TsvScheme scheme = new TsvScheme(fields, delimiter); - assertArrayEquals(o.toArray(), scheme.deserialize(byteBuffer).toArray()); - } - -} diff --git a/sql/storm-sql-runtime/src/test/org/apache/storm/sql/TestUtils.java b/sql/storm-sql-runtime/src/test/org/apache/storm/sql/TestUtils.java deleted file mode 100644 index a1aa55214c8..00000000000 --- a/sql/storm-sql-runtime/src/test/org/apache/storm/sql/TestUtils.java +++ /dev/null @@ -1,363 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ -package org.apache.storm.sql; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.PriorityQueue; - -import org.apache.storm.spout.SpoutOutputCollector; -import org.apache.storm.sql.runtime.ISqlStreamsDataSource; -import org.apache.storm.streams.Pair; -import org.apache.storm.task.OutputCollector; -import org.apache.storm.task.TopologyContext; -import org.apache.storm.topology.IRichBolt; -import org.apache.storm.topology.IRichSpout; -import org.apache.storm.topology.OutputFieldsDeclarer; -import org.apache.storm.topology.base.BaseRichBolt; -import org.apache.storm.topology.base.BaseRichSpout; -import org.apache.storm.tuple.Fields; -import org.apache.storm.tuple.Tuple; -import org.apache.storm.tuple.Values; -import org.junit.jupiter.api.extension.BeforeEachCallback; -import org.junit.jupiter.api.extension.ExtensionContext; - -public class TestUtils { - public static final class MockInsertBoltExtension implements BeforeEachCallback { - @Override - public void beforeEach(ExtensionContext ctx) throws Exception { - MockInsertBolt.getCollectedValues().clear(); - } - } - - public static final class MockBoltExtension implements BeforeEachCallback { - @Override - public void beforeEach(ExtensionContext arg0) throws Exception { - MockBolt.getCollectedValues().clear(); - } - } - - public static class MyPlus { - public static Integer evaluate(Integer x, Integer y) { - return x + y; - } - } - - public static class MyConcat { - public static String init() { - return ""; - } - public static String add(String accumulator, String val) { - return accumulator + val; - } - public static String result(String accumulator) { - return accumulator; - } - } - - public static class TopN { - public static PriorityQueue init() { - return new PriorityQueue<>(); - } - public static PriorityQueue add(PriorityQueue accumulator, Integer n, Integer val) { - if (n <= 0) { - return accumulator; - } - if (accumulator.size() >= n) { - if (val > accumulator.peek()) { - accumulator.remove(); - accumulator.add(val); - } - } else { - accumulator.add(val); - } - return accumulator; - } - public static List result(PriorityQueue accumulator) { - List res = new ArrayList<>(accumulator); - Collections.reverse(res); - return res; - } - } - - public static class MockSpout extends BaseRichSpout { - - private final List records; - private final Fields outputFields; - private boolean emitted = false; - private SpoutOutputCollector collector; - - public MockSpout(List records, Fields outputFields) { - this.records = records; - this.outputFields = outputFields; - } - - @Override - public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) { - this.collector = collector; - } - - @Override - public void nextTuple() { - if (emitted) { - return; - } - - for (Values r : records) { - collector.emit(r); - } - - emitted = true; - } - - @Override - public void declareOutputFields(OutputFieldsDeclarer declarer) { - declarer.declare(outputFields); - } - } - - public static class MockBolt extends BaseRichBolt { - /** - * Collect all values in a static variable as the instance will go through serialization and deserialization. - * NOTE: This should be cleared before or after running each test. - */ - private transient static final List VALUES = new ArrayList<>(); - - public static List getCollectedValues() { - return VALUES; - } - - @Override - public void prepare(Map topoConf, TopologyContext context, OutputCollector collector) { - - } - - @Override - public void execute(Tuple input) { - VALUES.add((Values) input.getValue(0)); - } - - @Override - public void declareOutputFields(OutputFieldsDeclarer declarer) { - - } - } - - public static class MockInsertBolt extends BaseRichBolt { - /** - * Collect all values in a static variable as the instance will go through serialization and deserialization. - * NOTE: This should be cleared before or after running each test. - */ - private transient static final List> VALUES = new ArrayList<>(); - - public static List> getCollectedValues() { - return VALUES; - } - - @Override - public void prepare(Map topoConf, TopologyContext context, OutputCollector collector) { - - } - - @Override - public void execute(Tuple input) { - VALUES.add(Pair.of(input.getValue(0), (Values) input.getValue(1))); - } - - @Override - public void declareOutputFields(OutputFieldsDeclarer declarer) { - - } - } - - public static class MockSqlExprDataSource implements ISqlStreamsDataSource { - @Override - public IRichSpout getProducer() { - throw new UnsupportedOperationException("Not supported."); - } - - @Override - public IRichBolt getConsumer() { - return new MockBolt(); - } - } - - public static class MockSqlStreamsOutputDataSource implements ISqlStreamsDataSource { - - @Override - public IRichSpout getProducer() { - throw new UnsupportedOperationException("Not supported."); - } - - @Override - public IRichBolt getConsumer() { - return new MockInsertBolt(); - } - - } - - public static class MockSqlStreamsDataSource implements ISqlStreamsDataSource { - @Override - public IRichSpout getProducer() { - List records = new ArrayList<>(); - records.add(new Values(0, "a", "y")); - records.add(new Values(1, "ab", "y")); - records.add(new Values(2, "abc", "y")); - records.add(new Values(3, "abcd", "y")); - records.add(new Values(4, "abcde", "y")); - - Fields outputFields = new Fields("ID", "NAME", "ADDR"); - return new MockSpout(records, outputFields); - } - - @Override - public IRichBolt getConsumer() { - return new MockBolt(); - } - - } - - public static class MockSqlStreamsInsertDataSource extends MockSqlStreamsNestedDataSource { - @Override - public IRichBolt getConsumer() { - return new MockInsertBolt(); - } - } - - public static class MockSqlStreamsGroupedDataSource implements ISqlStreamsDataSource { - @Override - public IRichSpout getProducer() { - List records = new ArrayList<>(); - for (int i = 0; i < 5; ++i) { - records.add(new Values(i, 0, "x", "y", 5 - i, i * 10)); - } - - Fields outputFields = new Fields("ID", "GRPID", "NAME", "ADDR", "AGE", "SCORE"); - return new MockSpout(records, outputFields); - } - - @Override - public IRichBolt getConsumer() { - return new MockBolt(); - } - } - - public static class MockSqlStreamsInsertGroupedDataSource extends MockSqlStreamsGroupedDataSource { - @Override - public IRichBolt getConsumer() { - return new MockInsertBolt(); - } - } - - public static class MockSqlStreamsJoinDataSourceEmp implements ISqlStreamsDataSource { - @Override - public IRichSpout getProducer() { - List records = new ArrayList<>(); - Fields outputFields = new Fields("EMPID", "EMPNAME", "DEPTID"); - - for (int i = 0; i < 5; ++i) { - records.add(new Values(i, "emp-" + i, i % 2)); - } - for (int i = 10; i < 15; ++i) { - records.add(new Values(i, "emp-" + i, i)); - } - - return new MockSpout(records, outputFields); - } - - @Override - public IRichBolt getConsumer() { - return new MockBolt(); - } - } - - public static class MockSqlStreamsInsertJoinDataSourceEmp extends MockSqlStreamsJoinDataSourceEmp { - @Override - public IRichBolt getConsumer() { - return new MockInsertBolt(); - } - } - - public static class MockSqlStreamsJoinDataSourceDept implements ISqlStreamsDataSource { - @Override - public IRichSpout getProducer() { - List records = new ArrayList<>(); - Fields outputFields = new Fields("DEPTID", "DEPTNAME"); - - for (int i = 0; i < 5; ++i) { - records.add(new Values(i, "dept-" + i)); - } - - return new MockSpout(records, outputFields); - } - - @Override - public IRichBolt getConsumer() { - return new MockBolt(); - } - } - - public static class MockSqlStreamsInsertJoinDataSourceDept extends MockSqlStreamsJoinDataSourceDept { - @Override - public IRichBolt getConsumer() { - return new MockInsertBolt(); - } - } - - public static class MockSqlStreamsNestedDataSource implements ISqlStreamsDataSource { - @Override - public IRichSpout getProducer() { - List records = new ArrayList<>(); - Fields outputFields = new Fields("ID", "MAPFIELD", "NESTEDMAPFIELD", "ARRAYFIELD"); - - List ints = Arrays.asList(100, 200, 300); - for (int i = 0; i < 5; ++i) { - Map map = new HashMap<>(); - map.put("b", i); - map.put("c", i*i); - Map> mm = new HashMap<>(); - mm.put("a", map); - records.add(new Values(i, map, mm, ints)); - } - - return new MockSpout(records, outputFields); - } - - @Override - public IRichBolt getConsumer() { - return new MockBolt(); - } - } - - public static class MockSqlStreamsInsertNestedDataSource extends MockSqlStreamsNestedDataSource { - @Override - public IRichBolt getConsumer() { - return new MockInsertBolt(); - } - } - - public static long monotonicNow() { - final long NANOSECONDS_PER_MILLISECOND = 1000000; - return System.nanoTime() / NANOSECONDS_PER_MILLISECOND; - } -} \ No newline at end of file diff --git a/storm-client/test/py/test_storm_cli.py b/storm-client/test/py/test_storm_cli.py index 104baa60f5b..2815de42f1c 100644 --- a/storm-client/test/py/test_storm_cli.py +++ b/storm-client/test/py/test_storm_cli.py @@ -136,24 +136,6 @@ def test_local_command(self): ]) ) - def test_sql_command(self): - self.base_test( - ['storm', 'sql', 'apache_log_error_filtering.sql', 'apache_log_error_filtering', '--artifacts', - '"org.apache.storm:storm-sql-kafka:2.0.0-SNAPSHOT,org.apache.storm:storm-kafka:2.0.0-SNAPSHOT,org.apache.kafka:kafka_2.10:0.8.2.2^org.slf4j:slf4j-log4j12,org.apache.kafka:kafka-clients:0.8.2.2"' - ], self.mock_execvp, mock.call( - self.java_cmd, - [self.java_cmd, '-client', '-Ddaemon.name=', '-Dstorm.options=', - '-Dstorm.home=' + self.storm_dir, '-Dstorm.log.dir=' + self.storm_dir + "/logs", - '-Djava.library.path=', '-Dstorm.conf.file=', '-cp', - self.storm_dir + '/*:' + self.storm_dir + '/lib:' + self.storm_dir + '/extlib:' + - self.storm_dir + - '/conf:' + self.storm_dir + '/bin:' + self.storm_dir + '/lib-tools/sql/core',\ - '-Dstorm.dependency.jars=', '-Dstorm.dependency.artifacts={}', - 'org.apache.storm.sql.StormSqlRunner', '--file', 'apache_log_error_filtering.sql', - '--topology', 'apache_log_error_filtering'] - ) - ) - def test_kill_command(self): self.base_test([ 'storm', 'kill', 'doomed_topology' diff --git a/storm-dist/binary/final-package/src/main/assembly/binary.xml b/storm-dist/binary/final-package/src/main/assembly/binary.xml index d396a1f6ca5..a473da894ba 100644 --- a/storm-dist/binary/final-package/src/main/assembly/binary.xml +++ b/storm-dist/binary/final-package/src/main/assembly/binary.xml @@ -242,21 +242,6 @@ - - - ${project.basedir}/../storm-sql-core-bin/target/sql-core/sql-core/lib-sql-core - lib-tools/sql/core - - *jar - - - - ${project.basedir}/../storm-sql-runtime-bin/target/sql-runtime/sql-runtime/lib-sql-runtime - lib-tools/sql/runtime - - *jar - - ${project.basedir}/../storm-autocreds-bin/target/autocreds/autocreds/lib-autocreds diff --git a/storm-dist/binary/pom.xml b/storm-dist/binary/pom.xml index f849e31adc3..061ec40bb98 100644 --- a/storm-dist/binary/pom.xml +++ b/storm-dist/binary/pom.xml @@ -46,8 +46,6 @@ storm-client-bin storm-webapp-bin storm-autocreds-bin - storm-sql-core-bin - storm-sql-runtime-bin storm-submit-tools-bin storm-kafka-monitor-bin diff --git a/storm-dist/binary/storm-sql-core-bin/pom.xml b/storm-dist/binary/storm-sql-core-bin/pom.xml deleted file mode 100644 index 50a1a0b4295..00000000000 --- a/storm-dist/binary/storm-sql-core-bin/pom.xml +++ /dev/null @@ -1,71 +0,0 @@ - - - - 4.0.0 - - org.apache.storm - apache-storm-bin - 2.8.3-SNAPSHOT - - storm-sql-core-bin - pom - - - - 33.4.8-jre - - - - - org.apache.storm - storm-sql-core - ${project.version} - - - com.google.guava - guava - ${guava.version} - - - - - sql-core - - - org.apache.maven.plugins - maven-assembly-plugin - - - prepare-package - - single - - - - - false - false - - ${project.basedir}/src/main/assembly/storm-sql-core.xml - - false - - - - - diff --git a/storm-dist/binary/storm-sql-core-bin/src/main/assembly/storm-sql-core.xml b/storm-dist/binary/storm-sql-core-bin/src/main/assembly/storm-sql-core.xml deleted file mode 100644 index ecfcbe9911d..00000000000 --- a/storm-dist/binary/storm-sql-core-bin/src/main/assembly/storm-sql-core.xml +++ /dev/null @@ -1,33 +0,0 @@ - - - - storm-sql-core-bin - - dir - - - - false - lib-sql-core - false - - - diff --git a/storm-dist/binary/storm-sql-runtime-bin/pom.xml b/storm-dist/binary/storm-sql-runtime-bin/pom.xml deleted file mode 100644 index 6755e551661..00000000000 --- a/storm-dist/binary/storm-sql-runtime-bin/pom.xml +++ /dev/null @@ -1,71 +0,0 @@ - - - - 4.0.0 - - org.apache.storm - apache-storm-bin - 2.8.3-SNAPSHOT - - storm-sql-runtime-bin - pom - - - - 33.4.8-jre - - - - - org.apache.storm - storm-sql-runtime - ${project.version} - - - com.google.guava - guava - ${guava.version} - - - - - sql-runtime - - - org.apache.maven.plugins - maven-assembly-plugin - - - prepare-package - - single - - - - - false - false - - ${project.basedir}/src/main/assembly/storm-sql-runtime.xml - - false - - - - - diff --git a/storm-dist/binary/storm-sql-runtime-bin/src/main/assembly/storm-sql-runtime.xml b/storm-dist/binary/storm-sql-runtime-bin/src/main/assembly/storm-sql-runtime.xml deleted file mode 100644 index f0692c08df4..00000000000 --- a/storm-dist/binary/storm-sql-runtime-bin/src/main/assembly/storm-sql-runtime.xml +++ /dev/null @@ -1,33 +0,0 @@ - - - - storm-sql-runtime-bin - - dir - - - - false - lib-sql-runtime - false - - -