diff --git a/build/docker/deploySchemaAndTenant.sh b/build/docker/deploySchemaAndTenant.sh index d57a19a6c67..e95248ac74b 100755 --- a/build/docker/deploySchemaAndTenant.sh +++ b/build/docker/deploySchemaAndTenant.sh @@ -21,7 +21,7 @@ while [ "$not_ready" == "true" ] do EXIT_CODE="-1" java -jar schema/fhir-persistence-schema-*-cli.jar \ - --prop-file db2.properties --schema-name FHIRDATA --create-schemas | tee -a ${TMP_FILE} + --db-type db2 --prop-file db2.properties --schema-name FHIRDATA --create-schemas | tee -a ${TMP_FILE} EXIT_CODE="${PIPESTATUS[0]}" LOG_OUT=`cat ${TMP_FILE}` if [[ "$EXIT_CODE" == "0" ]] @@ -54,18 +54,18 @@ then fi java -jar schema/fhir-persistence-schema-*-cli.jar \ - --prop-file db2.properties --schema-name FHIRDATA --update-schema --pool-size 2 + --db-type db2 --prop-file db2.properties --schema-name FHIRDATA --update-schema --pool-size 2 java -jar schema/fhir-persistence-schema-*-cli.jar \ - --prop-file db2.properties --schema-name FHIRDATA --grant-to FHIRSERVER --pool-size 2 + --db-type db2 --prop-file db2.properties --schema-name FHIRDATA --grant-to FHIRSERVER --pool-size 2 java -jar schema/fhir-persistence-schema-*-cli.jar \ - --prop-file db2.properties --schema-name FHIRDATA --allocate-tenant default --pool-size 2 + --db-type db2 --prop-file db2.properties --schema-name FHIRDATA --allocate-tenant default --pool-size 2 # The regex in the following command will output the capture group between "key=" and "]" # With GNU grep, the following would work as well: grep -oP 'key=\K\S+(?=])' tenantKey=$(java -jar schema/fhir-persistence-schema-*-cli.jar \ - --prop-file db2.properties --schema-name FHIRDATA --add-tenant-key default 2>&1 \ + --db-type db2 --prop-file db2.properties --schema-name FHIRDATA --add-tenant-key default 2>&1 \ | grep "key=" | sed -e 's/.*key\=\(.*\)\].*/\1/') # Creating a backup file is the easiest way to make in-place sed portable across OSX and Linux diff --git a/build/docker/updateSchema.sh b/build/docker/updateSchema.sh index e8dc21f4849..df2c401286d 100755 --- a/build/docker/updateSchema.sh +++ b/build/docker/updateSchema.sh @@ -13,17 +13,17 @@ cd ${DIR} # For #1366 the migration hits deadlock issues if run in parallel, so # to avoid this, serialize the steps using --pool-size 1 java -jar schema/fhir-persistence-schema-*-cli.jar \ - --prop-file db2.properties --schema-name FHIRDATA --update-schema \ + --db-type db2 --prop-file db2.properties --schema-name FHIRDATA --update-schema \ --pool-size 1 # Rerun grants to cover any new tables added by the above migration step java -jar schema/fhir-persistence-schema-*-cli.jar \ - --prop-file db2.properties --schema-name FHIRDATA --grant-to FHIRSERVER --pool-size 2 + --db-type db2 --prop-file db2.properties --schema-name FHIRDATA --grant-to FHIRSERVER --pool-size 2 # And make sure that the new tables have partitions for existing tenants java -jar schema/fhir-persistence-schema-*-cli.jar \ - --prop-file db2.properties --refresh-tenants + --db-type db2 --prop-file db2.properties --refresh-tenants java -jar schema/fhir-persistence-schema-*-cli.jar \ - --prop-file db2.properties --schema-name FHIRDATA --grant-to FHIRSERVER \ + --db-type db2 --prop-file db2.properties --schema-name FHIRDATA --grant-to FHIRSERVER \ --pool-size 20 diff --git a/build/migration/bin/6_current-reindex.sh b/build/migration/bin/6_current-reindex.sh index 296abdd0a5e..07f18104c06 100644 --- a/build/migration/bin/6_current-reindex.sh +++ b/build/migration/bin/6_current-reindex.sh @@ -23,7 +23,7 @@ run_reindex(){ DATE_ISO=$(date +%Y-%m-%dT%H:%M:%SZ) status=$(curl -k -X POST -o reindex.json -i -w '%{http_code}' -u 'fhiruser:change-password' 'https://localhost:9443/fhir-server/api/v4/$reindex' \ -H 'Content-Type: application/fhir+json' -H 'X-FHIR-TENANT-ID: default' \ - -d "{\"resourceType\": \"Parameters\",\"parameter\":[{\"name\":\"resourceCount\",\"valueInteger\":100},{\"name\":\"tstamp\",\"valueString\":\"${DATE_ISO}\"}]}") + -d "{\"resourceType\": \"Parameters\",\"parameter\":[{\"name\":\"resourceCount\",\"valueInteger\":100},{\"name\":\"tstamp\",\"valueString\":\"${DATE_ISO}\"},{\"name\":\"force\",\"valueBoolean\":true}]}") echo "Status: ${status}" while [ $status -ne 200 ] @@ -57,7 +57,7 @@ run_reindex(){ fi status=$(curl -k -X POST -o reindex.json -i -w '%{http_code}' -u 'fhiruser:change-password' 'https://localhost:9443/fhir-server/api/v4/$reindex' \ -H 'Content-Type: application/fhir+json' -H 'X-FHIR-TENANT-ID: default' \ - -d "{\"resourceType\": \"Parameters\",\"parameter\":[{\"name\":\"resourceCount\",\"valueInteger\":100},{\"name\":\"tstamp\",\"valueString\":\"${DATE_ISO}\"}]}") + -d "{\"resourceType\": \"Parameters\",\"parameter\":[{\"name\":\"resourceCount\",\"valueInteger\":100},{\"name\":\"tstamp\",\"valueString\":\"${DATE_ISO}\"},{\"name\":\"force\",\"valueBoolean\":true}]}") echo "Status: ${status}" done fi @@ -77,4 +77,4 @@ run_reindex "${1}" popd > /dev/null # EOF -############################################################################### \ No newline at end of file +############################################################################### diff --git a/docs/src/pages/guides/FHIRSearchConfiguration.md b/docs/src/pages/guides/FHIRSearchConfiguration.md index 10768ef5366..618c463ade1 100644 --- a/docs/src/pages/guides/FHIRSearchConfiguration.md +++ b/docs/src/pages/guides/FHIRSearchConfiguration.md @@ -217,6 +217,7 @@ By default, the operation will select 10 resources and re-extract their search p |----|----|-----------| |`tstamp`|string|Reindex only resources not previously reindexed since this timestamp. Format as a date YYYY-MM-DD or time YYYY-MM-DDTHH:MM:SSZ.| |`resourceCount`|integer|The maximum number of resources to reindex in this call. If this number is too large, the processing time might exceed the transaction timeout and fail.| +|`force`|boolean|Force the parameters to be replaced even if the parameter hash matches. This is only required following a schema migration which changes how the parameters are stored in the database.| The IBM FHIR Server tracks when a resource was last reindexed and only resources with a reindex_tstamp value less than the given tstamp parameter will be processed. When a resource is reindexed, its reindex_tstamp is set to the given tstamp value. In most cases, using the current date (for example "2020-10-27") is the best option for this value. diff --git a/docs/src/pages/guides/FHIRServerUsersGuide.md b/docs/src/pages/guides/FHIRServerUsersGuide.md index 4052648628c..8c60804ebc0 100644 --- a/docs/src/pages/guides/FHIRServerUsersGuide.md +++ b/docs/src/pages/guides/FHIRServerUsersGuide.md @@ -27,6 +27,7 @@ permalink: /FHIRServerUsersGuide/ * [4.10 Bulk data operations](#410-bulk-data-operations) * [4.11 Audit logging service](#411-audit-logging-service) * [4.12 FHIR REST API](#412-fhir-rest-api) + * [4.13 Remote Index Service](#413-remote-index-service) - [5 Appendix](#5-appendix) * [5.1 Configuration properties reference](#51-configuration-properties-reference) * [5.2 Keystores, truststores, and the FHIR server](#52-keystores-truststores-and-the-fhir-server) @@ -2070,6 +2071,10 @@ For example, consider a FHIR Server that is listening at https://fhir:9443/fhir- The originalRequestUriHeader is expected to contain the full path of the original request. Values with no scheme (e.g. `https://`) will be handled like relative URLs, but full URL values (including scheme, hostname, optional port, and path) are recommended. Query string values can be included in the header value but will be ignored by the server; the server will use the query string of the actual request to process the request. +### 4.13 Remote Index Service + +To use the experimental remote index service feature, see the instructions documented in the [fhir-remote-index](https://github.com/IBM/FHIR/tree/main/operation/fhir-remote-index/README.md) project. + # 5 Appendix ## 5.1 Configuration properties reference @@ -2260,7 +2265,17 @@ This section contains reference information about each of the configuration prop |`fhirServer/operations/membermatch/strategy`|string|The key identifying the Member Match strategy| |`fhirServer/operations/membermatch/extendedProps`|object|The extended options for the extended member match implementation| |`fhirServer/operations/everything/includeTypes`|list|The list of related resource types to include alongside the patient compartment resource types. Instances of these resource types will only be returned when they are referenced from one or more resource instances from the target patient compartment. Example values are like `Location`, `Medication`, `Organization`, and `Practitioner`| - +|`fhirServer/remoteIndexService/type`|string| The type of service used to send remote index messages. Only `kafka` is currently supported| +|`fhirServer/remoteIndexService/instanceIdentifier`|string| A UUID or other identifier unique to this cluster of IBM FHIR Servers | +|`fhirServer/remoteIndexService/kafka/mode`|string| Current operation mode of the service. Specify `ACTIVE` to use the service| +|`fhirServer/remoteIndexService/kafka/topicName`|string| The Kafka topic name. Typically `FHIR_REMOTE_INDEX` | +|`fhirServer/remoteIndexService/kafka/connectionProperties/bootstrap.servers`|string| Bootstrap servers for the Kafka service | +|`fhirServer/remoteIndexService/kafka/connectionProperties/sasl.jaas.config`|string| Kafka service authentication | +|`fhirServer/remoteIndexService/kafka/connectionProperties/sasl.mechanism`|string| Kafka service authentication| +|`fhirServer/remoteIndexService/kafka/connectionProperties/security.protocol`|string| Kafka service security | +|`fhirServer/remoteIndexService/kafka/connectionProperties/ssl.protocol`|string| Kafka service SSL configuration | +|`fhirServer/remoteIndexService/kafka/connectionProperties/ssl.enabled.protocols`|string| Kafka service SSL configuration | +|`fhirServer/remoteIndexService/kafka/connectionProperties/ssl.endpoint.identification.algorithm`|string| Kafka service SSL configuration | ### 5.1.2 Default property values | Property Name | Default value | @@ -2406,6 +2421,17 @@ This section contains reference information about each of the configuration prop |`fhirServer/operations/membermatch/strategy`|default| |`fhirServer/operations/membermatch/extendedProps`|empty| |`fhirServer/operations/everything/includeTypes`|null| +|`fhirServer/remoteIndexService/type`|null| +|`fhirServer/remoteIndexService/instanceIdentifier`|null| +|`fhirServer/remoteIndexService/kafka/mode`|null| +|`fhirServer/remoteIndexService/kafka/topicName`|null| +|`fhirServer/remoteIndexService/kafka/connectionProperties/bootstrap.servers`|null| +|`fhirServer/remoteIndexService/kafka/connectionProperties/sasl.jaas.config`|null| +|`fhirServer/remoteIndexService/kafka/connectionProperties/sasl.mechanism`|null| +|`fhirServer/remoteIndexService/kafka/connectionProperties/security.protocol`|null| +|`fhirServer/remoteIndexService/kafka/connectionProperties/ssl.protocol`|null| +|`fhirServer/remoteIndexService/kafka/connectionProperties/ssl.enabled.protocols`|null| +|`fhirServer/remoteIndexService/kafka/connectionProperties/ssl.endpoint.identification.algorithm`|null| ### 5.1.3 Property attributes Depending on the context of their use, config properties can be: @@ -2588,6 +2614,17 @@ Cases where that behavior is not supported are marked below with an `N` in the ` |`fhirServer/operations/membermatch/strategy`|Y|Y|Y| |`fhirServer/operations/membermatch/extendedProps`|Y|Y|Y| |`fhirServer/operations/everything/includeTypes`|Y|Y|N| +|`fhirServer/remoteIndexService/type`|N|N|N| +|`fhirServer/remoteIndexService/instanceIdentifier`|N|N|N| +|`fhirServer/remoteIndexService/kafka/mode`|N|N|N| +|`fhirServer/remoteIndexService/kafka/topicName`|N|N|N| +|`fhirServer/remoteIndexService/kafka/connectionProperties/bootstrap.servers`|N|N|N| +|`fhirServer/remoteIndexService/kafka/connectionProperties/sasl.jaas.config`|N|N|N| +|`fhirServer/remoteIndexService/kafka/connectionProperties/sasl.mechanism`|N|N|N| +|`fhirServer/remoteIndexService/kafka/connectionProperties/security.protocol`|N|N|N| +|`fhirServer/remoteIndexService/kafka/connectionProperties/ssl.protocol`|N|N|N| +|`fhirServer/remoteIndexService/kafka/connectionProperties/ssl.enabled.protocols`|N|N|N| +|`fhirServer/remoteIndexService/kafka/connectionProperties/ssl.endpoint.identification.algorithm`|N|N|N| ## 5.2 Keystores, truststores, and the IBM FHIR server diff --git a/fhir-bucket/src/main/java/com/ibm/fhir/bucket/app/Main.java b/fhir-bucket/src/main/java/com/ibm/fhir/bucket/app/Main.java index dc9f503ef55..1860ef2124e 100644 --- a/fhir-bucket/src/main/java/com/ibm/fhir/bucket/app/Main.java +++ b/fhir-bucket/src/main/java/com/ibm/fhir/bucket/app/Main.java @@ -57,10 +57,13 @@ import com.ibm.fhir.database.utils.api.IDatabaseAdapter; import com.ibm.fhir.database.utils.api.IDatabaseTranslator; import com.ibm.fhir.database.utils.api.ILeaseManagerConfig; +import com.ibm.fhir.database.utils.api.ISchemaAdapter; import com.ibm.fhir.database.utils.api.ITransaction; import com.ibm.fhir.database.utils.api.ITransactionProvider; +import com.ibm.fhir.database.utils.api.SchemaApplyContext; import com.ibm.fhir.database.utils.api.UniqueConstraintViolationException; import com.ibm.fhir.database.utils.common.JdbcConnectionProvider; +import com.ibm.fhir.database.utils.common.PlainSchemaAdapter; import com.ibm.fhir.database.utils.db2.Db2Adapter; import com.ibm.fhir.database.utils.db2.Db2PropertyAdapter; import com.ibm.fhir.database.utils.db2.Db2Translator; @@ -147,6 +150,9 @@ public class Main { // The adapter configured for the type of database we're using private IDatabaseAdapter adapter; + // The (plain) schema adapter which wraps the database adapter + private ISchemaAdapter schemaAdapter; + // The number of threads to use for the schema creation step private int createSchemaThreads = 1; @@ -639,6 +645,7 @@ public void configure() { setupDerbyRepository(); break; case POSTGRESQL: + case CITUS: setupPostgresRepository(); break; } @@ -669,6 +676,7 @@ public void setupDerbyRepository() { this.connectionPool = new PoolConnectionProvider(cp, connectionPoolSize); this.connectionPool.setCloseOnAnyError(); this.adapter = new DerbyAdapter(connectionPool); + this.schemaAdapter = new PlainSchemaAdapter(adapter); this.transactionProvider = new SimpleTransactionProvider(connectionPool); } @@ -692,6 +700,7 @@ public void setupDb2Repository() { IConnectionProvider cp = new JdbcConnectionProvider(translator, propertyAdapter); this.connectionPool = new PoolConnectionProvider(cp, connectionPoolSize); this.adapter = new Db2Adapter(connectionPool); + this.schemaAdapter = new PlainSchemaAdapter(adapter); this.transactionProvider = new SimpleTransactionProvider(connectionPool); } @@ -715,6 +724,7 @@ public void setupPostgresRepository() { IConnectionProvider cp = new JdbcConnectionProvider(translator, propertyAdapter); this.connectionPool = new PoolConnectionProvider(cp, connectionPoolSize); this.adapter = new PostgresAdapter(connectionPool); + this.schemaAdapter = new PlainSchemaAdapter(adapter); this.transactionProvider = new SimpleTransactionProvider(connectionPool); } @@ -732,7 +742,7 @@ protected VersionHistoryService createVersionHistoryService() { // Create the version history table if it doesn't yet exist try (ITransaction tx = transactionProvider.getTransaction()) { try { - CreateVersionHistory.createTableIfNeeded(schemaName, this.adapter); + CreateVersionHistory.createTableIfNeeded(schemaName, this.schemaAdapter); } catch (Exception x) { logger.log(Level.SEVERE, "failed to create version history table", x); tx.setRollbackOnly(); @@ -761,8 +771,8 @@ public void bootstrapDb() { try (ITransaction tx = transactionProvider.getTransaction()) { try { adapter.createSchema(schemaName); - CreateControl.createTableIfNeeded(schemaName, adapter); - CreateWholeSchemaVersion.createTableIfNeeded(schemaName, adapter); + CreateControl.createTableIfNeeded(schemaName, schemaAdapter); + CreateWholeSchemaVersion.createTableIfNeeded(schemaName, schemaAdapter); success = true; } catch (Exception x) { logger.log(Level.SEVERE, "failed to create schema management tables", x); @@ -822,7 +832,8 @@ private void buildSchema() { TaskService taskService = new TaskService(); ExecutorService pool = Executors.newFixedThreadPool(this.createSchemaThreads); ITaskCollector collector = taskService.makeTaskCollector(pool); - pdm.collect(collector, adapter, this.transactionProvider, vhs); + SchemaApplyContext context = SchemaApplyContext.getDefault(); + pdm.collect(collector, schemaAdapter, context, this.transactionProvider, vhs); // FHIR in the hole! logger.info("Starting schema updates"); @@ -845,7 +856,7 @@ private void buildSchema() { try { Set resourceTypes = ResourceTypeHelper.getR4bResourceTypesFor(FHIRVersionParam.VERSION_43); - if (adapter.getTranslator().getType() == DbType.POSTGRESQL) { + if (adapter.getTranslator().isFamilyPostgreSQL()) { // Postgres doesn't support batched merges, so we go with a simpler UPSERT MergeResourceTypesPostgres mrt = new MergeResourceTypesPostgres(schemaName, resourceTypes); adapter.runStatement(mrt); diff --git a/fhir-bucket/src/main/java/com/ibm/fhir/bucket/persistence/AddBucketPath.java b/fhir-bucket/src/main/java/com/ibm/fhir/bucket/persistence/AddBucketPath.java index 93be50bedf7..76f2539e7a1 100644 --- a/fhir-bucket/src/main/java/com/ibm/fhir/bucket/persistence/AddBucketPath.java +++ b/fhir-bucket/src/main/java/com/ibm/fhir/bucket/persistence/AddBucketPath.java @@ -17,7 +17,6 @@ import com.ibm.fhir.database.utils.api.IDatabaseSupplier; import com.ibm.fhir.database.utils.api.IDatabaseTranslator; import com.ibm.fhir.database.utils.common.DataDefinitionUtil; -import com.ibm.fhir.database.utils.model.DbType; /** * DAO to encapsulate all the SQL/DML used to retrieve and persist data @@ -57,7 +56,7 @@ public Long run(IDatabaseTranslator translator, Connection c) { // try the old-fashioned way and handle duplicate key final String bucketPaths = DataDefinitionUtil.getQualifiedName(schemaName, "bucket_paths"); final String dml; - if (translator.getType() == DbType.POSTGRESQL) { + if (translator.isFamilyPostgreSQL()) { // For POSTGRES, if a statement fails it causes the whole transaction // to fail, so we need turn this into an UPSERT dml = "INSERT INTO " + bucketPaths + "(bucket_name, bucket_path) VALUES (?,?) ON CONFLICT(bucket_name, bucket_path) DO NOTHING"; diff --git a/fhir-bucket/src/main/java/com/ibm/fhir/bucket/persistence/AddResourceBundle.java b/fhir-bucket/src/main/java/com/ibm/fhir/bucket/persistence/AddResourceBundle.java index 5215d12650d..478a50bc706 100644 --- a/fhir-bucket/src/main/java/com/ibm/fhir/bucket/persistence/AddResourceBundle.java +++ b/fhir-bucket/src/main/java/com/ibm/fhir/bucket/persistence/AddResourceBundle.java @@ -20,7 +20,6 @@ import com.ibm.fhir.database.utils.api.IDatabaseSupplier; import com.ibm.fhir.database.utils.api.IDatabaseTranslator; import com.ibm.fhir.database.utils.common.DataDefinitionUtil; -import com.ibm.fhir.database.utils.model.DbType; /** * DAO to encapsulate all the SQL/DML used to retrieve and persist data @@ -76,7 +75,7 @@ public ResourceBundleData run(IDatabaseTranslator translator, Connection c) { int version = 1; final String resourceBundles = DataDefinitionUtil.getQualifiedName(schemaName, "resource_bundles"); final String dml; - if (translator.getType() == DbType.POSTGRESQL) { + if (translator.isFamilyPostgreSQL()) { // For PostgresSQL, make sure we don't break the current transaction // if the statement fails...annoying dml = "INSERT INTO " + resourceBundles + "(" diff --git a/fhir-bucket/src/main/java/com/ibm/fhir/bucket/persistence/FhirBucketSchema.java b/fhir-bucket/src/main/java/com/ibm/fhir/bucket/persistence/FhirBucketSchema.java index e61026d77f6..625a7de9347 100644 --- a/fhir-bucket/src/main/java/com/ibm/fhir/bucket/persistence/FhirBucketSchema.java +++ b/fhir-bucket/src/main/java/com/ibm/fhir/bucket/persistence/FhirBucketSchema.java @@ -1,16 +1,65 @@ /* - * (C) Copyright IBM Corp. 2020 + * (C) Copyright IBM Corp. 2020, 2022 * * SPDX-License-Identifier: Apache-2.0 */ package com.ibm.fhir.bucket.persistence; -import static com.ibm.fhir.bucket.persistence.SchemaConstants.*; - +import static com.ibm.fhir.bucket.persistence.SchemaConstants.ALLOCATION_ID; +import static com.ibm.fhir.bucket.persistence.SchemaConstants.BUCKET_NAME; +import static com.ibm.fhir.bucket.persistence.SchemaConstants.BUCKET_PATH; +import static com.ibm.fhir.bucket.persistence.SchemaConstants.BUCKET_PATHS; +import static com.ibm.fhir.bucket.persistence.SchemaConstants.BUCKET_PATH_ID; +import static com.ibm.fhir.bucket.persistence.SchemaConstants.CREATED_TSTAMP; +import static com.ibm.fhir.bucket.persistence.SchemaConstants.ERROR_TEXT; +import static com.ibm.fhir.bucket.persistence.SchemaConstants.ERROR_TEXT_LEN; +import static com.ibm.fhir.bucket.persistence.SchemaConstants.ERROR_TSTAMP; +import static com.ibm.fhir.bucket.persistence.SchemaConstants.ETAG; +import static com.ibm.fhir.bucket.persistence.SchemaConstants.FAILURE_COUNT; +import static com.ibm.fhir.bucket.persistence.SchemaConstants.FILE_TYPE; +import static com.ibm.fhir.bucket.persistence.SchemaConstants.FK; +import static com.ibm.fhir.bucket.persistence.SchemaConstants.HEARTBEAT_TSTAMP; +import static com.ibm.fhir.bucket.persistence.SchemaConstants.HOSTNAME; +import static com.ibm.fhir.bucket.persistence.SchemaConstants.HTTP_STATUS_CODE; +import static com.ibm.fhir.bucket.persistence.SchemaConstants.HTTP_STATUS_TEXT; +import static com.ibm.fhir.bucket.persistence.SchemaConstants.HTTP_STATUS_TEXT_LEN; +import static com.ibm.fhir.bucket.persistence.SchemaConstants.IDX; +import static com.ibm.fhir.bucket.persistence.SchemaConstants.JOB_ALLOCATION_SEQ; +import static com.ibm.fhir.bucket.persistence.SchemaConstants.LAST_MODIFIED; +import static com.ibm.fhir.bucket.persistence.SchemaConstants.LINE_NUMBER; +import static com.ibm.fhir.bucket.persistence.SchemaConstants.LOADER_INSTANCES; +import static com.ibm.fhir.bucket.persistence.SchemaConstants.LOADER_INSTANCE_ID; +import static com.ibm.fhir.bucket.persistence.SchemaConstants.LOADER_INSTANCE_KEY; +import static com.ibm.fhir.bucket.persistence.SchemaConstants.LOAD_COMPLETED; +import static com.ibm.fhir.bucket.persistence.SchemaConstants.LOAD_STARTED; +import static com.ibm.fhir.bucket.persistence.SchemaConstants.LOGICAL_ID; +import static com.ibm.fhir.bucket.persistence.SchemaConstants.LOGICAL_ID_BYTES; +import static com.ibm.fhir.bucket.persistence.SchemaConstants.LOGICAL_RESOURCES; +import static com.ibm.fhir.bucket.persistence.SchemaConstants.LOGICAL_RESOURCE_ID; +import static com.ibm.fhir.bucket.persistence.SchemaConstants.NOT_NULL; +import static com.ibm.fhir.bucket.persistence.SchemaConstants.NULLABLE; +import static com.ibm.fhir.bucket.persistence.SchemaConstants.OBJECT_NAME; +import static com.ibm.fhir.bucket.persistence.SchemaConstants.OBJECT_SIZE; +import static com.ibm.fhir.bucket.persistence.SchemaConstants.PID; +import static com.ibm.fhir.bucket.persistence.SchemaConstants.RESOURCE_BUNDLES; +import static com.ibm.fhir.bucket.persistence.SchemaConstants.RESOURCE_BUNDLE_ERRORS; +import static com.ibm.fhir.bucket.persistence.SchemaConstants.RESOURCE_BUNDLE_ID; +import static com.ibm.fhir.bucket.persistence.SchemaConstants.RESOURCE_BUNDLE_LOADS; +import static com.ibm.fhir.bucket.persistence.SchemaConstants.RESOURCE_BUNDLE_LOAD_ID; +import static com.ibm.fhir.bucket.persistence.SchemaConstants.RESOURCE_TYPE; +import static com.ibm.fhir.bucket.persistence.SchemaConstants.RESOURCE_TYPES; +import static com.ibm.fhir.bucket.persistence.SchemaConstants.RESOURCE_TYPE_ID; +import static com.ibm.fhir.bucket.persistence.SchemaConstants.RESPONSE_TIME_MS; +import static com.ibm.fhir.bucket.persistence.SchemaConstants.ROWS_PROCESSED; +import static com.ibm.fhir.bucket.persistence.SchemaConstants.SCAN_TSTAMP; +import static com.ibm.fhir.bucket.persistence.SchemaConstants.STATUS; +import static com.ibm.fhir.bucket.persistence.SchemaConstants.UNQ; +import static com.ibm.fhir.bucket.persistence.SchemaConstants.VERSION; import com.ibm.fhir.bucket.app.Main; -import com.ibm.fhir.database.utils.api.IDatabaseAdapter; +import com.ibm.fhir.database.utils.api.ISchemaAdapter; +import com.ibm.fhir.database.utils.api.SchemaApplyContext; import com.ibm.fhir.database.utils.model.Generated; import com.ibm.fhir.database.utils.model.PhysicalDataModel; import com.ibm.fhir.database.utils.model.Sequence; @@ -255,10 +304,11 @@ protected void addResourceBundleErrors(PhysicalDataModel pdm) { /** * Apply the model to the database. Will generate the DDL and execute it + * @param adapter + * @param context * @param pdm */ - protected void applyModel(IDatabaseAdapter adapter, PhysicalDataModel pdm) { - pdm.apply(adapter); - } - + protected void applyModel(ISchemaAdapter adapter, SchemaApplyContext context, PhysicalDataModel pdm) { + pdm.apply(adapter, context); + } } diff --git a/fhir-bucket/src/main/java/com/ibm/fhir/bucket/persistence/MergeResourceTypes.java b/fhir-bucket/src/main/java/com/ibm/fhir/bucket/persistence/MergeResourceTypes.java index a209424cc2a..d1da59eb82a 100644 --- a/fhir-bucket/src/main/java/com/ibm/fhir/bucket/persistence/MergeResourceTypes.java +++ b/fhir-bucket/src/main/java/com/ibm/fhir/bucket/persistence/MergeResourceTypes.java @@ -18,7 +18,6 @@ import com.ibm.fhir.database.utils.api.IDatabaseStatement; import com.ibm.fhir.database.utils.api.IDatabaseTranslator; import com.ibm.fhir.database.utils.common.DataDefinitionUtil; -import com.ibm.fhir.database.utils.model.DbType; /** * DAO to encapsulate all the SQL/DML used to retrieve and persist data @@ -62,7 +61,7 @@ public void run(IDatabaseTranslator translator, Connection c) { try (PreparedStatement ps = c.prepareStatement(merge)) { // Assume the list is small enough to process in one batch for (String resourceType: resourceTypes) { - if (translator.getType() == DbType.POSTGRESQL) { + if (translator.isFamilyPostgreSQL()) { ps.setString(1, resourceType); } else { ps.setString(1, resourceType); diff --git a/fhir-bucket/src/main/java/com/ibm/fhir/bucket/persistence/RecordLogicalId.java b/fhir-bucket/src/main/java/com/ibm/fhir/bucket/persistence/RecordLogicalId.java index f9b71bbc7a6..10f80c6d8fc 100644 --- a/fhir-bucket/src/main/java/com/ibm/fhir/bucket/persistence/RecordLogicalId.java +++ b/fhir-bucket/src/main/java/com/ibm/fhir/bucket/persistence/RecordLogicalId.java @@ -16,7 +16,6 @@ import com.ibm.fhir.database.utils.api.IDatabaseStatement; import com.ibm.fhir.database.utils.api.IDatabaseTranslator; import com.ibm.fhir.database.utils.common.DataDefinitionUtil; -import com.ibm.fhir.database.utils.model.DbType; /** * DAO to encapsulate all the SQL/DML used to retrieve and persist data @@ -67,7 +66,7 @@ public void run(IDatabaseTranslator translator, Connection c) { final String logicalResources = DataDefinitionUtil.getQualifiedName(schemaName, "logical_resources"); final String dml; - if (translator.getType() == DbType.POSTGRESQL) { + if (translator.isFamilyPostgreSQL()) { // Use UPSERT syntax for Postgres to avoid breaking the transaction when // a statement fails dml = diff --git a/fhir-bucket/src/test/java/com/ibm/fhir/bucket/persistence/test/FhirBucketSchemaTest.java b/fhir-bucket/src/test/java/com/ibm/fhir/bucket/persistence/test/FhirBucketSchemaTest.java index 740f17b872c..8b348d19794 100644 --- a/fhir-bucket/src/test/java/com/ibm/fhir/bucket/persistence/test/FhirBucketSchemaTest.java +++ b/fhir-bucket/src/test/java/com/ibm/fhir/bucket/persistence/test/FhirBucketSchemaTest.java @@ -51,11 +51,13 @@ import com.ibm.fhir.bucket.persistence.ResourceRec; import com.ibm.fhir.bucket.persistence.ResourceTypeRec; import com.ibm.fhir.bucket.persistence.ResourceTypesReader; +import com.ibm.fhir.database.utils.api.ISchemaAdapter; import com.ibm.fhir.core.FHIRVersionParam; import com.ibm.fhir.core.util.ResourceTypeHelper; import com.ibm.fhir.database.utils.api.ITransaction; import com.ibm.fhir.database.utils.api.ITransactionProvider; import com.ibm.fhir.database.utils.common.JdbcTarget; +import com.ibm.fhir.database.utils.common.PlainSchemaAdapter; import com.ibm.fhir.database.utils.derby.DerbyAdapter; import com.ibm.fhir.database.utils.derby.DerbyConnectionProvider; import com.ibm.fhir.database.utils.derby.DerbyMaster; @@ -364,7 +366,8 @@ protected VersionHistoryService createVersionHistoryService() throws SQLExceptio try { JdbcTarget target = new JdbcTarget(c); DerbyAdapter derbyAdapter = new DerbyAdapter(target); - CreateVersionHistory.createTableIfNeeded(ADMIN_SCHEMA_NAME, derbyAdapter); + ISchemaAdapter schemaAdapter = new PlainSchemaAdapter(derbyAdapter); + CreateVersionHistory.createTableIfNeeded(ADMIN_SCHEMA_NAME, schemaAdapter); c.commit(); } catch (SQLException x) { logger.log(Level.SEVERE, "failed to create version history table", x); diff --git a/fhir-bulkdata-webapp/src/main/java/com/ibm/fhir/bulkdata/export/patient/resource/PatientResourceHandler.java b/fhir-bulkdata-webapp/src/main/java/com/ibm/fhir/bulkdata/export/patient/resource/PatientResourceHandler.java index 2d1f91894ca..68542ceccc6 100644 --- a/fhir-bulkdata-webapp/src/main/java/com/ibm/fhir/bulkdata/export/patient/resource/PatientResourceHandler.java +++ b/fhir-bulkdata-webapp/src/main/java/com/ibm/fhir/bulkdata/export/patient/resource/PatientResourceHandler.java @@ -125,7 +125,7 @@ public List executeSearch(Set patientIds) throws Exception { do { searchContext.setPageNumber(compartmentPageNum); - FHIRPersistenceContext persistenceContext = FHIRPersistenceContextFactory.createPersistenceContext(null, searchContext); + FHIRPersistenceContext persistenceContext = FHIRPersistenceContextFactory.createPersistenceContext(null, searchContext, null); Date startTime = new Date(System.currentTimeMillis()); List> resourceResults = fhirPersistence.search(persistenceContext, resourceType).getResourceResults(); diff --git a/fhir-bulkdata-webapp/src/main/java/com/ibm/fhir/bulkdata/jbatch/export/patient/ChunkReader.java b/fhir-bulkdata-webapp/src/main/java/com/ibm/fhir/bulkdata/jbatch/export/patient/ChunkReader.java index 7807463beeb..5b0cca0ba30 100644 --- a/fhir-bulkdata-webapp/src/main/java/com/ibm/fhir/bulkdata/jbatch/export/patient/ChunkReader.java +++ b/fhir-bulkdata-webapp/src/main/java/com/ibm/fhir/bulkdata/jbatch/export/patient/ChunkReader.java @@ -201,7 +201,7 @@ public Object readItem() throws Exception { FHIRTransactionHelper txn = new FHIRTransactionHelper(fhirPersistence.getTransaction()); txn.begin(); try { - FHIRPersistenceContext persistenceContext = FHIRPersistenceContextFactory.createPersistenceContext(null, searchContext); + FHIRPersistenceContext persistenceContext = FHIRPersistenceContextFactory.createPersistenceContext(null, searchContext, null); Date startTime = new Date(System.currentTimeMillis()); List> resourceResults = fhirPersistence.search(persistenceContext, Patient.class).getResourceResults(); List patientResources = ResourceResult.toResourceList(resourceResults); diff --git a/fhir-bulkdata-webapp/src/main/java/com/ibm/fhir/bulkdata/jbatch/export/patient/PatientExportPartitionMapper.java b/fhir-bulkdata-webapp/src/main/java/com/ibm/fhir/bulkdata/jbatch/export/patient/PatientExportPartitionMapper.java index 7490e2bfb0c..2552a86c65b 100644 --- a/fhir-bulkdata-webapp/src/main/java/com/ibm/fhir/bulkdata/jbatch/export/patient/PatientExportPartitionMapper.java +++ b/fhir-bulkdata-webapp/src/main/java/com/ibm/fhir/bulkdata/jbatch/export/patient/PatientExportPartitionMapper.java @@ -83,7 +83,7 @@ public PartitionPlan mapPartitions() throws Exception { List target = new ArrayList<>(); try { for (String resourceType : resourceTypes) { - List resourceResults = fhirPersistence.changes(1, null, null, null, + List resourceResults = fhirPersistence.changes(null, 1, null, null, null, Arrays.asList(resourceType), false, HistorySortOrder.NONE); // Early Exit Logic diff --git a/fhir-bulkdata-webapp/src/main/java/com/ibm/fhir/bulkdata/jbatch/export/system/ChunkReader.java b/fhir-bulkdata-webapp/src/main/java/com/ibm/fhir/bulkdata/jbatch/export/system/ChunkReader.java index dd9991ac925..f803038a714 100644 --- a/fhir-bulkdata-webapp/src/main/java/com/ibm/fhir/bulkdata/jbatch/export/system/ChunkReader.java +++ b/fhir-bulkdata-webapp/src/main/java/com/ibm/fhir/bulkdata/jbatch/export/system/ChunkReader.java @@ -222,7 +222,7 @@ public Object readItem() throws Exception { Date startTime = new Date(System.currentTimeMillis()); try { // Execute the search query to obtain the page of resources - persistenceContext = FHIRPersistenceContextFactory.createPersistenceContext(null, searchContext); + persistenceContext = FHIRPersistenceContextFactory.createPersistenceContext(null, searchContext, null); List> resourceResults = fhirPersistence.search(persistenceContext, resourceType).getResourceResults(); resources = ResourceResult.toResourceList(resourceResults); diff --git a/fhir-bulkdata-webapp/src/main/java/com/ibm/fhir/bulkdata/jbatch/export/system/SystemExportPartitionMapper.java b/fhir-bulkdata-webapp/src/main/java/com/ibm/fhir/bulkdata/jbatch/export/system/SystemExportPartitionMapper.java index 817f49feae4..0e1277f56c2 100644 --- a/fhir-bulkdata-webapp/src/main/java/com/ibm/fhir/bulkdata/jbatch/export/system/SystemExportPartitionMapper.java +++ b/fhir-bulkdata-webapp/src/main/java/com/ibm/fhir/bulkdata/jbatch/export/system/SystemExportPartitionMapper.java @@ -78,7 +78,7 @@ public PartitionPlan mapPartitions() throws Exception { List target = new ArrayList<>(); try { for (String resourceType : resourceTypes) { - List resourceResults = fhirPersistence.changes(1, null, null, null, Arrays.asList(resourceType), false, HistorySortOrder.NONE); + List resourceResults = fhirPersistence.changes(null, 1, null, null, null, Arrays.asList(resourceType), false, HistorySortOrder.NONE); // Early Exit Logic if (!resourceResults.isEmpty()) { diff --git a/fhir-config/src/main/java/com/ibm/fhir/config/FHIRConfiguration.java b/fhir-config/src/main/java/com/ibm/fhir/config/FHIRConfiguration.java index cbb6fcadab6..48cbf4e6010 100644 --- a/fhir-config/src/main/java/com/ibm/fhir/config/FHIRConfiguration.java +++ b/fhir-config/src/main/java/com/ibm/fhir/config/FHIRConfiguration.java @@ -25,6 +25,7 @@ public class FHIRConfiguration { // Core server properties public static final String PROPERTY_ORIGINAL_REQUEST_URI_HEADER_NAME = "fhirServer/core/originalRequestUriHeaderName"; public static final String PROPERTY_TENANT_ID_HEADER_NAME = "fhirServer/core/tenantIdHeaderName"; + public static final String PROPERTY_SHARD_KEY_HEADER_NAME = "fhirServer/core/shardKeyHeaderName"; public static final String PROPERTY_DATASTORE_ID_HEADER_NAME = "fhirServer/core/datastoreIdHeaderName"; public static final String PROPERTY_DEFAULT_TENANT_ID = "fhirServer/core/defaultTenantId"; public static final String PROPERTY_DEFAULT_PRETTY_PRINT = "fhirServer/core/defaultPrettyPrint"; @@ -112,6 +113,13 @@ public class FHIRConfiguration { public static final String PROPERTY_NATS_KEYSTORE = "fhirServer/notifications/nats/keystoreLocation"; public static final String PROPERTY_NATS_KEYSTORE_PW = "fhirServer/notifications/nats/keystorePassword"; + // Configuration properties for the Kafka-based async index service + public static final String PROPERTY_REMOTE_INDEX_SERVICE_TYPE = "fhirServer/remoteIndexService/type"; + public static final String PROPERTY_REMOTE_INDEX_SERVICE_INSTANCEIDENTIFIER = "fhirServer/remoteIndexService/instanceIdentifier"; + public static final String PROPERTY_KAFKA_INDEX_SERVICE_TOPICNAME = "fhirServer/remoteIndexService/kafka/topicName"; + public static final String PROPERTY_KAFKA_INDEX_SERVICE_CONNECTIONPROPS = "fhirServer/remoteIndexService/kafka/connectionProperties"; + public static final String PROPERTY_KAFKA_INDEX_SERVICE_MODE = "fhirServer/remoteIndexService/kafka/mode"; + // Operations config properties public static final String PROPERTY_OPERATIONS_EVERYTHING = "fhirServer/operations/everything"; public static final String PROPERTY_OPERATIONS_EVERYTHING_INCLUDE_TYPES = "includeTypes"; @@ -135,6 +143,7 @@ public class FHIRConfiguration { public static final String DEFAULT_TENANT_ID_HEADER_NAME = "X-FHIR-TENANT-ID"; public static final String DEFAULT_DATASTORE_ID_HEADER_NAME = "X-FHIR-DSID"; public static final String DEFAULT_PRETTY_RESPONSE_HEADER_NAME = "X-FHIR-FORMATTED"; + public static final String DEFAULT_SHARD_KEY_HEADER_NAME = "X-FHIR-SHARD-KEY"; public static final String FHIR_SERVER_DEFAULT_CONFIG = "config/default/fhir-server-config.json"; diff --git a/fhir-config/src/main/java/com/ibm/fhir/config/FHIRRequestContext.java b/fhir-config/src/main/java/com/ibm/fhir/config/FHIRRequestContext.java index f7f6d8888a7..2675254c682 100644 --- a/fhir-config/src/main/java/com/ibm/fhir/config/FHIRRequestContext.java +++ b/fhir-config/src/main/java/com/ibm/fhir/config/FHIRRequestContext.java @@ -36,6 +36,10 @@ public class FHIRRequestContext { // The datastore to be used for this request. Usually "default" private String dataStoreId; + + // An optional shard key passed with the request for use with distributed schemas + private String requestShardKey; + private String requestUniqueId; private String originalRequestUri; private Map> httpHeaders; @@ -144,6 +148,24 @@ public void setDataStoreId(String dataStoreId) throws FHIRException { } } + /** + * Set the shard key string value provided by the request + * @param k + */ + public void setRequestShardKey(String k) { + this.requestShardKey = k; + } + + /** + * Get the shard key string value provided by the request. This value is + * not filtered in any way because the value eventually gets hashed into + * a short (2 byte integer number) before being used. + * @return + */ + public String getRequestShardKey() { + return this.requestShardKey; + } + /** * set an Operation Context property * @param name diff --git a/fhir-core/src/main/java/com/ibm/fhir/core/util/LogSupport.java b/fhir-core/src/main/java/com/ibm/fhir/core/util/LogSupport.java new file mode 100644 index 00000000000..1b17b6a2cbb --- /dev/null +++ b/fhir-core/src/main/java/com/ibm/fhir/core/util/LogSupport.java @@ -0,0 +1,52 @@ +/* + * (C) Copyright IBM Corp. 2022 + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.ibm.fhir.core.util; + +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * Static support functions related to logging + */ +public class LogSupport { + private static final String MASK = "*****"; + private static final Pattern PASSWORD_EQ_PATTERN = Pattern.compile("[^\"]password[= ]*\"([^\"]*)\"", Pattern.CASE_INSENSITIVE); + private static final Pattern PASSWORD_PATTERN = Pattern.compile("\"password\"[: ]*\"([^\"]*)\"", Pattern.CASE_INSENSITIVE); + + /** + * Hide any text in quotes following the token "password" to avoid writing secrets to log files + * @param input + * @return + */ + public static String hidePassword(String input) { + String result = hidePassword(input, PASSWORD_EQ_PATTERN); + result = hidePassword(result, PASSWORD_PATTERN); + return result; + } + + /** + * Replace any text matching the given pattern with the MASK value + * @param input + * @param pattern + * @return + */ + private static String hidePassword(String input, Pattern pattern) { + final Matcher m = pattern.matcher(input); + final StringBuffer result = new StringBuffer(); + while (m.find()) { + final String match = m.group(); + final int start = m.start(); + m.appendReplacement(result, + match.substring(0, + m.start(1) - start) + + MASK + + match.substring(m.end(1) - start, m.end() - start)); + } + m.appendTail(result); + return result.toString(); + } +} diff --git a/fhir-core/src/test/java/com/ibm/fhir/core/util/test/LogSupportTest.java b/fhir-core/src/test/java/com/ibm/fhir/core/util/test/LogSupportTest.java new file mode 100644 index 00000000000..dd925c11832 --- /dev/null +++ b/fhir-core/src/test/java/com/ibm/fhir/core/util/test/LogSupportTest.java @@ -0,0 +1,46 @@ +/* + * (C) Copyright IBM Corp. 2022 + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.ibm.fhir.core.util.test; + +import static org.testng.Assert.assertEquals; + +import org.testng.annotations.Test; + +import com.ibm.fhir.core.util.LogSupport; + +/** + * Unit tests for {@link LogSupport} methods + */ +public class LogSupportTest { + + @Test + public void testPassReplaceEquals() { + assertEquals(LogSupport.hidePassword("something password=\"change-password\" something else"), "something password=\"*****\" something else"); + } + + @Test + public void testPassReplaceEqualsWithSpace() { + assertEquals(LogSupport.hidePassword("something password = \"change-password\" something else"), "something password = \"*****\" something else"); + } + + @Test + public void testPassReplaceJson() { + assertEquals(LogSupport.hidePassword("something \"password\": \"change-password\" something else"), "something \"password\": \"*****\" something else"); + } + + @Test + public void testPassReplaceJsonCompact() { + assertEquals(LogSupport.hidePassword("something \"password\":\"change-password\" something else"), "something \"password\":\"*****\" something else"); + } + + @Test + public void testPassReplaceMixed() { + final String src = "org.apache.kafka.common.security.plain.PlainLoginModule required username=\"token\" password=\"change-password-\";"; + final String tgt = src.replace("change-password-", "*****"); + assertEquals(LogSupport.hidePassword(src), tgt); + } +} diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/api/DistributionContext.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/api/DistributionContext.java new file mode 100644 index 00000000000..4475d9c37e7 --- /dev/null +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/api/DistributionContext.java @@ -0,0 +1,42 @@ +/* + * (C) Copyright IBM Corp. 2022 + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.ibm.fhir.database.utils.api; + + +/** + * Carrier for the distribution context passed to some adapter methods + */ +public class DistributionContext { + // The type of distribution to be applied for a particular table + private final DistributionType distributionType; + // The column name to be used for distribution when the distributionType is DISTRIBUTED + private final String distributionColumnName; + + /** + * Public constructor + * @param distributionType + * @param distributionColumnName + */ + public DistributionContext(DistributionType distributionType, String distributionColumnName) { + this.distributionType = distributionType; + this.distributionColumnName = distributionColumnName; + } + + /** + * @return the distributionType + */ + public DistributionType getDistributionType() { + return distributionType; + } + + /** + * @return the distributionColumnName + */ + public String getDistributionColumnName() { + return distributionColumnName; + } +} diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/api/DistributionType.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/api/DistributionType.java new file mode 100644 index 00000000000..0442b40e42a --- /dev/null +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/api/DistributionType.java @@ -0,0 +1,16 @@ +/* + * (C) Copyright IBM Corp. 2022 + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.ibm.fhir.database.utils.api; + +/** + * The type of distribution to use for a table + */ +public enum DistributionType { + NONE, // table will not be distributed at all + REFERENCE, // table will be replicated + DISTRIBUTED // table will be sharded by a known column +} diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/api/IDatabaseAdapter.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/api/IDatabaseAdapter.java index 371b8bddce2..27d87d62597 100644 --- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/api/IDatabaseAdapter.java +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/api/IDatabaseAdapter.java @@ -1,5 +1,5 @@ /* - * (C) Copyright IBM Corp. 2019, 2021 + * (C) Copyright IBM Corp. 2019, 2022 * * SPDX-License-Identifier: Apache-2.0 */ @@ -79,9 +79,19 @@ public interface IDatabaseAdapter { * @param tablespaceName * @param withs * @param checkConstraints + * @param distributionContext */ public void createTable(String schemaName, String name, String tenantColumnName, List columns, - PrimaryKeyDef primaryKey, IdentityDef identity, String tablespaceName, List withs, List checkConstraints); + PrimaryKeyDef primaryKey, IdentityDef identity, String tablespaceName, List withs, List checkConstraints, + DistributionContext distributionContext); + + /** + * Apply any distribution rules configured for the named table + * @param schemaName + * @param tableName + * @param distributionContext + */ + public void applyDistributionRules(String schemaName, String tableName, DistributionContext distributionContext); /** * Add a new column to an existing table @@ -150,27 +160,29 @@ public void createTable(String schemaName, String name, String tenantColumnName, public void dropProcedure(String schemaName, String procedureName); /** - * + * Create a unique index * @param schemaName * @param tableName * @param indexName * @param tenantColumnName * @param indexColumns * @param includeColumns + * @param distributionContext */ public void createUniqueIndex(String schemaName, String tableName, String indexName, String tenantColumnName, - List indexColumns, List includeColumns); + List indexColumns, List includeColumns, DistributionContext distributionContext); /** - * + * Create a unique index * @param schemaName * @param tableName * @param indexName * @param tenantColumnName * @param indexColumns + * @param distributionContext */ public void createUniqueIndex(String schemaName, String tableName, String indexName, String tenantColumnName, - List indexColumns); + List indexColumns, DistributionContext distributionContext); /** * @@ -500,6 +512,14 @@ public default boolean useSessionVariable() { */ public void createOrReplaceFunction(String schemaName, String objectName, Supplier supplier); + /** + * For Citus, functions can be distributed by one of their parameters (typically the first) + * @param schemaName + * @param functionName + * @param distributeByParamNumber + */ + public void distributeFunction(String schemaName, String functionName, int distributeByParamNumber); + /** * drops a given function * @param schemaName @@ -545,6 +565,15 @@ public default boolean useSessionVariable() { */ public void enableForeignKey(String schemaName, String tableName, String constraintName); + /** + * Does the named foreign key constraint exist + * @param schemaName + * @param tableName + * @param constraintName + * @return + */ + public boolean doesForeignKeyConstraintExist(String schemaName, String tableName, String constraintName); + /** * * @param schemaName diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/api/IDatabaseTranslator.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/api/IDatabaseTranslator.java index a03484d4df7..19c726175c4 100644 --- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/api/IDatabaseTranslator.java +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/api/IDatabaseTranslator.java @@ -196,6 +196,12 @@ public interface IDatabaseTranslator { */ DbType getType(); + /** + * True if the database type is part of the PostgreSQL family (POSTGRESQL, CITUS) + * @return + */ + boolean isFamilyPostgreSQL(); + /** * The name of the "DUAL" table...that special table giving us one row/column. * @return the name of the "DUAL" table for the database, or null if not supported @@ -210,6 +216,13 @@ public interface IDatabaseTranslator { */ String dropForeignKeyConstraint(String qualifiedTableName, String constraintName); + /** + * Generate the DDL for dropping the named view + * @param qualifiedViewName + * @return + */ + String dropView(String qualifiedViewName); + /** * Does this database use the schema prefix when defining indexes * @return diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/api/IDatabaseTypeAdapter.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/api/IDatabaseTypeAdapter.java index 6860034c2a8..bab22adbc4d 100644 --- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/api/IDatabaseTypeAdapter.java +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/api/IDatabaseTypeAdapter.java @@ -35,6 +35,14 @@ public default String doubleClause() { return "DOUBLE"; } + /** + * Generate a clause for smallint data type + * @return + */ + public default String smallintClause() { + return "SMALLINT"; + } + /** * Generate a clause for VARCHAR * @param size diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/api/ISchemaAdapter.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/api/ISchemaAdapter.java new file mode 100644 index 00000000000..7af8243cca8 --- /dev/null +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/api/ISchemaAdapter.java @@ -0,0 +1,622 @@ +/* + * (C) Copyright IBM Corp. 2022 + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.ibm.fhir.database.utils.api; + +import java.util.Collection; +import java.util.List; +import java.util.function.Supplier; + +import com.ibm.fhir.database.utils.common.SchemaInfoObject; +import com.ibm.fhir.database.utils.model.CheckConstraint; +import com.ibm.fhir.database.utils.model.ColumnBase; +import com.ibm.fhir.database.utils.model.IdentityDef; +import com.ibm.fhir.database.utils.model.OrderedColumnDef; +import com.ibm.fhir.database.utils.model.PrimaryKeyDef; +import com.ibm.fhir.database.utils.model.Privilege; +import com.ibm.fhir.database.utils.model.Table; +import com.ibm.fhir.database.utils.model.With; + +/** + * Adapter to create a particular flavor of the FHIR schema + */ +public interface ISchemaAdapter { + + /** + * Create a new tablespace with the given name + * + * @param tablespaceName + */ + public void createTablespace(String tablespaceName); + + /** + * Create a new tablespace using the given extent size + * + * @param tablespaceName + * @param extentSizeKB + */ + public void createTablespace(String tablespaceName, int extentSizeKB); + + /** + * Drop an existing tablespace, including all of its contents + * + * @param tablespaceName + */ + public void dropTablespace(String tablespaceName); + + /** + * Detach the partition + * + * @param schemaName + * @param tableName + * @param partitionName + * @param newTableName + */ + public void detachPartition(String schemaName, String tableName, String partitionName, String newTableName); + + /** + * Build the create table DDL + * + * @param schemaName + * @param name + * @param tenantColumnName optional column name to enable multi-tenancy + * @param columns + * @param primaryKey + * @param identity + * @param tablespaceName + * @param withs + * @param checkConstraints + * @param distributionType + * @param distributionColumnName + */ + public void createTable(String schemaName, String name, String tenantColumnName, List columns, + PrimaryKeyDef primaryKey, IdentityDef identity, String tablespaceName, List withs, List checkConstraints, + DistributionType distributionType, String distributionColumnName); + + /** + * Apply any distribution rules configured for the named table + * @param schemaName + * @param tableName + * @param distributionType + * @param distributionColumnName + */ + public void applyDistributionRules(String schemaName, String tableName, DistributionType distributionType, String distributionColumnName); + + /** + * Add a new column to an existing table + * @param schemaName + * @param tableName + * @param column + */ + public void alterTableAddColumn(String schemaName, String tableName, ColumnBase column); + + /** + * Reorg the table if the underlying database supports it. Required after + * columns are added/removed from a table. + * @param schemaName + * @param tableName + */ + public void reorgTable(String schemaName, String tableName); + + /** + * Create ROW type used for passing values to stored procedures e.g.: + * + *
+     * CREATE OR REPLACE TYPE .t_str_values AS ROW (parameter_name_id INTEGER,
+     * str_value VARCHAR(511 OCTETS), str_value_lcase VARCHAR(511 OCTETS))
+     * 
+ * + * @param schemaName + * @param typeName + * @param columns + */ + public void createRowType(String schemaName, String typeName, List columns); + + /** + * Create ARRAY type used for passing values to stored procedures e.g.: CREATE + * OR REPLACE TYPE .t_str_values_arr AS .t_str_values ARRAY[256] + * + * @param schemaName + * @param typeName + * @param valueType + * @param arraySize + */ + public void createArrType(String schemaName, String typeName, String valueType, int arraySize); + + /** + * Drop the type object from the schema + * + * @param schemaName + * @param typeName + */ + public void dropType(String schemaName, String typeName); + + /** + * Create the stored procedure using the DDL text provided by the supplier + * + * @param schemaName + * @param procedureName + * @param supplier + */ + public void createOrReplaceProcedure(String schemaName, String procedureName, Supplier supplier); + + /** + * Drop the given procedure + * + * @param schemaName + * @param procedureName + */ + public void dropProcedure(String schemaName, String procedureName); + + /** + * Create a unique index + * @param schemaName + * @param tableName + * @param indexName + * @param tenantColumnName + * @param indexColumns + * @param includeColumns + * @param distributionType + * @param distributionColumnName + */ + public void createUniqueIndex(String schemaName, String tableName, String indexName, String tenantColumnName, + List indexColumns, List includeColumns, + DistributionType distributionType, String distributionColumnName); + + /** + * Create a unique index + * @param schemaName + * @param tableName + * @param indexName + * @param tenantColumnName + * @param indexColumns + * @param distributionRules + * @param distributionColumnName + */ + public void createUniqueIndex(String schemaName, String tableName, String indexName, String tenantColumnName, + List indexColumns, DistributionType distributionType, String distributionColumnName); + + /** + * Create an index on the named schema.table object + * @param schemaName + * @param tableName + * @param indexName + * @param tenantColumnName + * @param indexColumns + * @param distributionType + */ + public void createIndex(String schemaName, String tableName, String indexName, String tenantColumnName, + List indexColumns, DistributionType distributionType); + + /** + * + *
+     * CREATE VARIABLE ptng.session_tenant INT DEFAULT NULL;
+     * 
+ * + * @param schemaName + * @param variableName + */ + public void createIntVariable(String schemaName, String variableName); + + /** + * + *
+     * CREATE OR REPLACE PERMISSION ROW_ACCESS ON ptng.patients FOR ROWS WHERE patients.mt_id =
+     * ptng.session_tenant ENFORCED FOR ALL ACCESS ENABLE;
+     * 
+ * + * @param schemaName + * @param permissionName + * @param tableName + * @param predicate + */ + public void createOrReplacePermission(String schemaName, String permissionName, String tableName, String predicate); + + /** + * + * + *
 ALTER TABLE  ACTIVATE ROW ACCESS CONTROL
+     * 
+ * + * @param schemaName + * @param tableName + */ + public void activateRowAccessControl(String schemaName, String tableName); + + /** + * Deactivate row access control on a table ALTER TABLE tbl_name DEACTIVATE ROW + * ACCESS CONTROL + * + * @param schemaName + * @param tableName + */ + public void deactivateRowAccessControl(String schemaName, String tableName); + + /** + * Build the DML statement for setting a session variable + * + * @param schemaName + * @param variableName + * @param value + */ + public void setIntVariable(String schemaName, String variableName, int value); + + /** + * Drop table from the schema + * + * @param schemaName + * @param name + */ + public void dropTable(String schemaName, String name); + + /** + * Drop permission object from the schema + * + * @param schemaName + * @param permissionName + */ + public void dropPermission(String schemaName, String permissionName); + + /** + * @param schemaName + * @param variableName + */ + public void dropVariable(String schemaName, String variableName); + + /** + * + * @param constraintName + * @param schemaName + * @param name + * @param targetSchema + * @param targetTable + * @param targetColumnName + * @param tenantColumnName + * @param columns + * @param enforced + * @param distributionType distribution type of the source table + * @param targetIsReference + */ + public void createForeignKeyConstraint(String constraintName, String schemaName, String name, String targetSchema, + String targetTable, String targetColumnName, String tenantColumnName, List columns, + boolean enforced, DistributionType distributionType, boolean targetIsReference); + + /** + * Allocate a new tenant + * + * @param adminSchemaName + * @param schemaName + * @param tenantName + * @param tenantKey + * @param tenantSalt + * @param idSequenceName + * @return + */ + public int allocateTenant(String adminSchemaName, String schemaName, String tenantName, String tenantKey, + String tenantSalt, String idSequenceName); + + /** + * Delete all the metadata associated with the given tenant identifier, as long as the + * tenant status is DROPPED. + * @param tenantId + */ + public void deleteTenantMeta(String adminSchemaName, int tenantId); + + /** + * Get the tenant id for the given schema and tenant name + * + * @param adminSchemaName + * @param tenantName + * @return + */ + public int findTenantId(String adminSchemaName, String tenantName); + + /** + * Create the partitions on each of these tables + * + * @param tables + * @param schemaName + * @param newTenantId + * @param extentSizeKB + */ + public void createTenantPartitions(Collection tables, String schemaName, int newTenantId, int extentSizeKB); + + /** + * Add a new tenant partition to each of the tables in the collection. Idempotent, so can + * be run to add partitions for existing tenants to new tables + * @param tables + * @param schemaName + * @param newTenantId + */ + public void addNewTenantPartitions(Collection
tables, String schemaName, int newTenantId); + + /** + * Detach the partition associated with the tenantId from each of the given tables + * + * @param tables + * @param schemaName + * @param tenantId + * @param tenantStagingTable + */ + public void removeTenantPartitions(Collection
tables, String schemaName, int tenantId); + + /** + * Drop the tables which were created by the detach partition operation (as + * part of tenant deprovisioning). + * @param tables + * @param schemaName + * @param tenantId + */ + public void dropDetachedPartitions(Collection
tables, String schemaName, int tenantId); + + /** + * Update the tenant status + * + * @param adminSchemaName + * @param tenantId + * @param status + */ + public void updateTenantStatus(String adminSchemaName, int tenantId, TenantStatus status); + + /** + * + * @param schemaName + * @param sequenceName + * @param startWith the START WITH value for the sequence + * @param cache the sequence CACHE value + */ + public void createSequence(String schemaName, String sequenceName, long startWith, int cache, int incrementBy); + + /** + * + * @param schemaName + * @param sequenceName + */ + public void dropSequence(String schemaName, String sequenceName); + + /** + * Sets/resets the sequence to start with the given value. + * @param schemaName + * @param sequenceName + * @param restartWith + * @param cache + */ + public void alterSequenceRestartWith(String schemaName, String sequenceName, long restartWith, int cache, int incrementBy); + + /** + * Grant the list of privileges on the named object to the user. This is a + * general purpose method which can be used to specify privileges for any object + * type which doesn't need the object type to be specified in the grant DDL. + * + * @param schemaName + * @param tableName + * @param privileges + * @param toUser + */ + public void grantObjectPrivileges(String schemaName, String tableName, Collection privileges, String toUser); + + /** + * Grant the collection of privileges on the named procedure to the user + * + * @param schemaName + * @param procedureName + * @param privileges + * @param toUser + */ + public void grantProcedurePrivileges(String schemaName, String procedureName, Collection privileges, + String toUser); + + /** + * Grant the collection of privileges on the named variable to the user + * + * @param schemaName + * @param variableName + * @param privileges + * @param toUser + */ + public void grantVariablePrivileges(String schemaName, String variableName, Collection privileges, + String toUser); + + /** + * Grant the collection of privileges on the named variable to the user + * + * @param schemaName + * @param objectName + * @param group + * @param toUser + */ + public void grantSequencePrivileges(String schemaName, String objectName, Collection group, + String toUser); + + /** + * Grants USAGE on the given schemaName to the given user + * @param schemaName + */ + public void grantSchemaUsage(String schemaName, String grantToUser); + + /** + * Grant access to all sequences in the named schema + * @param schemaName + * @param grantToUser + */ + public void grantAllSequenceUsage(String schemaName, String grantToUser); + + /** + * Check if the table currently exists + * + * @param schemaName + * @param objectName + * @return + */ + public boolean doesTableExist(String schemaName, String objectName); + + /** + * Create a database schema + * + * @param schemaName + */ + public void createSchema(String schemaName); + + /** + * create a unique constraint on a table. + * + * @param constraintName + * @param columns + * @param schemaName + * @param name + */ + public void createUniqueConstraint(String constraintName, List columns, String schemaName, String name); + + /** + * checks connectivity to the database and that it is compatible + * @param adminSchema + * @return + */ + public boolean checkCompatibility(String adminSchema); + + /** + * @return a false, if not used, or true if used with the persistence layer. + */ + public boolean useSessionVariable(); + + /** + * creates or replaces the SQL function + * @param schemaName + * @param objectName + * @param supplier + */ + public void createOrReplaceFunction(String schemaName, String objectName, Supplier supplier); + + /** + * For Citus, functions can be distributed by one of their parameters (typically the first) + * @param schemaName + * @param functionName + * @param distributeByParamNumber + */ + public void distributeFunction(String schemaName, String functionName, int distributeByParamNumber); + + /** + * drops a given function + * @param schemaName + * @param functionName + */ + public void dropFunction(String schemaName, String functionName); + + /** + * grants permissions on a given function + * @param schemaName + * @param functionName + * @param privileges + * @param toUser + */ + public void grantFunctionPrivileges(String schemaName, String functionName, Collection privileges, String toUser); + + /** + * Drop the tablespace associated with the given tenantId + * @param tenantId + */ + public void dropTenantTablespace(int tenantId); + + /** + * Disable the FK with the given constraint name + * @param tableName + * @param constraintName + */ + public void disableForeignKey(String schemaName, String tableName, String constraintName); + + /** + * Drop the FK on the table with the given constraint name + * @param schemaName + * @param tableName + * @param constraintName + */ + public void dropForeignKey(String schemaName, String tableName, String constraintName); + + /** + * Enable the FK with the given constraint name + * @param schemaName + * @param tableName + * @param constraintName + */ + public void enableForeignKey(String schemaName, String tableName, String constraintName); + + /** + * Check to see if the named foreign key constraint already exists + * @param schemaName + * @param tableName + * @param constraintName + * @return + */ + public boolean doesForeignKeyConstraintExist(String schemaName, String tableName, String constraintName); + + /** + * + * @param schemaName + * @param tableName + */ + public void setIntegrityOff(String schemaName, String tableName); + + /** + * + * @param schemaName + * @param tableName + */ + public void setIntegrityUnchecked(String schemaName, String tableName); + + /** + * Change the CACHE value of the named identity generated always column + * @param schemaName + * @param objectName + * @param columnName + * @param cache + */ + public void alterTableColumnIdentityCache(String schemaName, String objectName, String columnName, int cache); + + /** + * Drop the named index + * @param schemaName + * @param indexName + */ + public void dropIndex(String schemaName, String indexName); + + /** + * Create the view as defined by the selectClause + * @param schemaName + * @param objectName + * @param selectClause + */ + public void createView(String schemaName, String objectName, String selectClause); + + /** + * Drop the view from the database + * @param schemaName + * @param objectName + */ + public void dropView(String schemaName, String objectName); + + /** + * Create or replace the view + * @param schemaName + * @param objectName + * @param selectClause + */ + public void createOrReplaceView(String schemaName, String objectName, String selectClause); + + /** + * List the objects present in the given schema + * @param schemaName + * @return + */ + List listSchemaObjects(String schemaName); + + /** + * Run the given statement against the database represented by this adapter + * + * @param statement + */ + public void runStatement(IDatabaseStatement statement); +} \ No newline at end of file diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/api/SchemaApplyContext.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/api/SchemaApplyContext.java new file mode 100644 index 00000000000..1c7cc32b2d9 --- /dev/null +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/api/SchemaApplyContext.java @@ -0,0 +1,72 @@ +/* + * (C) Copyright IBM Corp. 2022 + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.ibm.fhir.database.utils.api; + + +/** + * Used to control how the schema gets applied + */ +public class SchemaApplyContext { + private final boolean includeForeignKeys; + + /** + * Protected constructor + * @param includeForeignKeys + */ + protected SchemaApplyContext(boolean includeForeignKeys) { + this.includeForeignKeys = includeForeignKeys; + } + + /** + * Get the includeForeignKeys flag + * @return + */ + public boolean isIncludeForeignKeys() { + return this.includeForeignKeys; + } + + /** + * Create a new {@link Builder} instance + * @return + */ + public static Builder builder() { + return new Builder(); + } + + /** + * Builder for {@link SchemaApplyContext} + */ + public static class Builder { + private boolean includeForeignKeys; + + /** + * Setter for includeForeignKeys + * @param flag + * @return + */ + public Builder setIncludeForeignKeys(boolean flag) { + this.includeForeignKeys = flag; + return this; + } + /** + * Build an immutable instance of {@link SchemaApplyContext} using + * the current state of this + * @return + */ + public SchemaApplyContext build() { + return new SchemaApplyContext(this.includeForeignKeys); + } + } + + /** + * Get a default instance of the schema apply context + * @return + */ + public static SchemaApplyContext getDefault() { + return new SchemaApplyContext(true); + } +} diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/api/SchemaType.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/api/SchemaType.java new file mode 100644 index 00000000000..39fce034ae2 --- /dev/null +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/api/SchemaType.java @@ -0,0 +1,22 @@ +/* + * (C) Copyright IBM Corp. 2022 + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.ibm.fhir.database.utils.api; + + +/** + * The flavor of database schema + * PLAIN - the schema we typically deploy to Derby or PostgreSQL + * MULTITENANT - on Db2 supporting multiple tenants using partitioning and RBAC + * DISTRIBUTED - for use with distributed technologies like Citus DB + * SHARDED - explicitly sharded using an injected shard_key column + */ +public enum SchemaType { + PLAIN, + MULTITENANT, + DISTRIBUTED, + SHARDED +} diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/citus/CitusAdapter.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/citus/CitusAdapter.java new file mode 100644 index 00000000000..82257514882 --- /dev/null +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/citus/CitusAdapter.java @@ -0,0 +1,265 @@ +/* + * (C) Copyright IBM Corp. 2022 + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.ibm.fhir.database.utils.citus; + +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.Collection; +import java.util.List; +import java.util.Set; +import java.util.logging.Logger; +import java.util.stream.Collectors; + +import com.ibm.fhir.database.utils.api.DistributionContext; +import com.ibm.fhir.database.utils.api.DistributionType; +import com.ibm.fhir.database.utils.api.IConnectionProvider; +import com.ibm.fhir.database.utils.api.IDatabaseTarget; +import com.ibm.fhir.database.utils.common.DataDefinitionUtil; +import com.ibm.fhir.database.utils.model.CheckConstraint; +import com.ibm.fhir.database.utils.model.ColumnBase; +import com.ibm.fhir.database.utils.model.IdentityDef; +import com.ibm.fhir.database.utils.model.OrderedColumnDef; +import com.ibm.fhir.database.utils.model.PrimaryKeyDef; +import com.ibm.fhir.database.utils.model.With; +import com.ibm.fhir.database.utils.postgres.PostgresAdapter; + + +/** + * A database adapter implementation for Citus (distributed PostgreSQL) + */ +public class CitusAdapter extends PostgresAdapter { + private static final Logger logger = Logger.getLogger(CitusAdapter.class.getName()); + + /** + * Public constructor + * @param target + */ + public CitusAdapter(IDatabaseTarget target) { + super(target); + } + + /** + * Public constructor + * @param cp + */ + public CitusAdapter(IConnectionProvider cp) { + super(cp); + } + + @Override + public void createTable(String schemaName, String name, String tenantColumnName, List columns, PrimaryKeyDef primaryKey, + IdentityDef identity, String tablespaceName, List withs, List checkConstraints, + DistributionContext distributionContext) { + + // We don't use partitioning for multi-tenancy in our Citus implementation, so ignore the mt_id column + if (tenantColumnName != null) { + warnOnce(MessageKey.MULTITENANCY, "Citus does not support multi-tenancy: " + name); + } + + if (distributionContext != null) { + // Build a Citus-specific create table statement + String ddl = buildCitusCreateTableStatement(schemaName, name, columns, primaryKey, identity, withs, checkConstraints, distributionContext); + runStatement(ddl); + } else { + // building a plain schema, so we can use the standard PostgreSQL method + super.createTable(schemaName, name, tenantColumnName, columns, primaryKey, identity, tablespaceName, withs, checkConstraints, distributionContext); + } + } + + /** + * Construct a CREATE TABLE statement using some Citus-specific business + * logic. + * @param schema + * @param name + * @param columns + * @param pkDef + * @param identity + * @param withs + * @param checkConstraints + * @param distributionType + * @return + */ + private String buildCitusCreateTableStatement(String schema, String name, List columns, + PrimaryKeyDef pkDef, IdentityDef identity, List withs, + List checkConstraints, DistributionContext distributionContext) { + + final DistributionType distributionType = distributionContext.getDistributionType(); + final String distributionColumnName = distributionContext.getDistributionColumnName(); + if (identity != null && (distributionType == DistributionType.DISTRIBUTED || distributionType == DistributionType.REFERENCE)) { + logger.warning("Citus: Ignoring IDENTITY columns on distributed table: '" + name + "." + identity.getColumnName()); + identity = null; + } + + StringBuilder result = new StringBuilder(); + result.append("CREATE TABLE "); + result.append(getQualifiedName(schema, name)); + result.append('('); + result.append(buildColumns(columns, identity)); + + if (pkDef != null) { + // Add the primary key definition after the columns. For Citus, if the table is + // distributed (sharded) then the distribution key MUST be one of the columns + // of the primary key. Make sure we lower-case things first so we guarantee a + // match where expected + String pkColString = String.join(",", pkDef.getColumns()); + Set pkSet = pkDef.getColumns().stream().map(c -> c.toLowerCase()).collect(Collectors.toSet()); + final String ldc = distributionType == DistributionType.DISTRIBUTED ? distributionColumnName.toLowerCase() : null; + if (ldc == null || pkSet.contains(ldc)) { + result.append(", CONSTRAINT "); + result.append(pkDef.getConstraintName()); + result.append(" PRIMARY KEY ("); + result.append(pkColString); + result.append(')'); + } else { + // Hopefully this is an intended data model design decision. Because it's so + // fundamental, we always want to warn about it. + logger.warning("Skipping primary key for table '" + name + + "' because it does not include required distribution column: '" + ldc + + "', only (" + pkColString + ")"); + } + } + + // Add any CHECK constraints + for (CheckConstraint cc: checkConstraints) { + result.append(", CONSTRAINT "); + result.append(cc.constraintName); + result.append(" CHECK ("); + result.append(cc.getConstraintExpression()); + result.append(")"); + } + result.append(')'); + + // Creates WITH (fillfactor=70, key2=val2); + if (withs != null && !withs.isEmpty()) { + StringBuilder builder = new StringBuilder(" WITH ("); + builder.append( + withs.stream() + .map(with -> with.buildWithComponent()) + .collect(Collectors.joining(","))); + builder.append(")"); + result.append(builder.toString()); + } + + return result.toString(); + } + + @Override + public void createUniqueIndex(String schemaName, String tableName, String indexName, String tenantColumnName, + List indexColumns, DistributionContext distributionContext) { + // For Citus, we are prevented from creating a unique index unless the index contains + // the distribution column + final DistributionType distributionType = distributionContext.getDistributionType(); + final String distributionColumnName = distributionContext.getDistributionColumnName(); + List columnNames = indexColumns.stream().map(ocd -> ocd.getColumnName()).collect(Collectors.toList()); + if (distributionType == DistributionType.DISTRIBUTED && !includesDistributionColumn(distributionColumnName, columnNames)) { + // Can only a normal index because it isn't partitioned by the distributionColumn + String ddl = DataDefinitionUtil.createIndex(schemaName, tableName, indexName, indexColumns, !USE_SCHEMA_PREFIX); + runStatement(ddl); + } else { + // Index is partitioned by the distributionColumn, so it can be unique + String ddl = DataDefinitionUtil.createUniqueIndex(schemaName, tableName, indexName, indexColumns, !USE_SCHEMA_PREFIX); + runStatement(ddl); + } + } + + @Override + public void applyDistributionRules(String schemaName, String tableName, DistributionContext distributionContext) { + // Apply the distribution rules. Tables without distribution rules are created + // only on Citus controller nodes and never distributed to the worker nodes. + final String fullName = DataDefinitionUtil.getQualifiedName(schemaName, tableName); + CitusDistributionCheckDAO distributionCheck = new CitusDistributionCheckDAO(schemaName, tableName); + if (runStatement(distributionCheck)) { + logger.info("Table '" + fullName + "' is already distributed"); + return; + } + + final DistributionType distributionType = distributionContext.getDistributionType(); + final String distributionColumnName = distributionContext.getDistributionColumnName(); + if (distributionType == DistributionType.REFERENCE) { + // A table that is fully replicated for each worker node + logger.info("Citus: distributing reference table '" + fullName + "'"); + CreateReferenceTableDAO dao = new CreateReferenceTableDAO(schemaName, tableName); + runStatement(dao); + } else if (distributionType == DistributionType.DISTRIBUTED) { + // A table that is sharded using a hash on the distributionColumn value + logger.info("Citus: Sharding table '" + fullName + "' using '" + distributionColumnName + "'"); + CreateDistributedTableDAO dao = new CreateDistributedTableDAO(schemaName, tableName, distributionColumnName); + runStatement(dao); + } + } + + @Override + public void distributeFunction(String schemaName, String functionName, int distributeByParamNumber) { + if (distributeByParamNumber < 1) { + throw new IllegalArgumentException("invalid distributeByParamNumber value: " + distributeByParamNumber); + } + // Need to get the signature text first in order to build the create_distribution_function + // statement. Note the cast to ::regprocedure will return a string like this: + // "fhirdata.add_logical_resource_ident(integer,character varying)" + // which can be passed in to the Citus create_distributed_function procedure + final String objectName = DataDefinitionUtil.getQualifiedName(schemaName, functionName); + final String SELECT = + "SELECT p.oid::regprocedure " + + " FROM pg_catalog.pg_proc p " + + " WHERE p.oid::regproc::text = LOWER(?)"; + + if (connectionProvider != null) { + try (Connection c = connectionProvider.getConnection()) { + String functionSig = null; + try (PreparedStatement ps = c.prepareStatement(SELECT)) { + ps.setString(1, objectName); + ResultSet rs = ps.executeQuery(); + if (rs.next()) { + functionSig = rs.getString(1); + } + + if (rs.next()) { + final String fn = DataDefinitionUtil.getQualifiedName(schemaName, functionName); + logger.severe("Overloaded function signature: " + fn + " " + functionSig); + functionSig = rs.getString(1); + logger.severe("Overloaded function signature: " + fn + " " + functionSig); + throw new IllegalStateException("Overloading not supported for function '" + fn + "'"); + } + } + + if (functionSig != null) { + logger.info("Distributing function: " + functionSig); + final String DISTRIBUTE = "SELECT create_distributed_function(?::regprocedure, ?::text)"; + try (PreparedStatement ps = c.prepareStatement(DISTRIBUTE)) { + ps.setString(1, functionSig); + ps.setString(2, "$" + distributeByParamNumber); + ps.execute(); + } + } else { + logger.warning("No matching function found for '" + objectName + "'"); + } + } catch (SQLException x) { + throw getTranslator().translate(x); + } + } else { + throw new IllegalStateException("distributeFunction requires a connectionProvider"); + } + } + + /** + * Asks if the distributionColumnName is contained in the given collection of column names + * + * @implNote case-insensitive + * @param distributionColumnName + * @param columns + * @return + */ + public boolean includesDistributionColumn(String distributionColumnName, Collection columns) { + if (distributionColumnName != null) { + Set colSet = columns.stream().map(p -> p.toLowerCase()).collect(Collectors.toSet()); + return colSet.contains(distributionColumnName.toLowerCase()); + } + return false; + } +} \ No newline at end of file diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/citus/CitusDistributionCheckDAO.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/citus/CitusDistributionCheckDAO.java new file mode 100644 index 00000000000..3bb1b3395ce --- /dev/null +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/citus/CitusDistributionCheckDAO.java @@ -0,0 +1,62 @@ +/* + * (C) Copyright IBM Corp. 2022 + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.ibm.fhir.database.utils.citus; + +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.logging.Logger; + +import com.ibm.fhir.database.utils.api.IDatabaseSupplier; +import com.ibm.fhir.database.utils.api.IDatabaseTranslator; +import com.ibm.fhir.database.utils.common.DataDefinitionUtil; + +/** + * DAO to check if the table is already distributed + */ +public class CitusDistributionCheckDAO implements IDatabaseSupplier { + private static final Logger logger = Logger.getLogger(CitusDistributionCheckDAO.class.getName()); + + private final String schemaName; + private final String tableName; + + /** + * Public constructor + * + * @param schemaName + * @param tableName + */ + public CitusDistributionCheckDAO(String schemaName, String tableName) { + DataDefinitionUtil.assertValidName(schemaName); + DataDefinitionUtil.assertValidName(tableName); + this.schemaName = schemaName.toLowerCase(); + this.tableName = tableName.toLowerCase(); + } + + @Override + public Boolean run(IDatabaseTranslator translator, Connection c) { + Boolean result = Boolean.FALSE; + + final String relname = DataDefinitionUtil.getQualifiedName(schemaName, this.tableName); + final String SQL = "SELECT 1 FROM pg_dist_partition WHERE logicalrelid = ?::regclass"; + + try (PreparedStatement ps = c.prepareStatement(SQL)) { + ps.setString(1, relname); + ResultSet rs = ps.executeQuery(); + if (rs.next()) { + result = Boolean.TRUE; + } + } catch (SQLException x) { + // Translate the exception into something a little more meaningful + // for this database type and application + logger.severe("select failed: " + SQL + " for logicalrelid = '" + relname + "'"); + throw translator.translate(x); + } + return result; + } +} \ No newline at end of file diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/citus/CitusTranslator.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/citus/CitusTranslator.java new file mode 100644 index 00000000000..5933ec50dd1 --- /dev/null +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/citus/CitusTranslator.java @@ -0,0 +1,21 @@ +/* + * (C) Copyright IBM Corp. 2022 + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.ibm.fhir.database.utils.citus; + +import com.ibm.fhir.database.utils.model.DbType; +import com.ibm.fhir.database.utils.postgres.PostgresTranslator; + + +/** + * IDatabaseTranslator implementation supporting Citus + */ +public class CitusTranslator extends PostgresTranslator { + @Override + public DbType getType() { + return DbType.CITUS; + } +} diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/citus/ConfigureConnectionDAO.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/citus/ConfigureConnectionDAO.java new file mode 100644 index 00000000000..c75b26004ff --- /dev/null +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/citus/ConfigureConnectionDAO.java @@ -0,0 +1,52 @@ +/* + * (C) Copyright IBM Corp. 2022 + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.ibm.fhir.database.utils.citus; + +import java.sql.Connection; +import java.sql.SQLException; +import java.sql.Statement; + +import com.ibm.fhir.database.utils.api.IDatabaseStatement; +import com.ibm.fhir.database.utils.api.IDatabaseTranslator; + +/** + * DAO to configure the Citus database connection when performing schema build + * activities. This must be performed before any of the following UDFs are called: + *
    + *
  • create_distributed_table + *
  • create_reference_table + *
+ * to avoid the following error: + *
+ * org.postgresql.util.PSQLException: ERROR: cannot modify table "common_token_values" because there was a parallel operation on a distributed table in the transaction
+ * Detail: When there is a foreign key to a reference table, Citus needs to perform all operations over a single connection per node to ensure consistency.
+ * Hint: Try re-running the transaction with "SET LOCAL citus.multi_shard_modify_mode TO 'sequential';"
+ * 
+ */ +public class ConfigureConnectionDAO implements IDatabaseStatement { + + /** + * Public constructor + */ + public ConfigureConnectionDAO() { + } + + @Override + public void run(IDatabaseTranslator translator, Connection c) { + // we need this behavior for all transactions on this connection, so + // we use SET SESSION instead of SET LOCAL + final String SQL = "SET SESSION citus.multi_shard_modify_mode TO 'sequential'"; + + try (Statement s = c.createStatement()) { + s.executeUpdate(SQL); + } catch (SQLException x) { + // Translate the exception into something a little more meaningful + // for this database type and application + throw translator.translate(x); + } + } +} \ No newline at end of file diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/citus/CreateDistributedTableDAO.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/citus/CreateDistributedTableDAO.java new file mode 100644 index 00000000000..d9e6589d582 --- /dev/null +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/citus/CreateDistributedTableDAO.java @@ -0,0 +1,65 @@ +/* + * (C) Copyright IBM Corp. 2022 + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.ibm.fhir.database.utils.citus; + +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.SQLException; +import java.util.logging.Logger; + +import com.ibm.fhir.database.utils.api.IDatabaseStatement; +import com.ibm.fhir.database.utils.api.IDatabaseTranslator; +import com.ibm.fhir.database.utils.common.DataDefinitionUtil; + +/** + * DAO to add a new tenant key record + */ +public class CreateDistributedTableDAO implements IDatabaseStatement { + private static final Logger logger = Logger.getLogger(CreateDistributedTableDAO.class.getName()); + + private final String schemaName; + private final String tableName; + private final String distributionKey; + + /** + * Public constructor + * + * @param schemaName + * @param tableName + * @param distributionKey + */ + public CreateDistributedTableDAO(String schemaName, String tableName, String distributionKey) { + DataDefinitionUtil.assertValidName(schemaName); + DataDefinitionUtil.assertValidName(tableName); + DataDefinitionUtil.assertValidName(distributionKey); + this.schemaName = schemaName.toLowerCase(); + this.tableName = tableName.toLowerCase(); + this.distributionKey = distributionKey.toLowerCase(); + } + + @Override + public void run(IDatabaseTranslator translator, Connection c) { + // Run the Citus create_distributed_table UDF + final String table = DataDefinitionUtil.getQualifiedName(schemaName, this.tableName); + StringBuilder sql = new StringBuilder(); + sql.append("SELECT create_distributed_table("); + sql.append("'").append(table).append("'"); + sql.append(", "); + sql.append("'").append(distributionKey).append("'"); + sql.append(")"); + + try (PreparedStatement ps = c.prepareStatement(sql.toString())) { + // It's a SELECT statement, but we don't care about the ResultSet + ps.executeQuery(); + } catch (SQLException x) { + // Translate the exception into something a little more meaningful + // for this database type and application + logger.severe("Call failed: " + sql.toString()); + throw translator.translate(x); + } + } +} \ No newline at end of file diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/citus/CreateReferenceTableDAO.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/citus/CreateReferenceTableDAO.java new file mode 100644 index 00000000000..78023aba905 --- /dev/null +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/citus/CreateReferenceTableDAO.java @@ -0,0 +1,59 @@ +/* + * (C) Copyright IBM Corp. 2022 + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.ibm.fhir.database.utils.citus; + +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.SQLException; +import java.util.logging.Logger; + +import com.ibm.fhir.database.utils.api.IDatabaseStatement; +import com.ibm.fhir.database.utils.api.IDatabaseTranslator; +import com.ibm.fhir.database.utils.common.DataDefinitionUtil; + +/** + * DAO to add a new tenant key record + */ +public class CreateReferenceTableDAO implements IDatabaseStatement { + private static final Logger logger = Logger.getLogger(CreateReferenceTableDAO.class.getName()); + + private final String schemaName; + private final String tableName; + + /** + * Public constructor + * + * @param schemaName + * @param tableName + */ + public CreateReferenceTableDAO(String schemaName, String tableName) { + DataDefinitionUtil.assertValidName(schemaName); + DataDefinitionUtil.assertValidName(tableName); + this.schemaName = schemaName.toLowerCase(); + this.tableName = tableName.toLowerCase(); + } + + @Override + public void run(IDatabaseTranslator translator, Connection c) { + // Run the Citus create_reference_table UDF + final String table = DataDefinitionUtil.getQualifiedName(schemaName, this.tableName); + StringBuilder sql = new StringBuilder(); + sql.append("SELECT create_reference_table("); + sql.append("'").append(table).append("'"); + sql.append(")"); + + try (PreparedStatement ps = c.prepareStatement(sql.toString())) { + // It's a SELECT statement, but we don't care about the ResultSet + ps.executeQuery(); + } catch (SQLException x) { + // Translate the exception into something a little more meaningful + // for this database type and application + logger.severe("Call failed: " + sql.toString()); + throw translator.translate(x); + } + } +} \ No newline at end of file diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/common/AddColumn.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/common/AddColumn.java index 0d1b65deac5..e5303fb7606 100644 --- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/common/AddColumn.java +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/common/AddColumn.java @@ -17,7 +17,6 @@ import com.ibm.fhir.database.utils.db2.Db2Adapter; import com.ibm.fhir.database.utils.derby.DerbyAdapter; import com.ibm.fhir.database.utils.model.ColumnBase; -import com.ibm.fhir.database.utils.model.DbType; import com.ibm.fhir.database.utils.postgres.PostgresAdapter; /** @@ -46,14 +45,22 @@ public void run(IDatabaseTranslator translator, Connection c) { String qname = DataDefinitionUtil.getQualifiedName(schemaName, tableName); // DatabaseTypeAdapter is needed to find the correct data type for the column. - IDatabaseTypeAdapter dbAdapter = null; - String driveClassName = translator.getDriverClassName(); - if (driveClassName.contains(DbType.DB2.value())) { + final IDatabaseTypeAdapter dbAdapter; + switch (translator.getType()) { + case DB2: dbAdapter = new Db2Adapter(); - } else if (driveClassName.contains(DbType.DERBY.value())) { + break; + case DERBY: dbAdapter = new DerbyAdapter(); - } else if (driveClassName.contains(DbType.POSTGRESQL.value())) { + break; + case POSTGRESQL: dbAdapter = new PostgresAdapter(); + break; + case CITUS: + dbAdapter = new PostgresAdapter(); + break; + default: + throw new IllegalArgumentException("Unsupported database type: " + translator.getType().name()); } String ddl = "ALTER TABLE " + qname + " ADD COLUMN " + columnDef(column, dbAdapter); diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/common/CommonDatabaseAdapter.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/common/CommonDatabaseAdapter.java index a8b187527c4..40cde9f3997 100644 --- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/common/CommonDatabaseAdapter.java +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/common/CommonDatabaseAdapter.java @@ -1,5 +1,5 @@ /* - * (C) Copyright IBM Corp. 2019, 2021 + * (C) Copyright IBM Corp. 2019, 2022 * * SPDX-License-Identifier: Apache-2.0 */ @@ -19,6 +19,7 @@ import java.util.logging.Logger; import java.util.stream.Collectors; +import com.ibm.fhir.database.utils.api.DistributionContext; import com.ibm.fhir.database.utils.api.IConnectionProvider; import com.ibm.fhir.database.utils.api.IDatabaseAdapter; import com.ibm.fhir.database.utils.api.IDatabaseStatement; @@ -197,7 +198,7 @@ protected String buildCreateTableStatement(String schema, String name, List indexColumns, List includeColumns) { + List indexColumns, List includeColumns, DistributionContext distributionContext) { indexColumns = prefixTenantColumn(tenantColumnName, indexColumns); String ddl = DataDefinitionUtil.createUniqueIndex(schemaName, tableName, indexName, indexColumns, includeColumns, true); runStatement(ddl); @@ -205,7 +206,7 @@ public void createUniqueIndex(String schemaName, String tableName, String indexN @Override public void createUniqueIndex(String schemaName, String tableName, String indexName, String tenantColumnName, - List indexColumns) { + List indexColumns, DistributionContext distributionContext) { indexColumns = prefixTenantColumn(tenantColumnName, indexColumns); String ddl = DataDefinitionUtil.createUniqueIndex(schemaName, tableName, indexName, indexColumns, true); runStatement(ddl); @@ -748,6 +749,16 @@ public void reorgTable(String schemaName, String tableName) { // NOP, unless overridden by a subclass (Db2Adapter, for example) } + @Override + public void applyDistributionRules(String schemaName, String tableName, DistributionContext distributionContext) { + // NOP. Only used for distributed databases like Citus + } + + @Override + public void distributeFunction(String schemaName, String functionName, int distributeByParamNumber) { + // NOP. Only used for distributed databases like Citus + } + @Override public void grantSchemaUsage(String schemaName, String grantToUser) { logger.info("Grant usage on schema not required for '" + schemaName + "' on this database"); diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/common/DatabaseTranslatorFactory.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/common/DatabaseTranslatorFactory.java index 78c71e83832..07d8b2e83c5 100644 --- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/common/DatabaseTranslatorFactory.java +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/common/DatabaseTranslatorFactory.java @@ -7,6 +7,7 @@ package com.ibm.fhir.database.utils.common; import com.ibm.fhir.database.utils.api.IDatabaseTranslator; +import com.ibm.fhir.database.utils.citus.CitusTranslator; import com.ibm.fhir.database.utils.db2.Db2Translator; import com.ibm.fhir.database.utils.derby.DerbyTranslator; import com.ibm.fhir.database.utils.model.DbType; @@ -34,6 +35,9 @@ public static IDatabaseTranslator getTranslator(DbType type) { case POSTGRESQL: result = new PostgresTranslator(); break; + case CITUS: + result = new CitusTranslator(); + break; default: throw new IllegalStateException("DbType not supported: " + type); } diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/common/DropColumn.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/common/DropColumn.java index 0871d0a81b1..c35c7e77fc1 100644 --- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/common/DropColumn.java +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/common/DropColumn.java @@ -18,7 +18,6 @@ import com.ibm.fhir.database.utils.api.DataAccessException; import com.ibm.fhir.database.utils.api.IDatabaseStatement; import com.ibm.fhir.database.utils.api.IDatabaseTranslator; -import com.ibm.fhir.database.utils.model.DbType; /** * Drop columns from the schema.table @@ -68,7 +67,7 @@ public void run(IDatabaseTranslator translator, Connection c) { int dropCount = 0; for (String columnName : columnNames) { - if (translator.getType() == DbType.POSTGRESQL) { + if (translator.isFamilyPostgreSQL()) { if (postgresColumnExists(translator, c, columnName)) { ddl.append("\n\t" + "DROP COLUMN " + columnName); dropCount++; diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/common/DropPrimaryKey.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/common/DropPrimaryKey.java index 8da90c32c82..584afddfd53 100644 --- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/common/DropPrimaryKey.java +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/common/DropPrimaryKey.java @@ -16,7 +16,6 @@ import com.ibm.fhir.database.utils.api.DataAccessException; import com.ibm.fhir.database.utils.api.IDatabaseStatement; import com.ibm.fhir.database.utils.api.IDatabaseTranslator; -import com.ibm.fhir.database.utils.model.DbType; /** * Drop the primary key constraint on a table @@ -47,7 +46,7 @@ public void run(IDatabaseTranslator translator, Connection c) { // ought to be doing this via an adapter, which hides the differences between databases final String qname = DataDefinitionUtil.getQualifiedName(this.schemaName, this.tableName); final String ddl; - if (translator.getType() == DbType.POSTGRESQL) { + if (translator.isFamilyPostgreSQL()) { // There could be some schemas built between releases which don't have the ROW_ID PK // so for PostgreSQL we need to check if the constraint exists otherwise the whole // transaction fails. diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/common/DropView.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/common/DropView.java new file mode 100644 index 00000000000..787fc3b8bd7 --- /dev/null +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/common/DropView.java @@ -0,0 +1,49 @@ +/* + * (C) Copyright IBM Corp. 2022 + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.ibm.fhir.database.utils.common; + +import java.sql.Connection; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.logging.Logger; + +import com.ibm.fhir.database.utils.api.IDatabaseStatement; +import com.ibm.fhir.database.utils.api.IDatabaseTranslator; + +/** + * Drop the view identified by schema and view name + */ +public class DropView implements IDatabaseStatement { + private static final Logger logger = Logger.getLogger(DropView.class.getName()); + private final String schemaName; + private final String viewName; + + /** + * Public constructor + * @param schemaName + * @param viewName + */ + public DropView(String schemaName, String viewName) { + DataDefinitionUtil.assertValidName(schemaName); + DataDefinitionUtil.assertValidName(viewName); + this.schemaName = schemaName; + this.viewName = viewName; + } + + @Override + public void run(IDatabaseTranslator translator, Connection c) { + final String qname = DataDefinitionUtil.getQualifiedName(schemaName, viewName); + final String ddl = translator.dropView(qname); + + try (Statement s = c.createStatement()) { + s.executeUpdate(ddl.toString()); + } catch (SQLException x) { + // just log because this means that the view doesn't yet exist + logger.warning("Drop view statement failed: '" + ddl + "': " + x.getMessage()); + } + } +} diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/common/PlainSchemaAdapter.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/common/PlainSchemaAdapter.java new file mode 100644 index 00000000000..189a50cbd38 --- /dev/null +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/common/PlainSchemaAdapter.java @@ -0,0 +1,376 @@ +/* + * (C) Copyright IBM Corp. 2022 + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.ibm.fhir.database.utils.common; + +import java.util.Collection; +import java.util.List; +import java.util.function.Supplier; + +import com.ibm.fhir.database.utils.api.DistributionType; +import com.ibm.fhir.database.utils.api.IDatabaseAdapter; +import com.ibm.fhir.database.utils.api.IDatabaseStatement; +import com.ibm.fhir.database.utils.api.ISchemaAdapter; +import com.ibm.fhir.database.utils.api.TenantStatus; +import com.ibm.fhir.database.utils.model.CheckConstraint; +import com.ibm.fhir.database.utils.model.ColumnBase; +import com.ibm.fhir.database.utils.model.IdentityDef; +import com.ibm.fhir.database.utils.model.OrderedColumnDef; +import com.ibm.fhir.database.utils.model.PrimaryKeyDef; +import com.ibm.fhir.database.utils.model.Privilege; +import com.ibm.fhir.database.utils.model.Table; +import com.ibm.fhir.database.utils.model.With; + + +/** + * Adapter to build the plain version of the FHIR schema. Uses + * the IDatabaseAdapter to hide the specifics of a particular + * database flavor (like Db2, PostgreSQL, Derby etc). + */ +public class PlainSchemaAdapter implements ISchemaAdapter { + + // The adapter we use to execute database-specific DDL + protected final IDatabaseAdapter databaseAdapter; + + /** + * Public constructor + * + * @param databaseAdapter + */ + public PlainSchemaAdapter(IDatabaseAdapter databaseAdapter) { + this.databaseAdapter = databaseAdapter; + } + + @Override + public void createTablespace(String tablespaceName) { + databaseAdapter.createTablespace(tablespaceName); + } + + @Override + public void createTablespace(String tablespaceName, int extentSizeKB) { + databaseAdapter.createTablespace(tablespaceName, extentSizeKB); + } + + @Override + public void dropTablespace(String tablespaceName) { + databaseAdapter.dropTablespace(tablespaceName); + } + + @Override + public void detachPartition(String schemaName, String tableName, String partitionName, String newTableName) { + databaseAdapter.detachPartition(schemaName, tableName, partitionName, newTableName); + } + + @Override + public boolean useSessionVariable() { + return databaseAdapter.useSessionVariable(); + } + + @Override + public void createTable(String schemaName, String name, String tenantColumnName, List columns, PrimaryKeyDef primaryKey, IdentityDef identity, + String tablespaceName, List withs, List checkConstraints, DistributionType distributionType, String distributionColumnName) { + databaseAdapter.createTable(schemaName, name, tenantColumnName, columns, primaryKey, identity, tablespaceName, withs, checkConstraints, null); + } + + @Override + public void applyDistributionRules(String schemaName, String tableName, DistributionType distributionType, String distributionColumnName) { + databaseAdapter.applyDistributionRules(schemaName, tableName, null); + } + + @Override + public void alterTableAddColumn(String schemaName, String tableName, ColumnBase column) { + databaseAdapter.alterTableAddColumn(schemaName, tableName, column); + } + + @Override + public void reorgTable(String schemaName, String tableName) { + databaseAdapter.reorgTable(schemaName, tableName); + } + + @Override + public void createRowType(String schemaName, String typeName, List columns) { + databaseAdapter.createRowType(schemaName, typeName, columns); + } + + @Override + public void createArrType(String schemaName, String typeName, String valueType, int arraySize) { + databaseAdapter.createArrType(schemaName, typeName, valueType, arraySize); + } + + @Override + public void dropType(String schemaName, String typeName) { + databaseAdapter.dropType(schemaName, typeName); + } + + @Override + public void createOrReplaceProcedure(String schemaName, String procedureName, Supplier supplier) { + databaseAdapter.createOrReplaceProcedure(schemaName, procedureName, supplier); + } + + @Override + public void dropProcedure(String schemaName, String procedureName) { + databaseAdapter.dropProcedure(schemaName, procedureName); + } + + @Override + public void createUniqueIndex(String schemaName, String tableName, String indexName, String tenantColumnName, List indexColumns, + List includeColumns, DistributionType distributionType, String distributionColumnName) { + databaseAdapter.createUniqueIndex(schemaName, tableName, indexName, tenantColumnName, indexColumns, includeColumns, null); + } + + @Override + public void createUniqueIndex(String schemaName, String tableName, String indexName, String tenantColumnName, List indexColumns, + DistributionType distributionType, String distributionColumnName) { + databaseAdapter.createUniqueIndex(schemaName, tableName, indexName, tenantColumnName, indexColumns, null); + } + + @Override + public void createIndex(String schemaName, String tableName, String indexName, String tenantColumnName, List indexColumns, DistributionType distributionType) { + databaseAdapter.createIndex(schemaName, tableName, indexName, tenantColumnName, indexColumns); + } + + @Override + public void createIntVariable(String schemaName, String variableName) { + databaseAdapter.createIntVariable(schemaName, variableName); + } + + @Override + public void createOrReplacePermission(String schemaName, String permissionName, String tableName, String predicate) { + databaseAdapter.createOrReplacePermission(schemaName, permissionName, tableName, predicate); + } + + @Override + public void activateRowAccessControl(String schemaName, String tableName) { + databaseAdapter.activateRowAccessControl(schemaName, tableName); + } + + @Override + public void deactivateRowAccessControl(String schemaName, String tableName) { + databaseAdapter.deactivateRowAccessControl(schemaName, tableName); + } + + @Override + public void setIntVariable(String schemaName, String variableName, int value) { + databaseAdapter.setIntVariable(schemaName, variableName, value); + } + + @Override + public void dropTable(String schemaName, String name) { + databaseAdapter.dropTable(schemaName, name); + } + + @Override + public void dropPermission(String schemaName, String permissionName) { + databaseAdapter.dropPermission(schemaName, permissionName); + } + + @Override + public void dropVariable(String schemaName, String variableName) { + databaseAdapter.dropVariable(schemaName, variableName); + } + + @Override + public void createForeignKeyConstraint(String constraintName, String schemaName, String name, String targetSchema, String targetTable, + String targetColumnName, String tenantColumnName, List columns, boolean enforced, DistributionType distributionType, boolean targetIsReference) { + databaseAdapter.createForeignKeyConstraint(constraintName, schemaName, name, targetSchema, targetTable, targetColumnName, tenantColumnName, columns, enforced); + } + + @Override + public int allocateTenant(String adminSchemaName, String schemaName, String tenantName, String tenantKey, String tenantSalt, String idSequenceName) { + return databaseAdapter.allocateTenant(adminSchemaName, schemaName, tenantName, tenantKey, tenantSalt, idSequenceName); + } + + @Override + public void deleteTenantMeta(String adminSchemaName, int tenantId) { + databaseAdapter.deleteTenantMeta(adminSchemaName, tenantId); + } + + @Override + public int findTenantId(String adminSchemaName, String tenantName) { + return databaseAdapter.findTenantId(adminSchemaName, tenantName); + } + + @Override + public void createTenantPartitions(Collection
tables, String schemaName, int newTenantId, int extentSizeKB) { + databaseAdapter.createTenantPartitions(tables, schemaName, newTenantId, extentSizeKB); + } + + @Override + public void addNewTenantPartitions(Collection
tables, String schemaName, int newTenantId) { + databaseAdapter.addNewTenantPartitions(tables, schemaName, newTenantId); + } + + @Override + public void removeTenantPartitions(Collection
tables, String schemaName, int tenantId) { + databaseAdapter.removeTenantPartitions(tables, schemaName, tenantId); + } + + @Override + public void dropDetachedPartitions(Collection
tables, String schemaName, int tenantId) { + databaseAdapter.dropDetachedPartitions(tables, schemaName, tenantId); + } + + @Override + public void updateTenantStatus(String adminSchemaName, int tenantId, TenantStatus status) { + databaseAdapter.updateTenantStatus(adminSchemaName, tenantId, status); + } + + @Override + public void createSequence(String schemaName, String sequenceName, long startWith, int cache, int incrementBy) { + databaseAdapter.createSequence(schemaName, sequenceName, startWith, cache, incrementBy); + } + + @Override + public void dropSequence(String schemaName, String sequenceName) { + databaseAdapter.dropSequence(schemaName, sequenceName); + } + + @Override + public void alterSequenceRestartWith(String schemaName, String sequenceName, long restartWith, int cache, int incrementBy) { + databaseAdapter.alterSequenceRestartWith(schemaName, sequenceName, restartWith, cache, incrementBy); + } + + @Override + public void grantObjectPrivileges(String schemaName, String tableName, Collection privileges, String toUser) { + databaseAdapter.grantObjectPrivileges(schemaName, tableName, privileges, toUser); + } + + @Override + public void grantProcedurePrivileges(String schemaName, String procedureName, Collection privileges, String toUser) { + databaseAdapter.grantProcedurePrivileges(schemaName, procedureName, privileges, toUser); + } + + @Override + public void grantVariablePrivileges(String schemaName, String variableName, Collection privileges, String toUser) { + databaseAdapter.grantVariablePrivileges(schemaName, variableName, privileges, toUser); + } + + @Override + public void grantSequencePrivileges(String schemaName, String objectName, Collection group, String toUser) { + databaseAdapter.grantSequencePrivileges(schemaName, objectName, group, toUser); + } + + @Override + public void grantSchemaUsage(String schemaName, String grantToUser) { + databaseAdapter.grantSchemaUsage(schemaName, grantToUser); + + } + + @Override + public void grantAllSequenceUsage(String schemaName, String grantToUser) { + databaseAdapter.grantAllSequenceUsage(schemaName, grantToUser); + } + + @Override + public boolean doesTableExist(String schemaName, String objectName) { + return databaseAdapter.doesTableExist(schemaName, objectName); + } + + @Override + public void createSchema(String schemaName) { + databaseAdapter.createSchema(schemaName); + } + + @Override + public void createUniqueConstraint(String constraintName, List columns, String schemaName, String name) { + databaseAdapter.createUniqueConstraint(constraintName, columns, schemaName, name); + } + + @Override + public boolean checkCompatibility(String adminSchema) { + return databaseAdapter.checkCompatibility(adminSchema); + } + + @Override + public void createOrReplaceFunction(String schemaName, String objectName, Supplier supplier) { + databaseAdapter.createOrReplaceFunction(schemaName, objectName, supplier); + } + + @Override + public void distributeFunction(String schemaName, String functionName, int distributeByParamNumber) { + databaseAdapter.distributeFunction(schemaName, functionName, distributeByParamNumber); + } + + @Override + public void dropFunction(String schemaName, String functionName) { + databaseAdapter.dropFunction(schemaName, functionName); + } + + @Override + public void grantFunctionPrivileges(String schemaName, String functionName, Collection privileges, String toUser) { + databaseAdapter.grantFunctionPrivileges(schemaName, functionName, privileges, toUser); + } + + @Override + public void dropTenantTablespace(int tenantId) { + databaseAdapter.dropTenantTablespace(tenantId); + } + + @Override + public void disableForeignKey(String schemaName, String tableName, String constraintName) { + databaseAdapter.disableForeignKey(schemaName, tableName, constraintName); + } + + @Override + public void dropForeignKey(String schemaName, String tableName, String constraintName) { + databaseAdapter.dropForeignKey(schemaName, tableName, constraintName); + } + + @Override + public void enableForeignKey(String schemaName, String tableName, String constraintName) { + databaseAdapter.enableForeignKey(schemaName, tableName, constraintName); + } + + @Override + public boolean doesForeignKeyConstraintExist(String schemaName, String tableName, String constraintName) { + return databaseAdapter.doesForeignKeyConstraintExist(schemaName, tableName, constraintName); + } + + @Override + public void setIntegrityOff(String schemaName, String tableName) { + databaseAdapter.setIntegrityOff(schemaName, tableName); + } + + @Override + public void setIntegrityUnchecked(String schemaName, String tableName) { + databaseAdapter.setIntegrityUnchecked(schemaName, tableName); + } + + @Override + public void alterTableColumnIdentityCache(String schemaName, String objectName, String columnName, int cache) { + databaseAdapter.alterTableColumnIdentityCache(schemaName, objectName, columnName, cache); + } + + @Override + public void dropIndex(String schemaName, String indexName) { + databaseAdapter.dropIndex(schemaName, indexName); + } + + @Override + public void createView(String schemaName, String objectName, String selectClause) { + databaseAdapter.createView(schemaName, objectName, selectClause); + } + + @Override + public void dropView(String schemaName, String objectName) { + databaseAdapter.dropView(schemaName, objectName); + } + + @Override + public void createOrReplaceView(String schemaName, String objectName, String selectClause) { + databaseAdapter.createOrReplaceView(schemaName, objectName, selectClause); + } + + @Override + public List listSchemaObjects(String schemaName) { + return databaseAdapter.listSchemaObjects(schemaName); + } + + @Override + public void runStatement(IDatabaseStatement statement) { + databaseAdapter.runStatement(statement); + } +} diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/common/PreparedStatementHelper.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/common/PreparedStatementHelper.java new file mode 100644 index 00000000000..d281b44af66 --- /dev/null +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/common/PreparedStatementHelper.java @@ -0,0 +1,167 @@ +/* + * (C) Copyright IBM Corp. 2022 + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.ibm.fhir.database.utils.common; + +import java.io.InputStream; +import java.sql.CallableStatement; +import java.sql.PreparedStatement; +import java.sql.SQLException; +import java.sql.Timestamp; +import java.sql.Types; +import java.util.Calendar; + +/** + * Collection of utility functions to simply setting values on a PreparedStatement + */ +public class PreparedStatementHelper { + // The PreparedStatement we delegate everything to + private final PreparedStatement ps; + + // The calendar to make sure all times are treated as UTC + private final Calendar UTC = CalendarHelper.getCalendarForUTC(); + + // The current parameter index in the statement + private int index = 1; + + /** + * Public constructor + * @param ps + */ + public PreparedStatementHelper(PreparedStatement ps) { + this.ps = ps; + } + + /** + * Set the (possibly null) int value at the current position + * and increment the position by 1 + * @param value + * @return this instance + * @throws SQLException + */ + public PreparedStatementHelper setInt(Integer value) throws SQLException { + if (value != null) { + ps.setInt(index, value); + } else { + ps.setNull(index, Types.INTEGER); + } + index++; + return this; + } + + /** + * Set the (possibly null) long value at the current position + * and increment the position by 1 + * @param value + * @return this instance + * @throws SQLException + */ + public PreparedStatementHelper setLong(Long value) throws SQLException { + if (value != null) { + ps.setLong(index, value); + } else { + ps.setNull(index, Types.BIGINT); + } + index++; + return this; + } + + /** + * Set the (possibly null) long value at the current position + * and increment the position by 1 + * @param value + * @return this instance + * @throws SQLException + */ + public PreparedStatementHelper setShort(Short value) throws SQLException { + if (value != null) { + ps.setShort(index, value); + } else { + ps.setNull(index, Types.SMALLINT); + } + index++; + return this; + } + + /** + * Set the (possibly null) String value at the current position + * and increment the position by 1 + * @param value + * @return this instance + * @throws SQLException + */ + public PreparedStatementHelper setString(String value) throws SQLException { + if (value != null) { + ps.setString(index, value); + } else { + ps.setNull(index, Types.VARCHAR); + } + index++; + return this; + } + + /** + * Set the (possibly null) InputStream value at the current position + * and increment the position by 1 + * @param value + * @return this instance + * @throws SQLException + */ + public PreparedStatementHelper setBinaryStream(InputStream value) throws SQLException { + if (value != null) { + ps.setBinaryStream(index, value); + } else { + ps.setNull(index, Types.BINARY); + } + index++; + return this; + } + + /** + * Set the (possibly null) int value at the current position + * and increment the position by 1 + * @param value + * @return this instance + * @throws SQLException + */ + public PreparedStatementHelper setTimestamp(Timestamp value) throws SQLException { + if (value != null) { + ps.setTimestamp(index, value, UTC); + } else { + ps.setNull(index, Types.TIMESTAMP); + } + index++; + return this; + } + + /** + * Register an OUT parameter, assuming the delegate is a CallableStatement + * @param parameterType from {@link java.sql.Types} + * @return the parameter index of the OUT parameter + * @throws SQLException + */ + public int registerOutParameter(int parameterType) throws SQLException { + int idx = index++; + if (ps instanceof CallableStatement) { + CallableStatement cs = (CallableStatement)ps; + cs.registerOutParameter(idx, parameterType); + } else { + throw new IllegalStateException("Delegate is not a CallableStatement"); + } + return idx; + } + + /** + * Add a new batch entry based on the current state of the {@link PreparedStatement}. + * Note that we don't return this on purpose...because addBatch should be last in + * any sequence of setXX(...) calls. + * @throws SQLException + */ + public void addBatch() throws SQLException { + ps.addBatch(); + index = 1; + } +} diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/common/ResultSetReader.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/common/ResultSetReader.java new file mode 100644 index 00000000000..f74901ef08a --- /dev/null +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/common/ResultSetReader.java @@ -0,0 +1,130 @@ +/* + * (C) Copyright IBM Corp. 2022 + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.ibm.fhir.database.utils.common; + +import java.math.BigDecimal; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Timestamp; +import java.util.Calendar; + +/** + * Simplifies reading values from a {@link ResultSet} + */ +public class ResultSetReader { + private final Calendar UTC = CalendarHelper.getCalendarForUTC(); + + // The ResultSet we're reading from + private final ResultSet rs; + + private int index = 0; + + /** + * Canonical constructor + * @param rs + */ + public ResultSetReader(ResultSet rs) { + this.rs = rs; + } + + /** + * Invoke {@link ResultSet#next()} + * @return true if the ResultSet has a row + * @throws SQLException + */ + public boolean next() throws SQLException { + index = 1; + return rs.next(); + } + + /** + * Get a string column value and increment the column index + * @return + * @throws SQLException + */ + public String getString() throws SQLException { + return rs.getString(index++); + } + + /** + * Get a Short column value and increment the column index + * @return + * @throws SQLException + */ + public Short getShort() throws SQLException { + Short result = rs.getShort(index++); + if (rs.wasNull()) { + result = null; + } + return result; + } + + /** + * Get an Integer column value and increment the column index + * @return + * @throws SQLException + */ + public Integer getInt() throws SQLException { + Integer result = rs.getInt(index++); + if (rs.wasNull()) { + result = null; + } + return result; + } + + /** + * Get a Long column value and increment the column index + * @return + * @throws SQLException + */ + public Long getLong() throws SQLException { + Long result = rs.getLong(index++); + if (rs.wasNull()) { + result = null; + } + return result; + } + + /** + * Get a BigDecimal column value and increment the column index + * @return + * @throws SQLException + */ + public BigDecimal getBigDecimal() throws SQLException { + BigDecimal result = rs.getBigDecimal(index++); + if (rs.wasNull()) { + result = null; + } + return result; + } + + /** + * Get a Double column value and increment the column index + * @return + * @throws SQLException + */ + public Double getDouble() throws SQLException { + Double result = rs.getDouble(index++); + if (rs.wasNull()) { + result = null; + } + return result; + } + + /** + * Get a Timestamp column value and increment the column index + * @return + * @throws SQLException + */ + public Timestamp getTimestamp() throws SQLException { + Timestamp result = rs.getTimestamp(index++, UTC); + if (rs.wasNull()) { + result = null; + } + return result; + } +} diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/db2/Db2Adapter.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/db2/Db2Adapter.java index 8f5d927c783..a6af211b4a5 100644 --- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/db2/Db2Adapter.java +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/db2/Db2Adapter.java @@ -22,8 +22,10 @@ import java.util.function.Supplier; import java.util.logging.Level; import java.util.logging.Logger; +import java.util.stream.Collectors; import com.ibm.fhir.database.utils.api.DataAccessException; +import com.ibm.fhir.database.utils.api.DistributionContext; import com.ibm.fhir.database.utils.api.DuplicateNameException; import com.ibm.fhir.database.utils.api.DuplicateSchemaException; import com.ibm.fhir.database.utils.api.IConnectionProvider; @@ -72,7 +74,8 @@ public Db2Adapter() { @Override public void createTable(String schemaName, String name, String tenantColumnName, List columns, PrimaryKeyDef primaryKey, - IdentityDef identity, String tablespaceName, List withs, List checkConstraints) { + IdentityDef identity, String tablespaceName, List withs, List checkConstraints, + DistributionContext distributionContext) { // With DB2 we can implement support for multi-tenancy, which we do by injecting a MT_ID column // to the definition and partitioning on that column @@ -207,7 +210,7 @@ public void addNewTenantPartitions(Collection
tables, String schemaName, * @param newTenantId * @param tablespaceName */ - public void addNewTenantPartitions(Collection
tables, Map partitionInfoMap, int newTenantId, String tablespaceName) { + public void addNewTenantPartitions(Collection
allTables, Map partitionInfoMap, int newTenantId, String tablespaceName) { // Thread pool for parallelizing requests int poolSize = connectionProvider.getPoolSize(); if (poolSize == -1) { @@ -217,6 +220,10 @@ public void addNewTenantPartitions(Collection
tables, Map tables = allTables.stream().filter(t->t.isCreate()).collect(Collectors.toList()); + for (Table t: tables) { String qualifiedName = t.getQualifiedName(); PartitionInfo pi = partitionInfoMap.get(t.getObjectName()); @@ -604,6 +611,14 @@ public void enableForeignKey(String schemaName, String tableName, String constra runStatement(ddl); } + @Override + public boolean doesForeignKeyConstraintExist(String schemaName, String tableName, String constraintName) { + Db2DoesForeignKeyConstraintExist test = new Db2DoesForeignKeyConstraintExist(schemaName, tableName, constraintName); + // runStatement may return null in some unit-tests, so we need to protect against that + Boolean val = runStatement(test); + return val != null && val.booleanValue(); + } + /* (non-Javadoc) * @see com.ibm.fhir.database.utils.api.IDatabaseAdapter#setIntegrityOff(java.lang.String, java.lang.String) */ diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/db2/Db2DoesForeignKeyConstraintExist.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/db2/Db2DoesForeignKeyConstraintExist.java new file mode 100644 index 00000000000..e0ee134188c --- /dev/null +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/db2/Db2DoesForeignKeyConstraintExist.java @@ -0,0 +1,61 @@ +/* + * (C) Copyright IBM Corp. 2022 + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.ibm.fhir.database.utils.db2; + +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; + +import com.ibm.fhir.database.utils.api.IDatabaseSupplier; +import com.ibm.fhir.database.utils.api.IDatabaseTranslator; +import com.ibm.fhir.database.utils.common.DataDefinitionUtil; + +/** + * Check the Db2 catalog to see if the configured constraint exists + */ +public class Db2DoesForeignKeyConstraintExist implements IDatabaseSupplier { + + // The constraint identity + private final String schemaName; + private final String tableName; + private final String constraintName; + + /** + * Public constructor + * @param schemaName + */ + public Db2DoesForeignKeyConstraintExist(String schemaName, String tableName, String constraintName) { + this.schemaName = DataDefinitionUtil.assertValidName(schemaName).toUpperCase(); + this.tableName = DataDefinitionUtil.assertValidName(tableName).toUpperCase(); + this.constraintName = DataDefinitionUtil.assertValidName(constraintName).toUpperCase(); + } + + @Override + public Boolean run(IDatabaseTranslator translator, Connection c) { + Boolean result; + + // Grab the list of tables for the configured schema from the DB2 catalog + final String sql = "" + + "SELECT 1 FROM SYSCAT.REFERENCES " + + " WHERE tabschema = ? " + + " AND tabname = ? " + + " AND constname = ? "; + + try (PreparedStatement ps = c.prepareStatement(sql)) { + ps.setString(1, schemaName); + ps.setString(2, tableName); + ps.setString(3, constraintName); + ResultSet rs = ps.executeQuery(); + result = Boolean.valueOf(rs.next()); + } catch (SQLException x) { + throw translator.translate(x); + } + + return result; + } +} diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/db2/Db2Translator.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/db2/Db2Translator.java index 4f2009b59c1..4ee83a6a5b1 100644 --- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/db2/Db2Translator.java +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/db2/Db2Translator.java @@ -265,6 +265,11 @@ public String dropForeignKeyConstraint(String qualifiedTableName, String constra return "ALTER TABLE " + qualifiedTableName + " DROP FOREIGN KEY " + constraintName; } + @Override + public String dropView(String qualifiedViewName) { + return "DROP VIEW " + qualifiedViewName; + } + @Override public String nextValue(String schemaName, String sequenceName) { String qname = DataDefinitionUtil.getQualifiedName(schemaName, sequenceName); @@ -300,4 +305,9 @@ public Optional maximumQueryParameters() { // Maximum number of host variable references in a dynamic SQL statement 32,767 return Optional.of(Integer.valueOf(32767)); } + + @Override + public boolean isFamilyPostgreSQL() { + return false; + } } \ No newline at end of file diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/derby/DerbyAdapter.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/derby/DerbyAdapter.java index 3062e3a5b18..15f1058642a 100644 --- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/derby/DerbyAdapter.java +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/derby/DerbyAdapter.java @@ -1,5 +1,5 @@ /* - * (C) Copyright IBM Corp. 2019, 2021 + * (C) Copyright IBM Corp. 2019, 2022 * * SPDX-License-Identifier: Apache-2.0 */ @@ -15,6 +15,7 @@ import java.util.logging.Level; import java.util.logging.Logger; +import com.ibm.fhir.database.utils.api.DistributionContext; import com.ibm.fhir.database.utils.api.DuplicateNameException; import com.ibm.fhir.database.utils.api.DuplicateSchemaException; import com.ibm.fhir.database.utils.api.IConnectionProvider; @@ -79,7 +80,8 @@ public void warnOnce(MessageKey messageKey, String msg) { @Override public void createTable(String schemaName, String name, String tenantColumnName, List columns, PrimaryKeyDef primaryKey, - IdentityDef identity, String tablespaceName, List withs, List checkConstraints) { + IdentityDef identity, String tablespaceName, List withs, List checkConstraints, + DistributionContext distributionContext) { // Derby doesn't support partitioning, so we ignore tenantColumnName if (tenantColumnName != null) { warnOnce(MessageKey.MULTITENANCY, "Derby does not support multi-tenancy on: [" + name + "]"); @@ -92,10 +94,10 @@ public void createTable(String schemaName, String name, String tenantColumnName, @Override public void createUniqueIndex(String schemaName, String tableName, String indexName, String tenantColumnName, List indexColumns, - List includeColumns) { + List includeColumns, DistributionContext distributionContext) { // Derby doesn't support include columns, so we just have to create a normal index - createUniqueIndex(schemaName, tableName, indexName, tenantColumnName, indexColumns); + createUniqueIndex(schemaName, tableName, indexName, tenantColumnName, indexColumns, distributionContext); } @Override @@ -308,6 +310,14 @@ public void createForeignKeyConstraint(String constraintName, String schemaName, } } + @Override + public boolean doesForeignKeyConstraintExist(String schemaName, String tableName, String constraintName) { + DerbyDoesForeignKeyConstraintExist test = new DerbyDoesForeignKeyConstraintExist(schemaName, tableName, constraintName); + // runStatement may return null in some unit-tests, so we need to protect against that + Boolean val = runStatement(test); + return val != null && val.booleanValue(); + } + @Override protected List prefixTenantColumn(String tenantColumnName, List columns) { // No tenant support, so simply return the columns list unchanged, without prefixing diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/derby/DerbyDoesForeignKeyConstraintExist.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/derby/DerbyDoesForeignKeyConstraintExist.java new file mode 100644 index 00000000000..82f84fc2407 --- /dev/null +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/derby/DerbyDoesForeignKeyConstraintExist.java @@ -0,0 +1,66 @@ +/* + * (C) Copyright IBM Corp. 2022 + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.ibm.fhir.database.utils.derby; + +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; + +import com.ibm.fhir.database.utils.api.IDatabaseSupplier; +import com.ibm.fhir.database.utils.api.IDatabaseTranslator; +import com.ibm.fhir.database.utils.common.DataDefinitionUtil; + +/** + * Inspect the Derby catalog to see if the configured constraint exists + */ +public class DerbyDoesForeignKeyConstraintExist implements IDatabaseSupplier { + + // The constraint identity + private final String schemaName; + private final String tableName; + private final String constraintName; + + /** + * Public constructor + * @param schemaName + */ + public DerbyDoesForeignKeyConstraintExist(String schemaName, String tableName, String constraintName) { + this.schemaName = DataDefinitionUtil.assertValidName(schemaName).toUpperCase(); + this.tableName = DataDefinitionUtil.assertValidName(tableName).toUpperCase(); + this.constraintName = DataDefinitionUtil.assertValidName(constraintName).toUpperCase(); + } + + @Override + public Boolean run(IDatabaseTranslator translator, Connection c) { + Boolean result; + + // Check the catalog to see if the named constraint exists + final String sql = "" + + "SELECT 1 " + + " FROM sys.sysschemas s," + + " sys.sysconstraints c," + + " sys.systables t " + + " WHERE t.schemaid = s.schemaid " + + " AND c.tableid = t.tableid " + + " AND s.schemaname = ? " + + " AND t.tablename = ? " + + " AND c.constraintname = ? "; + + try (PreparedStatement ps = c.prepareStatement(sql)) { + ps.setString(1, schemaName); + ps.setString(2, tableName); + ps.setString(3, constraintName); + ResultSet rs = ps.executeQuery(); + result = Boolean.valueOf(rs.next()); + } catch (SQLException x) { + throw translator.translate(x); + } + + return result; + } +} diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/derby/DerbyMaster.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/derby/DerbyMaster.java index 5bc03f16667..f9e4ba0c308 100644 --- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/derby/DerbyMaster.java +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/derby/DerbyMaster.java @@ -1,5 +1,5 @@ /* - * (C) Copyright IBM Corp. 2019, 2020 + * (C) Copyright IBM Corp. 2019, 2022 * * SPDX-License-Identifier: Apache-2.0 */ @@ -22,10 +22,13 @@ import com.ibm.fhir.database.utils.api.IConnectionProvider; import com.ibm.fhir.database.utils.api.IDatabaseAdapter; import com.ibm.fhir.database.utils.api.IDatabaseTranslator; +import com.ibm.fhir.database.utils.api.ISchemaAdapter; import com.ibm.fhir.database.utils.api.IVersionHistoryService; +import com.ibm.fhir.database.utils.api.SchemaApplyContext; import com.ibm.fhir.database.utils.common.ConnectionProviderTarget; import com.ibm.fhir.database.utils.common.DataDefinitionUtil; import com.ibm.fhir.database.utils.common.JdbcTarget; +import com.ibm.fhir.database.utils.common.PlainSchemaAdapter; import com.ibm.fhir.database.utils.common.PrintTarget; import com.ibm.fhir.database.utils.model.PhysicalDataModel; @@ -224,7 +227,8 @@ public void createSchema(IConnectionProvider pool, PhysicalDataModel pdm) { * @param pdm the data model we want to create */ public void createSchema(IConnectionProvider pool, IVersionHistoryService vhs, PhysicalDataModel pdm) { - runWithAdapter(pool, target -> pdm.applyWithHistory(target, vhs)); + SchemaApplyContext context = SchemaApplyContext.getDefault(); + runWithAdapter(pool, target -> pdm.applyWithHistory(target, context, vhs)); } /** @@ -232,7 +236,7 @@ public void createSchema(IConnectionProvider pool, IVersionHistoryService vhs, P * @param pool provides database connections * @param fn the command to execute */ - public void runWithAdapter(IConnectionProvider pool, Consumer fn) { + public void runWithAdapter(IConnectionProvider pool, Consumer fn) { // We need to obtain connections from the same pool as the version history service // so we can avoid deadlocks for certain DDL like DROP INDEX @@ -242,7 +246,7 @@ public void runWithAdapter(IConnectionProvider pool, Consumer // call the Function we've been given using the adapter we just wrapped // around the connection. - fn.accept(adapter); + fn.accept(wrap(adapter)); } catch (DataAccessException x) { logger.log(Level.SEVERE, "Error while running", x); throw x; @@ -255,7 +259,7 @@ public void runWithAdapter(IConnectionProvider pool, Consumer * * @param fn */ - public void runWithAdapter(java.util.function.Consumer fn) { + public void runWithAdapter(java.util.function.Consumer fn) { IConnectionProvider cp = new DerbyConnectionProvider(this, null); ConnectionProviderTarget target = new ConnectionProviderTarget(cp); @@ -270,7 +274,18 @@ public void runWithAdapter(java.util.function.Consumer fn) { // call the Function we've been given using the adapter we just wrapped // around the connection. Each statement executes in its own connection/transaction. - fn.accept(adapter); + // We also need to wrap the DerbyAdapter in a plain schema adapter + fn.accept(wrap(adapter)); + } + + /** + * Utility method to wrap the database adapter in a plain schema adapter + * which acts as a pass-through to the underlying databaseAdapter + * @param databaseAdapter + * @return + */ + public static ISchemaAdapter wrap(IDatabaseAdapter databaseAdapter) { + return new PlainSchemaAdapter(databaseAdapter); } /** diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/derby/DerbyTranslator.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/derby/DerbyTranslator.java index 0f041f3731e..71e1ac774d8 100644 --- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/derby/DerbyTranslator.java +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/derby/DerbyTranslator.java @@ -193,6 +193,11 @@ public String dropForeignKeyConstraint(String qualifiedTableName, String constra return "ALTER TABLE " + qualifiedTableName + " DROP FOREIGN KEY " + constraintName; } + @Override + public String dropView(String qualifiedViewName) { + return "DROP VIEW " + qualifiedViewName; + } + @Override public String nextValue(String schemaName, String sequenceName) { final String qname = DataDefinitionUtil.getQualifiedName(schemaName, sequenceName); @@ -223,4 +228,9 @@ public String pagination(int offset, int rowsPerPage) { return result.toString(); } + @Override + public boolean isFamilyPostgreSQL() { + return false; + } + } \ No newline at end of file diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/AlterSequenceStartWith.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/AlterSequenceStartWith.java index b31ee17a5c8..58346d69130 100644 --- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/AlterSequenceStartWith.java +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/AlterSequenceStartWith.java @@ -1,5 +1,5 @@ /* - * (C) Copyright IBM Corp. 2020 + * (C) Copyright IBM Corp. 2020, 2022 * * SPDX-License-Identifier: Apache-2.0 */ @@ -8,7 +8,8 @@ import java.util.Set; -import com.ibm.fhir.database.utils.api.IDatabaseAdapter; +import com.ibm.fhir.database.utils.api.ISchemaAdapter; +import com.ibm.fhir.database.utils.api.SchemaApplyContext; /** * Modify an existing sequence to start with a higher value @@ -39,22 +40,22 @@ public AlterSequenceStartWith(String schemaName, String sequenceName, int versio } @Override - public void apply(IDatabaseAdapter target) { + public void apply(ISchemaAdapter target, SchemaApplyContext context) { target.alterSequenceRestartWith(getSchemaName(), getObjectName(), startWith, this.cache, this.incrementBy); } @Override - public void apply(Integer priorVersion, IDatabaseAdapter target) { - apply(target); + public void apply(Integer priorVersion, ISchemaAdapter target, SchemaApplyContext context) { + apply(target, context); } @Override - public void drop(IDatabaseAdapter target) { + public void drop(ISchemaAdapter target) { // NOP. Sequence will be dropped by the object initially creating it } @Override - protected void grantGroupPrivileges(IDatabaseAdapter target, Set group, String toUser) { + protected void grantGroupPrivileges(ISchemaAdapter target, Set group, String toUser) { target.grantSequencePrivileges(getSchemaName(), getObjectName(), group, toUser); } diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/AlterTableAddColumn.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/AlterTableAddColumn.java index 639032a74f8..92c6ef64df5 100644 --- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/AlterTableAddColumn.java +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/AlterTableAddColumn.java @@ -1,5 +1,5 @@ /* - * (C) Copyright IBM Corp. 2020 + * (C) Copyright IBM Corp. 2020, 2022 * * SPDX-License-Identifier: Apache-2.0 */ @@ -8,11 +8,11 @@ import java.util.ArrayList; import java.util.Arrays; -import java.util.Collections; import java.util.List; import java.util.Set; -import com.ibm.fhir.database.utils.api.IDatabaseAdapter; +import com.ibm.fhir.database.utils.api.ISchemaAdapter; +import com.ibm.fhir.database.utils.api.SchemaApplyContext; /** * Add new columns to an existing table. This alter will change the version history of the underlying table. @@ -62,7 +62,7 @@ public String getTypeNameVersion() { } @Override - public void apply(IDatabaseAdapter target) { + public void apply(ISchemaAdapter target, SchemaApplyContext context) { // To keep things simple, just add each column in its own statement for (ColumnBase c: columns) { target.alterTableAddColumn(getSchemaName(), getObjectName(), c); @@ -70,17 +70,17 @@ public void apply(IDatabaseAdapter target) { } @Override - public void apply(Integer priorVersion, IDatabaseAdapter target) { - apply(target); + public void apply(Integer priorVersion, ISchemaAdapter target, SchemaApplyContext context) { + apply(target, context); } @Override - public void drop(IDatabaseAdapter target) { + public void drop(ISchemaAdapter target) { // NOP } @Override - protected void grantGroupPrivileges(IDatabaseAdapter target, Set group, String toUser) { + protected void grantGroupPrivileges(ISchemaAdapter target, Set group, String toUser) { // NOP } diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/AlterTableIdentityCache.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/AlterTableIdentityCache.java index f30c40b65d2..81f3ac67d4c 100644 --- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/AlterTableIdentityCache.java +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/AlterTableIdentityCache.java @@ -1,5 +1,5 @@ /* - * (C) Copyright IBM Corp. 2020 + * (C) Copyright IBM Corp. 2020, 2022 * * SPDX-License-Identifier: Apache-2.0 */ @@ -8,7 +8,8 @@ import java.util.Set; -import com.ibm.fhir.database.utils.api.IDatabaseAdapter; +import com.ibm.fhir.database.utils.api.ISchemaAdapter; +import com.ibm.fhir.database.utils.api.SchemaApplyContext; /** * Modify the CACHE property of an AS IDENTITY column (changes @@ -44,22 +45,22 @@ public String getTypeNameVersion() { } @Override - public void apply(IDatabaseAdapter target) { + public void apply(ISchemaAdapter target, SchemaApplyContext context) { target.alterTableColumnIdentityCache(getSchemaName(), getObjectName(), this.columnName, this.cache); } @Override - public void apply(Integer priorVersion, IDatabaseAdapter target) { - apply(target); + public void apply(Integer priorVersion, ISchemaAdapter target, SchemaApplyContext context) { + apply(target, context); } @Override - public void drop(IDatabaseAdapter target) { + public void drop(ISchemaAdapter target) { // NOP } @Override - protected void grantGroupPrivileges(IDatabaseAdapter target, Set group, String toUser) { + protected void grantGroupPrivileges(ISchemaAdapter target, Set group, String toUser) { // NOP } diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/BaseObject.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/BaseObject.java index f0c52dd0050..709518f2d27 100644 --- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/BaseObject.java +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/BaseObject.java @@ -1,5 +1,5 @@ /* - * (C) Copyright IBM Corp. 2019, 2021 + * (C) Copyright IBM Corp. 2019, 2022 * * SPDX-License-Identifier: Apache-2.0 */ @@ -19,11 +19,12 @@ import java.util.logging.Level; import java.util.logging.Logger; -import com.ibm.fhir.database.utils.api.IDatabaseAdapter; +import com.ibm.fhir.database.utils.api.ISchemaAdapter; import com.ibm.fhir.database.utils.api.ITransaction; import com.ibm.fhir.database.utils.api.ITransactionProvider; import com.ibm.fhir.database.utils.api.IVersionHistoryService; import com.ibm.fhir.database.utils.api.LockException; +import com.ibm.fhir.database.utils.api.SchemaApplyContext; import com.ibm.fhir.database.utils.common.DataDefinitionUtil; import com.ibm.fhir.database.utils.thread.ThreadHandler; import com.ibm.fhir.task.api.ITaskCollector; @@ -186,24 +187,24 @@ public String toString() { } @Override - public ITaskGroup collect(final ITaskCollector tc, final IDatabaseAdapter target, final ITransactionProvider tp, final IVersionHistoryService vhs) { + public ITaskGroup collect(final ITaskCollector tc, final ISchemaAdapter target, final SchemaApplyContext context, final ITransactionProvider tp, final IVersionHistoryService vhs) { // Make sure that anything we depend on gets processed first List children = null; if (!this.dependencies.isEmpty()) { children = new ArrayList<>(this.dependencies.size()); for (IDatabaseObject obj: dependencies) { - children.add(obj.collect(tc, target, tp, vhs)); + children.add(obj.collect(tc, target, context, tp, vhs)); } } // create a new task group representing this node, pointing to any dependencies // we collected above. We need to use the type and name for the task group, to // ensure we allow for the different namespaces (e.g. procedures vs tables). - return tc.makeTaskGroup(this.getTypeNameVersion(), () -> applyTx(target, tp, vhs), children); + return tc.makeTaskGroup(this.getTypeNameVersion(), () -> applyTx(target, context, tp, vhs), children); } @Override - public void applyTx(IDatabaseAdapter target, ITransactionProvider tp, IVersionHistoryService vhs) { + public void applyTx(ISchemaAdapter target, SchemaApplyContext context, ITransactionProvider tp, IVersionHistoryService vhs) { // Wrap the apply operation in its own transaction, as this is likely // being executed from a thread-pool. DB2 has some issues with deadlocks // on its catalog tables (SQLCODE=-911, SQLSTATE=40001, SQLERRMC=2) when @@ -212,7 +213,7 @@ public void applyTx(IDatabaseAdapter target, ITransactionProvider tp, IVersionHi while (remainingAttempts-- > 0) { try (ITransaction tx = tp.getTransaction()) { try { - applyVersion(target, vhs); + applyVersion(target, context, vhs); remainingAttempts = 0; // exit the retry loop } catch (LockException x) { @@ -249,21 +250,15 @@ public void applyTx(IDatabaseAdapter target, ITransactionProvider tp, IVersionHi } } - /** - * Apply the change, but only if it has a newer version than we already have - * recorded in the database - * @param target - * @param vhs the service used to manage the version history table - */ @Override - public void applyVersion(IDatabaseAdapter target, IVersionHistoryService vhs) { + public void applyVersion(ISchemaAdapter target, SchemaApplyContext context, IVersionHistoryService vhs) { // Only for Procedures do we skip the Version History Service check, and apply. if (vhs.applies(getSchemaName(), getObjectType().name(), getObjectName(), version) || getObjectType() == DatabaseObjectType.PROCEDURE) { logger.fine("Applying change [v" + version + "]: " + this.getTypeNameVersion()); // Apply this change to the target database - apply(vhs.getVersion(getSchemaName(), getObjectType().name(), getObjectName()), target); + apply(vhs.getVersion(getSchemaName(), getObjectType().name(), getObjectName()), target, context); // Check if the PROCEDURE is this exact version (Applies to FunctionDef and ProcedureDef) if (DatabaseObjectType.PROCEDURE.equals(getObjectType()) @@ -283,7 +278,7 @@ public Map getTags() { } @Override - public void grant(IDatabaseAdapter target, String groupName, String toUser) { + public void grant(ISchemaAdapter target, String groupName, String toUser) { // The group is optional. Some objects may not have a group corresponding with // the requested groupName, in which case no privileges will be granted @@ -300,7 +295,7 @@ public void grant(IDatabaseAdapter target, String groupName, String toUser) { * @param group * @param toUser */ - protected void grantGroupPrivileges(IDatabaseAdapter target, Set group, String toUser) { + protected void grantGroupPrivileges(ISchemaAdapter target, Set group, String toUser) { target.grantObjectPrivileges(this.schemaName, this.objectName, group, toUser); } @@ -323,4 +318,9 @@ public void addPrivilege(String groupName, Privilege p) { public void visit(Consumer c) { c.accept(this); } + + @Override + public void applyDistributionRules(ISchemaAdapter target, int pass) { + // NOP. Only applies to Table + } } \ No newline at end of file diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/ColumnDefBuilder.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/ColumnDefBuilder.java index d6ac9edb090..e4cc22806b5 100644 --- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/ColumnDefBuilder.java +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/ColumnDefBuilder.java @@ -209,6 +209,9 @@ public List buildColumns() { case SMALLINT: column = new SmallIntColumn(cd.getName(), cd.isNullable(), cd.getDefaultVal()); break; + case SMALLINT_BOOLEAN: + column = new SmallIntBooleanColumn(cd.getName(), cd.isNullable(), cd.getDefaultVal()); + break; case DOUBLE: column = new DoubleColumn(cd.getName(), cd.isNullable()); break; diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/ColumnType.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/ColumnType.java index 4a74fb17ecc..2b91bf0b37f 100644 --- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/ColumnType.java +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/ColumnType.java @@ -20,5 +20,6 @@ public enum ColumnType { TIMESTAMP, BLOB, CLOB, - SMALLINT + SMALLINT, + SMALLINT_BOOLEAN // for JavaBatch } diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/CreateIndex.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/CreateIndex.java index 36a6bd7c1d4..6140d6cd043 100644 --- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/CreateIndex.java +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/CreateIndex.java @@ -1,5 +1,5 @@ /* - * (C) Copyright IBM Corp. 2020 + * (C) Copyright IBM Corp. 2020, 2022 * * SPDX-License-Identifier: Apache-2.0 */ @@ -11,7 +11,9 @@ import java.util.logging.Level; import java.util.logging.Logger; -import com.ibm.fhir.database.utils.api.IDatabaseAdapter; +import com.ibm.fhir.database.utils.api.DistributionType; +import com.ibm.fhir.database.utils.api.ISchemaAdapter; +import com.ibm.fhir.database.utils.api.SchemaApplyContext; import com.ibm.fhir.database.utils.common.CreateIndexStatement; /** @@ -26,20 +28,30 @@ public class CreateIndex extends BaseObject { // The name of the tenant column when used for multi-tenant databases private final String tenantColumnName; - + + // The table name the index will be created on private final String tableName; + // Distribution rules if the associated table is distributed + private final DistributionType distributionType; + private final String distributionColumnName; + /** * Protected constructor. Use the Builder to create instance. * @param schemaName * @param indexName * @param version + * @param distributionType */ - protected CreateIndex(String schemaName, String versionTrackingName, String tableName, int version, IndexDef indexDef, String tenantColumnName) { + protected CreateIndex(String schemaName, String versionTrackingName, String tableName, int version, IndexDef indexDef, String tenantColumnName, + DistributionType distributionType, String distributionColumnName) { super(schemaName, versionTrackingName, DatabaseObjectType.INDEX, version); this.tableName = tableName; this.indexDef = indexDef; this.tenantColumnName = tenantColumnName; + this.distributionType = distributionType; + this.distributionColumnName = distributionColumnName; + } /** @@ -84,9 +96,9 @@ public String getTypeNameVersion() { @Override - public void apply(IDatabaseAdapter target) { + public void apply(ISchemaAdapter target, SchemaApplyContext context) { long start = System.nanoTime(); - indexDef.apply(getSchemaName(), getTableName(), tenantColumnName, target); + indexDef.apply(getSchemaName(), getTableName(), tenantColumnName, target, distributionType, distributionColumnName); if (logger.isLoggable(Level.FINE)) { long end = System.nanoTime(); @@ -96,12 +108,12 @@ public void apply(IDatabaseAdapter target) { } @Override - public void apply(Integer priorVersion, IDatabaseAdapter target) { - apply(target); + public void apply(Integer priorVersion, ISchemaAdapter target, SchemaApplyContext context) { + apply(target, context); } @Override - public void drop(IDatabaseAdapter target) { + public void drop(ISchemaAdapter target) { long start = System.nanoTime(); indexDef.drop(getSchemaName(), target); @@ -151,7 +163,10 @@ public static class Builder { // Special case to handle a previous defect where indexes were tracked using tableName in version_history private String versionTrackingName; - + // Set if the table is distributed + private DistributionType distributionType = DistributionType.NONE; + private String distributionColumnName; + /** * @param schemaName the schemaName to set */ @@ -182,7 +197,27 @@ public Builder setVersionTrackingName(String name) { this.versionTrackingName = name; return this; } - + + /** + * Setter for distributionType + * @param dt + * @return + */ + public Builder setDistributionType(DistributionType dt) { + this.distributionType = dt; + return this; + } + + /** + * Setter for distributionColumnName + * @param distributionColumnName + * @return + */ + public Builder setDistributionColumnName(String distributionColumnName) { + this.distributionColumnName = distributionColumnName; + return this; + } + /** * @param version the version to set */ @@ -190,7 +225,6 @@ public Builder setVersion(int version) { this.version = version; return this; } - /** * @param unique the unique to set @@ -236,8 +270,9 @@ public CreateIndex build() { if (versionTrackingName == null) { versionTrackingName = this.indexName; } + return new CreateIndex(schemaName, versionTrackingName, tableName, version, - new IndexDef(indexName, indexCols, unique), tenantColumnName); + new IndexDef(indexName, indexCols, unique), tenantColumnName, distributionType, distributionColumnName); } /** diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/DatabaseObject.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/DatabaseObject.java index 42012c047a5..3c2707443ac 100644 --- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/DatabaseObject.java +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/DatabaseObject.java @@ -1,5 +1,5 @@ /* - * (C) Copyright IBM Corp. 2019, 2020 + * (C) Copyright IBM Corp. 2019, 2022 * * SPDX-License-Identifier: Apache-2.0 */ @@ -17,11 +17,12 @@ import java.util.logging.Level; import java.util.logging.Logger; -import com.ibm.fhir.database.utils.api.IDatabaseAdapter; +import com.ibm.fhir.database.utils.api.ISchemaAdapter; import com.ibm.fhir.database.utils.api.ITransaction; import com.ibm.fhir.database.utils.api.ITransactionProvider; import com.ibm.fhir.database.utils.api.IVersionHistoryService; import com.ibm.fhir.database.utils.api.LockException; +import com.ibm.fhir.database.utils.api.SchemaApplyContext; import com.ibm.fhir.database.utils.thread.ThreadHandler; /** @@ -141,7 +142,7 @@ public String toString() { } @Override - public void applyTx(IDatabaseAdapter target, ITransactionProvider tp, IVersionHistoryService vhs) { + public void applyTx(ISchemaAdapter target, SchemaApplyContext context, ITransactionProvider tp, IVersionHistoryService vhs) { // Wrap the apply operation in its own transaction, as this is likely // being executed from a thread-pool. DB2 has some issues with deadlocks // on its catalog tables (SQLCODE=-911, SQLSTATE=40001, SQLERRMC=2) when @@ -150,7 +151,7 @@ public void applyTx(IDatabaseAdapter target, ITransactionProvider tp, IVersionHi while (remainingAttempts-- > 0) { try (ITransaction tx = tp.getTransaction()) { try { - applyVersion(target, vhs); + applyVersion(target, context, vhs); remainingAttempts = 0; // exit the retry loop } catch (LockException x) { @@ -192,13 +193,13 @@ public void applyTx(IDatabaseAdapter target, ITransactionProvider tp, IVersionHi * @param vhs the service used to manage the version history table */ @Override - public void applyVersion(IDatabaseAdapter target, IVersionHistoryService vhs) { + public void applyVersion(ISchemaAdapter target, SchemaApplyContext context, IVersionHistoryService vhs) { // TODO find a better way to track database-level type stuff (not schema-specific) if (vhs.applies("__DATABASE__", getObjectType().name(), getObjectName(), version)) { logger.info("Applying change [v" + version + "]: "+ this.getTypeNameVersion()); // Apply this change to the target database - apply(vhs.getVersion("__DATABASE__", getObjectType().name(), getObjectName()), target); + apply(vhs.getVersion("__DATABASE__", getObjectType().name(), getObjectName()), target, context); // call back to the version history service to add the new version to the table // being used to track the change history @@ -215,4 +216,9 @@ public Map getTags() { public void visit(Consumer c) { c.accept(this); } + + @Override + public void applyDistributionRules(ISchemaAdapter target, int pass) { + // NOP + } } \ No newline at end of file diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/DbType.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/DbType.java index 312f1e6a021..cc413af6307 100644 --- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/DbType.java +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/DbType.java @@ -20,6 +20,11 @@ public enum DbType { */ POSTGRESQL("postgresql"), + /** + * Citus (Distributed PostgreSQL) + */ + CITUS("citus"), + /** * IBM Db2 */ diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/ForeignKeyConstraint.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/ForeignKeyConstraint.java index 374b4120250..68ca87906ae 100644 --- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/ForeignKeyConstraint.java +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/ForeignKeyConstraint.java @@ -1,5 +1,5 @@ /* - * (C) Copyright IBM Corp. 2019, 2020 + * (C) Copyright IBM Corp. 2019, 2022 * * SPDX-License-Identifier: Apache-2.0 */ @@ -10,7 +10,8 @@ import java.util.Arrays; import java.util.List; -import com.ibm.fhir.database.utils.api.IDatabaseAdapter; +import com.ibm.fhir.database.utils.api.DistributionType; +import com.ibm.fhir.database.utils.api.ISchemaAdapter; import com.ibm.fhir.database.utils.common.DataDefinitionUtil; /** @@ -25,6 +26,9 @@ public class ForeignKeyConstraint extends Constraint { private final String targetColumnName; private final List columns = new ArrayList<>(); + // Flag to indicate that the target is a REFERENCE table when using distribution (like Citus) + private boolean targetIsReference; + /** * @param constraintName * @param enforced @@ -68,6 +72,22 @@ public String getTargetColumnName() { public boolean isSelf() { return self; } + + /** + * Is the target table distributed as a REFERENCE table (Citus) + * @return + */ + public boolean isTargetReference() { + return this.targetIsReference; + } + + /** + * Set the flag to indicate if the target table is a reference type + * @param flag + */ + public void setTargetReference(boolean flag) { + this.targetIsReference = flag; + } /** * Getter for the target table name * @return @@ -105,10 +125,34 @@ public String getQualifiedTargetName() { } /** + * Apply the FK constraint to the given target + * @param schemaName * @param name + * @param tenantColumnName * @param target + * @param sourceDistributionType + */ + public void apply(String schemaName, String name, String tenantColumnName, ISchemaAdapter target, DistributionType sourceDistributionType) { + // make this idempotent to support upgrade scenarios + if (!target.doesForeignKeyConstraintExist(schemaName, name, getConstraintName())) { + target.createForeignKeyConstraint(getConstraintName(), schemaName, name, targetSchema, targetTable, targetColumnName, tenantColumnName, columns, enforced, sourceDistributionType, + targetIsReference); + } + } + + /** + * Return true if the list of columns includes the column name, ignoring case + * @param distributionColumnName + * @return */ - public void apply(String schemaName, String name, String tenantColumnName, IDatabaseAdapter target) { - target.createForeignKeyConstraint(getConstraintName(), schemaName, name, targetSchema, targetTable, targetColumnName, tenantColumnName, columns, enforced); + public boolean includesColumn(String columnName) { + // Linear search is OK because the list is very small and probably + // will be cheaper than maintaining both a list and set of values + for (String cn: this.columns) { + if (cn.equalsIgnoreCase(columnName)) { + return true; + } + } + return false; } } \ No newline at end of file diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/FunctionDef.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/FunctionDef.java index d57cc4514d4..35b22a8c6a0 100644 --- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/FunctionDef.java +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/FunctionDef.java @@ -1,5 +1,5 @@ /* - * (C) Copyright IBM Corp. 2019, 2021 + * (C) Copyright IBM Corp. 2019, 2022 * * SPDX-License-Identifier: Apache-2.0 */ @@ -10,7 +10,8 @@ import java.util.function.Supplier; import java.util.logging.Logger; -import com.ibm.fhir.database.utils.api.IDatabaseAdapter; +import com.ibm.fhir.database.utils.api.ISchemaAdapter; +import com.ibm.fhir.database.utils.api.SchemaApplyContext; /** * The definition of a function, whose content is provided by a Supplier function @@ -21,40 +22,49 @@ public class FunctionDef extends BaseObject { // supplier provides the procedure body when requested private Supplier supplier; + // When >0, indicates that this function should be distributed + private final int distributeByParamNum; + /** - * Public constructor + * Public constructor. Supports distribution of the function by the given parameter number + * * @param schemaName * @param procedureName * @param version * @param supplier + * @param distributeByParamNum */ - public FunctionDef(String schemaName, String procedureName, int version, Supplier supplier) { + public FunctionDef(String schemaName, String procedureName, int version, Supplier supplier, int distributeByParamNum) { super(schemaName, procedureName, DatabaseObjectType.PROCEDURE, version); this.supplier = supplier; + this.distributeByParamNum = distributeByParamNum; } @Override - public void apply(IDatabaseAdapter target) { + public void apply(ISchemaAdapter target, SchemaApplyContext context) { target.createOrReplaceFunction(getSchemaName(), getObjectName(), supplier); + if (distributeByParamNum > 0) { + target.distributeFunction(getSchemaName(), getObjectName(), distributeByParamNum); + } } @Override - public void apply(Integer priorVersion, IDatabaseAdapter target) { + public void apply(Integer priorVersion, ISchemaAdapter target, SchemaApplyContext context) { if (priorVersion != null && priorVersion > 0 && this.getVersion() > priorVersion && !migrations.isEmpty()) { logger.warning("Found '" + migrations.size() + "' migration steps, but performing 'create or replace' instead"); } // Functions are applied with "Create or replace", so just do a regular apply - apply(target); + apply(target, context); } @Override - public void drop(IDatabaseAdapter target) { + public void drop(ISchemaAdapter target) { target.dropFunction(getSchemaName(), getObjectName()); } @Override - protected void grantGroupPrivileges(IDatabaseAdapter target, Set group, String toUser) { + protected void grantGroupPrivileges(ISchemaAdapter target, Set group, String toUser) { target.grantFunctionPrivileges(getSchemaName(), getObjectName(), group, toUser); } diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/IDataModel.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/IDataModel.java index 9e8537d2dcb..186504524a3 100644 --- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/IDataModel.java +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/IDataModel.java @@ -1,5 +1,5 @@ /* - * (C) Copyright IBM Corp. 2019 + * (C) Copyright IBM Corp. 2019, 2022 * * SPDX-License-Identifier: Apache-2.0 */ @@ -18,4 +18,10 @@ public interface IDataModel { * @return */ public Table findTable(String schemaName, String tableName); + + /** + * Is the target database distributed (e.g. with sharding)? + * @return + */ + public boolean isDistributed(); } diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/IDatabaseObject.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/IDatabaseObject.java index 091ce64bff6..1ea54e4170e 100644 --- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/IDatabaseObject.java +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/IDatabaseObject.java @@ -1,5 +1,5 @@ /* - * (C) Copyright IBM Corp. 2019, 2020 + * (C) Copyright IBM Corp. 2019, 2022 * * SPDX-License-Identifier: Apache-2.0 */ @@ -10,9 +10,10 @@ import java.util.Map; import java.util.function.Consumer; -import com.ibm.fhir.database.utils.api.IDatabaseAdapter; +import com.ibm.fhir.database.utils.api.ISchemaAdapter; import com.ibm.fhir.database.utils.api.ITransactionProvider; import com.ibm.fhir.database.utils.api.IVersionHistoryService; +import com.ibm.fhir.database.utils.api.SchemaApplyContext; import com.ibm.fhir.task.api.ITaskCollector; import com.ibm.fhir.task.api.ITaskGroup; @@ -31,37 +32,48 @@ public interface IDatabaseObject { * Apply the DDL for this object to the target database * @param priorVersion * @param target the database target + * @param context context to control the schema apply process */ - public void apply(IDatabaseAdapter target); + public void apply(ISchemaAdapter target, SchemaApplyContext context); /** * Apply migration logic to bring the target database to the current level of this object * @param priorVersion * @param target the database target + * @param context to control the schema apply process */ - public void apply(Integer priorVersion, IDatabaseAdapter target); + public void apply(Integer priorVersion, ISchemaAdapter target, SchemaApplyContext context); /** * Apply the DDL, but within its own transaction * @param target the target database we apply to + * @param context the context used to modify how the schema objects are applied * @param cp of thread-specific transactions * @param vhs the service interface for adding this object to the version history table */ - public void applyTx(IDatabaseAdapter target, ITransactionProvider cp, IVersionHistoryService vhs); + public void applyTx(ISchemaAdapter target, SchemaApplyContext context, ITransactionProvider cp, IVersionHistoryService vhs); + + /** + * Apply any distribution rules associated with the object (usually a table) + * @param target the target database we apply the operation to + * @param pass multiple pass number + */ + public void applyDistributionRules(ISchemaAdapter target, int pass); /** * Apply the change, but only if it has a newer version than we already have * recorded in the database * @param target + * @param context * @param vhs the service used to manage the version history table */ - public void applyVersion(IDatabaseAdapter target, IVersionHistoryService vhs); + public void applyVersion(ISchemaAdapter target, SchemaApplyContext context, IVersionHistoryService vhs); /** * DROP this object from the target database * @param target */ - public void drop(IDatabaseAdapter target); + public void drop(ISchemaAdapter target); /** * Grant the given privileges to the user @@ -69,7 +81,7 @@ public interface IDatabaseObject { * @param groupName * @param toUser */ - public void grant(IDatabaseAdapter target, String groupName, String toUser); + public void grant(ISchemaAdapter target, String groupName, String toUser); /** * Visit this object, calling the consumer for itself, or its children if any @@ -96,10 +108,11 @@ public interface IDatabaseObject { * executed concurrently (but in the right order) * @param tc * @param target + * @param context * @param tp * @param vhs */ - public ITaskGroup collect(ITaskCollector tc, IDatabaseAdapter target, ITransactionProvider tp, IVersionHistoryService vhs); + public ITaskGroup collect(ITaskCollector tc, ISchemaAdapter target, SchemaApplyContext context, ITransactionProvider tp, IVersionHistoryService vhs); /** * Return the qualified name for this object (e.g. schema.name). diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/IndexDef.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/IndexDef.java index 372e519ad50..d6e900123a5 100644 --- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/IndexDef.java +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/IndexDef.java @@ -1,5 +1,5 @@ /* - * (C) Copyright IBM Corp. 2019 + * (C) Copyright IBM Corp. 2019, 2022 * * SPDX-License-Identifier: Apache-2.0 */ @@ -11,7 +11,8 @@ import java.util.List; import java.util.stream.Collectors; -import com.ibm.fhir.database.utils.api.IDatabaseAdapter; +import com.ibm.fhir.database.utils.api.DistributionType; +import com.ibm.fhir.database.utils.api.ISchemaAdapter; import com.ibm.fhir.database.utils.common.CreateIndexStatement; /** @@ -66,18 +67,24 @@ public boolean isUnique() { /** * Apply this object to the given database target + * + * @param schemaName * @param tableName + * @param tenantColumnName * @param target + * @param distributionType + * @param distributionColumn */ - public void apply(String schemaName, String tableName, String tenantColumnName, IDatabaseAdapter target) { + public void apply(String schemaName, String tableName, String tenantColumnName, ISchemaAdapter target, + DistributionType distributionType, String distributionColumn) { if (includeColumns != null && includeColumns.size() > 0) { - target.createUniqueIndex(schemaName, tableName, indexName, tenantColumnName, indexColumns, includeColumns); + target.createUniqueIndex(schemaName, tableName, indexName, tenantColumnName, indexColumns, includeColumns, distributionType, distributionColumn); } else if (unique) { - target.createUniqueIndex(schemaName, tableName, indexName, tenantColumnName, indexColumns); + target.createUniqueIndex(schemaName, tableName, indexName, tenantColumnName, indexColumns, distributionType, distributionColumn); } else { - target.createIndex(schemaName, tableName, indexName, tenantColumnName, indexColumns); + target.createIndex(schemaName, tableName, indexName, tenantColumnName, indexColumns, distributionType); } } @@ -86,7 +93,7 @@ else if (unique) { * @param schemaName * @param target */ - public void drop(String schemaName, IDatabaseAdapter target) { + public void drop(String schemaName, ISchemaAdapter target) { target.dropIndex(schemaName, indexName); } diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/NopObject.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/NopObject.java index 576afa4c0a3..feb0186c140 100644 --- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/NopObject.java +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/NopObject.java @@ -1,14 +1,15 @@ /* - * (C) Copyright IBM Corp. 2019 + * (C) Copyright IBM Corp. 2019, 2022 * * SPDX-License-Identifier: Apache-2.0 */ package com.ibm.fhir.database.utils.model; -import com.ibm.fhir.database.utils.api.IDatabaseAdapter; +import com.ibm.fhir.database.utils.api.ISchemaAdapter; import com.ibm.fhir.database.utils.api.ITransactionProvider; import com.ibm.fhir.database.utils.api.IVersionHistoryService; +import com.ibm.fhir.database.utils.api.SchemaApplyContext; /** * A NOP (no operation) object which can be used to simplify dependencies @@ -29,22 +30,22 @@ public NopObject(String schemaName, String objectName) { } @Override - public void apply(IDatabaseAdapter target) { + public void apply(ISchemaAdapter target, SchemaApplyContext context) { // We're NOP so we do nothing on purpose } @Override - public void apply(Integer priorVersion, IDatabaseAdapter target) { + public void apply(Integer priorVersion, ISchemaAdapter target, SchemaApplyContext context) { // We're NOP so we do nothing on purpose } @Override - public void drop(IDatabaseAdapter target) { + public void drop(ISchemaAdapter target) { // We're NOP so we do nothing on purpose } @Override - public void applyTx(IDatabaseAdapter target, ITransactionProvider tp, IVersionHistoryService vhs) { + public void applyTx(ISchemaAdapter target, SchemaApplyContext context, ITransactionProvider tp, IVersionHistoryService vhs) { // We're NOP so we do nothing on purpose } diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/ObjectGroup.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/ObjectGroup.java index f365393b011..407f8679406 100644 --- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/ObjectGroup.java +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/ObjectGroup.java @@ -1,5 +1,5 @@ /* - * (C) Copyright IBM Corp. 2019, 2020 + * (C) Copyright IBM Corp. 2019, 2022 * * SPDX-License-Identifier: Apache-2.0 */ @@ -13,8 +13,9 @@ import java.util.Set; import java.util.function.Consumer; -import com.ibm.fhir.database.utils.api.IDatabaseAdapter; +import com.ibm.fhir.database.utils.api.ISchemaAdapter; import com.ibm.fhir.database.utils.api.IVersionHistoryService; +import com.ibm.fhir.database.utils.api.SchemaApplyContext; /** * A collection of {@link IDatabaseObject} which are applied in order within one transaction @@ -59,17 +60,17 @@ public ObjectGroup(String schemaName, String name, Collection g } @Override - public void applyVersion(IDatabaseAdapter target, IVersionHistoryService vhs) { + public void applyVersion(ISchemaAdapter target, SchemaApplyContext context, IVersionHistoryService vhs) { // Apply each member of our group to the target if it is a new version. // Version tracking is done at the individual level, not the group. for (IDatabaseObject obj: this.group) { - obj.applyVersion(target, vhs); + obj.applyVersion(target, context, vhs); } } @Override - public void drop(IDatabaseAdapter target) { + public void drop(ISchemaAdapter target) { // Apply each member of the group, but going in reverse for (int i=group.size()-1; i>=0; i--) { group.get(i).drop(target); @@ -77,7 +78,7 @@ public void drop(IDatabaseAdapter target) { } @Override - public void grant(IDatabaseAdapter target, String groupName, String toUser) { + public void grant(ISchemaAdapter target, String groupName, String toUser) { // Override the BaseObject behavior because we need to propagate the grant request // to the indivual objects we have aggregated for (IDatabaseObject obj: this.group) { @@ -86,18 +87,18 @@ public void grant(IDatabaseAdapter target, String groupName, String toUser) { } @Override - public void apply(IDatabaseAdapter target) { + public void apply(ISchemaAdapter target, SchemaApplyContext context) { // Plain old apply, used to apply all changes, regardless of version - e.g. for testing for (IDatabaseObject obj: this.group) { - obj.apply(target); + obj.apply(target, context); } } @Override - public void apply(Integer priorVersion, IDatabaseAdapter target) { + public void apply(Integer priorVersion, ISchemaAdapter target, SchemaApplyContext context) { // Plain old apply, used to apply all changes, regardless of version - e.g. for testing for (IDatabaseObject obj: this.group) { - obj.apply(priorVersion, target); + obj.apply(priorVersion, target, context); } } @@ -134,4 +135,11 @@ public void visitReverse(DataModelVisitor v) { group.get(i).visit(v); } } -} + + @Override + public void applyDistributionRules(ISchemaAdapter target, int pass) { + for (IDatabaseObject obj: this.group) { + obj.applyDistributionRules(target, pass); + } + } +} \ No newline at end of file diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/PhysicalDataModel.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/PhysicalDataModel.java index 4265012f5f8..dba497299fc 100644 --- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/PhysicalDataModel.java +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/PhysicalDataModel.java @@ -1,5 +1,5 @@ /* - * (C) Copyright IBM Corp. 2019, 2021 + * (C) Copyright IBM Corp. 2019, 2022 * * SPDX-License-Identifier: Apache-2.0 */ @@ -20,8 +20,11 @@ import java.util.logging.Logger; import com.ibm.fhir.database.utils.api.IDatabaseAdapter; +import com.ibm.fhir.database.utils.api.ISchemaAdapter; +import com.ibm.fhir.database.utils.api.ITransaction; import com.ibm.fhir.database.utils.api.ITransactionProvider; import com.ibm.fhir.database.utils.api.IVersionHistoryService; +import com.ibm.fhir.database.utils.api.SchemaApplyContext; import com.ibm.fhir.database.utils.common.DataDefinitionUtil; import com.ibm.fhir.task.api.ITaskCollector; @@ -51,11 +54,14 @@ public class PhysicalDataModel implements IDataModel { // Common models that we rely on (e.g. for FK constraints) private final List federatedModels = new ArrayList<>(); + // Is this model configured to operate with a distributed (sharded) database? + private final boolean distributed; + /** * Default constructor. No federated models */ - public PhysicalDataModel() { - // No Op + public PhysicalDataModel(boolean distributed) { + this.distributed = distributed; } /** @@ -63,7 +69,19 @@ public PhysicalDataModel() { * @param federatedModels */ public PhysicalDataModel(PhysicalDataModel... federatedModels) { - this.federatedModels.addAll(Arrays.asList(federatedModels)); + boolean dist = false; + if (federatedModels != null) { + this.federatedModels.addAll(Arrays.asList(federatedModels)); + + // If any of the federated models are distributed, then assume we must be + for (PhysicalDataModel dm: federatedModels) { + if (dm.isDistributed()) { + dist = true; + break; + } + } + } + this.distributed = dist; } /** @@ -97,25 +115,78 @@ public void addObject(IDatabaseObject obj) { * time it takes to provision a schema. * @param tc collects and manages the object creation tasks and their dependencies * @param target the target database adapter + * @param context to control how the schema is built * @param tp * @param vhs */ - public void collect(ITaskCollector tc, IDatabaseAdapter target, ITransactionProvider tp, IVersionHistoryService vhs) { + public void collect(ITaskCollector tc, ISchemaAdapter target, SchemaApplyContext context, ITransactionProvider tp, IVersionHistoryService vhs) { for (IDatabaseObject obj: allObjects) { - obj.collect(tc, target, tp, vhs); + obj.collect(tc, target, context, tp, vhs); } } /** * Apply the entire model to the target in order * @param target + * @param context */ - public void apply(IDatabaseAdapter target) { + public void apply(ISchemaAdapter target, SchemaApplyContext context) { int total = allObjects.size(); int count = 1; for (IDatabaseObject obj: allObjects) { logger.fine(String.format("Creating [%d/%d] %s", count++, total, obj.toString())); - obj.apply(target); + obj.apply(target, context); + } + } + + /** + * Make a pass over all the objects and apply any distribution rules they + * may have (e.g. for Citus). We have to process a large number of tables, + * which can cause shared memory issues for Citus if we try and do this in + * a single transaction, hence the need for a transactionSupplier + * @param target + */ + public void applyDistributionRules(ISchemaAdapter target, Supplier transactionSupplier) { + + // takes a long time, so track progress + int total = allObjects.size() * 2; + int count = 0; + int objectsPerMessage = total / 100; // 1% increments + int nextCount = objectsPerMessage; + // make a first pass to apply reference rules + for (IDatabaseObject obj: allObjects) { + try (ITransaction tx = transactionSupplier.get()) { + try { + obj.applyDistributionRules(target, 0); + + if (++count >= nextCount) { + int pc = 100 * nextCount / total; + logger.info("Progress: [" + pc + "% complete]"); + nextCount += objectsPerMessage; + } + } catch (RuntimeException x) { + tx.setRollbackOnly(); + throw x; + } + } + } + + // and another pass to apply sharding rules + for (IDatabaseObject obj: allObjects) { + try (ITransaction tx = transactionSupplier.get()) { + try { + obj.applyDistributionRules(target, 1); + + if (++count >= nextCount) { + int pc = 100 * nextCount / total; + logger.info("Progress: [" + pc + "% complete]"); + nextCount += objectsPerMessage; + } + } catch (RuntimeException x) { + tx.setRollbackOnly(); + throw x; + } + } } } @@ -125,12 +196,12 @@ public void apply(IDatabaseAdapter target) { * @param target * @param vhs */ - public void applyWithHistory(IDatabaseAdapter target, IVersionHistoryService vhs) { + public void applyWithHistory(ISchemaAdapter target, SchemaApplyContext context, IVersionHistoryService vhs) { int total = allObjects.size(); int count = 1; for (IDatabaseObject obj: allObjects) { logger.fine(String.format("Creating [%d/%d] %s", count++, total, obj.toString())); - obj.applyVersion(target, vhs); + obj.applyVersion(target, context, vhs); } } @@ -138,13 +209,13 @@ public void applyWithHistory(IDatabaseAdapter target, IVersionHistoryService vhs * Apply all the procedures in the order in which they were added to the model * @param adapter */ - public void applyProcedures(IDatabaseAdapter adapter) { + public void applyProcedures(ISchemaAdapter adapter, SchemaApplyContext context) { int total = procedures.size(); int count = 1; for (ProcedureDef obj: procedures) { logger.fine(String.format("Applying [%d/%d] %s", count++, total, obj.toString())); obj.drop(adapter); - obj.apply(adapter); + obj.apply(adapter, context); } } @@ -152,12 +223,12 @@ public void applyProcedures(IDatabaseAdapter adapter) { * Apply all the functions in the order in which they were added to the model * @param adapter */ - public void applyFunctions(IDatabaseAdapter adapter) { + public void applyFunctions(ISchemaAdapter adapter, SchemaApplyContext context) { int total = functions.size(); int count = 1; for (FunctionDef obj: functions) { logger.fine(String.format("Applying [%d/%d] %s", count++, total, obj.toString())); - obj.apply(adapter); + obj.apply(adapter, context); } } @@ -169,7 +240,7 @@ public void applyFunctions(IDatabaseAdapter adapter) { * @param tagGroup * @param tag */ - public void drop(IDatabaseAdapter target, String tagGroup, String tag) { + public void drop(ISchemaAdapter target, String tagGroup, String tag) { // The simplest way to reverse the list is add everything into an array list // which we then simply traverse end to start ArrayList copy = new ArrayList<>(); @@ -190,6 +261,43 @@ public void drop(IDatabaseAdapter target, String tagGroup, String tag) { } } + + /** + * Split the drop in multiple (smaller) transactions, which can be helpful to + * reduce memory utilization in some scenarios + * @param target + * @param transactionProvider + * @param tagGroup + * @param tag + */ + public void dropSplitTransaction(ISchemaAdapter target, ITransactionProvider transactionProvider, String tagGroup, String tag) { + + ArrayList copy = new ArrayList<>(); + copy.addAll(allObjects); + + int total = allObjects.size(); + int count = 1; + for (int i=total-1; i>=0; i--) { + IDatabaseObject obj = copy.get(i); + + // Each object (which often represents a group of tables) will be dropped + // in its own transaction...so clearly this needs to be an idempotent + // operation + try (ITransaction tx = transactionProvider.getTransaction()) { + try { + if (tag == null || obj.getTags().get(tagGroup) != null && tag.equals(obj.getTags().get(tagGroup))) { + logger.info(String.format("Dropping [%d/%d] %s", count++, total, obj.toString())); + obj.drop(target); + } else { + logger.info(String.format("Skipping [%d/%d] %s", count++, total, obj.toString())); + } + } catch (RuntimeException x) { + tx.setRollbackOnly(); + throw x; + } + } + } + } /** * Drop all foreign key constraints on tables in this model. Typically done prior to dropping @@ -198,7 +306,7 @@ public void drop(IDatabaseAdapter target, String tagGroup, String tag) { * @param tagGroup * @param tag */ - public void dropForeignKeyConstraints(IDatabaseAdapter target, String tagGroup, String tag) { + public void dropForeignKeyConstraints(ISchemaAdapter target, String tagGroup, String tag) { // The simplest way to reverse the list is add everything into an array list // which we then simply traverse end to start ArrayList copy = new ArrayList<>(); @@ -225,12 +333,42 @@ public void dropForeignKeyConstraints(IDatabaseAdapter target, String tagGroup, * @param v * @param tagGroup * @param tag + * @param transactionSupplier */ - public void visit(DataModelVisitor v, final String tagGroup, final String tag) { - // visit just the matching subset of objects - this.allObjects.stream() - .filter(obj -> tag == null || obj.getTags().get(tagGroup) != null && tag.equals(obj.getTags().get(tagGroup))) - .forEach(obj -> obj.visit(v)); + public void visit(DataModelVisitor v, final String tagGroup, final String tag, Supplier transactionSupplier) { + // visit just the matching subset of objects. If a transactionSupplier has been provided, we break up the + // operation into multiple transactions to avoid transaction size limitations (e.g. with Citus FK creation) + if (transactionSupplier != null) { + ITransaction tx = transactionSupplier.get(); + try { + int count = 0; + for (IDatabaseObject obj: allObjects) { + if (tag == null || obj.getTags().get(tagGroup) != null && tag.equals(obj.getTags().get(tagGroup))) { + if (++count == 10) { + // commit the current transaction and start a fresh one + tx.close(); + tx = transactionSupplier.get(); + count = 0; + } + + try { + obj.visit(v); + } catch (RuntimeException x) { + tx.setRollbackOnly(); + throw x; + } + } + + } + } finally { + tx.close(); + } + } else { + // the old way, which will visit everything in the scope of one transaction + this.allObjects.stream() + .filter(obj -> tag == null || obj.getTags().get(tagGroup) != null && tag.equals(obj.getTags().get(tagGroup))) + .forEach(obj -> obj.visit(v)); + } } /** @@ -261,7 +399,7 @@ public void visitReverse(DataModelVisitor v) { * Drop the lot * @param target */ - public void drop(IDatabaseAdapter target) { + public void drop(ISchemaAdapter target) { drop(target, null, null); } @@ -394,7 +532,24 @@ public ProcedureDef addProcedure(String schemaName, String objectName, int versi */ public FunctionDef addFunction(String schemaName, String objectName, int version, Supplier templateProvider, Collection dependencies, Collection privileges) { - FunctionDef func = new FunctionDef(schemaName, objectName, version, templateProvider); + return addFunction(schemaName, objectName, version, templateProvider, dependencies, privileges, 0); + } + + /** + * adds the function to the model. + * + * @param schemaName + * @param objectName + * @param version + * @param templateProvider + * @param dependencies + * @param privileges + * @param distributeByParamNum + * @return + */ + public FunctionDef addFunction(String schemaName, String objectName, int version, Supplier templateProvider, + Collection dependencies, Collection privileges, int distributeByParamNum) { + FunctionDef func = new FunctionDef(schemaName, objectName, version, templateProvider, distributeByParamNum); privileges.forEach(p -> p.addToObject(func)); if (dependencies != null) { @@ -506,7 +661,7 @@ public void processObjectsWithTag(String tagName, String tagValue, Consumer function @@ -34,7 +35,7 @@ public ProcedureDef(String schemaName, String procedureName, int version, Suppli } @Override - public void apply(IDatabaseAdapter target) { + public void apply(ISchemaAdapter target, SchemaApplyContext context) { // Serialize the execution of the procedure, to try and avoid the // horrible deadlocks we keep getting synchronized (this) { @@ -44,23 +45,23 @@ public void apply(IDatabaseAdapter target) { } @Override - public void apply(Integer priorVersion, IDatabaseAdapter target) { + public void apply(Integer priorVersion, ISchemaAdapter target, SchemaApplyContext context) { if (priorVersion != null && priorVersion > 0 && this.getVersion() > priorVersion && !migrations.isEmpty()) { logger.warning("Found '" + migrations.size() + "' migration steps, but performing 'create or replace' instead"); } // we need to drop and then apply. drop(target); - apply(target); + apply(target, context); } @Override - public void drop(IDatabaseAdapter target) { + public void drop(ISchemaAdapter target) { target.dropProcedure(getSchemaName(), getObjectName()); } @Override - protected void grantGroupPrivileges(IDatabaseAdapter target, Set group, String toUser) { + protected void grantGroupPrivileges(ISchemaAdapter target, Set group, String toUser) { target.grantProcedurePrivileges(getSchemaName(), getObjectName(), group, toUser); } diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/RowArrayType.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/RowArrayType.java index d90e57bd1ad..9be488c4e7b 100644 --- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/RowArrayType.java +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/RowArrayType.java @@ -1,12 +1,13 @@ /* - * (C) Copyright IBM Corp. 2019, 2020 + * (C) Copyright IBM Corp. 2019, 2022 * * SPDX-License-Identifier: Apache-2.0 */ package com.ibm.fhir.database.utils.model; -import com.ibm.fhir.database.utils.api.IDatabaseAdapter; +import com.ibm.fhir.database.utils.api.ISchemaAdapter; +import com.ibm.fhir.database.utils.api.SchemaApplyContext; import com.ibm.fhir.database.utils.common.DataDefinitionUtil; /** @@ -35,20 +36,20 @@ public String toString() { } @Override - public void apply(IDatabaseAdapter target) { + public void apply(ISchemaAdapter target, SchemaApplyContext context) { target.createArrType(getSchemaName(), getObjectName(), rowTypeName, arraySize); } @Override - public void apply(Integer priorVersion, IDatabaseAdapter target) { + public void apply(Integer priorVersion, ISchemaAdapter target, SchemaApplyContext context) { if (priorVersion != null && priorVersion > 0 && this.version > priorVersion) { throw new UnsupportedOperationException("Upgrading row array types is not supported"); } - apply(target); + apply(target, context); } @Override - public void drop(IDatabaseAdapter target) { + public void drop(ISchemaAdapter target) { target.dropType(getSchemaName(), getObjectName()); } diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/RowType.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/RowType.java index b2910828f7a..6f0ae923192 100644 --- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/RowType.java +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/RowType.java @@ -1,5 +1,5 @@ /* - * (C) Copyright IBM Corp. 2019, 2020 + * (C) Copyright IBM Corp. 2019, 2022 * * SPDX-License-Identifier: Apache-2.0 */ @@ -10,7 +10,8 @@ import java.util.Collection; import java.util.List; -import com.ibm.fhir.database.utils.api.IDatabaseAdapter; +import com.ibm.fhir.database.utils.api.ISchemaAdapter; +import com.ibm.fhir.database.utils.api.SchemaApplyContext; /** * Represents the ROW type used to pass parameters to the add_resource stored procedures @@ -25,20 +26,20 @@ public RowType(String schemaName, String typeName, int version, Collection 0 && this.version > priorVersion) { throw new UnsupportedOperationException("Upgrading row types is not supported"); } - apply(target); + apply(target, context); } @Override - public void drop(IDatabaseAdapter target) { + public void drop(ISchemaAdapter target) { target.dropType(getSchemaName(), getObjectName()); } diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/Sequence.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/Sequence.java index e0cd210a475..42da00b22a2 100644 --- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/Sequence.java +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/Sequence.java @@ -1,5 +1,5 @@ /* - * (C) Copyright IBM Corp. 2019, 2020 + * (C) Copyright IBM Corp. 2019, 2022 * * SPDX-License-Identifier: Apache-2.0 */ @@ -8,7 +8,8 @@ import java.util.Set; -import com.ibm.fhir.database.utils.api.IDatabaseAdapter; +import com.ibm.fhir.database.utils.api.ISchemaAdapter; +import com.ibm.fhir.database.utils.api.SchemaApplyContext; /** * Sequence related to the SQL sequence @@ -44,12 +45,12 @@ public Sequence(String schemaName, String sequenceName, int version, long startW } @Override - public void apply(IDatabaseAdapter target) { + public void apply(ISchemaAdapter target, SchemaApplyContext context) { target.createSequence(getSchemaName(), getObjectName(), this.startWith, this.cache, this.incrementBy); } @Override - public void apply(Integer priorVersion, IDatabaseAdapter target) { + public void apply(Integer priorVersion, ISchemaAdapter target, SchemaApplyContext context) { if (priorVersion != null && priorVersion > 0 && this.version > priorVersion) { throw new UnsupportedOperationException("Upgrading sequences is not supported"); } @@ -57,17 +58,17 @@ public void apply(Integer priorVersion, IDatabaseAdapter target) { // Only if VERSION1 then we want to apply, else fall through // Re-creating a sequence can have unintended consequences. if (this.version == 1 && (priorVersion == null || priorVersion == 0)) { - apply(target); + apply(target, context); } } @Override - public void drop(IDatabaseAdapter target) { + public void drop(ISchemaAdapter target) { target.dropSequence(getSchemaName(), getObjectName()); } @Override - protected void grantGroupPrivileges(IDatabaseAdapter target, Set group, String toUser) { + protected void grantGroupPrivileges(ISchemaAdapter target, Set group, String toUser) { target.grantSequencePrivileges(getSchemaName(), getObjectName(), group, toUser); } diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/SessionVariableDef.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/SessionVariableDef.java index e37ed61fb54..f3bd594d21a 100644 --- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/SessionVariableDef.java +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/SessionVariableDef.java @@ -1,5 +1,5 @@ /* - * (C) Copyright IBM Corp. 2019 + * (C) Copyright IBM Corp. 2019, 2022 * * SPDX-License-Identifier: Apache-2.0 */ @@ -8,7 +8,8 @@ import java.util.Set; -import com.ibm.fhir.database.utils.api.IDatabaseAdapter; +import com.ibm.fhir.database.utils.api.ISchemaAdapter; +import com.ibm.fhir.database.utils.api.SchemaApplyContext; /** * Adds a session variable to the database @@ -20,22 +21,22 @@ public SessionVariableDef(String schemaName, String variableName, int version) { } @Override - public void apply(IDatabaseAdapter target) { + public void apply(ISchemaAdapter target, SchemaApplyContext context) { target.createIntVariable(getSchemaName(), getObjectName()); } @Override - public void apply(Integer priorVersion, IDatabaseAdapter target) { + public void apply(Integer priorVersion, ISchemaAdapter target, SchemaApplyContext context) { target.createIntVariable(getSchemaName(), getObjectName()); } @Override - public void drop(IDatabaseAdapter target) { + public void drop(ISchemaAdapter target) { target.dropVariable(getSchemaName(), getObjectName()); } @Override - protected void grantGroupPrivileges(IDatabaseAdapter target, Set group, String toUser) { + protected void grantGroupPrivileges(ISchemaAdapter target, Set group, String toUser) { if (target.useSessionVariable()) { target.grantVariablePrivileges(getSchemaName(), getObjectName(), group, toUser); } diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/SmallIntBooleanColumn.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/SmallIntBooleanColumn.java new file mode 100644 index 00000000000..2ff6c98c7f1 --- /dev/null +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/SmallIntBooleanColumn.java @@ -0,0 +1,35 @@ +/* + * (C) Copyright IBM Corp. 2020 + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.ibm.fhir.database.utils.model; + +import com.ibm.fhir.database.utils.api.IDatabaseTypeAdapter; +import com.ibm.fhir.database.utils.postgres.PostgresAdapter; + +/** + * Column acting as either a boolean or smallint depending on the underlying + * database type + */ +public class SmallIntBooleanColumn extends ColumnBase { + /** + * @param name + * @param nullable + * @param defaultValue + */ + public SmallIntBooleanColumn(String name, boolean nullable, String defaultValue) { + super(name, nullable, defaultValue); + } + + @Override + public String getTypeInfo(IDatabaseTypeAdapter adapter) { + if (adapter instanceof PostgresAdapter) { + this.resetDefaultValue(); + return "BOOLEAN"; + } else { + return "SMALLINT"; + } + } +} \ No newline at end of file diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/SmallIntColumn.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/SmallIntColumn.java index a44f1b14152..2aa9075ba10 100644 --- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/SmallIntColumn.java +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/SmallIntColumn.java @@ -1,5 +1,5 @@ /* - * (C) Copyright IBM Corp. 2020 + * (C) Copyright IBM Corp. 2022 * * SPDX-License-Identifier: Apache-2.0 */ @@ -7,10 +7,9 @@ package com.ibm.fhir.database.utils.model; import com.ibm.fhir.database.utils.api.IDatabaseTypeAdapter; -import com.ibm.fhir.database.utils.postgres.PostgresAdapter; /** - * Small Int Column + * Small Int Column (2 bytes signed integer) */ public class SmallIntColumn extends ColumnBase { /** @@ -24,11 +23,6 @@ public SmallIntColumn(String name, boolean nullable, String defaultValue) { @Override public String getTypeInfo(IDatabaseTypeAdapter adapter) { - if (adapter instanceof PostgresAdapter) { - this.resetDefaultValue(); - return "BOOLEAN"; - } else { - return "SMALLINT"; - } + return adapter.smallintClause(); } } \ No newline at end of file diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/Table.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/Table.java index 21db63313ef..9c4428b486d 100644 --- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/Table.java +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/Table.java @@ -1,5 +1,5 @@ /* - * (C) Copyright IBM Corp. 2019, 2021 + * (C) Copyright IBM Corp. 2019, 2022 * * SPDX-License-Identifier: Apache-2.0 */ @@ -17,7 +17,9 @@ import java.util.Set; import java.util.stream.Collectors; -import com.ibm.fhir.database.utils.api.IDatabaseAdapter; +import com.ibm.fhir.database.utils.api.DistributionType; +import com.ibm.fhir.database.utils.api.ISchemaAdapter; +import com.ibm.fhir.database.utils.api.SchemaApplyContext; import com.ibm.fhir.database.utils.common.DataDefinitionUtil; /** @@ -47,11 +49,20 @@ public class Table extends BaseObject { // The column to use when making this table multi-tenant (if supported by the the target) private final String tenantColumnName; + // The rules to distribute the table in a distributed RDBMS implementation (Citus) + private final DistributionType distributionType; + + // If set, overrides the column used to distribute the data in a distributed database + private final String distributionColumnName; + // The With parameters on the table private final List withs; private final List checkConstraints = new ArrayList<>(); + // Do we still want to create this table? + private final boolean create; + /** * Public constructor * @@ -72,11 +83,16 @@ public class Table extends BaseObject { * @param migrations * @param withs * @param checkConstraints + * @param distributionType + * @param distributionColumnName + * @param create */ - public Table(String schemaName, String name, int version, String tenantColumnName, Collection columns, PrimaryKeyDef pk, + public Table(String schemaName, String name, int version, String tenantColumnName, + Collection columns, PrimaryKeyDef pk, IdentityDef identity, Collection indexes, Collection fkConstraints, SessionVariableDef accessControlVar, Tablespace tablespace, List dependencies, Map tags, - Collection privileges, List migrations, List withs, List checkConstraints) { + Collection privileges, List migrations, List withs, List checkConstraints, + DistributionType distributionType, String distributionColumnName, boolean create) { super(schemaName, name, DatabaseObjectType.TABLE, version, migrations); this.tenantColumnName = tenantColumnName; this.columns.addAll(columns); @@ -88,6 +104,9 @@ public Table(String schemaName, String name, int version, String tenantColumnNam this.tablespace = tablespace; this.withs = withs; this.checkConstraints.addAll(checkConstraints); + this.distributionType = distributionType; + this.distributionColumnName = distributionColumnName; + this.create = create; // Adds all dependencies which aren't null. // The only circumstances where it is null is when it is self referencial (an FK on itself). @@ -121,20 +140,44 @@ public String getTenantColumnName() { return this.tenantColumnName; } + /** + * Getter for the create flag + * @return + */ + public boolean isCreate() { + return this.create; + } + + /** + * Getter for the table's distributionType + * @return + */ + public DistributionType getDistributionType() { + return this.distributionType; + } + @Override - public void apply(IDatabaseAdapter target) { + public void apply(ISchemaAdapter target, SchemaApplyContext context) { + if (!create) { + // Skip creation for tables we no longer want + return; + } + final String tsName = this.tablespace == null ? null : this.tablespace.getName(); target.createTable(getSchemaName(), getObjectName(), this.tenantColumnName, this.columns, - this.primaryKey, this.identity, tsName, this.withs, this.checkConstraints); + this.primaryKey, this.identity, tsName, this.withs, this.checkConstraints, + this.distributionType, this.distributionColumnName); // Now add any indexes associated with this table for (IndexDef idx: this.indexes) { - idx.apply(getSchemaName(), getObjectName(), this.tenantColumnName, target); + idx.apply(getSchemaName(), getObjectName(), this.tenantColumnName, target, this.distributionType, this.distributionColumnName); } - // Foreign key constraints - for (ForeignKeyConstraint fkc: this.fkConstraints) { - fkc.apply(getSchemaName(), getObjectName(), this.tenantColumnName, target); + if (context.isIncludeForeignKeys()) { + // Foreign key constraints + for (ForeignKeyConstraint fkc: this.fkConstraints) { + fkc.apply(getSchemaName(), getObjectName(), this.tenantColumnName, target, this.distributionType); + } } // Apply tenant access control if required @@ -151,15 +194,15 @@ public void apply(IDatabaseAdapter target) { } @Override - public void apply(Integer priorVersion, IDatabaseAdapter target) { + public void apply(Integer priorVersion, ISchemaAdapter target, SchemaApplyContext context) { if (priorVersion == null || priorVersion == 0) { - apply(target); + apply(target, context); } else if (this.getVersion() > priorVersion) { for (Migration step : migrations) { step.migrateFrom(priorVersion).stream().forEachOrdered(target::runStatement); } // Re-apply tenant access control if required - if (this.accessControlVar != null) { + if (this.accessControlVar != null && this.create) { // The accessControlVar represents a DB2 session variable. Programs must set this value // for the current tenant when executing any SQL (both reads and writes) on // tables with this access control enabled @@ -173,7 +216,15 @@ public void apply(Integer priorVersion, IDatabaseAdapter target) { } @Override - public void drop(IDatabaseAdapter target) { + protected void grantGroupPrivileges(ISchemaAdapter target, Set group, String toUser) { + if (create) { + // only issue the grant if we have created this object + super.grantGroupPrivileges(target, group, toUser); + } + } + + @Override + public void drop(ISchemaAdapter target) { if (this.accessControlVar != null) { target.deactivateRowAccessControl(getSchemaName(), getObjectName()); @@ -241,6 +292,15 @@ public static class Builder extends VersionedSchemaObject { // Check constraints added to the table private List checkConstraints = new ArrayList<>(); + // The type of distribution to use for this table when using a distributed database + private DistributionType distributionType = DistributionType.NONE; + + // Allows the standard distribution column to be overridden + private String distributionColumnName; + + // Do we still want to create this table + private boolean create = true; + /** * Private constructor to force creation through factory method * @param schemaName @@ -270,6 +330,36 @@ public Builder setTablespace(Tablespace ts) { return this; } + /** + * Setter for the create flag + * @param ts + * @return + */ + public Builder setCreate(boolean flag) { + this.create = flag; + return this; + } + + /** + * Setter for the distributionType + * @param cn + * @return + */ + public Builder setDistributionType(DistributionType dt) { + this.distributionType = dt; + return this; + } + + /** + * Setter for the distributionColumnName value + * @param columnName + * @return + */ + public Builder setDistributionColumnName(String columnName) { + this.distributionColumnName = columnName; + return this; + } + public Builder addIntColumn(String columnName, boolean nullable) { ColumnDef cd = new ColumnDef(columnName); if (columns.contains(cd)) { @@ -299,6 +389,30 @@ public Builder addSmallIntColumn(String columnName, Integer defaultValue, boolea return this; } + /** + * Variant used by JavaBatch which is BOOLEAN in PostgreSQL but SMALLINT elsewhere + * @param columnName + * @param defaultValue + * @param nullable + * @return + */ + public Builder addSmallIntBooleanColumn(String columnName, Integer defaultValue, boolean nullable) { + ColumnDef cd = new ColumnDef(columnName); + if (columns.contains(cd)) { + throw new IllegalArgumentException("Duplicate column: " + columnName); + } + + cd.setNullable(nullable); + + if (defaultValue != null) { + cd.setDefaultVal(Integer.toString(defaultValue)); + } + + cd.setColumnType(ColumnType.SMALLINT_BOOLEAN); + columns.add(cd); + return this; + } + public Builder addBigIntColumn(String columnName, boolean nullable) { addBigIntColumn(columnName, nullable, null); return this; @@ -689,15 +803,58 @@ public Table build(IDataModel dataModel) { // Check the FK references are valid List allDependencies = new ArrayList<>(); + // The list of FK constraints we are able to apply + List enabledFKConstraints = new ArrayList<>(); allDependencies.addAll(this.dependencies); - for (ForeignKeyConstraint c: this.fkConstraints.values()) { + // Filter the foreign key constraints to those allowed for the model. Distribution (e.g. Citus) adds + // certain restrictions on which foreign keys are supported, so we have no choice but to ignore them + for (ForeignKeyConstraint c: this.fkConstraints.values()) { Table target = dataModel.findTable(c.getTargetSchema(), c.getTargetTable()); if (target == null && !c.isSelf()) { String targetName = DataDefinitionUtil.getQualifiedName(c.getTargetSchema(), c.getTargetTable()); throw new IllegalArgumentException("Invalid foreign key constraint " + c.getConstraintName() + ": target table does not exist: " + targetName); } - allDependencies.add(target); + + // Determine the distribution type of the FK target. If target is null, it must mean that this is a FK to self + DistributionType targetDistributionType = target != null ? target.getDistributionType() : this.distributionType; + if (targetDistributionType == DistributionType.REFERENCE) { + // Mark the constraint as pointing to a REFERENCE table (which won't include a shard + // column). FK relationships are therefore local to each distributed node (Citus) + c.setTargetReference(true); + } + + if (!dataModel.isDistributed()) { + // ignore any distribution configuration because the target database is a plain RDBMS + if (target != null) { + // only add dependency if target is something else. If target is null, it means + // a FK reference to self and so no dependency is needed + allDependencies.add(target); + } + enabledFKConstraints.add(c); + } else { + // Make sure that FK references adhere to the restrictions imposed by distribution (replication or sharding) + if (distributionType == DistributionType.NONE) { + // this table is not distributed, so we can handle the FK relationship as long + // as the target isn't sharded (replicated is OK) + if (targetDistributionType == DistributionType.REFERENCE) { + allDependencies.add(target); + enabledFKConstraints.add(c); + } + } else if (distributionType == DistributionType.REFERENCE) { + // This table is a reference (replicated) table. We can create FK relationships + // to other replicated tables + if (targetDistributionType == DistributionType.REFERENCE) { + allDependencies.add(target); + enabledFKConstraints.add(c); + } + } else if (targetDistributionType != DistributionType.NONE) { + // This table is distributed, and the target is either distributed or a reference + // table. In either case we can support the FK because the tables will be co-located + allDependencies.add(target); + enabledFKConstraints.add(c); + } + } } if (this.tablespace != null) { @@ -707,8 +864,8 @@ public Table build(IDataModel dataModel) { // Our schema objects are immutable by design, so all initialization takes place // through the constructor return new Table(getSchemaName(), getObjectName(), this.version, this.tenantColumnName, buildColumns(), this.primaryKey, this.identity, this.indexes.values(), - this.fkConstraints.values(), this.accessControlVar, this.tablespace, allDependencies, tags, privileges, migrations, withs, checkConstraints); - + enabledFKConstraints, this.accessControlVar, this.tablespace, allDependencies, tags, privileges, migrations, withs, checkConstraints, distributionType, + distributionColumnName, create); } /** @@ -733,6 +890,9 @@ protected List buildColumns() { case SMALLINT: column = new SmallIntColumn(cd.getName(), cd.isNullable(), cd.getDefaultVal()); break; + case SMALLINT_BOOLEAN: + column = new SmallIntBooleanColumn(cd.getName(), cd.isNullable(), cd.getDefaultVal()); + break; case DOUBLE: column = new DoubleColumn(cd.getName(), cd.isNullable()); break; @@ -841,14 +1001,16 @@ public Builder addWiths(List withs) { * @param target * @return */ - public boolean exists(IDatabaseAdapter target) { + public boolean exists(ISchemaAdapter target) { return target.doesTableExist(getSchemaName(), getObjectName()); } @Override public void visit(DataModelVisitor v) { - v.visited(this); - this.fkConstraints.forEach(fk -> v.visited(this, fk)); + if (this.create) { + v.visited(this); + this.fkConstraints.forEach(fk -> v.visited(this, fk)); + } } @Override @@ -857,4 +1019,20 @@ public void visitReverse(DataModelVisitor v) { this.fkConstraints.forEach(fk -> v.visited(this, fk)); v.visited(this); } + + @Override + public void applyDistributionRules(ISchemaAdapter target, int pass) { + if (!this.create) { + // skip if we no longer create this table + return; + } + + // make sure all the reference tables are distributed first before + // we attempt to distribute any of the sharded (DISTRIBUTED) tables + if (pass == 0 && this.distributionType == DistributionType.REFERENCE) { + target.applyDistributionRules(getSchemaName(), getObjectName(), this.distributionType, null); + } else if (pass == 1 && this.distributionType == DistributionType.DISTRIBUTED) { + target.applyDistributionRules(getSchemaName(), getObjectName(), this.distributionType, this.distributionColumnName); + } + } } \ No newline at end of file diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/Tablespace.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/Tablespace.java index ca1e6e44fde..5da8395af35 100644 --- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/Tablespace.java +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/Tablespace.java @@ -1,5 +1,5 @@ /* - * (C) Copyright IBM Corp. 2019, 2020 + * (C) Copyright IBM Corp. 2019, 2022 * * SPDX-License-Identifier: Apache-2.0 */ @@ -9,9 +9,10 @@ import java.util.Collection; import java.util.List; -import com.ibm.fhir.database.utils.api.IDatabaseAdapter; +import com.ibm.fhir.database.utils.api.ISchemaAdapter; import com.ibm.fhir.database.utils.api.ITransactionProvider; import com.ibm.fhir.database.utils.api.IVersionHistoryService; +import com.ibm.fhir.database.utils.api.SchemaApplyContext; import com.ibm.fhir.task.api.ITaskCollector; import com.ibm.fhir.task.api.ITaskGroup; @@ -34,7 +35,7 @@ public Tablespace(String tablespaceName, int version, int extentSizeKB) { } @Override - public void apply(IDatabaseAdapter target) { + public void apply(ISchemaAdapter target, SchemaApplyContext context) { if (this.extentSizeKB > 0) { target.createTablespace(getName(), this.extentSizeKB); } @@ -45,23 +46,23 @@ public void apply(IDatabaseAdapter target) { } @Override - public void apply(Integer priorVersion, IDatabaseAdapter target) { + public void apply(Integer priorVersion, ISchemaAdapter target, SchemaApplyContext context) { if (priorVersion != null && priorVersion > 0) { throw new UnsupportedOperationException("Modifying tablespaces is not supported"); } - apply(target); + apply(target, context); } @Override - public void drop(IDatabaseAdapter target) { + public void drop(ISchemaAdapter target) { target.dropTablespace(getName()); } @Override - public ITaskGroup collect(ITaskCollector tc, IDatabaseAdapter target, ITransactionProvider tp, IVersionHistoryService vhs) { + public ITaskGroup collect(ITaskCollector tc, ISchemaAdapter target, SchemaApplyContext context, ITransactionProvider tp, IVersionHistoryService vhs) { // no dependencies, so no need to recurse down List children = null; - return tc.makeTaskGroup(this.getTypeNameVersion(), () -> applyTx(target, tp, vhs), children); + return tc.makeTaskGroup(this.getTypeNameVersion(), () -> applyTx(target, context, tp, vhs), children); } @Override @@ -75,7 +76,7 @@ public void fetchDependenciesTo(Collection out) { } @Override - public void grant(IDatabaseAdapter target, String groupName, String toUser) { + public void grant(ISchemaAdapter target, String groupName, String toUser) { // NOP } @@ -94,5 +95,4 @@ public void visit(DataModelVisitor v) { public void visitReverse(DataModelVisitor v) { v.visited(this); } - } diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/View.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/View.java index 273bd9de2a0..2b7f07a50db 100644 --- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/View.java +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/model/View.java @@ -1,5 +1,5 @@ /* - * (C) Copyright IBM Corp. 2020 + * (C) Copyright IBM Corp. 2020, 2022 * * SPDX-License-Identifier: Apache-2.0 */ @@ -15,7 +15,8 @@ import java.util.Set; import java.util.stream.Collectors; -import com.ibm.fhir.database.utils.api.IDatabaseAdapter; +import com.ibm.fhir.database.utils.api.ISchemaAdapter; +import com.ibm.fhir.database.utils.api.SchemaApplyContext; /** @@ -47,17 +48,17 @@ protected View(String schemaName, String objectName, int version, String selectC } @Override - public void apply(IDatabaseAdapter target) { + public void apply(ISchemaAdapter target, SchemaApplyContext context) { target.createOrReplaceView(getSchemaName(), getObjectName(), this.selectClause); } @Override - public void apply(Integer priorVersion, IDatabaseAdapter target) { - apply(target); + public void apply(Integer priorVersion, ISchemaAdapter target, SchemaApplyContext context) { + apply(target, context); } @Override - public void drop(IDatabaseAdapter target) { + public void drop(ISchemaAdapter target) { target.dropView(getSchemaName(), getObjectName()); } diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/pool/DatabaseSupport.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/pool/DatabaseSupport.java index a968b234a25..859d52957d2 100644 --- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/pool/DatabaseSupport.java +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/pool/DatabaseSupport.java @@ -72,6 +72,7 @@ public void init() { configureForDerby(); break; case POSTGRESQL: + case CITUS: configureForPostgresql(); break; default: diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/pool/PoolConnectionProvider.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/pool/PoolConnectionProvider.java index ebe2d8a0c56..f6699829f48 100644 --- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/pool/PoolConnectionProvider.java +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/pool/PoolConnectionProvider.java @@ -1,5 +1,5 @@ /* - * (C) Copyright IBM Corp. 2019, 2020 + * (C) Copyright IBM Corp. 2019, 2022 * * SPDX-License-Identifier: Apache-2.0 */ @@ -14,6 +14,7 @@ import java.util.concurrent.locks.Condition; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; +import java.util.function.Consumer; import java.util.logging.Logger; import com.ibm.fhir.database.utils.api.DataAccessException; @@ -54,6 +55,7 @@ public class PoolConnectionProvider implements IConnectionProvider { // Should we reuse connections after an exception, or close them instead of returning them to the pool private boolean closeOnAnyError = false; + private Consumer newConnectionHandler; /** * Public constructor * @param cp @@ -73,6 +75,24 @@ public void setCloseOnAnyError() { this.closeOnAnyError = true; } + /** + * Setter for the newConnectionHandler + * @param handler + */ + public void setNewConnectionHandler(Consumer handler) { + this.newConnectionHandler = handler; + } + + /** + * Apply an configuration steps to a new connection + * @param c + */ + protected void configureConnection(Connection c) { + if (newConnectionHandler != null) { + newConnectionHandler.accept(c); + } + } + @Override public Connection getConnection() throws SQLException { // We use the same connection on a given thread each time it is requested @@ -148,6 +168,9 @@ public Connection getConnection() throws SQLException { } } + // Apply any configuration we want for a new connection + configureConnection(c); + long endTime = System.nanoTime(); double elapsed = (endTime-startTime) / 1e9; if (elapsed > 1.0) { diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/postgres/PostgresAdapter.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/postgres/PostgresAdapter.java index 8f735266107..b1ba13d5431 100644 --- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/postgres/PostgresAdapter.java +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/postgres/PostgresAdapter.java @@ -1,5 +1,5 @@ /* - * (C) Copyright IBM Corp. 2020, 2021 + * (C) Copyright IBM Corp. 2020, 2022 * * SPDX-License-Identifier: Apache-2.0 */ @@ -19,6 +19,7 @@ import java.util.logging.Level; import java.util.logging.Logger; +import com.ibm.fhir.database.utils.api.DistributionContext; import com.ibm.fhir.database.utils.api.DuplicateNameException; import com.ibm.fhir.database.utils.api.DuplicateSchemaException; import com.ibm.fhir.database.utils.api.IConnectionProvider; @@ -46,7 +47,7 @@ public class PostgresAdapter extends CommonDatabaseAdapter { private static final Logger logger = Logger.getLogger(PostgresAdapter.class.getName()); // Different warning messages we track so that we only have to report them once - private enum MessageKey { + protected enum MessageKey { MULTITENANCY, CREATE_VAR, CREATE_PERM, @@ -64,6 +65,9 @@ private enum MessageKey { DROP_VARIABLE } + // Constant for better readability in method calls + protected static final boolean USE_SCHEMA_PREFIX = true; + // Just warn once for each unique message key. This cleans up build logs a lot private static final Set warned = ConcurrentHashMap.newKeySet(); @@ -96,7 +100,8 @@ public void warnOnce(MessageKey messageKey, String msg) { @Override public void createTable(String schemaName, String name, String tenantColumnName, List columns, PrimaryKeyDef primaryKey, - IdentityDef identity, String tablespaceName, List withs, List checkConstraints) { + IdentityDef identity, String tablespaceName, List withs, List checkConstraints, + DistributionContext distributionContext) { // PostgreSql doesn't support partitioning, so we ignore tenantColumnName if (tenantColumnName != null) { @@ -110,9 +115,9 @@ public void createTable(String schemaName, String name, String tenantColumnName, @Override public void createUniqueIndex(String schemaName, String tableName, String indexName, String tenantColumnName, List indexColumns, - List includeColumns) { + List includeColumns, DistributionContext distributionContext) { // PostgreSql doesn't support include columns, so we just have to create a normal index - createUniqueIndex(schemaName, tableName, indexName, tenantColumnName, indexColumns); + createUniqueIndex(schemaName, tableName, indexName, tenantColumnName, indexColumns, distributionContext); } @Override @@ -298,10 +303,10 @@ public void runStatement(IDatabaseStatement stmt) { @Override public void createUniqueIndex(String schemaName, String tableName, String indexName, String tenantColumnName, - List indexColumns) { + List indexColumns, DistributionContext distributionContext) { indexColumns = prefixTenantColumn(tenantColumnName, indexColumns); // Postgresql doesn't support index name prefixed with the schema name. - String ddl = DataDefinitionUtil.createUniqueIndex(schemaName, tableName, indexName, indexColumns, false); + String ddl = DataDefinitionUtil.createUniqueIndex(schemaName, tableName, indexName, indexColumns, !USE_SCHEMA_PREFIX); runStatement(ddl); } @@ -310,7 +315,7 @@ public void createIndex(String schemaName, String tableName, String indexName, S List indexColumns) { indexColumns = prefixTenantColumn(tenantColumnName, indexColumns); // Postgresql doesn't support index name prefixed with the schema name. - String ddl = DataDefinitionUtil.createIndex(schemaName, tableName, indexName, indexColumns, false); + String ddl = DataDefinitionUtil.createIndex(schemaName, tableName, indexName, indexColumns, !USE_SCHEMA_PREFIX); runStatement(ddl); } @@ -380,6 +385,15 @@ public void enableForeignKey(String schemaName, String tableName, String constra throw new UnsupportedOperationException("Disable FK currently not supported for this adapter."); } + @Override + public boolean doesForeignKeyConstraintExist(String schemaName, String tableName, String constraintName) { + // check the catalog to see if the named constraint exists + PostgresDoesForeignKeyConstraintExist fkExists = new PostgresDoesForeignKeyConstraintExist(schemaName, constraintName); + // runStatement may return null in some unit-tests, so we need to protect against that + Boolean val = runStatement(fkExists); + return val != null && val.booleanValue(); + } + @Override public void setIntegrityOff(String schemaName, String tableName) { // not expecting this to be called for this adapter diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/postgres/PostgresDoesConstraintExist.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/postgres/PostgresDoesConstraintExist.java new file mode 100644 index 00000000000..dc2c2203fcd --- /dev/null +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/postgres/PostgresDoesConstraintExist.java @@ -0,0 +1,66 @@ +/* + * (C) Copyright IBM Corp. 2021 + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.ibm.fhir.database.utils.postgres; + +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; + +import com.ibm.fhir.database.utils.api.IDatabaseSupplier; +import com.ibm.fhir.database.utils.api.IDatabaseTranslator; +import com.ibm.fhir.database.utils.common.DataDefinitionUtil; + +/** + * PostgreSQL catalog query to determine if the named constraint exists for the given + * schema and table + */ +public class PostgresDoesConstraintExist implements IDatabaseSupplier { + + // Identity of the constraint + private final String schemaName; + private final String tableName; + private final String constraintName; + + /** + * Public constructor + * @param schemaName + * @param tableName + * @param constraintName + */ + public PostgresDoesConstraintExist(String schemaName, String tableName, String constraintName) { + this.schemaName = DataDefinitionUtil.assertValidName(schemaName).toLowerCase(); + this.tableName = DataDefinitionUtil.assertValidName(tableName).toLowerCase(); + this.constraintName = DataDefinitionUtil.assertValidName(constraintName).toLowerCase(); + } + + @Override + public Boolean run(IDatabaseTranslator translator, Connection c) { + Boolean result; + + final String SQL = "" + + "SELECT 1 FROM " + + " pg_catalog.pg_constraint con " + + " JOIN pg_catalog.pg_class rel ON rel.oid = con.conrelid " + + " JOIN pg_catalog.pg_namespace nsp ON nsp.oid = connamespace " + + " WHERE nsp.nspname = ? " + + " AND rel.relname = ? " + + " AND con.conname = ? "; + + try (PreparedStatement ps = c.prepareStatement(SQL)) { + ps.setString(1, schemaName); + ps.setString(2, tableName); + ps.setString(3, constraintName); + ResultSet rs = ps.executeQuery(); + result = Boolean.valueOf(rs.next()); + } catch (SQLException x) { + throw translator.translate(x); + } + + return result; + } +} \ No newline at end of file diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/postgres/PostgresDoesForeignKeyConstraintExist.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/postgres/PostgresDoesForeignKeyConstraintExist.java index 7e578671e30..f29f4c340db 100644 --- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/postgres/PostgresDoesForeignKeyConstraintExist.java +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/postgres/PostgresDoesForeignKeyConstraintExist.java @@ -31,16 +31,16 @@ public class PostgresDoesForeignKeyConstraintExist implements IDatabaseSupplier< * @param schemaName */ public PostgresDoesForeignKeyConstraintExist(String schemaName, String constraintName) { - this.schemaName = DataDefinitionUtil.assertValidName(schemaName); - this.constraintName = DataDefinitionUtil.assertValidName(constraintName); + this.schemaName = DataDefinitionUtil.assertValidName(schemaName).toLowerCase(); + this.constraintName = DataDefinitionUtil.assertValidName(constraintName).toLowerCase(); } @Override public Boolean run(IDatabaseTranslator translator, Connection c) { - Boolean result = Boolean.FALSE; + Boolean result; final String sql = "" - + "SELECT 1 " - + " FROM pg_constraint " + + "SELECT 1 FROM " + + " pg_constraint " + " WHERE contype = 'f' " + " AND connamespace = ?::regnamespace " + " AND conname = ? "; @@ -49,9 +49,7 @@ public Boolean run(IDatabaseTranslator translator, Connection c) { ps.setString(1, schemaName); ps.setString(2, constraintName); ResultSet rs = ps.executeQuery(); - if(rs.next()) { - result = Boolean.TRUE; - } + result = Boolean.valueOf(rs.next()); } catch (SQLException x) { throw translator.translate(x); diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/postgres/PostgresFillfactorSettingDAO.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/postgres/PostgresFillfactorSettingDAO.java index 3a7ea2cd50e..1ce289c2148 100644 --- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/postgres/PostgresFillfactorSettingDAO.java +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/postgres/PostgresFillfactorSettingDAO.java @@ -10,14 +10,10 @@ import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.SQLException; -import java.util.ArrayList; -import java.util.List; import java.util.logging.Logger; -import java.util.stream.Collectors; import com.ibm.fhir.database.utils.api.IDatabaseStatement; import com.ibm.fhir.database.utils.api.IDatabaseTranslator; -import com.ibm.fhir.database.utils.model.DbType; import com.ibm.fhir.database.utils.model.With; /** @@ -55,7 +51,7 @@ public PostgresFillfactorSettingDAO(String schema, String tableName, int fillfac @Override public void run(IDatabaseTranslator translator, Connection c) { - if (translator.getType() == DbType.POSTGRESQL) { + if (translator.isFamilyPostgreSQL()) { // assume we need to set it unless it's been configured already boolean isFillfactor = true; LOG.fine(() -> "Checking the table fillfactor settings"); diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/postgres/PostgresTranslator.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/postgres/PostgresTranslator.java index 1d6b4c85ef0..accf4062287 100644 --- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/postgres/PostgresTranslator.java +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/postgres/PostgresTranslator.java @@ -242,6 +242,11 @@ public String dropForeignKeyConstraint(String qualifiedTableName, String constra return "ALTER TABLE " + qualifiedTableName + " DROP CONSTRAINT IF EXISTS " + constraintName; } + @Override + public String dropView(String qualifiedViewName) { + return "DROP VIEW IF EXISTS " + qualifiedViewName; + } + @Override public String nextValue(String schemaName, String sequenceName) { String qname = DataDefinitionUtil.getQualifiedName(schemaName, sequenceName); @@ -274,4 +279,9 @@ public Optional maximumQueryParameters() { // it's Short.MAX_VALUE return Optional.of(Integer.valueOf(32767)); } + + @Override + public boolean isFamilyPostgreSQL() { + return true; + } } \ No newline at end of file diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/postgres/PostgresVacuumSettingDAO.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/postgres/PostgresVacuumSettingDAO.java index f0856868b83..af7b8079e38 100644 --- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/postgres/PostgresVacuumSettingDAO.java +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/postgres/PostgresVacuumSettingDAO.java @@ -17,7 +17,6 @@ import com.ibm.fhir.database.utils.api.IDatabaseStatement; import com.ibm.fhir.database.utils.api.IDatabaseTranslator; -import com.ibm.fhir.database.utils.model.DbType; import com.ibm.fhir.database.utils.model.With; /** @@ -60,7 +59,7 @@ public PostgresVacuumSettingDAO(String schema, String tableName, int vacuumCostL @Override public void run(IDatabaseTranslator translator, Connection c) { - if (translator.getType() == DbType.POSTGRESQL) { + if (translator.isFamilyPostgreSQL()) { boolean isScaleFactor = true; boolean isVacuumThreshold = true; boolean isVacuumCostLimit = true; diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/query/FromAdapter.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/query/FromAdapter.java index 0e2b050675d..dc61f7c3ae1 100644 --- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/query/FromAdapter.java +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/query/FromAdapter.java @@ -59,6 +59,18 @@ public FromAdapter innerJoin(String tableName, Alias alias, WhereFragment joinOn return this; } + /** + * Add an INNER JOIN for the given sub select + * @param sub + * @param alias + * @param joinOnPredicate + * @return + */ + public FromAdapter innerJoin(Select sub, Alias alias, WhereFragment joinOnPredicate) { + this.select.addInnerJoin(sub, alias, joinOnPredicate.getExpression()); + return this; + } + /** * Add a LEFT OUTER JOIN for the given table * @param tableName diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/query/FromClause.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/query/FromClause.java index d2f2ece3bf9..631b9a89616 100644 --- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/query/FromClause.java +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/query/FromClause.java @@ -121,6 +121,17 @@ public void addInnerJoin(String tableName, Alias alias, ExpNode joinOnPredicate) items.add(new FromJoin(JoinType.INNER_JOIN, trs, alias, joinOnPredicate)); } + /** + * Add an inner join clause to the FROM items list + * @param sub + * @param alias + * @param joinOnPredicate + */ + public void addInnerJoin(Select sub, Alias alias, ExpNode joinOnPredicate) { + SelectRowSource srs = new SelectRowSource(sub); + items.add(new FromJoin(JoinType.INNER_JOIN, srs, alias, joinOnPredicate)); + } + /** * Add a left outer join clause to the FROM items list * @param tableName diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/query/Select.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/query/Select.java index 6397c514cc7..c1de626d076 100644 --- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/query/Select.java +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/query/Select.java @@ -288,6 +288,16 @@ public T render(StatementRenderer renderer) { public void addInnerJoin(String tableName, Alias alias, ExpNode joinOnPredicate) { fromClause.addInnerJoin(tableName, alias, joinOnPredicate); } + /** + * Add an inner join to the from clause for this select statement + * where the joining row source is a sub-query + * @param sub + * @param alias + * @param joinOnPredicate + */ + public void addInnerJoin(Select sub, Alias alias, ExpNode joinOnPredicate) { + fromClause.addInnerJoin(sub, alias, joinOnPredicate); + } /** * Add a left outer join to the from clause for this select statement. diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/schema/LeaseManager.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/schema/LeaseManager.java index 27e5cceea6b..6b4ddbe3048 100644 --- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/schema/LeaseManager.java +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/schema/LeaseManager.java @@ -237,6 +237,7 @@ private GetLease getLeaseDAO() { final GetLease result; switch (this.translator.getType()) { case POSTGRESQL: + case CITUS: result = new GetLeasePostgresql(adminSchema, schemaName, config.getHost(), leaseId, leaseUntil); break; default: diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/schema/SchemaVersionsManager.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/schema/SchemaVersionsManager.java index 8ee6dd03e84..c84e4103cd7 100644 --- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/schema/SchemaVersionsManager.java +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/schema/SchemaVersionsManager.java @@ -70,6 +70,7 @@ public void updateSchemaVersionId(int versionId) { final UpdateSchemaVersion cmd; switch (this.translator.getType()) { case POSTGRESQL: + case CITUS: cmd = new UpdateSchemaVersionPostgresql(schemaName, versionId); break; default: diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/version/CreateControl.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/version/CreateControl.java index d5e22ec2ade..bc88066a46a 100644 --- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/version/CreateControl.java +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/version/CreateControl.java @@ -1,5 +1,5 @@ /* - * (C) Copyright IBM Corp. 2021 + * (C) Copyright IBM Corp. 2021, 2022 * * SPDX-License-Identifier: Apache-2.0 */ @@ -8,7 +8,8 @@ import java.util.logging.Logger; -import com.ibm.fhir.database.utils.api.IDatabaseAdapter; +import com.ibm.fhir.database.utils.api.ISchemaAdapter; +import com.ibm.fhir.database.utils.api.SchemaApplyContext; import com.ibm.fhir.database.utils.model.PhysicalDataModel; import com.ibm.fhir.database.utils.model.Table; @@ -59,7 +60,8 @@ public static Table buildTableDef(PhysicalDataModel dataModel, String adminSchem * @param adminSchemaName * @param target */ - public static void createTableIfNeeded(String adminSchemaName, IDatabaseAdapter target) { + public static void createTableIfNeeded(String adminSchemaName, ISchemaAdapter target) { + SchemaApplyContext context = SchemaApplyContext.getDefault(); PhysicalDataModel dataModel = new PhysicalDataModel(); Table t = buildTableDef(dataModel, adminSchemaName, false); @@ -73,7 +75,7 @@ public static void createTableIfNeeded(String adminSchemaName, IDatabaseAdapter // update tool could try to build the table. The solution is to make it // idempotent...if the table exists already, that's success try { - dataModel.apply(target); + dataModel.apply(target, context); } catch (Exception x) { if (t.exists(target)) { logger.info("Table '" + t.getQualifiedName() + "' already exists; skipping create"); diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/version/CreateVersionHistory.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/version/CreateVersionHistory.java index 9c429eb19db..6a66e508328 100644 --- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/version/CreateVersionHistory.java +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/version/CreateVersionHistory.java @@ -1,12 +1,13 @@ /* - * (C) Copyright IBM Corp. 2019, 2021 + * (C) Copyright IBM Corp. 2019, 2022 * * SPDX-License-Identifier: Apache-2.0 */ package com.ibm.fhir.database.utils.version; -import com.ibm.fhir.database.utils.api.IDatabaseAdapter; +import com.ibm.fhir.database.utils.api.ISchemaAdapter; +import com.ibm.fhir.database.utils.api.SchemaApplyContext; import com.ibm.fhir.database.utils.model.PhysicalDataModel; import com.ibm.fhir.database.utils.model.Table; @@ -58,15 +59,16 @@ public static Table generateTable(PhysicalDataModel dataModel, String adminSchem * @param adminSchemaName * @param target */ - public static void createTableIfNeeded(String adminSchemaName, IDatabaseAdapter target) { + public static void createTableIfNeeded(String adminSchemaName, ISchemaAdapter target) { PhysicalDataModel dataModel = new PhysicalDataModel(); + SchemaApplyContext context = SchemaApplyContext.getDefault(); Table t = generateTable(dataModel, adminSchemaName, false); // apply this data model to the target if necessary - note - this bypasses the // version history table...because this is the table we're trying to create! if (!t.exists(target)) { - dataModel.apply(target); + dataModel.apply(target, context); } } diff --git a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/version/CreateWholeSchemaVersion.java b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/version/CreateWholeSchemaVersion.java index acad4246cf3..8b45b692512 100644 --- a/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/version/CreateWholeSchemaVersion.java +++ b/fhir-database-utils/src/main/java/com/ibm/fhir/database/utils/version/CreateWholeSchemaVersion.java @@ -1,5 +1,5 @@ /* - * (C) Copyright IBM Corp. 2021 + * (C) Copyright IBM Corp. 2021, 2022 * * SPDX-License-Identifier: Apache-2.0 */ @@ -8,7 +8,8 @@ import java.util.logging.Logger; -import com.ibm.fhir.database.utils.api.IDatabaseAdapter; +import com.ibm.fhir.database.utils.api.ISchemaAdapter; +import com.ibm.fhir.database.utils.api.SchemaApplyContext; import com.ibm.fhir.database.utils.model.PhysicalDataModel; import com.ibm.fhir.database.utils.model.Privilege; import com.ibm.fhir.database.utils.model.Table; @@ -69,8 +70,9 @@ public static Table buildTableDef(PhysicalDataModel dataModel, String schemaName * @param schemaName * @param target */ - public static void createTableIfNeeded(String schemaName, IDatabaseAdapter target) { + public static void createTableIfNeeded(String schemaName, ISchemaAdapter target) { PhysicalDataModel dataModel = new PhysicalDataModel(); + SchemaApplyContext context = SchemaApplyContext.getDefault(); Table t = buildTableDef(dataModel, schemaName, false); @@ -82,7 +84,7 @@ public static void createTableIfNeeded(String schemaName, IDatabaseAdapter targe // update tool could try to build the table. The solution is to make it // idempotent...if the table exists already, that's success try { - dataModel.apply(target); + dataModel.apply(target, context); } catch (Exception x) { if (t.exists(target)) { logger.info("Table '" + t.getQualifiedName() + "' already exists; skipping create"); @@ -98,7 +100,7 @@ public static void createTableIfNeeded(String schemaName, IDatabaseAdapter targe * @param schemaName * @param target */ - public static void dropTable(String schemaName, IDatabaseAdapter target) { + public static void dropTable(String schemaName, ISchemaAdapter target) { PhysicalDataModel dataModel = new PhysicalDataModel(); Table t = buildTableDef(dataModel, schemaName, false); @@ -118,7 +120,7 @@ public static void dropTable(String schemaName, IDatabaseAdapter target) { * @param groupName * @param toUser */ - public static void grantPrivilegesTo(IDatabaseAdapter target, String schemaName, String groupName, String toUser) { + public static void grantPrivilegesTo(ISchemaAdapter target, String schemaName, String groupName, String toUser) { PhysicalDataModel dataModel = new PhysicalDataModel(); Table t = buildTableDef(dataModel, schemaName, false); t.grant(target, groupName, toUser); diff --git a/fhir-persistence-cassandra/src/main/java/com/ibm/fhir/persistence/cassandra/payload/CqlDeletePayload.java b/fhir-persistence-cassandra/src/main/java/com/ibm/fhir/persistence/cassandra/payload/CqlDeletePayload.java index 90a7d619c60..458b5919040 100644 --- a/fhir-persistence-cassandra/src/main/java/com/ibm/fhir/persistence/cassandra/payload/CqlDeletePayload.java +++ b/fhir-persistence-cassandra/src/main/java/com/ibm/fhir/persistence/cassandra/payload/CqlDeletePayload.java @@ -25,8 +25,8 @@ import com.datastax.oss.driver.api.core.cql.Row; import com.datastax.oss.driver.api.core.cql.SimpleStatement; import com.datastax.oss.driver.api.querybuilder.select.Select; +import com.ibm.fhir.persistence.exception.FHIRPersistenceDataAccessException; import com.ibm.fhir.persistence.exception.FHIRPersistenceException; -import com.ibm.fhir.persistence.jdbc.exception.FHIRPersistenceDataAccessException; /** * DAO to delete all the records associated with this resource payload diff --git a/fhir-persistence-cassandra/src/main/java/com/ibm/fhir/persistence/cassandra/payload/CqlStorePayload.java b/fhir-persistence-cassandra/src/main/java/com/ibm/fhir/persistence/cassandra/payload/CqlStorePayload.java index e8837783779..060384e1ffb 100644 --- a/fhir-persistence-cassandra/src/main/java/com/ibm/fhir/persistence/cassandra/payload/CqlStorePayload.java +++ b/fhir-persistence-cassandra/src/main/java/com/ibm/fhir/persistence/cassandra/payload/CqlStorePayload.java @@ -28,8 +28,8 @@ import com.datastax.oss.driver.api.core.cql.PreparedStatement; import com.datastax.oss.driver.api.core.cql.SimpleStatement; import com.datastax.oss.driver.api.querybuilder.insert.RegularInsert; +import com.ibm.fhir.persistence.exception.FHIRPersistenceDataAccessException; import com.ibm.fhir.persistence.exception.FHIRPersistenceException; -import com.ibm.fhir.persistence.jdbc.exception.FHIRPersistenceDataAccessException; import com.ibm.fhir.persistence.util.InputOutputByteStream; /** diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/FHIRPersistenceJDBCCache.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/FHIRPersistenceJDBCCache.java index 09c23f0ab8c..613310ebf09 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/FHIRPersistenceJDBCCache.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/FHIRPersistenceJDBCCache.java @@ -8,6 +8,7 @@ import com.ibm.fhir.persistence.jdbc.dao.api.ICommonTokenValuesCache; import com.ibm.fhir.persistence.jdbc.dao.api.IIdNameCache; +import com.ibm.fhir.persistence.jdbc.dao.api.ILogicalResourceIdentCache; import com.ibm.fhir.persistence.jdbc.dao.api.INameIdCache; /** @@ -34,6 +35,12 @@ public interface FHIRPersistenceJDBCCache { */ ICommonTokenValuesCache getResourceReferenceCache(); + /** + * Getter for the cache handling lookups for logical_resource_id values + * @return + */ + ILogicalResourceIdentCache getLogicalResourceIdentCache(); + /** * Getter for the cache of resource types used to look up resource type id * @return diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/FHIRResourceDAOFactory.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/FHIRResourceDAOFactory.java index 30ffa211339..1a63a2fabb3 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/FHIRResourceDAOFactory.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/FHIRResourceDAOFactory.java @@ -1,5 +1,5 @@ /* - * (C) Copyright IBM Corp. 2020, 2021 + * (C) Copyright IBM Corp. 2020, 2022 * * SPDX-License-Identifier: Apache-2.0 */ @@ -11,11 +11,14 @@ import javax.transaction.TransactionSynchronizationRegistry; import com.ibm.fhir.database.utils.api.IDatabaseTranslator; +import com.ibm.fhir.database.utils.citus.CitusTranslator; import com.ibm.fhir.database.utils.common.DatabaseTranslatorFactory; import com.ibm.fhir.database.utils.db2.Db2Translator; import com.ibm.fhir.database.utils.derby.DerbyTranslator; import com.ibm.fhir.database.utils.postgres.PostgresTranslator; import com.ibm.fhir.persistence.exception.FHIRPersistenceException; +import com.ibm.fhir.persistence.jdbc.citus.CitusResourceDAO; +import com.ibm.fhir.persistence.jdbc.citus.CitusResourceReferenceDAO; import com.ibm.fhir.persistence.jdbc.connection.FHIRDbFlavor; import com.ibm.fhir.persistence.jdbc.dao.ReindexResourceDAO; import com.ibm.fhir.persistence.jdbc.dao.api.FhirSequenceDAO; @@ -41,17 +44,21 @@ public class FHIRResourceDAOFactory { /** * Construct a new ResourceDAO implementation matching the database type * @param connection valid connection to the database + * @param adminSchemaName * @param schemaName the name of the schema containing the FHIR resource tables * @param flavor the type and capability of the database and schema * @param trxSynchRegistry + * @param cache + * @param ptdi + * @param shardKey * @return a concrete implementation of {@link ResourceDAO} * @throws IllegalArgumentException * @throws FHIRPersistenceException */ public static ResourceDAO getResourceDAO(Connection connection, String adminSchemaName, String schemaName, FHIRDbFlavor flavor, TransactionSynchronizationRegistry trxSynchRegistry, - FHIRPersistenceJDBCCache cache, ParameterTransactionDataImpl ptdi) + FHIRPersistenceJDBCCache cache, ParameterTransactionDataImpl ptdi, Short shardKey) throws IllegalArgumentException, FHIRPersistenceException { - ResourceDAO resourceDAO = null; + final ResourceDAO resourceDAO; IResourceReferenceDAO rrd = getResourceReferenceDAO(connection, adminSchemaName, schemaName, flavor, cache); switch (flavor.getType()) { @@ -62,8 +69,13 @@ public static ResourceDAO getResourceDAO(Connection connection, String adminSche resourceDAO = new DerbyResourceDAO(connection, schemaName, flavor, trxSynchRegistry, cache, rrd, ptdi); break; case POSTGRESQL: - resourceDAO = new PostgresResourceDAO(connection, schemaName, flavor, trxSynchRegistry, cache, rrd, ptdi); + resourceDAO = new PostgresResourceDAO(connection, schemaName, flavor, trxSynchRegistry, cache, rrd, ptdi, shardKey); + break; + case CITUS: + resourceDAO = new CitusResourceDAO(connection, schemaName, flavor, trxSynchRegistry, cache, rrd, ptdi, shardKey); break; + default: + throw new IllegalArgumentException("Unsupported database type: " + flavor.getType().name()); } return resourceDAO; } @@ -81,8 +93,8 @@ public static ResourceDAO getResourceDAO(Connection connection, String adminSche public static ReindexResourceDAO getReindexResourceDAO(Connection connection, String adminSchemaName, String schemaName, FHIRDbFlavor flavor, TransactionSynchronizationRegistry trxSynchRegistry, FHIRPersistenceJDBCCache cache, ParameterDAO parameterDao) { - IDatabaseTranslator translator = null; - ReindexResourceDAO result = null; + final IDatabaseTranslator translator; + final ReindexResourceDAO result; IResourceReferenceDAO rrd = getResourceReferenceDAO(connection, adminSchemaName, schemaName, flavor, cache); switch (flavor.getType()) { @@ -95,9 +107,12 @@ public static ReindexResourceDAO getReindexResourceDAO(Connection connection, St result = new ReindexResourceDAO(connection, translator, parameterDao, schemaName, flavor, cache, rrd); break; case POSTGRESQL: + case CITUS: translator = new PostgresTranslator(); result = new PostgresReindexResourceDAO(connection, translator, parameterDao, schemaName, flavor, cache, rrd); break; + default: + throw new IllegalArgumentException("Unsupported database type: " + flavor.getType().name()); } return result; } @@ -113,16 +128,20 @@ public static IDatabaseTranslator getTranslatorForFlavor(FHIRDbFlavor flavor) { /** * Construct a new ResourceDAO implementation matching the database type + * * @param connection valid connection to the database + * @param adminSchemaName * @param schemaName the name of the schema containing the FHIR resource tables * @param flavor the type and capability of the database and schema + * @param cache + * @param shardKey * @return a concrete implementation of {@link ResourceDAO} * @throws IllegalArgumentException * @throws FHIRPersistenceException */ public static ResourceDAO getResourceDAO(Connection connection, String adminSchemaName, String schemaName, FHIRDbFlavor flavor, - FHIRPersistenceJDBCCache cache) throws IllegalArgumentException, FHIRPersistenceException { - ResourceDAO resourceDAO = null; + FHIRPersistenceJDBCCache cache, Short shardKey) throws IllegalArgumentException, FHIRPersistenceException { + final ResourceDAO resourceDAO; IResourceReferenceDAO rrd = getResourceReferenceDAO(connection, adminSchemaName, schemaName, flavor, cache); switch (flavor.getType()) { @@ -133,8 +152,13 @@ public static ResourceDAO getResourceDAO(Connection connection, String adminSche resourceDAO = new DerbyResourceDAO(connection, schemaName, flavor, cache, rrd); break; case POSTGRESQL: - resourceDAO = new PostgresResourceDAO(connection, schemaName, flavor, cache, rrd); + resourceDAO = new PostgresResourceDAO(connection, schemaName, flavor, cache, rrd, shardKey); break; + case CITUS: + resourceDAO = new CitusResourceDAO(connection, schemaName, flavor, cache, rrd, shardKey); + break; + default: + throw new IllegalArgumentException("Unsupported database type: " + flavor.getType().name()); } return resourceDAO; } @@ -151,17 +175,22 @@ public static ResourceDAO getResourceDAO(Connection connection, String adminSche public static ResourceReferenceDAO getResourceReferenceDAO(Connection connection, String adminSchemaName, String schemaName, FHIRDbFlavor flavor, FHIRPersistenceJDBCCache cache) { - ResourceReferenceDAO rrd = null; + final ResourceReferenceDAO rrd; switch (flavor.getType()) { case DB2: - rrd = new Db2ResourceReferenceDAO(new Db2Translator(), connection, schemaName, cache.getResourceReferenceCache(), adminSchemaName, cache.getParameterNameCache()); + rrd = new Db2ResourceReferenceDAO(new Db2Translator(), connection, schemaName, cache.getResourceReferenceCache(), adminSchemaName, cache.getParameterNameCache(), cache.getLogicalResourceIdentCache()); break; case DERBY: - rrd = new DerbyResourceReferenceDAO(new DerbyTranslator(), connection, schemaName, cache.getResourceReferenceCache(), cache.getParameterNameCache()); + rrd = new DerbyResourceReferenceDAO(new DerbyTranslator(), connection, schemaName, cache.getResourceReferenceCache(), cache.getParameterNameCache(), cache.getLogicalResourceIdentCache()); break; case POSTGRESQL: - rrd = new PostgresResourceReferenceDAO(new PostgresTranslator(), connection, schemaName, cache.getResourceReferenceCache(), cache.getParameterNameCache()); + rrd = new PostgresResourceReferenceDAO(new PostgresTranslator(), connection, schemaName, cache.getResourceReferenceCache(), cache.getParameterNameCache(), cache.getLogicalResourceIdentCache()); + break; + case CITUS: + rrd = new CitusResourceReferenceDAO(new CitusTranslator(), connection, schemaName, cache.getResourceReferenceCache(), cache.getParameterNameCache(), cache.getLogicalResourceIdentCache()); break; + default: + throw new IllegalArgumentException("Unsupported database type: " + flavor.getType().name()); } return rrd; } @@ -174,7 +203,7 @@ public static ResourceReferenceDAO getResourceReferenceDAO(Connection connection * @return */ public static FhirSequenceDAO getSequenceDAO(Connection connection, FHIRDbFlavor flavor) { - FhirSequenceDAO result = null; + final FhirSequenceDAO result; switch (flavor.getType()) { case DB2: // Derby syntax also works for Db2 @@ -184,8 +213,11 @@ public static FhirSequenceDAO getSequenceDAO(Connection connection, FHIRDbFlavor result = new com.ibm.fhir.persistence.jdbc.derby.FhirSequenceDAOImpl(connection); break; case POSTGRESQL: + case CITUS: result = new com.ibm.fhir.persistence.jdbc.postgres.FhirSequenceDAOImpl(connection); break; + default: + throw new IllegalArgumentException("Unsupported database type: " + flavor.getType().name()); } return result; } diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/JDBCConstants.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/JDBCConstants.java index b700b42e14f..58c37a91177 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/JDBCConstants.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/JDBCConstants.java @@ -6,11 +6,9 @@ package com.ibm.fhir.persistence.jdbc; import java.util.Arrays; -import java.util.Calendar; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.TimeZone; import com.ibm.fhir.search.SearchConstants.Modifier; import com.ibm.fhir.search.SearchConstants.Type; @@ -40,6 +38,9 @@ public class JDBCConstants { public static final String _RESOURCES = "_RESOURCES"; public static final String _LOGICAL_RESOURCES = "_LOGICAL_RESOURCES"; public static final String RESOURCE_ID = "RESOURCE_ID"; + public static final String RESOURCE_TYPE_ID = "RESOURCE_TYPE_ID"; + public static final String REF_VALUE = "REF_VALUE"; + public static final String REF_LOGICAL_RESOURCE_ID = "REF_LOGICAL_RESOURCE_ID"; public static final String LOGICAL_ID = "LOGICAL_ID"; public static final String LOGICAL_RESOURCE_ID = "LOGICAL_RESOURCE_ID"; public static final String CURRENT_RESOURCE_ID = "CURRENT_RESOURCE_ID"; @@ -118,6 +119,9 @@ public class JDBCConstants { // Default code_system_id value public static final String DEFAULT_TOKEN_SYSTEM = "default-token-system"; + // Default resource type for references without a resource type + public static final String RESOURCE = "Resource"; + /** * This Calendar object is not thread-safe! Use CalendarHelper#getCalendarForUTC() instead. */ diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/cache/FHIRPersistenceJDBCCacheImpl.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/cache/FHIRPersistenceJDBCCacheImpl.java index 5ae9580e1d7..5dfe120aa76 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/cache/FHIRPersistenceJDBCCacheImpl.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/cache/FHIRPersistenceJDBCCacheImpl.java @@ -1,5 +1,5 @@ /* - * (C) Copyright IBM Corp. 2020, 2021 + * (C) Copyright IBM Corp. 2020, 2022 * * SPDX-License-Identifier: Apache-2.0 */ @@ -12,6 +12,7 @@ import com.ibm.fhir.persistence.jdbc.FHIRPersistenceJDBCCache; import com.ibm.fhir.persistence.jdbc.dao.api.ICommonTokenValuesCache; import com.ibm.fhir.persistence.jdbc.dao.api.IIdNameCache; +import com.ibm.fhir.persistence.jdbc.dao.api.ILogicalResourceIdentCache; import com.ibm.fhir.persistence.jdbc.dao.api.INameIdCache; /** @@ -28,6 +29,8 @@ public class FHIRPersistenceJDBCCacheImpl implements FHIRPersistenceJDBCCache { private final ICommonTokenValuesCache resourceReferenceCache; + private final ILogicalResourceIdentCache logicalResourceIdentCache; + // flag to allow one lucky caller to get the opportunity to prefill private final AtomicBoolean needToPrefillFlag = new AtomicBoolean(true); @@ -37,13 +40,15 @@ public class FHIRPersistenceJDBCCacheImpl implements FHIRPersistenceJDBCCache { * @param resourceTypeNameCache * @param parameterNameCache * @param resourceReferenceCache + * @param logicalResourceIdentCache */ public FHIRPersistenceJDBCCacheImpl(INameIdCache resourceTypeCache, IIdNameCache resourceTypeNameCache, - INameIdCache parameterNameCache, ICommonTokenValuesCache resourceReferenceCache) { + INameIdCache parameterNameCache, ICommonTokenValuesCache resourceReferenceCache, ILogicalResourceIdentCache logicalResourceIdentCache) { this.resourceTypeCache = resourceTypeCache; this.resourceTypeNameCache = resourceTypeNameCache; this.parameterNameCache = parameterNameCache; this.resourceReferenceCache = resourceReferenceCache; + this.logicalResourceIdentCache = logicalResourceIdentCache; } /** @@ -76,6 +81,11 @@ public INameIdCache getParameterNameCache() { return parameterNameCache; } + @Override + public ILogicalResourceIdentCache getLogicalResourceIdentCache() { + return logicalResourceIdentCache; + } + @Override public void transactionCommitted() { logger.fine("Transaction committed - updating cache shared maps"); @@ -83,6 +93,7 @@ public void transactionCommitted() { resourceTypeNameCache.updateSharedMaps(); parameterNameCache.updateSharedMaps(); resourceReferenceCache.updateSharedMaps(); + logicalResourceIdentCache.updateSharedMaps(); } @Override @@ -92,6 +103,7 @@ public void transactionRolledBack() { resourceTypeNameCache.clearLocalMaps(); parameterNameCache.clearLocalMaps(); resourceReferenceCache.clearLocalMaps(); + logicalResourceIdentCache.clearLocalMaps(); } @Override diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/cache/FHIRPersistenceJDBCCacheUtil.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/cache/FHIRPersistenceJDBCCacheUtil.java index d7d43a6a00f..111006639d3 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/cache/FHIRPersistenceJDBCCacheUtil.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/cache/FHIRPersistenceJDBCCacheUtil.java @@ -1,5 +1,5 @@ /* - * (C) Copyright IBM Corp. 2020, 2021 + * (C) Copyright IBM Corp. 2020, 2022 * * SPDX-License-Identifier: Apache-2.0 */ @@ -12,6 +12,7 @@ import com.ibm.fhir.persistence.exception.FHIRPersistenceException; import com.ibm.fhir.persistence.jdbc.FHIRPersistenceJDBCCache; import com.ibm.fhir.persistence.jdbc.dao.api.ICommonTokenValuesCache; +import com.ibm.fhir.persistence.jdbc.dao.api.ILogicalResourceIdentCache; import com.ibm.fhir.persistence.jdbc.dao.api.ParameterDAO; import com.ibm.fhir.persistence.jdbc.dao.api.ResourceDAO; @@ -24,9 +25,10 @@ public class FHIRPersistenceJDBCCacheUtil { * Factory function to create a new cache instance * @return */ - public static FHIRPersistenceJDBCCache create(int codeSystemCacheSize, int tokenValueCacheSize, int canonicalCacheSize) { + public static FHIRPersistenceJDBCCache create(int codeSystemCacheSize, int tokenValueCacheSize, int canonicalCacheSize, int logicalResourceIdentCacheSize) { ICommonTokenValuesCache rrc = new CommonTokenValuesCacheImpl(codeSystemCacheSize, tokenValueCacheSize, canonicalCacheSize); - return new FHIRPersistenceJDBCCacheImpl(new NameIdCache(), new IdNameCache(), new NameIdCache(), rrc); + ILogicalResourceIdentCache lric = new LogicalResourceIdentCacheImpl(logicalResourceIdentCacheSize); + return new FHIRPersistenceJDBCCacheImpl(new NameIdCache(), new IdNameCache(), new NameIdCache(), rrc, lric); } /** diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/cache/FHIRPersistenceJDBCTenantCache.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/cache/FHIRPersistenceJDBCTenantCache.java index fd378abddd5..853a3616ab2 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/cache/FHIRPersistenceJDBCTenantCache.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/cache/FHIRPersistenceJDBCTenantCache.java @@ -1,5 +1,5 @@ /* - * (C) Copyright IBM Corp. 2020, 2021 + * (C) Copyright IBM Corp. 2020, 2022 * * SPDX-License-Identifier: Apache-2.0 */ @@ -74,7 +74,8 @@ protected FHIRPersistenceJDBCCache createCache(String cacheKey) { int externalSystemCacheSize = pg.getIntProperty("externalSystemCacheSize", 1000); int externalValueCacheSize = pg.getIntProperty("externalValueCacheSize", 100000); int canonicalCacheSize = pg.getIntProperty("canonicalCacheSize", 1000); - return FHIRPersistenceJDBCCacheUtil.create(externalSystemCacheSize, externalValueCacheSize, canonicalCacheSize); + int logicalResourceIdentCacheSize = pg.getIntProperty("logicalResourceIdentCacheSize", 100000); + return FHIRPersistenceJDBCCacheUtil.create(externalSystemCacheSize, externalValueCacheSize, canonicalCacheSize, logicalResourceIdentCacheSize); } } catch (IllegalStateException ise) { throw ise; diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/cache/LogicalResourceIdentCacheImpl.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/cache/LogicalResourceIdentCacheImpl.java new file mode 100644 index 00000000000..84f240b5c50 --- /dev/null +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/cache/LogicalResourceIdentCacheImpl.java @@ -0,0 +1,172 @@ +/* + * (C) Copyright IBM Corp. 2022 + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.ibm.fhir.persistence.jdbc.cache; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +import com.ibm.fhir.persistence.jdbc.dao.api.ILogicalResourceIdentCache; +import com.ibm.fhir.persistence.jdbc.dao.api.LogicalResourceIdentKey; +import com.ibm.fhir.persistence.jdbc.dao.impl.ResourceReferenceValueRec; + + +/** + * Implementation of a cache used for lookups of entities related + * to local and external resource references + */ +public class LogicalResourceIdentCacheImpl implements ILogicalResourceIdentCache { + + // We use LinkedHashMap for the local map because we also need to maintain order + // of insertion to make sure we have correct LRU behavior when updating the shared cache + private final ThreadLocal> localLogicalResourceIdents = new ThreadLocal<>(); + + // The lru token values cache shared at the server level + private final LRUCache cache; + + /** + * Public constructor + * @param logicalResourceCacheSize + */ + public LogicalResourceIdentCacheImpl(int logicalResourceCacheSize) { + + // LRU cache for quick lookup of code-systems and token-values + cache = new LRUCache<>(logicalResourceCacheSize); + } + + /** + * Called after a transaction commit() to transfer all the staged (thread-local) data + * over to the shared LRU cache. + */ + @Override + public void updateSharedMaps() { + + LinkedHashMap localMap = localLogicalResourceIdents.get(); + if (localMap != null) { + synchronized(this.cache) { + cache.update(localMap); + } + + // clear the thread-local cache + localMap.clear(); + } + } + + @Override + public Long getLogicalResourceId(int resourceTypeId, String logicalId) { + // check the thread-local map first + Long result = null; + + LogicalResourceIdentKey key = new LogicalResourceIdentKey(resourceTypeId, logicalId); + Map localMap = this.localLogicalResourceIdents.get(); + if (localMap != null) { + result = localMap.get(key); + + if (result != null) { + return result; + } + } + + // See if it's in the shared cache + synchronized (this.cache) { + result = cache.get(key); + } + + if (result != null) { + // We found it in the shared cache, so update our thread-local + // cache. + addRecord(key, result); + } + + return result; + } + + @Override + public void resolveReferenceValues(Collection values, List misses) { + // Make one pass over the collection and resolve as much as we can in one go. Anything + // we can't resolve gets put into the corresponding missing lists. Worst case is two passes, when + // there's nothing in the local cache and we have to then look up everything in the shared cache + + // See what we have currently in our thread-local cache + LinkedHashMap valMap = localLogicalResourceIdents.get(); + + List foundKeys = new ArrayList<>(values.size()); // for updating LRU + List needToFindValues = new ArrayList<>(values.size()); // for the ref values we haven't yet found + for (ResourceReferenceValueRec tv: values) { + if (valMap != null) { + LogicalResourceIdentKey key = new LogicalResourceIdentKey(tv.getRefResourceTypeId(), tv.getRefLogicalId()); + Long id = valMap.get(key); + if (id != null) { + foundKeys.add(tv); + tv.setRefLogicalResourceId(id); + } else { + // not found, so add to the cache miss list + needToFindValues.add(tv); + } + } else { + needToFindValues.add(tv); + } + } + + // If we still have keys to find, look them up in the shared cache (which we need to lock first) + if (needToFindValues.size() > 0) { + synchronized (this.cache) { + for (ResourceReferenceValueRec tv: needToFindValues) { + LogicalResourceIdentKey key = new LogicalResourceIdentKey(tv.getRefResourceTypeId(), tv.getRefLogicalId()); + Long id = cache.get(key); + if (id != null) { + tv.setRefLogicalResourceId(id); + + // Update the local cache with this value + addRecord(key, id); + } else { + // cache miss so add this record to the miss list for further processing + misses.add(tv); + } + } + } + } + } + + + @Override + public void addRecord(LogicalResourceIdentKey key, long id) { + LinkedHashMap map = localLogicalResourceIdents.get(); + + if (map == null) { + map = new LinkedHashMap<>(); + localLogicalResourceIdents.set(map); + } + + // add the id to the thread-local cache. The shared cache is updated + // only if a call is made to #updateSharedMaps() + map.put(key, id); + } + + @Override + public void reset() { + localLogicalResourceIdents.remove(); + + // clear the shared caches too + synchronized (this.cache) { + this.cache.clear(); + } + } + + @Override + public void clearLocalMaps() { + // clear the maps, but keep the maps in place because they'll be used again + // the next time this thread is picked from the pool + LinkedHashMap map = localLogicalResourceIdents.get(); + + if (map != null) { + map.clear(); + } + } +} \ No newline at end of file diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/citus/CitusResourceDAO.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/citus/CitusResourceDAO.java new file mode 100644 index 00000000000..76be6e2c4fb --- /dev/null +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/citus/CitusResourceDAO.java @@ -0,0 +1,324 @@ +/* + * (C) Copyright IBM Corp. 2020, 2022 + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.ibm.fhir.persistence.jdbc.citus; + +import java.sql.CallableStatement; +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.SQLIntegrityConstraintViolationException; +import java.sql.Types; +import java.util.List; +import java.util.Objects; +import java.util.UUID; +import java.util.logging.Level; +import java.util.logging.Logger; + +import javax.transaction.TransactionSynchronizationRegistry; + +import com.ibm.fhir.database.utils.common.PreparedStatementHelper; +import com.ibm.fhir.persistence.InteractionStatus; +import com.ibm.fhir.persistence.exception.FHIRPersistenceDataAccessException; +import com.ibm.fhir.persistence.exception.FHIRPersistenceException; +import com.ibm.fhir.persistence.exception.FHIRPersistenceVersionIdMismatchException; +import com.ibm.fhir.persistence.index.FHIRRemoteIndexService; +import com.ibm.fhir.persistence.jdbc.FHIRPersistenceJDBCCache; +import com.ibm.fhir.persistence.jdbc.connection.FHIRDbFlavor; +import com.ibm.fhir.persistence.jdbc.dao.api.FHIRDAOConstants; +import com.ibm.fhir.persistence.jdbc.dao.api.IResourceReferenceDAO; +import com.ibm.fhir.persistence.jdbc.dao.api.JDBCIdentityCache; +import com.ibm.fhir.persistence.jdbc.dao.api.ParameterDAO; +import com.ibm.fhir.persistence.jdbc.dao.impl.JDBCIdentityCacheImpl; +import com.ibm.fhir.persistence.jdbc.dao.impl.ParameterVisitorBatchDAO; +import com.ibm.fhir.persistence.jdbc.dto.ExtractedParameterValue; +import com.ibm.fhir.persistence.jdbc.dto.Resource; +import com.ibm.fhir.persistence.jdbc.exception.FHIRPersistenceDBConnectException; +import com.ibm.fhir.persistence.jdbc.exception.FHIRPersistenceFKVException; +import com.ibm.fhir.persistence.jdbc.impl.ParameterTransactionDataImpl; +import com.ibm.fhir.persistence.jdbc.postgres.PostgresResourceDAO; + +/** + * Data access object for writing FHIR resources to Citus database using + * the stored procedure (or function, in this case) + */ +public class CitusResourceDAO extends PostgresResourceDAO { + private static final String CLASSNAME = CitusResourceDAO.class.getName(); + private static final Logger log = Logger.getLogger(CLASSNAME); + + // @formatter:off + // 0 1 + // 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 + // @formatter:on + // Don't forget that we must account for IN and OUT parameters. + private static final String SQL_INSERT_WITH_PARAMETERS = "{ CALL %s.add_any_resource(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?) }"; + private static final String SQL_LOGICAL_RESOURCE_IDENT = "{ CALL %s.add_logical_resource_ident(?,?,?) }"; + + // Read the current version of the resource (even if the resource has been deleted) + private static final String SQL_READ = "" + + "SELECT R.RESOURCE_ID, R.LOGICAL_RESOURCE_ID, R.VERSION_ID, R.LAST_UPDATED, R.IS_DELETED, R.DATA, LR.LOGICAL_ID, R.RESOURCE_PAYLOAD_KEY " + + " FROM %s_RESOURCES R, " + + " %s_LOGICAL_RESOURCES LR " + + " WHERE R.RESOURCE_ID = LR.CURRENT_RESOURCE_ID " + + " AND R.LOGICAL_RESOURCE_ID = LR.LOGICAL_RESOURCE_ID " // join must use common Citus distribution column + + " AND LR.LOGICAL_RESOURCE_ID = ? "; // lookup using logical_resource_id + + // Read a specific version of the resource + private static final String SQL_VERSION_READ = "" + + "SELECT R.RESOURCE_ID, R.LOGICAL_RESOURCE_ID, R.VERSION_ID, R.LAST_UPDATED, R.IS_DELETED, R.DATA, LR.LOGICAL_ID, R.RESOURCE_PAYLOAD_KEY " + + " FROM %s_RESOURCES R, " + + " %s_LOGICAL_RESOURCES LR " + + " WHERE LR.LOGICAL_RESOURCE_ID = ? " + + " AND R.LOGICAL_RESOURCE_ID = LR.LOGICAL_RESOURCE_ID " + + " AND R.VERSION_ID = ?"; + + /** + * Public constructor + * + * @param connection + * @param schemaName + * @param flavor + * @param cache + * @param rrd + */ + public CitusResourceDAO(Connection connection, String schemaName, FHIRDbFlavor flavor, FHIRPersistenceJDBCCache cache, IResourceReferenceDAO rrd, Short shardKey) { + super(connection, schemaName, flavor, cache, rrd, shardKey); + } + + /** + * Public constructor + * + * @param connection + * @param schemaName + * @param flavor + * @param trxSynchRegistry + * @param cache + * @param rrd + * @param ptdi + */ + public CitusResourceDAO(Connection connection, String schemaName, FHIRDbFlavor flavor, TransactionSynchronizationRegistry trxSynchRegistry, FHIRPersistenceJDBCCache cache, IResourceReferenceDAO rrd, + ParameterTransactionDataImpl ptdi, Short shardKey) { + super(connection, schemaName, flavor, trxSynchRegistry, cache, rrd, ptdi, shardKey); + } + + /** + * Read the logical_resource_id value from logical_resource_ident + * @param resourceType + * @param logicalId + * @return + */ + private Long getLogicalResourceIdentId(String resourceType, String logicalId) throws FHIRPersistenceDataAccessException { + final int resourceTypeId = getCache().getResourceTypeCache().getId(resourceType); + final Long logicalResourceId; + final String selectLogicalResourceIdent = "" + + "SELECT logical_resource_id " + + " FROM logical_resource_ident " + + " WHERE resource_type_id = ? " + + " AND logical_id = ? "; // distribution key + try (PreparedStatement ps = getConnection().prepareStatement(selectLogicalResourceIdent)) { + ps.setInt(1, resourceTypeId); + ps.setString(2, logicalId); + ResultSet rs = ps.executeQuery(); + if (rs.next()) { + logicalResourceId = rs.getLong(1); + } else { + logicalResourceId = null; + } + } catch (SQLException x) { + log.log(Level.SEVERE, "read '" + resourceType + "/" + logicalId + "'", x); + throw new FHIRPersistenceDataAccessException("read failed for logical resource ident record"); + } + return logicalResourceId; + } + + @Override + public Resource read(String logicalId, String resourceType) throws FHIRPersistenceDataAccessException, FHIRPersistenceDBConnectException { + final String METHODNAME = "read"; + log.entering(CLASSNAME, METHODNAME); + + // For Citus we want to first query the logical_resource_ident table because it is + // distributed by the logicalId. This gets us the logical_resource_id value which + // we can then use to access the logical_resource tables which are distributed by + // logical_resource_id + Long logicalResourceId = getLogicalResourceIdentId(resourceType, logicalId); + if (logicalResourceId == null) { + return null; + } + + Resource resource = null; + List resources; + String stmtString = null; + + try { + stmtString = String.format(SQL_READ, resourceType, resourceType); + resources = this.runQuery(stmtString, logicalResourceId); + if (!resources.isEmpty()) { + resource = resources.get(0); + } + } finally { + log.exiting(CLASSNAME, METHODNAME); + } + return resource; + } + + @Override + public Resource versionRead(String logicalId, String resourceType, int versionId) throws FHIRPersistenceDataAccessException, FHIRPersistenceDBConnectException { + final String METHODNAME = "versionRead"; + log.entering(CLASSNAME, METHODNAME); + + // For Citus we want to first query the logical_resource_ident table because it is + // distributed by the logicalId. This gets us the logical_resource_id value which + // we can then use to access the logical_resource tables which are distributed by + // logical_resource_id + Long logicalResourceId = getLogicalResourceIdentId(resourceType, logicalId); + if (logicalResourceId == null) { + return null; + } + + Resource resource = null; + List resources; + String stmtString = null; + + try { + stmtString = String.format(SQL_VERSION_READ, resourceType, resourceType); + resources = this.runQuery(stmtString, logicalResourceId, versionId); + if (!resources.isEmpty()) { + resource = resources.get(0); + } + } finally { + log.exiting(CLASSNAME, METHODNAME); + } + return resource; + + } + + @Override + public Resource insert(Resource resource, List parameters, String parameterHashB64, + ParameterDAO parameterDao, Integer ifNoneMatch) + throws FHIRPersistenceException { + final String METHODNAME = "insert(Resource, List"; + log.entering(CLASSNAME, METHODNAME); + + final Connection connection = getConnection(); // do not close + long dbCallStartTime; + double dbCallDuration; + + try { + // Just make sure this resource type is known to the database before we + // hit the procedure + Integer resourceTypeId = getResourceTypeId(resource.getResourceType()); + Objects.requireNonNull(resourceTypeId); + + // For Citus, we first make a call to establish the logical_resource_ident record + long logicalResourceId = createOrLockLogicalResourceIdent(resourceTypeId, resource.getLogicalId()); + + final String stmtString = String.format(SQL_INSERT_WITH_PARAMETERS, getSchemaName()); + try (CallableStatement stmt = connection.prepareCall(stmtString)) { + PreparedStatementHelper psh = new PreparedStatementHelper(stmt); + + psh.setLong(logicalResourceId); + psh.setInt(resourceTypeId); + psh.setString(resource.getResourceType()); + psh.setString(resource.getLogicalId()); + psh.setBinaryStream(resource.getDataStream() != null ? resource.getDataStream().inputStream() : null); + psh.setTimestamp(resource.getLastUpdated()); + psh.setString(resource.isDeleted() ? "Y": "N"); + psh.setString(UUID.randomUUID().toString()); + psh.setInt(resource.getVersionId()); + psh.setString(parameterHashB64); + psh.setInt(ifNoneMatch); + psh.setString(resource.getResourcePayloadKey()); + + final int oldParameterHashIndex = psh.registerOutParameter(Types.VARCHAR); + final int interactionStatusIndex = psh.registerOutParameter(Types.INTEGER); + final int ifNoneMatchVersionIndex = psh.registerOutParameter(Types.INTEGER); + + dbCallStartTime = System.nanoTime(); + stmt.execute(); + dbCallDuration = (System.nanoTime()-dbCallStartTime)/1e6; + + resource.setLogicalResourceId(logicalResourceId); + if (stmt.getInt(interactionStatusIndex) == 1) { // interaction status + // no change, so skip parameter updates + resource.setInteractionStatus(InteractionStatus.IF_NONE_MATCH_EXISTED); + resource.setIfNoneMatchVersion(stmt.getInt(ifNoneMatchVersionIndex)); // current version + } else { + resource.setInteractionStatus(InteractionStatus.MODIFIED); + + // Parameter time + // To keep things simple for the postgresql use-case, we just use a visitor to + // handle inserts of parameters directly in the resource parameter tables. + // Note we don't get any parameters for the resource soft-delete operation + // Bypass the parameter insert here if we have the remoteIndexService configured + FHIRRemoteIndexService remoteIndexService = FHIRRemoteIndexService.getServiceInstance(); + final String currentParameterHash = stmt.getString(oldParameterHashIndex); + if (remoteIndexService == null + && parameters != null && (parameterHashB64 == null || parameterHashB64.isEmpty() + || !parameterHashB64.equals(currentParameterHash))) { + // postgresql doesn't support partitioned multi-tenancy, so we disable it on the DAO: + JDBCIdentityCache identityCache = new JDBCIdentityCacheImpl(getCache(), this, parameterDao, getResourceReferenceDAO()); + try (ParameterVisitorBatchDAO pvd = new ParameterVisitorBatchDAO(connection, null, resource.getResourceType(), false, resource.getLogicalResourceId(), 100, + identityCache, getResourceReferenceDAO(), getTransactionData())) { + for (ExtractedParameterValue p: parameters) { + p.accept(pvd); + } + } + } + } + if (log.isLoggable(Level.FINE)) { + log.fine("Successfully inserted Resource. logicalResourceId=" + resource.getLogicalResourceId() + " executionTime=" + dbCallDuration + "ms"); + } + } + } catch(FHIRPersistenceDBConnectException | FHIRPersistenceDataAccessException e) { + throw e; + } catch(SQLIntegrityConstraintViolationException e) { + FHIRPersistenceFKVException fx = new FHIRPersistenceFKVException("Encountered FK violation while inserting Resource."); + throw severe(log, fx, e); + } catch(SQLException e) { + if (FHIRDAOConstants.SQLSTATE_WRONG_VERSION.equals(e.getSQLState())) { + // this is just a concurrency update, so there's no need to log the SQLException here + throw new FHIRPersistenceVersionIdMismatchException("Encountered version id mismatch while inserting Resource"); + } else { + FHIRPersistenceDataAccessException fx = new FHIRPersistenceDataAccessException("SQLException encountered while inserting Resource."); + throw severe(log, fx, e); + } + } catch(Throwable e) { + FHIRPersistenceDataAccessException fx = new FHIRPersistenceDataAccessException("Failure inserting Resource."); + throw severe(log, fx, e); + } finally { + log.exiting(CLASSNAME, METHODNAME); + } + + return resource; + } + + /** + * Call the ADD_LOGICAL_RESOURCE_IDENT procedure to create or lock (select for update) + * the logical_resource_ident record. For Citus we run this step first because this + * function is distributed by the logical_id parameter. + * @param resourceTypeId + * @param logicalId + * @return + * @throws SQLException + */ + protected long createOrLockLogicalResourceIdent(int resourceTypeId, String logicalId) throws SQLException { + long logicalResourceId; + + final String stmtString = String.format(SQL_LOGICAL_RESOURCE_IDENT, getSchemaName()); + try (CallableStatement cs = getConnection().prepareCall(stmtString)) { + PreparedStatementHelper psh = new PreparedStatementHelper(cs); + psh.setInt(resourceTypeId); + psh.setString(logicalId); + int idxLogicalResourceId = psh.registerOutParameter(Types.BIGINT); + cs.execute(); + logicalResourceId = cs.getLong(idxLogicalResourceId); + } + + // At this point the logical_resource_ident record will be locked for update + return logicalResourceId; + } +} \ No newline at end of file diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/citus/CitusResourceReferenceDAO.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/citus/CitusResourceReferenceDAO.java new file mode 100644 index 00000000000..903749419db --- /dev/null +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/citus/CitusResourceReferenceDAO.java @@ -0,0 +1,169 @@ +/* + * (C) Copyright IBM Corp. 2022 + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.ibm.fhir.persistence.jdbc.citus; + +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.logging.Level; +import java.util.logging.Logger; + +import com.ibm.fhir.database.utils.api.IDatabaseTranslator; +import com.ibm.fhir.persistence.jdbc.dao.api.ICommonTokenValuesCache; +import com.ibm.fhir.persistence.jdbc.dao.api.ILogicalResourceIdentCache; +import com.ibm.fhir.persistence.jdbc.dao.api.INameIdCache; +import com.ibm.fhir.persistence.jdbc.dao.impl.ResourceReferenceDAO; +import com.ibm.fhir.persistence.jdbc.dto.CommonTokenValue; +import com.ibm.fhir.persistence.jdbc.postgres.PostgresResourceReferenceDAO; + +/** + * Citus-specific extension of the {@link ResourceReferenceDAO} to work around + * some Citus distribution limitations + */ +public class CitusResourceReferenceDAO extends PostgresResourceReferenceDAO { + private static final Logger logger = Logger.getLogger(CitusResourceReferenceDAO.class.getName()); + + /** + * Public constructor + * + * @param t + * @param c + * @param schemaName + * @param cache + * @param parameterNameCache + * @param logicalResourceIdentCache + */ + public CitusResourceReferenceDAO(IDatabaseTranslator t, Connection c, String schemaName, ICommonTokenValuesCache cache, INameIdCache parameterNameCache, + ILogicalResourceIdentCache logicalResourceIdentCache) { + super(t, c, schemaName, cache, parameterNameCache, logicalResourceIdentCache); + } + + @Override + public void doCodeSystemsUpsert(String paramList, Collection sortedSystemNames) { + // If we try using the PostgreSQL insert-as-select variant, Citus + // rejects the statement, so instead we simplify things by grabbing + // the id values from the sequence first, then simply submit as a + // batch. + List sequenceValues = new ArrayList<>(sortedSystemNames.size()); + final String nextVal = getTranslator().nextValue(getSchemaName(), "fhir_ref_sequence"); + final String SELECT = "" + + "SELECT " + nextVal + + " FROM generate_series(1, ?)"; + try (PreparedStatement ps = getConnection().prepareStatement(SELECT)) { + ps.setInt(1, sortedSystemNames.size()); + ResultSet rs = ps.executeQuery(); + while (rs.next()) { + sequenceValues.add(rs.getInt(1)); + } + } catch (SQLException x) { + logger.log(Level.SEVERE, SELECT, x); + throw getTranslator().translate(x); + } + + final String INSERT = "" + + " INSERT INTO code_systems (code_system_id, code_system_name) " + + " VALUES (?, ?) " + + " ON CONFLICT DO NOTHING "; + + try (PreparedStatement ps = getConnection().prepareStatement(INSERT)) { + int index=0; + for (String csn: sortedSystemNames) { + ps.setInt(1, sequenceValues.get(index++)); + ps.setString(2, csn); + ps.addBatch(); + } + ps.executeBatch(); + } catch (SQLException x) { + logger.log(Level.SEVERE, INSERT, x); + throw getTranslator().translate(x); + } + } + + @Override + public void doCanonicalValuesUpsert(String paramList, Collection sortedURLS) { + // Because of how PostgreSQL MVCC implementation, the insert from negative outer + // join pattern doesn't work...you still hit conflicts. The PostgreSQL pattern + // for upsert is ON CONFLICT DO NOTHING, which is what we use here: + List sequenceValues = new ArrayList<>(sortedURLS.size()); + final String nextVal = getTranslator().nextValue(getSchemaName(), "fhir_ref_sequence"); + final String SELECT = "" + + "SELECT " + nextVal + + " FROM generate_series(1, ?)"; + try (PreparedStatement ps = getConnection().prepareStatement(SELECT)) { + ps.setInt(1, sortedURLS.size()); + ResultSet rs = ps.executeQuery(); + while (rs.next()) { + sequenceValues.add(rs.getInt(1)); + } + } catch (SQLException x) { + logger.log(Level.SEVERE, SELECT, x); + throw getTranslator().translate(x); + } + + final String INSERT = "" + + " INSERT INTO common_canonical_values (canonical_id, url) " + + " VALUES (?, ?) " + + " ON CONFLICT DO NOTHING "; + + try (PreparedStatement ps = getConnection().prepareStatement(INSERT)) { + int index=0; + for (String csn: sortedURLS) { + ps.setInt(1, sequenceValues.get(index++)); + ps.setString(2, csn); + ps.addBatch(); + } + ps.executeBatch(); + } catch (SQLException x) { + logger.log(Level.SEVERE, INSERT, x); + throw getTranslator().translate(x); + } + } + + @Override + protected void doCommonTokenValuesUpsert(String paramList, Collection sortedTokenValues) { + // In Citus, we can no longer use a generated id column, so we have to use + // values from the fhir-sequence and insert the values directly + List sequenceValues = new ArrayList<>(sortedTokenValues.size()); + final String nextVal = getTranslator().nextValue(getSchemaName(), "fhir_ref_sequence"); + final String SELECT = "" + + "SELECT " + nextVal + + " FROM generate_series(1, ?)"; + try (PreparedStatement ps = getConnection().prepareStatement(SELECT)) { + ps.setInt(1, sortedTokenValues.size()); + ResultSet rs = ps.executeQuery(); + while (rs.next()) { + sequenceValues.add(rs.getInt(1)); + } + } catch (SQLException x) { + logger.log(Level.SEVERE, SELECT, x); + throw getTranslator().translate(x); + } + + final String INSERT = "" + + " INSERT INTO common_token_values (common_token_value_id, token_value, code_system_id) " + + " VALUES (?, ?, ?) " + + " ON CONFLICT DO NOTHING "; + + try (PreparedStatement ps = getConnection().prepareStatement(INSERT)) { + int index=0; + for (CommonTokenValue ctv: sortedTokenValues) { + ps.setInt(1, sequenceValues.get(index++)); + ps.setString(2, ctv.getTokenValue()); + ps.setInt(3, ctv.getCodeSystemId()); + ps.addBatch(); + } + ps.executeBatch(); + } catch (SQLException x) { + logger.log(Level.SEVERE, INSERT, x); + throw getTranslator().translate(x); + } + } +} \ No newline at end of file diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/connection/FHIRDbConnectionStrategy.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/connection/FHIRDbConnectionStrategy.java index 71ae4223cf5..3855bff078e 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/connection/FHIRDbConnectionStrategy.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/connection/FHIRDbConnectionStrategy.java @@ -9,9 +9,9 @@ import java.sql.Connection; import com.ibm.fhir.config.FHIRRequestContext; +import com.ibm.fhir.persistence.exception.FHIRPersistenceDataAccessException; import com.ibm.fhir.persistence.jdbc.dao.impl.FHIRDbDAOImpl; import com.ibm.fhir.persistence.jdbc.exception.FHIRPersistenceDBConnectException; -import com.ibm.fhir.persistence.jdbc.exception.FHIRPersistenceDataAccessException; /** * Abstraction used to obtain JDBC connections. The database being connected * is determined by the datasource currently referenced by the {@link FHIRRequestContext} diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/connection/FHIRDbConnectionStrategyBase.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/connection/FHIRDbConnectionStrategyBase.java index bb4a72ca5e0..9dabc9bf543 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/connection/FHIRDbConnectionStrategyBase.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/connection/FHIRDbConnectionStrategyBase.java @@ -18,9 +18,10 @@ import com.ibm.fhir.config.FHIRConfiguration; import com.ibm.fhir.config.FHIRRequestContext; import com.ibm.fhir.config.PropertyGroup; +import com.ibm.fhir.database.utils.api.SchemaType; import com.ibm.fhir.database.utils.model.DbType; +import com.ibm.fhir.persistence.exception.FHIRPersistenceDataAccessException; import com.ibm.fhir.persistence.exception.FHIRPersistenceException; -import com.ibm.fhir.persistence.jdbc.exception.FHIRPersistenceDataAccessException; import com.ibm.fhir.persistence.jdbc.postgresql.SetPostgresOptimizerOptions; /** @@ -97,17 +98,17 @@ private FHIRDbFlavor createFlavor() throws FHIRPersistenceDataAccessException { if (dsPG != null) { try { - boolean multitenant = false; String typeValue = dsPG.getStringProperty("type"); + SchemaType schemaType = SchemaType.PLAIN; DbType type = DbType.from(typeValue); if (type == DbType.DB2) { // We make this absolute for now. May change in the future if we // support a single-tenant schema in DB2. - multitenant = true; + schemaType = SchemaType.MULTITENANT; } - result = new FHIRDbFlavorImpl(type, multitenant); + result = new FHIRDbFlavorImpl(type, schemaType); } catch (Exception x) { log.log(Level.SEVERE, "No type property found for datastore '" + datastoreId + "'", x); throw new FHIRPersistenceDataAccessException("Datastore configuration issue. Details in server logs"); @@ -195,6 +196,7 @@ public void applySearchOptimizerOptions(Connection c, boolean isCompartment) { switch (this.flavor.getType()) { case POSTGRESQL: + case CITUS: // PostgreSQL needs optimizer options set to address search performance issues // as described in issue 1911 final String pgName = FHIRConfiguration.PROPERTY_DATASOURCES + "/" + datastoreId + "/" + FHIRConfiguration.PROPERTY_JDBC_SEARCH_OPTIMIZER_OPTIONS; diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/connection/FHIRDbFlavor.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/connection/FHIRDbFlavor.java index 6a3471f5286..30eb68cd212 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/connection/FHIRDbFlavor.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/connection/FHIRDbFlavor.java @@ -1,11 +1,12 @@ /* - * (C) Copyright IBM Corp. 2020 + * (C) Copyright IBM Corp. 2020, 2022 * * SPDX-License-Identifier: Apache-2.0 */ package com.ibm.fhir.persistence.jdbc.connection; +import com.ibm.fhir.database.utils.api.SchemaType; import com.ibm.fhir.database.utils.model.DbType; /** @@ -21,9 +22,21 @@ public interface FHIRDbFlavor { */ public boolean isMultitenant(); + /** + * What type of schema is this + * @return + */ + public SchemaType getSchemaType(); + /** * What type of database is this? * @return */ public DbType getType(); + + /** + * Is the dbType from the PostgreSQL family? + * @return + */ + public boolean isFamilyPostgreSQL(); } diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/connection/FHIRDbFlavorImpl.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/connection/FHIRDbFlavorImpl.java index 269635154b6..c1c14c3df2f 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/connection/FHIRDbFlavorImpl.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/connection/FHIRDbFlavorImpl.java @@ -1,11 +1,12 @@ /* - * (C) Copyright IBM Corp. 2020 + * (C) Copyright IBM Corp. 2020, 2022 * * SPDX-License-Identifier: Apache-2.0 */ package com.ibm.fhir.persistence.jdbc.connection; +import com.ibm.fhir.database.utils.api.SchemaType; import com.ibm.fhir.database.utils.model.DbType; /** @@ -14,24 +15,38 @@ */ public class FHIRDbFlavorImpl implements FHIRDbFlavor { - // does the database schema support multi-tenancy - private final boolean multitenant; - // basic type of the database (DB2, Derby etc) private final DbType type; - public FHIRDbFlavorImpl(DbType type, boolean multitenant) { + private final SchemaType schemaType; + + /** + * Public constructor + * @param type + * @param schemaType + */ + public FHIRDbFlavorImpl(DbType type, SchemaType schemaType) { this.type = type; - this.multitenant = multitenant; + this.schemaType = schemaType; } @Override public boolean isMultitenant() { - return this.multitenant; + return this.schemaType == SchemaType.MULTITENANT; } @Override public DbType getType() { return this.type; } -} + + @Override + public boolean isFamilyPostgreSQL() { + return this.type == DbType.POSTGRESQL || this.type == DbType.CITUS; + } + + @Override + public SchemaType getSchemaType() { + return this.schemaType; + } +} \ No newline at end of file diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/connection/FHIRDbHelper.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/connection/FHIRDbHelper.java index 6a97cd6a08c..093089dcee2 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/connection/FHIRDbHelper.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/connection/FHIRDbHelper.java @@ -13,10 +13,10 @@ import com.ibm.fhir.model.resource.OperationOutcome.Issue; import com.ibm.fhir.model.type.code.IssueType; import com.ibm.fhir.model.util.FHIRUtil; +import com.ibm.fhir.persistence.exception.FHIRPersistenceDataAccessException; import com.ibm.fhir.persistence.exception.FHIRPersistenceException; import com.ibm.fhir.persistence.jdbc.exception.FHIRPersistenceDBCleanupException; import com.ibm.fhir.persistence.jdbc.exception.FHIRPersistenceDBConnectException; -import com.ibm.fhir.persistence.jdbc.exception.FHIRPersistenceDataAccessException; /** * Helper functions used for managing FHIR database interactions diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/connection/FHIRDbTenantDatasourceConnectionStrategy.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/connection/FHIRDbTenantDatasourceConnectionStrategy.java index dc31f4899fd..9f73a16a182 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/connection/FHIRDbTenantDatasourceConnectionStrategy.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/connection/FHIRDbTenantDatasourceConnectionStrategy.java @@ -20,12 +20,13 @@ import com.ibm.fhir.config.FHIRConfiguration; import com.ibm.fhir.config.FHIRRequestContext; import com.ibm.fhir.config.PropertyGroup; +import com.ibm.fhir.database.utils.api.SchemaType; import com.ibm.fhir.database.utils.model.DbType; import com.ibm.fhir.exception.FHIRException; +import com.ibm.fhir.persistence.exception.FHIRPersistenceDataAccessException; import com.ibm.fhir.persistence.exception.FHIRPersistenceException; import com.ibm.fhir.persistence.jdbc.dao.impl.FHIRDbDAOImpl; import com.ibm.fhir.persistence.jdbc.exception.FHIRPersistenceDBConnectException; -import com.ibm.fhir.persistence.jdbc.exception.FHIRPersistenceDataAccessException; /** @@ -212,17 +213,26 @@ private FHIRDbFlavor createFlavor() throws FHIRPersistenceDataAccessException { if (dsPG != null) { try { - boolean multitenant = false; - String typeValue = dsPG.getStringProperty("type"); + SchemaType schemaType = SchemaType.PLAIN; + String schemaTypeValue = dsPG.getStringProperty("schemaType", null); + if (schemaTypeValue != null) { + schemaType = SchemaType.valueOf(schemaTypeValue.toUpperCase()); + } + String typeValue = dsPG.getStringProperty("type"); DbType type = DbType.from(typeValue); if (type == DbType.DB2) { - // We make this absolute for now. May change in the future if we - // support a single-tenant schema in DB2. - multitenant = true; + // For Db2 we currently only support MULTITENANT so we force the schemaType + schemaType = SchemaType.MULTITENANT; + } else { + // Make sure for any other database of type we're not being asked to use the + // multitenant variant + if (schemaType == SchemaType.MULTITENANT) { + throw new FHIRPersistenceDataAccessException("schemaType MULTITENANT is only supported for Db2"); + } } - result = new FHIRDbFlavorImpl(type, multitenant); + result = new FHIRDbFlavorImpl(type, schemaType); } catch (Exception x) { log.log(Level.SEVERE, "No type property found for datastore '" + datastoreId + "'", x); diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/connection/FHIRDbTestConnectionStrategy.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/connection/FHIRDbTestConnectionStrategy.java index 0553bad6011..5583613a409 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/connection/FHIRDbTestConnectionStrategy.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/connection/FHIRDbTestConnectionStrategy.java @@ -11,8 +11,9 @@ import java.util.logging.Logger; import com.ibm.fhir.database.utils.api.IConnectionProvider; +import com.ibm.fhir.database.utils.api.SchemaType; +import com.ibm.fhir.persistence.exception.FHIRPersistenceDataAccessException; import com.ibm.fhir.persistence.jdbc.exception.FHIRPersistenceDBConnectException; -import com.ibm.fhir.persistence.jdbc.exception.FHIRPersistenceDataAccessException; /** * Hides the logic behind obtaining a JDBC {@link Connection} from the DAO code. @@ -51,8 +52,9 @@ public FHIRDbTestConnectionStrategy(IConnectionProvider cp, Action action) { this.connectionProvider = cp; this.action = action; - // we don't support multi-tenancy in our unit-test database - flavor = new FHIRDbFlavorImpl(cp.getTranslator().getType(), false); + // we don't support multi-tenancy or distribution in our unit-test database, + // so we use PLAIN for the schema type + flavor = new FHIRDbFlavorImpl(cp.getTranslator().getType(), SchemaType.PLAIN); } @Override diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/connection/FHIRTestTransactionAdapter.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/connection/FHIRTestTransactionAdapter.java index 65789cca91c..19d7df6eb81 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/connection/FHIRTestTransactionAdapter.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/connection/FHIRTestTransactionAdapter.java @@ -14,8 +14,8 @@ import com.ibm.fhir.database.utils.api.ITransaction; import com.ibm.fhir.database.utils.transaction.SimpleTransactionProvider; import com.ibm.fhir.persistence.FHIRPersistenceTransaction; +import com.ibm.fhir.persistence.exception.FHIRPersistenceDataAccessException; import com.ibm.fhir.persistence.exception.FHIRPersistenceException; -import com.ibm.fhir.persistence.jdbc.exception.FHIRPersistenceDataAccessException; /** * Hides the logic behind obtaining a JDBC {@link Connection} from the DAO code. diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/connection/FHIRUserTransactionAdapter.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/connection/FHIRUserTransactionAdapter.java index ed242e55fd6..318ac3b4100 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/connection/FHIRUserTransactionAdapter.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/connection/FHIRUserTransactionAdapter.java @@ -17,9 +17,9 @@ import com.ibm.fhir.config.FHIRRequestContext; import com.ibm.fhir.persistence.FHIRPersistenceTransaction; +import com.ibm.fhir.persistence.exception.FHIRPersistenceDataAccessException; import com.ibm.fhir.persistence.exception.FHIRPersistenceException; import com.ibm.fhir.persistence.jdbc.FHIRPersistenceJDBCCache; -import com.ibm.fhir.persistence.jdbc.exception.FHIRPersistenceDataAccessException; import com.ibm.fhir.persistence.jdbc.impl.CacheTransactionSync; diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/connection/SetMultiShardModifyModeAction.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/connection/SetMultiShardModifyModeAction.java new file mode 100644 index 00000000000..3a2b7b5e094 --- /dev/null +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/connection/SetMultiShardModifyModeAction.java @@ -0,0 +1,86 @@ +/* + * (C) Copyright IBM Corp. 2022 + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.ibm.fhir.persistence.jdbc.connection; + +import java.sql.Connection; +import java.util.logging.Logger; + +import com.ibm.fhir.database.utils.api.IDatabaseStatement; +import com.ibm.fhir.database.utils.citus.CitusTranslator; +import com.ibm.fhir.database.utils.citus.ConfigureConnectionDAO; +import com.ibm.fhir.database.utils.derby.DerbyAdapter; +import com.ibm.fhir.database.utils.model.DbType; +import com.ibm.fhir.persistence.jdbc.derby.CreateCanonicalValuesTmp; +import com.ibm.fhir.persistence.jdbc.derby.CreateCodeSystemsTmp; +import com.ibm.fhir.persistence.jdbc.derby.CreateCommonTokenValuesTmp; +import com.ibm.fhir.persistence.jdbc.exception.FHIRPersistenceDBConnectException; + +/** + * For Citus connections, SET LOCAL citus.multi_shard_modify_mode TO 'sequential' + */ +public class SetMultiShardModifyModeAction extends ChainedAction { + private static final Logger log = Logger.getLogger(CreateTempTablesAction.class.getName()); + + /** + * Public constructor. No next action, so this will be the last action applied + */ + public SetMultiShardModifyModeAction() { + super(); + } + + /** + * Public constructor + * @param next the next action in the chain + */ + public SetMultiShardModifyModeAction(Action next) { + super(next); + } + + @Override + public void performOn(FHIRDbFlavor flavor, Connection connection) throws FHIRPersistenceDBConnectException { + + if (flavor.getType() == DbType.CITUS) { + // This is only used for Citus databases + log.fine("SET LOCAL citus.multi_shard_modify_mode TO 'sequential'"); + ConfigureConnectionDAO dao = new ConfigureConnectionDAO(); + dao.run(new CitusTranslator(), connection); + } + + // perform next action in the chain + super.performOn(flavor, connection); + } + + /** + * Create the declared global temporary table COMMON_TOKEN_VALUES_TMP + * @param adapter + * @throws FHIRPersistenceDBConnectException + */ + public void createCommonTokenValuesTmp(DerbyAdapter adapter) throws FHIRPersistenceDBConnectException { + IDatabaseStatement cmd = new CreateCommonTokenValuesTmp(); + adapter.runStatement(cmd); + } + + /** + * Create the declared global temporary table CODE_SYSTEMS_TMP + * @param adapter + * @throws FHIRPersistenceDBConnectException + */ + public void createCodeSystemsTmp(DerbyAdapter adapter) throws FHIRPersistenceDBConnectException { + IDatabaseStatement cmd = new CreateCodeSystemsTmp(); + adapter.runStatement(cmd); + } + + /** + * Create the declared global temporary table COMMON_TOKEN_VALUES_TMP + * @param adapter + * @throws FHIRPersistenceDBConnectException + */ + public void createCanonicalValuesTmp(DerbyAdapter adapter) throws FHIRPersistenceDBConnectException { + IDatabaseStatement cmd = new CreateCanonicalValuesTmp(); + adapter.runStatement(cmd); + } +} \ No newline at end of file diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/EraseResourceDAO.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/EraseResourceDAO.java index 6f637bd2291..5f5c74e6d38 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/EraseResourceDAO.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/EraseResourceDAO.java @@ -369,7 +369,7 @@ public long erase(ResourceEraseRecord eraseRecord, EraseDTO eraseDto) throws Exc if (DbType.DB2.equals(getFlavor().getType()) && eraseDto.getVersion() == null) { runCallableStatement(CALL_DB2, erasedResourceGroupId); - } else if (DbType.POSTGRESQL.equals(getFlavor().getType()) && eraseDto.getVersion() == null) { + } else if (getFlavor().isFamilyPostgreSQL() && eraseDto.getVersion() == null) { runCallableStatement(CALL_POSTGRES, erasedResourceGroupId); } else { // Uses the Native Java to execute a Resource Erase @@ -389,7 +389,7 @@ public List getErasedResourceRecords(long erasedResourceGroup List result = new ArrayList<>(); final String SELECT_RECORDS = - "SELECT erased_resource_id, resource_type_id, logical_id, version_id " + + "SELECT resource_type_id, logical_id, version_id " + " FROM erased_resources " + " WHERE erased_resource_group_id = ?"; @@ -397,10 +397,10 @@ public List getErasedResourceRecords(long erasedResourceGroup stmt.setLong(1, erasedResourceGroupId); ResultSet rs = stmt.executeQuery(); while (rs.next()) { - long erasedResourceId = rs.getLong(1); - int resourceTypeId = rs.getInt(2); - String logicalId = rs.getString(3); - Integer versionId = rs.getInt(4); + long erasedResourceId = -1; // no longer used + int resourceTypeId = rs.getInt(1); + String logicalId = rs.getString(2); + Integer versionId = rs.getInt(3); if (rs.wasNull()) { versionId = null; } diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/api/CodeSystemDAO.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/api/CodeSystemDAO.java index 6bc1b3b3dd5..05412f3b01c 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/api/CodeSystemDAO.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/api/CodeSystemDAO.java @@ -8,8 +8,8 @@ import java.util.Map; +import com.ibm.fhir.persistence.exception.FHIRPersistenceDataAccessException; import com.ibm.fhir.persistence.jdbc.exception.FHIRPersistenceDBConnectException; -import com.ibm.fhir.persistence.jdbc.exception.FHIRPersistenceDataAccessException; /** * This Data Access Object interface defines APIs specific to parameter_names table. diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/api/ILogicalResourceIdentCache.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/api/ILogicalResourceIdentCache.java new file mode 100644 index 00000000000..19df4d9fda0 --- /dev/null +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/api/ILogicalResourceIdentCache.java @@ -0,0 +1,63 @@ +/* + * (C) Copyright IBM Corp. 2022 + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.ibm.fhir.persistence.jdbc.dao.api; + +import java.util.Collection; +import java.util.List; + +import com.ibm.fhir.persistence.jdbc.dao.impl.ResourceReferenceValueRec; + +/** + * An interface for a cache of logical_resource_ident records. The + * cache is specialized in that it supports some specific operations to + * process list of objects with minimal locking. + * + */ +public interface ILogicalResourceIdentCache { + + /** + * Take the records we've touched in the current thread and update the + * shared LRU maps. + */ + void updateSharedMaps(); + + /** + * Lookup all the database values we have cached for the given collection. + * Put any objects with cache misses into the corresponding + * miss lists (so that we know which records we need to generate inserts for) + * @param referenceValues + * @param misses the objects we couldn't find in the cache + */ + void resolveReferenceValues(Collection referenceValues, + List misses); + + /** + * Add the LogicalResourceIdent key and id to the local cache + * @param key + * @param id + */ + public void addRecord(LogicalResourceIdentKey key, long id); + + /** + * Clear any thread-local cache maps (probably because a transaction was rolled back) + */ + void clearLocalMaps(); + + /** + * Clear the thread-local and shared caches (for test purposes) + */ + void reset(); + + /** + * Get the database logical_resourc_id for the given resource type + * and logicalId + * @param resourceTypeId + * @param logicalId + * @return + */ + Long getLogicalResourceId(int resourceTypeId, String logicalId); +} diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/api/IResourceReferenceDAO.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/api/IResourceReferenceDAO.java index fb9eae123f9..2c0ea73047e 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/api/IResourceReferenceDAO.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/api/IResourceReferenceDAO.java @@ -12,6 +12,7 @@ import com.ibm.fhir.persistence.exception.FHIRPersistenceException; import com.ibm.fhir.persistence.jdbc.dao.impl.ResourceProfileRec; +import com.ibm.fhir.persistence.jdbc.dao.impl.ResourceReferenceValueRec; import com.ibm.fhir.persistence.jdbc.dao.impl.ResourceTokenValueRec; import com.ibm.fhir.persistence.jdbc.dto.CommonTokenValue; import com.ibm.fhir.persistence.jdbc.dto.CommonTokenValueResult; @@ -40,20 +41,22 @@ public interface IResourceReferenceDAO { * as necessary * @param resourceType * @param xrefs + * @param refValues * @param profileRecs * @param tagRecs * @param securityRecs */ - void addNormalizedValues(String resourceType, Collection xrefs, Collection profileRecs, Collection tagRecs, Collection securityRecs) throws FHIRPersistenceException; + void addNormalizedValues(String resourceType, Collection xrefs, Collection refValues, Collection profileRecs, Collection tagRecs, Collection securityRecs) throws FHIRPersistenceException; /** * Persist the records, which may span multiple resource types * @param records + * @param referenceRecords * @param profileRecs * @param tagRecs * @param securityRecs */ - void persist(Collection records, Collection profileRecs, Collection tagRecs, Collection securityRecs) throws FHIRPersistenceException; + void persist(Collection records, Collection referenceRecords, Collection profileRecs, Collection tagRecs, Collection securityRecs) throws FHIRPersistenceException; /** * Find the database id for the given token value and system diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/api/JDBCIdentityCache.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/api/JDBCIdentityCache.java index 834f41a6f58..746cbb3f925 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/api/JDBCIdentityCache.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/api/JDBCIdentityCache.java @@ -12,6 +12,7 @@ import com.ibm.fhir.persistence.exception.FHIRPersistenceException; import com.ibm.fhir.persistence.jdbc.dto.CommonTokenValue; +import com.ibm.fhir.persistence.jdbc.dto.ResourceReferenceValue; /** * Provides access to all the identity information we need when processing @@ -54,6 +55,16 @@ public interface JDBCIdentityCache { */ Integer getCanonicalId(String canonicalValue) throws FHIRPersistenceException; + /** + * Get the database id for the given (resourceType, logicalId) tuple. This + * represents records in logical_resource_ident which may be created before + * the actual resource is created. + * @param resourceType + * @param logicalId + * @return + */ + Long getLogicalResourceId(String resourceType, String logicalId) throws FHIRPersistenceException; + /** * Get the database id for the given parameter name. Creates new records if necessary. * @param parameterName @@ -81,6 +92,15 @@ public interface JDBCIdentityCache { */ Set getCommonTokenValueIds(Collection tokenValues); + /** + * Get the logical_resource_ids for the given referenceValues. Reads from + * a cache, or the database if not found in the cache. Values with no + * corresponding record in the database will be omitted from the result set. + * @param referenceValues + * @return a non-null, possibly empty set of logical_resource_ids. + */ + Set getLogicalResourceIds(Collection referenceValues) throws FHIRPersistenceException; + /** * Get a list of matching common_token_value_id values. Implementations may decide * to cache, but only if the cache can be invalidated when the list changes due to @@ -93,6 +113,20 @@ public interface JDBCIdentityCache { */ List getCommonTokenValueIdList(String tokenValue); + /** + * Get a list of logical_resource_id values matching the given logicalId without + * knowing the resource type. This means we could get back multiple ids, one per + * resource type, such as: + *
    + *
  • Claim/foo + *
  • Observation/foo + *
  • Patient/foo + *
+ * @param tokenValue + * @return + */ + List getLogicalResourceIdList(String logicalId) throws FHIRPersistenceException; + /** * Get the list of all resource type names. * @return diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/api/LogicalResourceIdentKey.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/api/LogicalResourceIdentKey.java new file mode 100644 index 00000000000..494a824f004 --- /dev/null +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/api/LogicalResourceIdentKey.java @@ -0,0 +1,65 @@ +/* + * (C) Copyright IBM Corp. 2022 + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.ibm.fhir.persistence.jdbc.dao.api; + +import java.util.Objects; + +/** + * A DTO representing a mapping of a logical_resource identity to its database + * logical_resource_id value. + * @implNote use record in Java 17 + */ +public class LogicalResourceIdentKey { + + private final int resourceTypeId; + private final String logicalId; + + /** + * Public constructor + * @param resourceTypeId + * @param logicalId + */ + public LogicalResourceIdentKey(int resourceTypeId, String logicalId) { + this.resourceTypeId = resourceTypeId; + this.logicalId = logicalId; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + + if (obj instanceof LogicalResourceIdentKey) { + LogicalResourceIdentKey that = (LogicalResourceIdentKey)obj; + return this.resourceTypeId == that.resourceTypeId + && this.logicalId.equals(that.logicalId); + } + + throw new IllegalArgumentException("invalid type"); + } + + @Override + public int hashCode() { + return Objects.hash(this.resourceTypeId, this.logicalId); + } + + /** + * @return the resourceTypeId + */ + public int getResourceTypeId() { + return resourceTypeId; + } + + + /** + * @return the logicalId + */ + public String getLogicalId() { + return logicalId; + } +} diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/api/LogicalResourceIdentValue.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/api/LogicalResourceIdentValue.java new file mode 100644 index 00000000000..2749fc70291 --- /dev/null +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/api/LogicalResourceIdentValue.java @@ -0,0 +1,41 @@ +/* + * (C) Copyright IBM Corp. 2022 + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.ibm.fhir.persistence.jdbc.dao.api; + + +/** + * Represents a record in logical_resource_ident + * @implNote no need to override hashCode or equals because logicalResourceId + * does not contribute to the identity of the record - it is just an + * attribute. + */ +public class LogicalResourceIdentValue extends LogicalResourceIdentKey { + private Long logicalResourceId; + + /** + * Public constructor + * @param resourceTypeId + * @param logicalId + */ + public LogicalResourceIdentValue(int resourceTypeId, String logicalId) { + super(resourceTypeId, logicalId); + } + + /** + * @return the logicalResourceId + */ + public Long getLogicalResourceId() { + return logicalResourceId; + } + + /** + * @param logicalResourceId the logicalResourceId to set + */ + public void setLogicalResourceId(Long logicalResourceId) { + this.logicalResourceId = logicalResourceId; + } +} diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/api/ParameterDAO.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/api/ParameterDAO.java index 16437517034..c2a24c1f7b5 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/api/ParameterDAO.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/api/ParameterDAO.java @@ -8,9 +8,9 @@ import java.util.Map; +import com.ibm.fhir.persistence.exception.FHIRPersistenceDataAccessException; import com.ibm.fhir.persistence.exception.FHIRPersistenceException; import com.ibm.fhir.persistence.jdbc.exception.FHIRPersistenceDBConnectException; -import com.ibm.fhir.persistence.jdbc.exception.FHIRPersistenceDataAccessException; /** * This Data Access Object interface defines methods for creating, updating, diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/api/ParameterNameDAO.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/api/ParameterNameDAO.java index 68247ab66c4..36607b0394b 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/api/ParameterNameDAO.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/api/ParameterNameDAO.java @@ -8,7 +8,7 @@ import java.util.Map; -import com.ibm.fhir.persistence.jdbc.exception.FHIRPersistenceDataAccessException; +import com.ibm.fhir.persistence.exception.FHIRPersistenceDataAccessException; /** * This Data Access Object interface defines APIs specific to parameter_names table. diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/api/ResourceDAO.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/api/ResourceDAO.java index 34d50abd725..8e777957c44 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/api/ResourceDAO.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/api/ResourceDAO.java @@ -12,12 +12,12 @@ import com.ibm.fhir.database.utils.query.Select; import com.ibm.fhir.persistence.context.FHIRPersistenceContext; +import com.ibm.fhir.persistence.exception.FHIRPersistenceDataAccessException; import com.ibm.fhir.persistence.exception.FHIRPersistenceException; import com.ibm.fhir.persistence.exception.FHIRPersistenceVersionIdMismatchException; import com.ibm.fhir.persistence.jdbc.dto.ExtractedParameterValue; import com.ibm.fhir.persistence.jdbc.dto.Resource; import com.ibm.fhir.persistence.jdbc.exception.FHIRPersistenceDBConnectException; -import com.ibm.fhir.persistence.jdbc.exception.FHIRPersistenceDataAccessException; /** * This Data Access Object interface provides methods creating, updating, and retrieving rows in the FHIR Resource tables. @@ -187,4 +187,21 @@ List search(String sqlSelect) Resource insert(Resource resource, List parameters, String parameterHashB64, ParameterDAO parameterDao, Integer ifNoneMatch) throws FHIRPersistenceException; + + /** + * Look up the value of the logical_resource_id from the logical_resource_ident table + * @param resourceTypeId + * @param logicalId + * @return + */ + Long readLogicalResourceId(int resourceTypeId, String logicalId) throws FHIRPersistenceDBConnectException, FHIRPersistenceDataAccessException; + + /** + * Read all the matching logical_resource_id values for the given logicalId + * @param logicalId + * @return + * @throws FHIRPersistenceDBConnectException + * @throws FHIRPersistenceDataAccessException + */ + List readLogicalResourceIdList(String logicalId) throws FHIRPersistenceDBConnectException, FHIRPersistenceDataAccessException; } \ No newline at end of file diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/impl/CodeSystemDAOImpl.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/impl/CodeSystemDAOImpl.java index bc0916f5a8a..0eb1b8983dc 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/impl/CodeSystemDAOImpl.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/impl/CodeSystemDAOImpl.java @@ -16,8 +16,8 @@ import java.util.logging.Level; import java.util.logging.Logger; +import com.ibm.fhir.persistence.exception.FHIRPersistenceDataAccessException; import com.ibm.fhir.persistence.jdbc.dao.api.CodeSystemDAO; -import com.ibm.fhir.persistence.jdbc.exception.FHIRPersistenceDataAccessException; /** * This DAO uses a connection provided to its constructor. It's therefore diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/impl/FHIRDbDAOImpl.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/impl/FHIRDbDAOImpl.java index faa10d7ad16..0b741a5fdda 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/impl/FHIRDbDAOImpl.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/impl/FHIRDbDAOImpl.java @@ -25,13 +25,13 @@ import com.ibm.fhir.model.resource.OperationOutcome.Issue; import com.ibm.fhir.model.type.code.IssueType; import com.ibm.fhir.model.util.FHIRUtil; +import com.ibm.fhir.persistence.exception.FHIRPersistenceDataAccessException; import com.ibm.fhir.persistence.exception.FHIRPersistenceException; import com.ibm.fhir.persistence.jdbc.connection.FHIRDbFlavor; import com.ibm.fhir.persistence.jdbc.dao.api.FHIRDbDAO; import com.ibm.fhir.persistence.jdbc.dto.Resource; import com.ibm.fhir.persistence.jdbc.exception.FHIRPersistenceDBCleanupException; import com.ibm.fhir.persistence.jdbc.exception.FHIRPersistenceDBConnectException; -import com.ibm.fhir.persistence.jdbc.exception.FHIRPersistenceDataAccessException; /** * This class is a root Data Access Object for managing JDBC access to the FHIR database. diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/impl/FetchResourcePayloadsDAO.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/impl/FetchResourcePayloadsDAO.java index ff67d2317f7..d3526c2dc53 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/impl/FetchResourcePayloadsDAO.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/impl/FetchResourcePayloadsDAO.java @@ -24,8 +24,8 @@ import com.ibm.fhir.database.utils.api.IDatabaseTranslator; import com.ibm.fhir.database.utils.common.CalendarHelper; import com.ibm.fhir.persistence.ResourcePayload; +import com.ibm.fhir.persistence.exception.FHIRPersistenceDataAccessException; import com.ibm.fhir.persistence.exception.FHIRPersistenceException; -import com.ibm.fhir.persistence.jdbc.exception.FHIRPersistenceDataAccessException; /** * DAO to fetch resource ids using a time range and optional current resource id as a filter. diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/impl/JDBCIdentityCacheImpl.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/impl/JDBCIdentityCacheImpl.java index 61ce7545ba5..6517e0882cc 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/impl/JDBCIdentityCacheImpl.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/impl/JDBCIdentityCacheImpl.java @@ -16,15 +16,18 @@ import java.util.logging.Logger; import java.util.stream.Collectors; +import com.ibm.fhir.persistence.exception.FHIRPersistenceDataAccessException; import com.ibm.fhir.persistence.exception.FHIRPersistenceException; import com.ibm.fhir.persistence.jdbc.FHIRPersistenceJDBCCache; +import com.ibm.fhir.persistence.jdbc.dao.api.ILogicalResourceIdentCache; import com.ibm.fhir.persistence.jdbc.dao.api.IResourceReferenceDAO; import com.ibm.fhir.persistence.jdbc.dao.api.JDBCIdentityCache; +import com.ibm.fhir.persistence.jdbc.dao.api.LogicalResourceIdentKey; import com.ibm.fhir.persistence.jdbc.dao.api.ParameterDAO; import com.ibm.fhir.persistence.jdbc.dao.api.ResourceDAO; import com.ibm.fhir.persistence.jdbc.dto.CommonTokenValue; import com.ibm.fhir.persistence.jdbc.dto.CommonTokenValueResult; -import com.ibm.fhir.persistence.jdbc.exception.FHIRPersistenceDataAccessException; +import com.ibm.fhir.persistence.jdbc.dto.ResourceReferenceValue; /** @@ -212,4 +215,64 @@ public List getResourceTypeNames() throws FHIRPersistenceException { public List getResourceTypeIds() throws FHIRPersistenceException { return new ArrayList<>(cache.getResourceTypeCache().getAllIds()); } + + @Override + public Long getLogicalResourceId(String resourceType, String logicalId) throws FHIRPersistenceException { + Integer resourceTypeId = cache.getResourceTypeCache().getId(resourceType); + if (resourceTypeId == null) { + throw new IllegalArgumentException("Invalid resource type: " + resourceType); + } + Long result = cache.getLogicalResourceIdentCache().getLogicalResourceId(resourceTypeId, logicalId); + if (result == null) { + if (logger.isLoggable(Level.FINE)) { + logger.fine("Cache miss. Fetching logical_resource_id from database: '" + resourceType + "/" + logicalId + "'"); + } + result = resourceDAO.readLogicalResourceId(resourceTypeId, logicalId); + if (result != null) { + // Value exists in the database, so we can add this to our cache. Note that we still + // choose to add it the thread-local cache - this avoids any locking. The values will + // be promoted to the shared cache at the end of the transaction. This avoids unnecessary + // contention. + if (logger.isLoggable(Level.FINE)) { + logger.fine("Adding logical_resource_id to cache: '" + resourceType + "/" + logicalId + "' = " + result); + } + cache.getLogicalResourceIdentCache().addRecord(new LogicalResourceIdentKey(resourceTypeId, logicalId), result); + } + } + return result; + } + + @Override + public Set getLogicalResourceIds(Collection referenceValues) throws FHIRPersistenceException { + // Pull values from the cache where we can. For anything unresolved, need to hit + // the database. It may be more efficient to collect all the misses and read in + // a single query, if the referenceValues collections are typically small, this + // won't make much difference. + ILogicalResourceIdentCache idCache = cache.getLogicalResourceIdentCache(); + Set result = new HashSet<>(referenceValues.size()); + for (ResourceReferenceValue rrv: referenceValues) { + Long logicalResourceId = idCache.getLogicalResourceId(rrv.getResourceTypeId(), rrv.getLogicalId()); + if (logicalResourceId != null) { + result.add(logicalResourceId); + } else { + // Just read directly from the database + logicalResourceId = resourceDAO.readLogicalResourceId(rrv.getResourceTypeId(), rrv.getLogicalId()); + if (logicalResourceId != null) { + result.add(logicalResourceId); + idCache.addRecord(new LogicalResourceIdentKey(rrv.getResourceTypeId(), rrv.getLogicalId()), logicalResourceId); + } + } + } + return result; + } + + @Override + public List getLogicalResourceIdList(String logicalId) throws FHIRPersistenceException { + // The implementation for this one is a little more interesting. We can + // leverage the fact that the PK on logical_resource_ident is + // logical_id, resource_type_id. This means that we get a simple + // index range scan (although where logical ids are derived from UUIDs, + // they are unique so we'll be getting just one row back) + return resourceDAO.readLogicalResourceIdList(logicalId); + } } \ No newline at end of file diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/impl/ParameterDAOImpl.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/impl/ParameterDAOImpl.java index 1be7a08cbe2..ca7908b81e5 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/impl/ParameterDAOImpl.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/impl/ParameterDAOImpl.java @@ -12,6 +12,7 @@ import javax.transaction.TransactionSynchronizationRegistry; +import com.ibm.fhir.persistence.exception.FHIRPersistenceDataAccessException; import com.ibm.fhir.persistence.exception.FHIRPersistenceException; import com.ibm.fhir.persistence.jdbc.connection.FHIRDbFlavor; import com.ibm.fhir.persistence.jdbc.dao.api.CodeSystemDAO; @@ -20,7 +21,6 @@ import com.ibm.fhir.persistence.jdbc.derby.DerbyCodeSystemDAO; import com.ibm.fhir.persistence.jdbc.derby.DerbyParameterNamesDAO; import com.ibm.fhir.persistence.jdbc.exception.FHIRPersistenceDBConnectException; -import com.ibm.fhir.persistence.jdbc.exception.FHIRPersistenceDataAccessException; import com.ibm.fhir.persistence.jdbc.postgres.PostgresCodeSystemDAO; import com.ibm.fhir.persistence.jdbc.postgres.PostgresParameterNamesDAO; @@ -117,6 +117,7 @@ public int readOrAddParameterNameId(String parameterName) throws FHIRPersistence pnd = new DerbyParameterNamesDAO(connection, getSchemaName()); break; case POSTGRESQL: + case CITUS: pnd = new PostgresParameterNamesDAO(connection, getSchemaName()); break; default: @@ -154,6 +155,7 @@ public int readOrAddCodeSystemId(String codeSystemName) throws FHIRPersistenceDB csd = new DerbyCodeSystemDAO(connection, getSchemaName()); break; case POSTGRESQL: + case CITUS: csd = new PostgresCodeSystemDAO(connection, getSchemaName()); break; default: diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/impl/ParameterNameDAOImpl.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/impl/ParameterNameDAOImpl.java index 952d81ba666..443f9ba5d58 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/impl/ParameterNameDAOImpl.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/impl/ParameterNameDAOImpl.java @@ -16,8 +16,8 @@ import java.util.logging.Level; import java.util.logging.Logger; +import com.ibm.fhir.persistence.exception.FHIRPersistenceDataAccessException; import com.ibm.fhir.persistence.jdbc.dao.api.ParameterNameDAO; -import com.ibm.fhir.persistence.jdbc.exception.FHIRPersistenceDataAccessException; /** * Database interaction for parameter_names. Caching etc is handled diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/impl/ParameterTransportVisitor.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/impl/ParameterTransportVisitor.java new file mode 100644 index 00000000000..595fa1b344a --- /dev/null +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/impl/ParameterTransportVisitor.java @@ -0,0 +1,165 @@ +/* + * (C) Copyright IBM Corp. 2022 + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.ibm.fhir.persistence.jdbc.dao.impl; + +import java.time.Instant; +import java.util.logging.Logger; + +import com.ibm.fhir.persistence.exception.FHIRPersistenceException; +import com.ibm.fhir.persistence.index.CanonicalSupport; +import com.ibm.fhir.persistence.index.ParameterValueVisitorAdapter; +import com.ibm.fhir.persistence.index.ProfileParameter; +import com.ibm.fhir.persistence.jdbc.JDBCConstants; +import com.ibm.fhir.persistence.jdbc.dto.CompositeParmVal; +import com.ibm.fhir.persistence.jdbc.dto.DateParmVal; +import com.ibm.fhir.persistence.jdbc.dto.ExtractedParameterValue; +import com.ibm.fhir.persistence.jdbc.dto.ExtractedParameterValueVisitor; +import com.ibm.fhir.persistence.jdbc.dto.LocationParmVal; +import com.ibm.fhir.persistence.jdbc.dto.NumberParmVal; +import com.ibm.fhir.persistence.jdbc.dto.QuantityParmVal; +import com.ibm.fhir.persistence.jdbc.dto.ReferenceParmVal; +import com.ibm.fhir.persistence.jdbc.dto.StringParmVal; +import com.ibm.fhir.persistence.jdbc.dto.TokenParmVal; +import com.ibm.fhir.search.SearchConstants; +import com.ibm.fhir.search.util.ReferenceValue; +import com.ibm.fhir.search.util.ReferenceValue.ReferenceType; + + +/** + * A visitor to map parameters to a format suitable for transport to another + * system (e.g. for remote indexing) + */ +public class ParameterTransportVisitor implements ExtractedParameterValueVisitor { + private static final Logger logger = Logger.getLogger(ParameterTransportVisitor.class.getName()); + private static final Boolean IS_WHOLE_SYSTEM = Boolean.TRUE; + + // The adapter to which we delegate each of our visit calls + private final ParameterValueVisitorAdapter adapter; + + // tracks the number of composites so we know what next composite_id to use + private int compositeIdCounter = 0; + + // Tracks the name of the composite parameter currently being processed + private String currentCompositeParameterName = null; + + /** + * Public constructor + * @param adapter + */ + public ParameterTransportVisitor(ParameterValueVisitorAdapter adapter) { + this.adapter = adapter; + } + + @Override + public void visit(StringParmVal stringParameter) throws FHIRPersistenceException { + + if (SearchConstants.PROFILE.equals(stringParameter.getName())) { + // special case to store profile parameters in their own table + ProfileParameter pp = CanonicalSupport.createProfileParameter(stringParameter.getName(), stringParameter.getValueString()); + adapter.profileValue(pp.getName(), pp.getUrl(), pp.getVersion(), pp.getFragment(), IS_WHOLE_SYSTEM); + } else { + Integer compositeId = this.currentCompositeParameterName != null ? this.compositeIdCounter : null; + adapter.stringValue(stringParameter.getName(), stringParameter.getValueString(), compositeId, stringParameter.isWholeSystem()); + } + } + + @Override + public void visit(NumberParmVal numberParameter) throws FHIRPersistenceException { + Integer compositeId = this.currentCompositeParameterName != null ? this.compositeIdCounter : null; + adapter.numberValue(numberParameter.getName(), + numberParameter.getValueNumber(), + numberParameter.getValueNumberLow(), + numberParameter.getValueNumberHigh(), + compositeId); + } + + @Override + public void visit(DateParmVal dateParameter) throws FHIRPersistenceException { + Integer compositeId = this.currentCompositeParameterName != null ? this.compositeIdCounter : null; + Instant dateStart = dateParameter.getValueDateStart().toInstant(); + Instant dateEnd = dateParameter.getValueDateEnd().toInstant(); + adapter.dateValue(dateParameter.getName(), dateStart, dateEnd, compositeId, dateParameter.isWholeSystem()); + } + + @Override + public void visit(TokenParmVal tokenParameter) throws FHIRPersistenceException { + Integer compositeId = this.currentCompositeParameterName != null ? this.compositeIdCounter : null; + // tag and profile search params are often low-selectivity (many resources sharing the same value) so + // we put them into their own tables to allow better cardinality estimation by the query + // optimizer + switch (tokenParameter.getName()) { + case SearchConstants.TAG: + adapter.tagValue(tokenParameter.getName(), tokenParameter.getValueSystem(), tokenParameter.getValueCode(), IS_WHOLE_SYSTEM); + break; + case SearchConstants.SECURITY: + adapter.securityValue(tokenParameter.getName(), tokenParameter.getValueSystem(), tokenParameter.getValueCode(), IS_WHOLE_SYSTEM); + break; + default: + adapter.tokenValue(tokenParameter.getName(), tokenParameter.getValueSystem(), tokenParameter.getValueCode(), compositeId); + } + } + + @Override + public void visit(QuantityParmVal quantityParameter) throws FHIRPersistenceException { + Integer compositeId = this.currentCompositeParameterName != null ? this.compositeIdCounter : null; + adapter.quantityValue(quantityParameter.getName(), quantityParameter.getValueSystem(), quantityParameter.getValueCode(), quantityParameter.getValueNumber(), + quantityParameter.getValueNumberLow(), quantityParameter.getValueNumberHigh(), compositeId); + + } + + @Override + public void visit(LocationParmVal locationParameter) throws FHIRPersistenceException { + Integer compositeId = this.currentCompositeParameterName != null ? this.compositeIdCounter : null; + adapter.locationValue(locationParameter.getName(), locationParameter.getValueLatitude(), locationParameter.getValueLongitude(), compositeId); + } + + @Override + public void visit(ReferenceParmVal rpv) throws FHIRPersistenceException { + Integer compositeId = this.currentCompositeParameterName != null ? this.compositeIdCounter : null; + + // The ReferenceValue has already been processed to convert the reference to + // the required standard form, ready for insertion as a token value. + ReferenceValue refValue = rpv.getRefValue(); + String resourceType = rpv.getResourceType(); + String refResourceType = refValue.getTargetResourceType(); + String refLogicalId = refValue.getValue(); + Integer refVersion = refValue.getVersion(); + if (refValue.getType() == ReferenceType.DISPLAY_ONLY || refValue.getType() == ReferenceType.INVALID) { + // protect against code regression. Invalid/improper references should be + // filtered out already. + logger.warning("Invalid reference parameter type: '" + resourceType + "." + rpv.getName() + "' type=" + refValue.getType().name()); + throw new IllegalArgumentException("Invalid reference parameter value. See server log for details."); + } + + if (refResourceType == null) { + // Prior to V0027, references without a target resource type would be assigned the + // DEFAULT_TOKEN_SYSTEM (having a valid system makes queries faster). For V0027, + // all reference values get an entry in logical_resource_ident so in order to use + // a valid resource type we use "Resource" instead. + refResourceType = JDBCConstants.RESOURCE; + } + adapter.referenceValue(rpv.getName(), refResourceType, refLogicalId, refVersion, compositeId); + } + + @Override + public void visit(CompositeParmVal compositeParameter) throws FHIRPersistenceException { + if (this.currentCompositeParameterName != null) { + throw new FHIRPersistenceException("found nested composite parameter which isn't supported. " + + "current:[" + currentCompositeParameterName + "]" + + " nested:[" + compositeParameter.getName() + "]"); + } + + // Each parameter contained within this composite will be assigned the same + // compositeIdCounter value + this.compositeIdCounter++; + this.currentCompositeParameterName = compositeParameter.getName(); + for (ExtractedParameterValue epv: compositeParameter.getComponent()) { + epv.accept(this); + } + this.currentCompositeParameterName = null; + } +} diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/impl/ParameterVisitorBatchDAO.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/impl/ParameterVisitorBatchDAO.java index 0aa713b84e7..f32b1be9981 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/impl/ParameterVisitorBatchDAO.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/impl/ParameterVisitorBatchDAO.java @@ -25,8 +25,8 @@ import com.ibm.fhir.config.FHIRConfigHelper; import com.ibm.fhir.database.utils.common.CalendarHelper; +import com.ibm.fhir.persistence.exception.FHIRPersistenceDataAccessException; import com.ibm.fhir.persistence.exception.FHIRPersistenceException; -import com.ibm.fhir.persistence.jdbc.JDBCConstants; import com.ibm.fhir.persistence.jdbc.dao.api.IResourceReferenceDAO; import com.ibm.fhir.persistence.jdbc.dao.api.JDBCIdentityCache; import com.ibm.fhir.persistence.jdbc.dto.CompositeParmVal; @@ -39,7 +39,6 @@ import com.ibm.fhir.persistence.jdbc.dto.ReferenceParmVal; import com.ibm.fhir.persistence.jdbc.dto.StringParmVal; import com.ibm.fhir.persistence.jdbc.dto.TokenParmVal; -import com.ibm.fhir.persistence.jdbc.exception.FHIRPersistenceDataAccessException; import com.ibm.fhir.persistence.jdbc.impl.ParameterTransactionDataImpl; import com.ibm.fhir.persistence.jdbc.util.CanonicalSupport; import com.ibm.fhir.schema.control.FhirSchemaConstants; @@ -102,6 +101,7 @@ public class ParameterVisitorBatchDAO implements ExtractedParameterValueVisitor, // Collect a list of token values to process in one go private final List tokenValueRecs = new ArrayList<>(); + private final List referenceValueRecs = new ArrayList<>(); // Tags are now stored in their own tables private final List tagTokenRecs = new ArrayList<>(); @@ -647,7 +647,7 @@ public void close() throws Exception { if (this.transactionData == null) { // Not using transaction data, so we need to process collected values right here - this.resourceReferenceDAO.addNormalizedValues(this.tablePrefix, tokenValueRecs, profileRecs, tagTokenRecs, securityTokenRecs); + this.resourceReferenceDAO.addNormalizedValues(this.tablePrefix, tokenValueRecs, referenceValueRecs, profileRecs, tagTokenRecs, securityTokenRecs); } closeStatement(strings); @@ -703,7 +703,6 @@ public void visit(ReferenceParmVal rpv) throws FHIRPersistenceException { String refResourceType = refValue.getTargetResourceType(); String refLogicalId = refValue.getValue(); Integer refVersion = refValue.getVersion(); - ResourceTokenValueRec rec; if (refValue.getType() == ReferenceType.DISPLAY_ONLY || refValue.getType() == ReferenceType.INVALID) { // protect against code regression. Invalid/improper references should be @@ -712,20 +711,22 @@ public void visit(ReferenceParmVal rpv) throws FHIRPersistenceException { throw new IllegalArgumentException("Invalid reference parameter value. See server log for details."); } - // reference params are never system-level - final boolean isSystemParam = false; - if (refResourceType != null) { - // Store a token value configured as a reference to another resource - rec = new ResourceTokenValueRec(parameterName, resourceType, resourceTypeId, logicalResourceId, refResourceType, refLogicalId, refVersion, this.currentCompositeId, isSystemParam); - } else { - // stored as a token with the default system - rec = new ResourceTokenValueRec(parameterName, resourceType, resourceTypeId, logicalResourceId, JDBCConstants.DEFAULT_TOKEN_SYSTEM, refLogicalId, this.currentCompositeId, isSystemParam); + // V0027. Absolute references won't have a resource type, but in order to store them + // in the LOGICAL_RESOURCE_IDENT table we need to have a valid LOGICAL_RESOURCE_ID. For + // that we use "Resource" + if (refResourceType == null) { + refResourceType = "Resource"; } - + // Store a reference value configured as a reference to another resource (reference params + // are never system-level). + int refResourceTypeId = identityCache.getResourceTypeId(refResourceType); + ResourceReferenceValueRec rec = new ResourceReferenceValueRec(parameterName, resourceType, resourceTypeId, logicalResourceId, + refResourceType, refResourceTypeId, + refLogicalId, refVersion, this.currentCompositeId); if (this.transactionData != null) { - this.transactionData.addValue(rec); + this.transactionData.addReferenceValue(rec); } else { - this.tokenValueRecs.add(rec); + this.referenceValueRecs.add(rec); } } } \ No newline at end of file diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/impl/ResourceDAOImpl.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/impl/ResourceDAOImpl.java index c8f227f682c..7645db9f9ad 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/impl/ResourceDAOImpl.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/impl/ResourceDAOImpl.java @@ -35,8 +35,10 @@ import com.ibm.fhir.database.utils.query.Select; import com.ibm.fhir.persistence.InteractionStatus; import com.ibm.fhir.persistence.context.FHIRPersistenceContext; +import com.ibm.fhir.persistence.exception.FHIRPersistenceDataAccessException; import com.ibm.fhir.persistence.exception.FHIRPersistenceException; import com.ibm.fhir.persistence.exception.FHIRPersistenceVersionIdMismatchException; +import com.ibm.fhir.persistence.index.FHIRRemoteIndexService; import com.ibm.fhir.persistence.jdbc.FHIRPersistenceJDBCCache; import com.ibm.fhir.persistence.jdbc.connection.FHIRDbFlavor; import com.ibm.fhir.persistence.jdbc.dao.api.FHIRDAOConstants; @@ -47,7 +49,6 @@ import com.ibm.fhir.persistence.jdbc.dto.ExtractedParameterValue; import com.ibm.fhir.persistence.jdbc.dto.Resource; import com.ibm.fhir.persistence.jdbc.exception.FHIRPersistenceDBConnectException; -import com.ibm.fhir.persistence.jdbc.exception.FHIRPersistenceDataAccessException; import com.ibm.fhir.persistence.jdbc.exception.FHIRPersistenceFKVException; import com.ibm.fhir.persistence.jdbc.impl.ParameterTransactionDataImpl; import com.ibm.fhir.persistence.util.InputOutputByteStream; @@ -131,6 +132,18 @@ public class ResourceDAOImpl extends FHIRDbDAOImpl implements ResourceDAO { "FROM %s_RESOURCES R, %s_LOGICAL_RESOURCES LR WHERE R.LOGICAL_RESOURCE_ID = LR.LOGICAL_RESOURCE_ID AND " + "R.RESOURCE_ID IN "; + private static final String SQL_GET_LOGICAL_RESOURCE_IDENT = "" + + "SELECT logical_resource_id " + + " FROM logical_resource_ident " + + " WHERE resource_type_id = ? " + + " AND logical_id = ?"; + + // Get all records matching the given logical_id (multiple resource types) + private static final String SQL_GET_LOGICAL_RESOURCE_IDENT_LIST = "" + + " SELECT logical_resource_id " + + " FROM logical_resource_ident " + + " WHERE logical_id = ?"; + private static final String SQL_ORDER_BY_IDS = "ORDER BY CASE R.RESOURCE_ID "; private static final String DERBY_PAGINATION_PARMS = "OFFSET ? ROWS FETCH NEXT ? ROWS ONLY"; @@ -266,7 +279,7 @@ protected Resource createDTO(ResultSet resultSet, boolean hasResourceTypeId) thr if (payloadData != null) { resource.setDataStream(new InputOutputByteStream(payloadData, payloadData.length)); } - resource.setId(resultSet.getLong(IDX_RESOURCE_ID)); + resource.setResourceId(resultSet.getLong(IDX_RESOURCE_ID)); resource.setLogicalResourceId(resultSet.getLong(IDX_LOGICAL_RESOURCE_ID)); resource.setLastUpdated(resultSet.getTimestamp(IDX_LAST_UPDATED, CalendarHelper.getCalendarForUTC())); resource.setLogicalId(resultSet.getString(IDX_LOGICAL_ID)); @@ -528,7 +541,7 @@ public Resource insert(Resource resource, List paramete long latestTime = System.nanoTime(); double dbCallDuration = (latestTime-dbCallStartTime)/1e6; - resource.setId(stmt.getLong(10)); + resource.setLogicalResourceId(stmt.getLong(10)); final long versionedResourceRowId = stmt.getLong(11); final String currentHash = stmt.getString(12); final int interactionStatus = stmt.getInt(13); @@ -560,11 +573,14 @@ public Resource insert(Resource resource, List paramete // TODO FHIR_ADMIN schema name needs to come from the configuration/context // We can skip the parameter insert if we've been given parameterHashB64 and // it matches the current value just returned by the stored procedure call + FHIRRemoteIndexService remoteIndexService = FHIRRemoteIndexService.getServiceInstance(); long paramInsertStartTime = latestTime; - if (parameters != null && (parameterHashB64 == null || !parameterHashB64.equals(currentHash))) { + if (remoteIndexService == null + && parameters != null && (parameterHashB64 == null || parameterHashB64.isEmpty() + || !parameterHashB64.equals(currentHash))) { JDBCIdentityCache identityCache = new JDBCIdentityCacheImpl(cache, this, parameterDao, getResourceReferenceDAO()); try (ParameterVisitorBatchDAO pvd = new ParameterVisitorBatchDAO(connection, "FHIR_ADMIN", resource.getResourceType(), true, - resource.getId(), 100, identityCache, resourceReferenceDAO, this.transactionData)) { + resource.getLogicalResourceId(), 100, identityCache, resourceReferenceDAO, this.transactionData)) { for (ExtractedParameterValue p: parameters) { p.accept(pvd); } @@ -575,7 +591,7 @@ public Resource insert(Resource resource, List paramete latestTime = System.nanoTime(); double totalDuration = (latestTime - dbCallStartTime) / 1e6; double paramInsertDuration = (latestTime-paramInsertStartTime)/1e6; - log.fine("Successfully inserted Resource. id=" + resource.getId() + " total=" + totalDuration + "ms, proc=" + dbCallDuration + "ms, param=" + paramInsertDuration + "ms"); + log.fine("Successfully inserted Resource. logicalResourceId=" + resource.getLogicalResourceId() + " total=" + totalDuration + "ms, proc=" + dbCallDuration + "ms, param=" + paramInsertDuration + "ms"); } } } catch (FHIRPersistenceDBConnectException | @@ -762,7 +778,7 @@ public List searchForIds(Select dataQuery) throws FHIRPersistenceDataAcces * @param logicalResourceId * @throws SQLException */ - protected void deleteFromParameterTable(Connection conn, String tableName, long logicalResourceId) throws SQLException { + private void deleteFromParameterTable(Connection conn, String tableName, long logicalResourceId) throws SQLException { final String delStrValues = "DELETE FROM " + tableName + " WHERE logical_resource_id = ?"; try (PreparedStatement stmt = conn.prepareStatement(delStrValues)) { // bind parameters @@ -837,4 +853,39 @@ protected void setString(PreparedStatement ps, int index, String value) throws S ps.setString(index, value); } } + + @Override + public Long readLogicalResourceId(int resourceTypeId, String logicalId) throws FHIRPersistenceDBConnectException, FHIRPersistenceDataAccessException { + Long result = null; + try (PreparedStatement ps = getConnection().prepareStatement(SQL_GET_LOGICAL_RESOURCE_IDENT)) { + ps.setInt(1, resourceTypeId); + ps.setString(2, logicalId); + ResultSet rs = ps.executeQuery(); + if (rs.next()) { + result = rs.getLong(1); + } + } catch (Throwable e) { + FHIRPersistenceDataAccessException fx = new FHIRPersistenceDataAccessException("Failure retrieving logical_resource_id"); + final String errMsg = "Failure retrieving logical_resource_id from logical_resource_ident for '" + resourceTypeId + "/" + logicalId + "'"; + throw severe(log, fx, errMsg, e); + } + return result; + } + + @Override + public List readLogicalResourceIdList(String logicalId) throws FHIRPersistenceDBConnectException, FHIRPersistenceDataAccessException { + List result = new ArrayList<>(); + try (PreparedStatement ps = getConnection().prepareStatement(SQL_GET_LOGICAL_RESOURCE_IDENT_LIST)) { + ps.setString(1, logicalId); + ResultSet rs = ps.executeQuery(); + while (rs.next()) { + result.add(rs.getLong(1)); + } + } catch (Throwable e) { + FHIRPersistenceDataAccessException fx = new FHIRPersistenceDataAccessException("Failure retrieving logical_resource_id"); + final String errMsg = "Failure retrieving logical_resource_id list from logical_resource_ident for '" + logicalId + "'"; + throw severe(log, fx, errMsg, e); + } + return result; + } } \ No newline at end of file diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/impl/ResourceReferenceDAO.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/impl/ResourceReferenceDAO.java index 3abca7a9001..d2519379e64 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/impl/ResourceReferenceDAO.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/impl/ResourceReferenceDAO.java @@ -27,14 +27,18 @@ import com.ibm.fhir.database.utils.api.IDatabaseTranslator; import com.ibm.fhir.database.utils.common.DataDefinitionUtil; +import com.ibm.fhir.database.utils.common.PreparedStatementHelper; +import com.ibm.fhir.persistence.exception.FHIRPersistenceDataAccessException; import com.ibm.fhir.persistence.exception.FHIRPersistenceException; import com.ibm.fhir.persistence.jdbc.dao.api.ICommonTokenValuesCache; +import com.ibm.fhir.persistence.jdbc.dao.api.ILogicalResourceIdentCache; import com.ibm.fhir.persistence.jdbc.dao.api.INameIdCache; import com.ibm.fhir.persistence.jdbc.dao.api.IResourceReferenceDAO; +import com.ibm.fhir.persistence.jdbc.dao.api.LogicalResourceIdentKey; +import com.ibm.fhir.persistence.jdbc.dao.api.LogicalResourceIdentValue; import com.ibm.fhir.persistence.jdbc.dto.CommonTokenValue; import com.ibm.fhir.persistence.jdbc.dto.CommonTokenValueResult; import com.ibm.fhir.persistence.jdbc.exception.FHIRPersistenceDBConnectException; -import com.ibm.fhir.persistence.jdbc.exception.FHIRPersistenceDataAccessException; import com.ibm.fhir.schema.control.FhirSchemaConstants; /** @@ -70,6 +74,9 @@ public abstract class ResourceReferenceDAO implements IResourceReferenceDAO, Aut // Cache of parameter names to id private final INameIdCache parameterNameCache; + // Cache of the logical resource id values from logical_resource_ident + private final ILogicalResourceIdentCache logicalResourceIdentCache; + // The translator for the type of database we are connected to private final IDatabaseTranslator translator; @@ -78,14 +85,21 @@ public abstract class ResourceReferenceDAO implements IResourceReferenceDAO, Aut /** * Public constructor + * + * @param t * @param c + * @param schemaName + * @param cache + * @param parameterNameCache + * @param logicalResourceIdentCache */ - public ResourceReferenceDAO(IDatabaseTranslator t, Connection c, String schemaName, ICommonTokenValuesCache cache, INameIdCache parameterNameCache) { + public ResourceReferenceDAO(IDatabaseTranslator t, Connection c, String schemaName, ICommonTokenValuesCache cache, INameIdCache parameterNameCache, ILogicalResourceIdentCache logicalResourceIdentCache) { this.translator = t; + this.schemaName = schemaName; this.connection = c; this.cache = cache; this.parameterNameCache = parameterNameCache; - this.schemaName = schemaName; + this.logicalResourceIdentCache = logicalResourceIdentCache; } /** @@ -231,10 +245,10 @@ public Integer readCanonicalId(String canonicalValue) { } @Override - public void addNormalizedValues(String resourceType, Collection xrefs, Collection profileRecs, Collection tagRecs, Collection securityRecs) throws FHIRPersistenceException { + public void addNormalizedValues(String resourceType, Collection xrefs, Collection resourceRefs, Collection profileRecs, Collection tagRecs, Collection securityRecs) throws FHIRPersistenceException { // This method is only called when we're not using transaction data logger.fine("Persist parameters for this resource - no transaction data available"); - persist(xrefs, profileRecs, tagRecs, securityRecs); + persist(xrefs, resourceRefs, profileRecs, tagRecs, securityRecs); } /** @@ -250,8 +264,8 @@ protected void insertResourceTokenRefs(String resourceType, Collection xrefs) { + protected void insertRefValues(String resourceType, Collection xrefs) { // Now all the values should have ids assigned so we can go ahead and insert them // as a batch - final String tableName = "RESOURCE_TOKEN_REFS"; + final String tableName = resourceType + "_REF_VALUES"; DataDefinitionUtil.assertValidName(tableName); final String insert = "INSERT INTO " + tableName + "(" - + "parameter_name_id, logical_resource_id, common_token_value_id, ref_version_id) " - + "VALUES (?, ?, ?, ?)"; + + "parameter_name_id, logical_resource_id, ref_logical_resource_id, ref_version_id, composite_id) " + + "VALUES (?, ?, ?, ?, ?)"; try (PreparedStatement ps = connection.prepareStatement(insert)) { int count = 0; - for (ResourceTokenValueRec xr: xrefs) { - if (xr.isSystemLevel()) { - ps.setInt(1, xr.getParameterNameId()); - ps.setLong(2, xr.getLogicalResourceId()); - - // common token value can be null - if (xr.getCommonTokenValueId() != null) { - ps.setLong(3, xr.getCommonTokenValueId()); - } else { - ps.setNull(3, Types.BIGINT); - } - - // version can be null - if (xr.getRefVersionId() != null) { - ps.setInt(4, xr.getRefVersionId()); - } else { - ps.setNull(4, Types.INTEGER); - } + PreparedStatementHelper psh = new PreparedStatementHelper(ps); + for (ResourceReferenceValueRec xr: xrefs) { + psh.setInt(xr.getParameterNameId()) + .setLong(xr.getLogicalResourceId()) + .setLong(xr.getRefLogicalResourceId()) + .setInt(xr.getRefVersionId()) + .setInt(xr.getCompositeId()) + .addBatch(); - ps.addBatch(); - if (++count == BATCH_SIZE) { - ps.executeBatch(); - count = 0; - } + if (++count == BATCH_SIZE) { + ps.executeBatch(); + count = 0; } } @@ -345,7 +342,6 @@ protected void insertSystemResourceTokenRefs(String resourceType, Collection profileValues) { } } + /** + * Insert any whole-system parameters to the token_refs table + * @param resourceType + * @param xrefs + */ + protected void insertSystemResourceTokenRefs(String resourceType, Collection xrefs) { + // Now all the values should have ids assigned so we can go ahead and insert them + // as a batch + final String tableName = "RESOURCE_TOKEN_REFS"; + DataDefinitionUtil.assertValidName(tableName); + final String insert = "INSERT INTO " + tableName + "(" + + "parameter_name_id, logical_resource_id, common_token_value_id) " + + "VALUES (?, ?, ?)"; + try (PreparedStatement ps = connection.prepareStatement(insert)) { + int count = 0; + for (ResourceTokenValueRec xr: xrefs) { + if (xr.isSystemLevel()) { + ps.setInt(1, xr.getParameterNameId()); + ps.setLong(2, xr.getLogicalResourceId()); + + // common token value can be null + if (xr.getCommonTokenValueId() != null) { + ps.setLong(3, xr.getCommonTokenValueId()); + } else { + ps.setNull(3, Types.BIGINT); + } + + ps.addBatch(); + if (++count == BATCH_SIZE) { + ps.executeBatch(); + count = 0; + } + } + } + + if (count > 0) { + ps.executeBatch(); + } + } catch (SQLException x) { + logger.log(Level.SEVERE, insert, x); + throw translator.translate(x); + } + } + protected void insertResourceProfiles(String resourceType, Collection profiles) { // Now all the values should have ids assigned so we can go ahead and insert them // as a batch @@ -868,9 +908,13 @@ public void upsertCommonTokenValues(List values) { protected abstract void doCommonTokenValuesUpsert(String paramList, Collection sortedTokenValues); @Override - public void persist(Collection records, Collection profileRecs, Collection tagRecs, Collection securityRecs) throws FHIRPersistenceException { + public void persist(Collection records, Collection referenceRecords, Collection profileRecs, Collection tagRecs, Collection securityRecs) throws FHIRPersistenceException { - collectAndResolveParameterNames(records, profileRecs, tagRecs, securityRecs); + boolean gotSomething = collectAndResolveParameterNames(records, referenceRecords, profileRecs, tagRecs, securityRecs); + if (!gotSomething) { + // nothing to do + return; + } // Grab the ids for all the code-systems, and upsert any misses List systemMisses = new ArrayList<>(); @@ -887,6 +931,11 @@ public void persist(Collection records, Collection referenceMisses = new ArrayList<>(); + logicalResourceIdentCache.resolveReferenceValues(referenceRecords, referenceMisses); + upsertLogicalResourceIdents(referenceMisses); + // Process all the common canonical values List canonicalMisses = new ArrayList<>(); cache.resolveCanonicalValues(profileRecs, canonicalMisses); @@ -904,6 +953,18 @@ public void persist(Collection records, Collection> referenceRecordMap = new HashMap<>(); + for (ResourceReferenceValueRec rtv: referenceRecords) { + List list = referenceRecordMap.computeIfAbsent(rtv.getResourceType(), k -> { return new ArrayList<>(); }); + list.add(rtv); + } + + // process each list of reference values by resource type + for (Map.Entry> entry: referenceRecordMap.entrySet()) { + insertRefValues(entry.getKey(), entry.getValue()); + } + // Split profile values by resource type Map> profileMap = new HashMap<>(); for (ResourceProfileRec rtv: profileRecs) { @@ -945,15 +1006,18 @@ public void persist(Collection records, Collection records, Collection profileRecs, + private boolean collectAndResolveParameterNames(Collection records, Collection referenceRecords, Collection profileRecs, Collection tagRecs, Collection securityRecs) throws FHIRPersistenceException { List recList = new ArrayList<>(); recList.addAll(records); + recList.addAll(referenceRecords); recList.addAll(profileRecs); recList.addAll(tagRecs); recList.addAll(securityRecs); @@ -973,6 +1037,7 @@ private void collectAndResolveParameterNames(Collection r for (ResourceRefRec rec: recList) { rec.setParameterNameId(getParameterNameId(rec.getParameterName())); } + return recList.size() > 0; } @Override @@ -1021,4 +1086,149 @@ protected int getParameterNameId(String parameterName) throws FHIRPersistenceDBC * @throws FHIRPersistenceDataAccessException */ protected abstract int readOrAddParameterNameId(String parameterName) throws FHIRPersistenceDBConnectException, FHIRPersistenceDataAccessException; + + protected void upsertLogicalResourceIdents(List unresolved) throws FHIRPersistenceException { + if (unresolved.isEmpty()) { + return; + } + + // Build a unique set of logical_resource_ident keys + Set keys = unresolved.stream().map(v -> new LogicalResourceIdentValue(v.getRefResourceTypeId(), v.getRefLogicalId())).collect(Collectors.toSet()); + List missing = new ArrayList<>(keys); + // Sort the list in logicalId,resourceTypeId order + missing.sort((a,b) -> { + int result = a.getLogicalId().compareTo(b.getLogicalId()); + if (result == 0) { + result = Integer.compare(a.getResourceTypeId(), b.getResourceTypeId()); + } + return result; + }); + addMissingLogicalResourceIdents(missing); + + // Now fetch all the identity records we just created so that we can + // process the unresolved list of ResourceReferenceValueRec records + Map lrIdentMap = new HashMap<>(); + fetchLogicalResourceIdentIds(lrIdentMap, missing); + + // Now we can use the map to find the logical_resource_id for each of the unresolved + // ResourceReferenceValueRec records + for (ResourceReferenceValueRec rec: unresolved) { + LogicalResourceIdentKey key = new LogicalResourceIdentKey(rec.getRefResourceTypeId(), rec.getRefLogicalId()); + LogicalResourceIdentValue val = lrIdentMap.get(key); + if (val != null) { + rec.setRefLogicalResourceId(val.getLogicalResourceId()); + } else { + // Shouldn't happen, but be defensive in case someone breaks something + throw new FHIRPersistenceException("logical_resource_idents still missing after upsert"); + } + } + } + + + /** + * Build and prepare a statement to fetch the code_system_id and code_system_name + * from the code_systems table for all the given (unresolved) code system values + * @param values + * @return + * @throws SQLException + */ + protected PreparedStatement buildLogicalResourceIdentSelectStatement(List values) throws SQLException { + StringBuilder query = new StringBuilder(); + query.append("SELECT lri.resource_type_id, lri.logical_id, lri.logical_resource_id "); + query.append(" FROM logical_resource_ident AS lri "); + query.append(" JOIN (VALUES "); + for (int i=0; i 0) { + query.append(","); + } + query.append("(?,?)"); + } + query.append(") AS v(resource_type_id, logical_id) "); + query.append(" ON (lri.resource_type_id = v.resource_type_id AND lri.logical_id = v.logical_id)"); + PreparedStatement ps = connection.prepareStatement(query.toString()); + // bind the parameter values + int param = 1; + for (LogicalResourceIdentValue val: values) { + ps.setInt(param++, val.getResourceTypeId()); + ps.setString(param++, val.getLogicalId()); + } + + if (logger.isLoggable(Level.FINE)) { + String params = String.join(",", values.stream().map(v -> "(" + v.getResourceTypeId() + "," + v.getLogicalId() + ")").collect(Collectors.toList())); + logger.fine("ident fetch: " + query.toString() + "; params: " + params); + } + + return ps; + } + + /** + * These logical_resource_ident values weren't found in the database, so we need to try and add them. + * We have to deal with concurrency here - there's a chance another thread could also + * be trying to add them. To avoid deadlocks, it's important to do any inserts in a + * consistent order. At the end, we should be able to read back values for each entry + * @param missing + */ + protected void addMissingLogicalResourceIdents(List missing) throws FHIRPersistenceException { + + // simplified implementation which handles inserts individually + final String nextVal = translator.nextValue(schemaName, "fhir_sequence"); + StringBuilder insert = new StringBuilder(); + insert.append("INSERT INTO logical_resource_ident (resource_type_id, logical_id, logical_resource_id) VALUES (?,?,"); + insert.append(nextVal); // next sequence value + insert.append(")"); + + logger.fine(() -> "ident insert: " + insert.toString()); + try (PreparedStatement ps = connection.prepareStatement(insert.toString())) { + for (LogicalResourceIdentKey value: missing) { + ps.setInt(1, value.getResourceTypeId()); + ps.setString(2, value.getLogicalId()); + try { + ps.executeUpdate(); + } catch (SQLException x) { + if (getTranslator().isDuplicate(x)) { + // do nothing + } else { + throw x; + } + } + } + } catch (SQLException x) { + logger.log(Level.SEVERE, "logical_resource_ident insert failed: " + insert.toString(), x); + throw new FHIRPersistenceException("logical_resource_ident insert failed"); + } + } + + protected void fetchLogicalResourceIdentIds(Map lrIdentMap, List unresolved) throws FHIRPersistenceException { + + int resultCount = 0; + final int maxValuesPerStatement = 512; + int offset = 0; + while (offset < unresolved.size()) { + int remaining = unresolved.size() - offset; + int subSize = Math.min(remaining, maxValuesPerStatement); + List sub = unresolved.subList(offset, offset+subSize); // remember toIndex is exclusive + offset += subSize; // set up for the next iteration + try (PreparedStatement ps = buildLogicalResourceIdentSelectStatement(sub)) { + ResultSet rs = ps.executeQuery(); + // We can't rely on the order of result rows matching the order of the in-list, + // so we have to go back to our map to look up each LogicalResourceIdentValue + while (rs.next()) { + resultCount++; + final int resourceTypeId = rs.getInt(1); + final String logicalId = rs.getString(2); + LogicalResourceIdentKey key = new LogicalResourceIdentKey(resourceTypeId, logicalId); + LogicalResourceIdentValue identValue = new LogicalResourceIdentValue(resourceTypeId, logicalId); + identValue.setLogicalResourceId(rs.getLong(3)); + lrIdentMap.put(key, identValue); + } + } catch (SQLException x) { + logger.log(Level.SEVERE, "logical resource ident fetch failed", x); + throw new FHIRPersistenceException("logical resource ident fetch failed"); + } + } + // quick check to make sure we got everything we expected + if (resultCount < unresolved.size()) { + throw new FHIRPersistenceException("logical_resource_ident fetch did not fetch everything expected"); + } + } } \ No newline at end of file diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/impl/ResourceReferenceValueRec.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/impl/ResourceReferenceValueRec.java new file mode 100644 index 00000000000..08cac7115b1 --- /dev/null +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/impl/ResourceReferenceValueRec.java @@ -0,0 +1,100 @@ +/* + * (C) Copyright IBM Corp. 2020, 2021 + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.ibm.fhir.persistence.jdbc.dao.impl; + + +/** + * A DTO representing a mapping of a resource and reference value. The + * record is used to drive the population of the xx_ref_values tables + */ +public class ResourceReferenceValueRec extends ResourceRefRec { + + private final String refResourceType; + private final int refResourceTypeId; + + // The external ref value and its normalized database id (when we have it) + private final String refLogicalId; + private Long refLogicalResourceId; + private final Integer refVersionId; + + // Issue 1683 - optional composite id used to correlate parameters + private final Integer compositeId; + + /** + * Public constructor. Used to create a versioned reference record + * @param parameterName + * @param resourceType + * @param resourceTypeId + * @param logicalResourceId + * @param refResourceType + * @param refResourceTypeId + * @param refLogicalId + * @param refVersionId + * @param compositeId + */ + public ResourceReferenceValueRec(String parameterName, String resourceType, long resourceTypeId, long logicalResourceId, + String refResourceType, int refResourceTypeId, String refLogicalId, Integer refVersionId, Integer compositeId) { + super(parameterName, resourceType, resourceTypeId, logicalResourceId); + this.refResourceType = refResourceType; + this.refResourceTypeId = refResourceTypeId; + this.refLogicalId = refLogicalId; + this.refVersionId = refVersionId; + this.compositeId = compositeId; + } + + /** + * Get the refLogicalResourceId + * @return + */ + public Long getRefLogicalResourceId() { + return refLogicalResourceId; + } + + /** + * Sets the database id for the referenced logical resource + * @param refLogicalResourceId to set + */ + public void setRefLogicalResourceId(long refLogicalResourceId) { + // because we're setting this, it can no longer be null + this.refLogicalResourceId = refLogicalResourceId; + } + + /** + * @return the refVersionId + */ + public Integer getRefVersionId() { + return refVersionId; + } + + /** + * @return the compositeId + */ + public Integer getCompositeId() { + return compositeId; + } + /** + * @return the refResourceType + */ + public String getRefResourceType() { + return refResourceType; + } + + + /** + * @return the refLogicalId + */ + public String getRefLogicalId() { + return refLogicalId; + } + + /** + * @return the refResourceTypeId + */ + public int getRefResourceTypeId() { + return refResourceTypeId; + } +} diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/impl/ResourceTokenValueRec.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/impl/ResourceTokenValueRec.java index 5e08dd18efc..5f63e4d9616 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/impl/ResourceTokenValueRec.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dao/impl/ResourceTokenValueRec.java @@ -21,7 +21,6 @@ public class ResourceTokenValueRec extends ResourceRefRec { // The external ref value and its normalized database id (when we have it) private final String tokenValue; private Long commonTokenValueId; - private final Integer refVersionId; // Issue 1683 - optional composite id used to correlate parameters private final Integer compositeId; @@ -30,39 +29,21 @@ public class ResourceTokenValueRec extends ResourceRefRec { private final boolean systemLevel; /** - * Public constructor - * @param parameterName - * @param resourceType - * @param resourceTypeId - * @param logicalResourceId - * @param codeSystem - * @param externalRefValue - * @param compositeId - * @param systemLevel - */ - public ResourceTokenValueRec(String parameterName, String resourceType, long resourceTypeId, long logicalResourceId, - String codeSystem, String externalRefValue, Integer compositeId, boolean systemLevel) { - this(parameterName, resourceType, resourceTypeId, logicalResourceId, codeSystem, externalRefValue, null, compositeId, systemLevel); - } - - /** - * Public constructor. Used to create a versioned resource reference + * Public constructor. * @param parameterName * @param resourceType * @param resourceTypeId * @param logicalResourceId * @param externalSystemName * @param externalRefValue - * @param refVersionId * @param compositeId * @param systemLevel */ public ResourceTokenValueRec(String parameterName, String resourceType, long resourceTypeId, long logicalResourceId, - String externalSystemName, String externalRefValue, Integer refVersionId, Integer compositeId, boolean systemLevel) { + String externalSystemName, String externalRefValue, Integer compositeId, boolean systemLevel) { super(parameterName, resourceType, resourceTypeId, logicalResourceId); this.codeSystemValue = externalSystemName; this.tokenValue = externalRefValue; - this.refVersionId = refVersionId; this.compositeId = compositeId; this.systemLevel = systemLevel; } @@ -111,13 +92,6 @@ public void setCommonTokenValueId(long commonTokenValueId) { this.commonTokenValueId = commonTokenValueId; } - /** - * @return the refVersionId - */ - public Integer getRefVersionId() { - return refVersionId; - } - /** * @return the compositeId */ diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/db2/Db2ResourceReferenceDAO.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/db2/Db2ResourceReferenceDAO.java index 15f737af91e..837026d6f65 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/db2/Db2ResourceReferenceDAO.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/db2/Db2ResourceReferenceDAO.java @@ -10,7 +10,6 @@ import java.sql.PreparedStatement; import java.sql.SQLException; import java.sql.Types; -import java.util.ArrayList; import java.util.Collection; import java.util.List; import java.util.logging.Level; @@ -18,17 +17,22 @@ import com.ibm.fhir.database.utils.api.IDatabaseTranslator; import com.ibm.fhir.database.utils.common.DataDefinitionUtil; +import com.ibm.fhir.database.utils.common.PreparedStatementHelper; +import com.ibm.fhir.persistence.exception.FHIRPersistenceDataAccessException; +import com.ibm.fhir.persistence.exception.FHIRPersistenceException; import com.ibm.fhir.persistence.jdbc.dao.api.ICommonTokenValuesCache; +import com.ibm.fhir.persistence.jdbc.dao.api.ILogicalResourceIdentCache; import com.ibm.fhir.persistence.jdbc.dao.api.INameIdCache; -import com.ibm.fhir.persistence.jdbc.dao.api.JDBCIdentityCache; +import com.ibm.fhir.persistence.jdbc.dao.api.LogicalResourceIdentKey; +import com.ibm.fhir.persistence.jdbc.dao.api.LogicalResourceIdentValue; import com.ibm.fhir.persistence.jdbc.dao.api.ParameterNameDAO; import com.ibm.fhir.persistence.jdbc.dao.impl.ParameterNameDAOImpl; import com.ibm.fhir.persistence.jdbc.dao.impl.ResourceProfileRec; import com.ibm.fhir.persistence.jdbc.dao.impl.ResourceReferenceDAO; +import com.ibm.fhir.persistence.jdbc.dao.impl.ResourceReferenceValueRec; import com.ibm.fhir.persistence.jdbc.dao.impl.ResourceTokenValueRec; import com.ibm.fhir.persistence.jdbc.dto.CommonTokenValue; import com.ibm.fhir.persistence.jdbc.exception.FHIRPersistenceDBConnectException; -import com.ibm.fhir.persistence.jdbc.exception.FHIRPersistenceDataAccessException; /** @@ -48,8 +52,9 @@ public class Db2ResourceReferenceDAO extends ResourceReferenceDAO { * @param schemaName * @param cache */ - public Db2ResourceReferenceDAO(IDatabaseTranslator t, Connection c, String schemaName, ICommonTokenValuesCache cache, String adminSchemaName, INameIdCache parameterNameCache) { - super(t, c, schemaName, cache, parameterNameCache); + public Db2ResourceReferenceDAO(IDatabaseTranslator t, Connection c, String schemaName, ICommonTokenValuesCache cache, String adminSchemaName, INameIdCache parameterNameCache, + ILogicalResourceIdentCache logicalResourceIdentCache) { + super(t, c, schemaName, cache, parameterNameCache, logicalResourceIdentCache); this.adminSchemaName = adminSchemaName; } @@ -171,8 +176,8 @@ protected void insertResourceTokenRefs(String resourceType, Collection missing) throws FHIRPersistenceException { + + // simplified implementation which handles inserts individually + final String nextVal = getTranslator().nextValue(getSchemaName(), "fhir_sequence"); + StringBuilder insert = new StringBuilder(); + insert.append("INSERT INTO logical_resource_ident (mt_id, resource_type_id, logical_id, logical_resource_id) VALUES ("); + insert.append(adminSchemaName).append(".SV_TENANT_ID, ?,?,"); + insert.append(nextVal); // next sequence value + insert.append(")"); + + logger.fine(() -> "ident insert: " + insert.toString()); + final String dml = insert.toString(); + try (PreparedStatement ps = getConnection().prepareStatement(dml)) { + for (LogicalResourceIdentKey value: missing) { + ps.setInt(1, value.getResourceTypeId()); + ps.setString(2, value.getLogicalId()); + try { + ps.executeUpdate(); + } catch (SQLException x) { + if (getTranslator().isDuplicate(x)) { + // do nothing + } else { + throw x; + } + } + } + } catch (SQLException x) { + logger.log(Level.SEVERE, "logical_resource_ident insert failed: " + dml, x); + throw new FHIRPersistenceException("logical_resource_ident insert failed"); + } + } + + @Override + protected void insertRefValues(String resourceType, Collection xrefs) { + // Now all the values should have ids assigned so we can go ahead and insert them + // as a batch. This is the multitenant variant, so we need to inject the mt_id value + final String tableName = resourceType + "_REF_VALUES"; + DataDefinitionUtil.assertValidName(tableName); + final String insert = "INSERT INTO " + tableName + "(mt_id, " + + "parameter_name_id, logical_resource_id, ref_logical_resource_id, ref_version_id, composite_id) " + + "VALUES (" + adminSchemaName + ".SV_TENANT_ID, ?, ?, ?, ?, ?)"; + try (PreparedStatement ps = getConnection().prepareStatement(insert)) { + int count = 0; + PreparedStatementHelper psh = new PreparedStatementHelper(ps); + for (ResourceReferenceValueRec xr: xrefs) { + psh.setInt(xr.getParameterNameId()) + .setLong(xr.getLogicalResourceId()) + .setLong(xr.getRefLogicalResourceId()) + .setInt(xr.getRefVersionId()) + .setInt(xr.getCompositeId()) + .addBatch(); + + if (++count == BATCH_SIZE) { + ps.executeBatch(); + count = 0; + } + } + + if (count > 0) { + ps.executeBatch(); + } + } catch (SQLException x) { + logger.log(Level.SEVERE, insert, x); + throw getTranslator().translate(x); + } + } } \ No newline at end of file diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/derby/DerbyCodeSystemDAO.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/derby/DerbyCodeSystemDAO.java index 831f084e3b6..33ec752c1f6 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/derby/DerbyCodeSystemDAO.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/derby/DerbyCodeSystemDAO.java @@ -11,9 +11,9 @@ import java.sql.ResultSet; import java.sql.SQLException; +import com.ibm.fhir.persistence.exception.FHIRPersistenceDataAccessException; import com.ibm.fhir.persistence.jdbc.dao.api.FhirRefSequenceDAO; import com.ibm.fhir.persistence.jdbc.dao.impl.CodeSystemDAOImpl; -import com.ibm.fhir.persistence.jdbc.exception.FHIRPersistenceDataAccessException; /** * Derby variant DAO used to manage code_systems records. Uses diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/derby/DerbyParameterNamesDAO.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/derby/DerbyParameterNamesDAO.java index 996965a81f3..450772adb2f 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/derby/DerbyParameterNamesDAO.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/derby/DerbyParameterNamesDAO.java @@ -11,9 +11,9 @@ import java.sql.ResultSet; import java.sql.SQLException; +import com.ibm.fhir.persistence.exception.FHIRPersistenceDataAccessException; import com.ibm.fhir.persistence.jdbc.dao.api.FhirRefSequenceDAO; import com.ibm.fhir.persistence.jdbc.dao.impl.ParameterNameDAOImpl; -import com.ibm.fhir.persistence.jdbc.exception.FHIRPersistenceDataAccessException; /** * For R4 we have replaced the old Derby (Java) stored procedure with diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/derby/DerbyResourceDAO.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/derby/DerbyResourceDAO.java index f9d7e70ad78..b157878c475 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/derby/DerbyResourceDAO.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/derby/DerbyResourceDAO.java @@ -28,8 +28,10 @@ import com.ibm.fhir.database.utils.derby.DerbyMaster; import com.ibm.fhir.database.utils.derby.DerbyTranslator; import com.ibm.fhir.persistence.InteractionStatus; +import com.ibm.fhir.persistence.exception.FHIRPersistenceDataAccessException; import com.ibm.fhir.persistence.exception.FHIRPersistenceException; import com.ibm.fhir.persistence.exception.FHIRPersistenceVersionIdMismatchException; +import com.ibm.fhir.persistence.index.FHIRRemoteIndexService; import com.ibm.fhir.persistence.jdbc.FHIRPersistenceJDBCCache; import com.ibm.fhir.persistence.jdbc.connection.FHIRDbFlavor; import com.ibm.fhir.persistence.jdbc.dao.api.FHIRDAOConstants; @@ -42,7 +44,6 @@ import com.ibm.fhir.persistence.jdbc.dto.ExtractedParameterValue; import com.ibm.fhir.persistence.jdbc.dto.Resource; import com.ibm.fhir.persistence.jdbc.exception.FHIRPersistenceDBConnectException; -import com.ibm.fhir.persistence.jdbc.exception.FHIRPersistenceDataAccessException; import com.ibm.fhir.persistence.jdbc.exception.FHIRPersistenceFKVException; import com.ibm.fhir.persistence.jdbc.impl.ParameterTransactionDataImpl; import com.ibm.fhir.persistence.jdbc.util.ParameterTableSupport; @@ -107,7 +108,7 @@ public Resource insert(Resource resource, List paramete AtomicInteger outInteractionStatus = new AtomicInteger(); AtomicInteger outIfNoneMatchVersion = new AtomicInteger(); - long resourceId = this.storeResource(resource.getResourceType(), + long logicalResourceId = this.storeResource(resource.getResourceType(), parameters, resource.getLogicalId(), resource.getDataStream() != null ? resource.getDataStream().inputStream() : null, @@ -131,11 +132,11 @@ public Resource insert(Resource resource, List paramete resource.setIfNoneMatchVersion(outIfNoneMatchVersion.get()); } else { resource.setInteractionStatus(InteractionStatus.MODIFIED); - resource.setId(resourceId); + resource.setLogicalResourceId(logicalResourceId); } if (logger.isLoggable(Level.FINE)) { - logger.fine("Successfully inserted Resource. id=" + resource.getId() + " executionTime=" + dbCallDuration + "ms"); + logger.fine("Successfully inserted Resource. logicalResourceId=" + resource.getLogicalResourceId() + " executionTime=" + dbCallDuration + "ms"); } } catch(FHIRPersistenceDBConnectException | FHIRPersistenceDataAccessException e) { throw e; @@ -198,7 +199,7 @@ public Resource insert(Resource resource, List paramete * @param resourcePayloadKey * @param outInteractionStatus * @param outIfNoneMatchVersion - * @return the resource_id for the entry we created + * @return the logical_resource_id for the entry we created * @throws Exception */ public long storeResource(String tablePrefix, List parameters, @@ -255,10 +256,10 @@ public long storeResource(String tablePrefix, List para // Get a lock at the system-wide logical resource level. Note the Derby-specific syntax if (logger.isLoggable(Level.FINEST)) { - logger.finest("Getting LOGICAL_RESOURCES row lock for: " + v_resource_type + "/" + p_logical_id); + logger.finest("Getting LOGICAL_RESOURCE_IDENT row lock for: " + v_resource_type + "/" + p_logical_id); } - final String SELECT_FOR_UPDATE = "SELECT logical_resource_id, parameter_hash, is_deleted" - + " FROM logical_resources" + final String SELECT_FOR_UPDATE = "SELECT logical_resource_id" + + " FROM logical_resource_ident " + " WHERE resource_type_id = ? AND logical_id = ?" + " FOR UPDATE WITH RS"; try (PreparedStatement stmt = conn.prepareStatement(SELECT_FOR_UPDATE)) { @@ -270,8 +271,6 @@ public long storeResource(String tablePrefix, List para logger.finest("Resource locked: " + v_resource_type + "/" + p_logical_id); } v_logical_resource_id = rs.getLong(1); - currentParameterHash = rs.getString(2); - v_currently_deleted = "Y".equals(rs.getString(3)); } else { if (logger.isLoggable(Level.FINEST)) { @@ -297,25 +296,21 @@ public long storeResource(String tablePrefix, List para } } } - - // insert the system-wide logical resource record - final String sql4 = "INSERT INTO logical_resources (logical_resource_id, resource_type_id, logical_id, reindex_tstamp, is_deleted, last_updated, parameter_hash) VALUES (?, ?, ?, ?, ?, ?, ?)"; + // insert the logical_resource_ident record (which we now do our locking on) + final String INS_IDENT = "INSERT INTO logical_resource_ident (resource_type_id, logical_id, logical_resource_id) VALUES (?, ?, ?)"; if (logger.isLoggable(Level.FINEST)) { - logger.finest("Creating new logical_resources row for: " + v_resource_type + "/" + p_logical_id); + logger.finest("Creating new logical_resource_ident row for: " + v_resource_type + "/" + p_logical_id + " => logical_resource_id=" + v_logical_resource_id); } - try (PreparedStatement stmt = conn.prepareStatement(sql4)) { + + try (PreparedStatement stmt = conn.prepareStatement(INS_IDENT)) { // bind parameters - stmt.setLong(1, v_logical_resource_id); - stmt.setInt(2, v_resource_type_id); - stmt.setString(3, p_logical_id); - stmt.setTimestamp(4, Timestamp.valueOf(DEFAULT_VALUE_REINDEX_TSTAMP), UTC); - stmt.setString(5, p_is_deleted ? "Y" : "N"); // from V0014 - stmt.setTimestamp(6, p_last_updated, UTC); // from V0014 - stmt.setString(7, p_parameterHashB64); // from V0015 + stmt.setInt(1, v_resource_type_id); + stmt.setString(2, p_logical_id); + stmt.setLong(3, v_logical_resource_id); stmt.executeUpdate(); if (logger.isLoggable(Level.FINEST)) { - logger.finest("Created logical_resources row for: " + v_resource_type + "/" + p_logical_id); + logger.finest("Created logical_resource_ident row for: " + v_resource_type + "/" + p_logical_id); } } catch (SQLException e) { if (translator.isDuplicate(e)) { @@ -348,41 +343,88 @@ public long storeResource(String tablePrefix, List para logger.finest("Resource locked: " + v_resource_type + "/" + p_logical_id); } v_logical_resource_id = res.getLong(1); - currentParameterHash = res.getString(2); - v_currently_deleted = "Y".equals(res.getString(3)); } else { // Extremely unlikely as we should never delete logical resource records throw new IllegalStateException("Logical resource was deleted: " + tablePrefix + "/" + p_logical_id); } } } - } else { - v_new_resource = true; + } + } - // Insert the resource-specific logical resource record. Remember that logical_id is denormalized - // so it gets stored again here for convenience + // At this point we have an exclusive lock at the logical resource level, so we + // no longer have to worry about concurrency issues. Let's see if we have a + // logical_resources entry: + final String SELECT_LOGICAL_RESOURCE = "" + + "SELECT parameter_hash, is_deleted" + + " FROM logical_resources " + + " WHERE logical_resource_id = ?"; + try (PreparedStatement stmt = conn.prepareStatement(SELECT_LOGICAL_RESOURCE)) { + stmt.setLong(1, v_logical_resource_id); + ResultSet rs = stmt.executeQuery(); + if (rs.next()) { if (logger.isLoggable(Level.FINEST)) { - logger.finest("Creating " + tablePrefix + "_logical_resources row: " + v_resource_type + "/" + p_logical_id); + logger.finest("Found logical_resources record for: " + v_resource_type + "/" + p_logical_id); } - final String sql5 = "INSERT INTO " + tablePrefix + "_logical_resources (logical_resource_id, logical_id, is_deleted, last_updated, version_id, current_resource_id) VALUES (?, ?, ?, ?, ?, ?)"; - try (PreparedStatement stmt = conn.prepareStatement(sql5)) { - // bind parameters - stmt.setLong(1, v_logical_resource_id); - stmt.setString(2, p_logical_id); - stmt.setString(3, p_is_deleted ? "Y" : "N"); - stmt.setTimestamp(4, p_last_updated, UTC); - stmt.setInt(5, p_version); // initial version - stmt.setLong(6, v_resource_id); - stmt.executeUpdate(); - if (logger.isLoggable(Level.FINEST)) { - logger.finest("Created " + tablePrefix + "_logical_resources row: " + v_resource_type + "/" + p_logical_id); - } + currentParameterHash = rs.getString(1); + v_currently_deleted = "Y".equals(rs.getString(2)); + v_not_found = false; + } + else { + if (logger.isLoggable(Level.FINEST)) { + logger.finest("Logical_resources record not found for: " + v_resource_type + "/" + p_logical_id); } + v_not_found = true; } } - // We have a lock at the logical resource level so no concurrency issues here + if (v_not_found) { + // insert the system-wide logical resource record + final String sql4 = "INSERT INTO logical_resources (logical_resource_id, resource_type_id, logical_id, reindex_tstamp, is_deleted, last_updated, parameter_hash) VALUES (?, ?, ?, ?, ?, ?, ?)"; + if (logger.isLoggable(Level.FINEST)) { + logger.finest("Creating new logical_resources row for: " + v_resource_type + "/" + p_logical_id); + } + try (PreparedStatement stmt = conn.prepareStatement(sql4)) { + // bind parameters + stmt.setLong(1, v_logical_resource_id); + stmt.setInt(2, v_resource_type_id); + stmt.setString(3, p_logical_id); + stmt.setTimestamp(4, Timestamp.valueOf(DEFAULT_VALUE_REINDEX_TSTAMP), UTC); + stmt.setString(5, p_is_deleted ? "Y" : "N"); // from V0014 + stmt.setTimestamp(6, p_last_updated, UTC); // from V0014 + stmt.setString(7, p_parameterHashB64); // from V0015 + stmt.executeUpdate(); + + if (logger.isLoggable(Level.FINEST)) { + logger.finest("Created logical_resources row for: " + v_resource_type + "/" + p_logical_id); + } + + v_new_resource = true; + } + + // Insert the resource-specific logical resource record. Remember that logical_id is denormalized + // so it gets stored again here for convenience + if (logger.isLoggable(Level.FINEST)) { + logger.finest("Creating " + tablePrefix + "_logical_resources row: " + v_resource_type + "/" + p_logical_id); + } + final String sql5 = "INSERT INTO " + tablePrefix + "_logical_resources (logical_resource_id, logical_id, is_deleted, last_updated, version_id, current_resource_id) VALUES (?, ?, ?, ?, ?, ?)"; + try (PreparedStatement stmt = conn.prepareStatement(sql5)) { + // bind parameters + stmt.setLong(1, v_logical_resource_id); + stmt.setString(2, p_logical_id); + stmt.setString(3, p_is_deleted ? "Y" : "N"); + stmt.setTimestamp(4, p_last_updated, UTC); + stmt.setInt(5, p_version); // initial version + stmt.setLong(6, v_resource_id); + stmt.executeUpdate(); + if (logger.isLoggable(Level.FINEST)) { + logger.finest("Created " + tablePrefix + "_logical_resources row: " + v_resource_type + "/" + p_logical_id); + } + } + + } + // For existing resources, we need to know the current resource version_id if (!v_new_resource) { // existing resource. We need to know the current version from the // resource-specific logical resources table. @@ -504,7 +546,8 @@ public long storeResource(String tablePrefix, List para // To keep things simple for the Derby use-case, we just use a visitor to // handle inserts of parameters directly in the resource parameter tables. // Note we don't get any parameters for the resource soft-delete operation - if (parameters != null && requireParameterUpdate) { + FHIRRemoteIndexService remoteIndexService = FHIRRemoteIndexService.getServiceInstance(); + if (remoteIndexService == null && parameters != null && requireParameterUpdate) { // Derby doesn't support partitioned multi-tenancy, so we disable it on the DAO: if (logger.isLoggable(Level.FINEST)) { logger.finest("Storing parameters for: " + v_resource_type + "/" + p_logical_id); @@ -534,7 +577,7 @@ identityCache, getResourceReferenceDAO(), getTransactionData())) { } logger.exiting(CLASSNAME, METHODNAME); - return v_resource_id; + return v_logical_resource_id; } /** diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/derby/DerbyResourceReferenceDAO.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/derby/DerbyResourceReferenceDAO.java index 11ab308c6a7..ac49d32151c 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/derby/DerbyResourceReferenceDAO.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/derby/DerbyResourceReferenceDAO.java @@ -25,15 +25,19 @@ import java.util.stream.Collectors; import com.ibm.fhir.database.utils.api.IDatabaseTranslator; +import com.ibm.fhir.persistence.exception.FHIRPersistenceDataAccessException; +import com.ibm.fhir.persistence.exception.FHIRPersistenceException; import com.ibm.fhir.persistence.jdbc.dao.api.ICommonTokenValuesCache; +import com.ibm.fhir.persistence.jdbc.dao.api.ILogicalResourceIdentCache; import com.ibm.fhir.persistence.jdbc.dao.api.INameIdCache; +import com.ibm.fhir.persistence.jdbc.dao.api.LogicalResourceIdentKey; +import com.ibm.fhir.persistence.jdbc.dao.api.LogicalResourceIdentValue; import com.ibm.fhir.persistence.jdbc.dao.api.ParameterNameDAO; import com.ibm.fhir.persistence.jdbc.dao.impl.ResourceReferenceDAO; import com.ibm.fhir.persistence.jdbc.dao.impl.ResourceTokenValueRec; import com.ibm.fhir.persistence.jdbc.dto.CommonTokenValue; import com.ibm.fhir.persistence.jdbc.dto.CommonTokenValueResult; import com.ibm.fhir.persistence.jdbc.exception.FHIRPersistenceDBConnectException; -import com.ibm.fhir.persistence.jdbc.exception.FHIRPersistenceDataAccessException; /** @@ -51,9 +55,12 @@ public class DerbyResourceReferenceDAO extends ResourceReferenceDAO { * @param c * @param schemaName * @param cache + * @param parameterNameCache + * @param logicalResourceIdentCache */ - public DerbyResourceReferenceDAO(IDatabaseTranslator t, Connection c, String schemaName, ICommonTokenValuesCache cache, INameIdCache parameterNameCache) { - super(t, c, schemaName, cache, parameterNameCache); + public DerbyResourceReferenceDAO(IDatabaseTranslator t, Connection c, String schemaName, ICommonTokenValuesCache cache, INameIdCache parameterNameCache, + ILogicalResourceIdentCache logicalResourceIdentCache) { + super(t, c, schemaName, cache, parameterNameCache, logicalResourceIdentCache); } @Override @@ -294,4 +301,66 @@ protected int readOrAddParameterNameId(String parameterName) throws FHIRPersiste final ParameterNameDAO pnd = new DerbyParameterNamesDAO(getConnection(), getSchemaName()); return pnd.readOrAddParameterNameId(parameterName); } + + @Override + protected PreparedStatement buildLogicalResourceIdentSelectStatement(List values) throws SQLException { + // Derby doesn't support a VALUES table list, so instead we simply build a big + // OR predicate + StringBuilder query = new StringBuilder(); + query.append("SELECT lri.resource_type_id, lri.logical_id, lri.logical_resource_id "); + query.append(" FROM logical_resource_ident AS lri "); + query.append(" WHERE "); + for (int i=0; i 0) { + query.append(" OR "); + } + query.append("(resource_type_id = ? AND logical_id = ?)"); + } + PreparedStatement ps = getConnection().prepareStatement(query.toString()); + // bind the parameter values + int param = 1; + for (LogicalResourceIdentValue val: values) { + ps.setInt(param++, val.getResourceTypeId()); + ps.setString(param++, val.getLogicalId()); + } + + if (logger.isLoggable(Level.FINE)) { + String params = String.join(",", values.stream().map(v -> "(" + v.getResourceTypeId() + "," + v.getLogicalId() + ")").collect(Collectors.toList())); + logger.fine("ident fetch: " + query.toString() + "; params: " + params); + } + + return ps; + } + + @Override + protected void fetchLogicalResourceIdentIds(Map lrIdentMap, List unresolved) throws FHIRPersistenceException { + // For Derby, we opt to do this row by row so that we can keep the selects in order which + // helps us to avoid deadlocks due to lock compatibility issues with Derby + StringBuilder query = new StringBuilder(); + query.append("SELECT lri.logical_resource_id "); + query.append(" FROM logical_resource_ident AS lri "); + query.append(" WHERE lri.resource_type_id = ? AND lri.logical_id = ?"); + final String sql = query.toString(); + try (PreparedStatement ps = getConnection().prepareStatement(sql)) { + for (LogicalResourceIdentValue value: unresolved) { + ps.setInt(1, value.getResourceTypeId()); + ps.setString(2, value.getLogicalId()); + ResultSet rs = ps.executeQuery(); + if (rs.next()) { + final long logicalResourceId = rs.getLong(1); + LogicalResourceIdentKey key = new LogicalResourceIdentKey(value.getResourceTypeId(), value.getLogicalId()); + value.setLogicalResourceId(logicalResourceId); + lrIdentMap.put(key, value); + } else { + // something wrong with our data handling code because we should already have values + // for every logical resource at this point + throw new FHIRPersistenceException("logical_resource_ident record missing: resourceTypeId[" + + value.getResourceTypeId() + "] logicalId[" + value.getLogicalId() + "]"); + } + } + } catch (SQLException x) { + logger.log(Level.SEVERE, "logical resource ident fetch failed", x); + throw new FHIRPersistenceException("logical resource ident fetch failed"); + } + } } \ No newline at end of file diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/domain/SearchQueryRenderer.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/domain/SearchQueryRenderer.java index beba46d627a..50eb1b4067a 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/domain/SearchQueryRenderer.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/domain/SearchQueryRenderer.java @@ -21,12 +21,16 @@ import static com.ibm.fhir.persistence.jdbc.JDBCConstants.ESCAPE_UNDERSCORE; import static com.ibm.fhir.persistence.jdbc.JDBCConstants.IS_DELETED; import static com.ibm.fhir.persistence.jdbc.JDBCConstants.LEFT_PAREN; +import static com.ibm.fhir.persistence.jdbc.JDBCConstants.LOGICAL_ID; import static com.ibm.fhir.persistence.jdbc.JDBCConstants.MAX; import static com.ibm.fhir.persistence.jdbc.JDBCConstants.MIN; import static com.ibm.fhir.persistence.jdbc.JDBCConstants.NUMBER_VALUE; import static com.ibm.fhir.persistence.jdbc.JDBCConstants.PARAMETER_NAME_ID; import static com.ibm.fhir.persistence.jdbc.JDBCConstants.PERCENT_WILDCARD; import static com.ibm.fhir.persistence.jdbc.JDBCConstants.QUANTITY_VALUE; +import static com.ibm.fhir.persistence.jdbc.JDBCConstants.REF_LOGICAL_RESOURCE_ID; +import static com.ibm.fhir.persistence.jdbc.JDBCConstants.REF_VALUE; +import static com.ibm.fhir.persistence.jdbc.JDBCConstants.RESOURCE_TYPE_ID; import static com.ibm.fhir.persistence.jdbc.JDBCConstants.RIGHT_PAREN; import static com.ibm.fhir.persistence.jdbc.JDBCConstants.TOKEN_VALUE; import static com.ibm.fhir.persistence.jdbc.JDBCConstants.UNDERSCORE_WILDCARD; @@ -69,14 +73,18 @@ import com.ibm.fhir.database.utils.query.expression.StringExpNodeVisitor; import com.ibm.fhir.database.utils.query.node.ExpNode; import com.ibm.fhir.model.resource.CodeSystem; +import com.ibm.fhir.model.resource.OperationOutcome.Issue; import com.ibm.fhir.model.resource.Resource; import com.ibm.fhir.model.type.Code; +import com.ibm.fhir.model.type.code.IssueSeverity; +import com.ibm.fhir.model.type.code.IssueType; import com.ibm.fhir.persistence.exception.FHIRPersistenceException; import com.ibm.fhir.persistence.exception.FHIRPersistenceNotSupportedException; import com.ibm.fhir.persistence.jdbc.JDBCConstants; import com.ibm.fhir.persistence.jdbc.dao.api.JDBCIdentityCache; import com.ibm.fhir.persistence.jdbc.dao.impl.ResourceProfileRec; import com.ibm.fhir.persistence.jdbc.dto.CommonTokenValue; +import com.ibm.fhir.persistence.jdbc.dto.ResourceReferenceValue; import com.ibm.fhir.persistence.jdbc.util.CanonicalSupport; import com.ibm.fhir.persistence.jdbc.util.CanonicalValue; import com.ibm.fhir.persistence.jdbc.util.NewUriModifierUtil; @@ -98,6 +106,7 @@ import com.ibm.fhir.search.parameters.QueryParameter; import com.ibm.fhir.search.parameters.QueryParameterValue; import com.ibm.fhir.search.sort.Sort.Direction; +import com.ibm.fhir.search.util.ReferenceUtil; import com.ibm.fhir.search.util.SearchHelper; import com.ibm.fhir.term.util.CodeSystemSupport; import com.ibm.fhir.term.util.ValueSetSupport; @@ -229,6 +238,25 @@ protected Set getCommonTokenValueIds(Collection tokenVal return this.identityCache.getCommonTokenValueIds(tokenValues); } + /** + * Obtain the logical_resource_id values for each of the given ResourceReferenceValues. + * @param referenceValues + * @return + * @throws FHIRPersistenceException + */ + protected Set getLogicalResourceIds(Collection referenceValues) throws FHIRPersistenceException { + return this.identityCache.getLogicalResourceIds(referenceValues); + } + + /** + * Obtain the list of logical_resource_id values that match the given logicalId. + * @param logicalId + * @return + */ + protected List getLogicalResourceIdList(String logicalId) throws FHIRPersistenceException { + return this.identityCache.getLogicalResourceIdList(logicalId); + } + /** * Get a list of common token values matching the given code * @param code @@ -713,6 +741,25 @@ private void addCommonTokenValueIdFilter(WhereFragment where, String paramAlias, addCommonTokenValueIdFilter(where, paramAlias, ctvs); } + /** + * Adds a filter predicate for ref_logical_resource_id. Fetches the list of posible matches (there's no resourceType, + * so there could be multiple matches. If no match, then -1 is used to make sure the row isn't produced. If there is + * single match, the predicate uses an equality, otherwise an IN-LIST. + * The query uses literal values not bind variables on purpose (better performance). + * @param where + * @param paramAlias + * @param searchValue + * @throws FHIRPersistenceException + */ + private void addLogicalResourceIdFilter(WhereFragment where, String paramAlias, String searchValue) throws FHIRPersistenceException { + // grab the list of all matching common_token_value_id values + Set ctvs = new HashSet<>(); + fetchLogicalResourceValues(ctvs, searchValue); + + // and add a filter expression paramAlias IN (...) for the values + addRefLogicalResourceIdFilter(where, paramAlias, ctvs); + } + /** * Add all common_token_value_id matching the given searchValue to the ctvs set. * @param ctvs @@ -724,6 +771,17 @@ private void fetchCommonTokenValues(Set ctvs, String searchValue) throws F ctvs.addAll(ctvList); } + /** + * All all matching logical_resource_id values to the given set + * @param lrids + * @param searchValue + * @throws FHIRPersistenceException + */ + private void fetchLogicalResourceValues(Set lrids, String searchValue) throws FHIRPersistenceException { + List tmpList = this.identityCache.getLogicalResourceIdList(searchValue); + lrids.addAll(tmpList); + } + /** * Adds a filter predicate for COMMON_TOKEN_VALUE_ID. If the ctvs list is empty, then -1 is used to make * sure the row isn't produced. If there is a single match, the predicate is COMMON_TOKEN_VALUE_ID = {n}. @@ -746,6 +804,28 @@ private void addCommonTokenValueIdFilter(WhereFragment where, String paramAlias, } } + /** + * Adds a filter predicate for REF_LOGICAL_RESOURCE_ID. If the ctvs list is empty, then -1 is used to make + * sure the row isn't produced. If there is a single match, the predicate is REF_LOGICAL_RESOURCE_ID = {n}. + * If there are multiple matches, the predicate is REF_LOGICAL_RESOURCE_ID IN (1, 2, 3, ...). + * The query uses literal values not bind variables on purpose (better performance). + * @param where + * @param paramAlias + * @param ctvs + * @throws FHIRPersistenceException + */ + private void addRefLogicalResourceIdFilter(WhereFragment where, String paramAlias, Collection ctvs) throws FHIRPersistenceException { + final List ctvList = new ArrayList<>(ctvs); + if (ctvList.isEmpty()) { + // use -1...resulting in no data + where.col(paramAlias, REF_LOGICAL_RESOURCE_ID).eq(-1L); + } else if (ctvList.size() == 1) { + where.col(paramAlias, REF_LOGICAL_RESOURCE_ID).eq(ctvList.get(0)); + } else { + where.col(paramAlias, REF_LOGICAL_RESOURCE_ID).inLiteralLong(ctvList); + } + } + /** * Builds an SQL segment which populates an IN clause with codes for a token search parameter * specifying the :in, :not-in, :above, or :below modifier. @@ -1008,6 +1088,8 @@ public String paramValuesTableName(String resourceType, QueryParameter queryParm name.append("LATLNG_VALUES"); break; case REFERENCE: + name.append("REF_VALUES"); + break; case TOKEN: if (!this.legacyWholeSystemSearchParamsEnabled && TAG.equals(queryParm.getCode())) { name.append(wholeSystemSearch ? "LOGICAL_RESOURCE_TAGS" : "TAGS"); @@ -1049,6 +1131,8 @@ public String paramValuesColumnName(Type paramType) { result = "LATLNG_VALUES"; break; case REFERENCE: + result = "REF_LOGICAL_RESOURCE_ID"; + break; case TOKEN: result = "TOKEN_VALUE"; break; @@ -1274,6 +1358,43 @@ protected String getTokenParamTable(ExpNode filter, String resourceType, String return xxTokenValues; } + /** + * Compute the reference parameter table name we want to use to join with. This method + * inspects the content of the given filter {@link ExpNode}. If the filter contains + * a reference to the LOGICAL_ID column, the returned table name will be based + * on xx_REF_VALUES_V, otherwise it will be based on xx_REF_VALUES. The + * latter is preferable because it eliminates an unnecessary join, improves cardinality + * estimation and (usually) results in a better execution plan. + * @param filter + * @param resourceType + * @param paramAlias + * @return + */ + protected String getRefParamTable(ExpNode filter, String resourceType, String paramAlias) { + ColumnExpNodeVisitor visitor = new ColumnExpNodeVisitor(); // gathers all columns used in the filter expression + Set columns = filter.visit(visitor); + boolean usesLogicalIdValue = columns.contains(DataDefinitionUtil.getQualifiedName(paramAlias, LOGICAL_ID)) || + columns.contains(DataDefinitionUtil.getQualifiedName(paramAlias, RESOURCE_TYPE_ID)); + + final String xxRefValues; + if (usesLogicalIdValue) { + // can't optimize because we filter on LOGICAL_ID + xxRefValues = resourceType + "_REF_VALUES_V"; + } else { + // only filters on REF_LOGICAL_RESOURCE_ID so we can optimize + xxRefValues = resourceType + "_REF_VALUES"; + } + return xxRefValues; + } + + protected WhereFragment getIdentifierFilter(QueryParameter queryParm, String paramAlias) throws FHIRPersistenceException { + WhereFragment whereClause = new WhereFragment(); + whereClause.leftParen(); + handleIdentifier(queryParm, paramAlias, whereClause); + whereClause.rightParen(); + return whereClause; + } + /** * Create a filter predicate for the given reference query parameter * @param queryParm @@ -1295,36 +1416,62 @@ protected WhereFragment getReferenceFilter(QueryParameter queryParm, String para resourceTypesAndIds.add(getResourceTypeAndId(queryParm, value)); } - List resourceReferenceTokenValues = new ArrayList<>(queryParm.getValues().size()); + List refValues = new ArrayList<>(queryParm.getValues().size()); List ambiguousResourceReferenceTokenValues = new ArrayList<>(); for (Pair resourceTypeAndId : resourceTypesAndIds) { String targetResourceType = resourceTypeAndId.getLeft(); String targetResourceId = resourceTypeAndId.getRight(); if (targetResourceType != null) { - Integer codeSystemIdForResourceType = getCodeSystemId(targetResourceType); + Integer resourceTypeId = identityCache.getResourceTypeId(targetResourceType); // targetResourceType is treated as the code-system for references - resourceReferenceTokenValues.add(new CommonTokenValue(targetResourceType, nullCheck(codeSystemIdForResourceType), targetResourceId)); + refValues.add(new ResourceReferenceValue(targetResourceType, resourceTypeId, targetResourceId)); } else { ambiguousResourceReferenceTokenValues.add(targetResourceId); } } - // For unambiguous resource references, look up the common token value ids - Set resourceReferenceTokenIds = getCommonTokenValueIds(resourceReferenceTokenValues); - addCommonTokenValueIdFilter(whereClause, paramAlias, resourceReferenceTokenIds); + // For unambiguous resource references, look up the logical_resource_ids + Set resourceReferenceTokenIds = getLogicalResourceIds(refValues); + addRefLogicalResourceIdFilter(whereClause, paramAlias, resourceReferenceTokenIds); for (String targetResourceId : ambiguousResourceReferenceTokenValues) { whereClause.or(); // grab the list of all matching common_token_value_id values - addCommonTokenValueIdFilter(whereClause, paramAlias, targetResourceId); + addLogicalResourceIdFilter(whereClause, paramAlias, targetResourceId); } whereClause.rightParen(); return whereClause; } + protected WhereFragment getReferenceFilter(QueryParameter queryParm, String paramAlias, List logicalResourceIdList) throws FHIRPersistenceException { + WhereFragment whereClause = new WhereFragment(); + whereClause.leftParen(); + + // For unambiguous resource references, look up the logical_resource_ids + addRefLogicalResourceIdFilter(whereClause, paramAlias, logicalResourceIdList); + + whereClause.rightParen(); + return whereClause; + } + + /** + * Create a filter predicate for the given reference query parameter using + * the ambiguous + * @param queryParm + * @param paramAlias + * @throws FHIRPersistenceException + */ + protected WhereFragment getReferenceStrFilter(QueryParameter queryParm, String paramAlias, List ambiguousResourceReferenceTokenValues) throws FHIRPersistenceException { + WhereFragment whereClause = new WhereFragment(); + whereClause.leftParen(); + whereClause.col(paramAlias, "str_value").in(ambiguousResourceReferenceTokenValues); + whereClause.rightParen(); + return whereClause; + } + private Pair getResourceTypeAndId(QueryParameter queryParm, QueryParameterValue value) { String targetResourceType = null; String searchValue = SqlParameterEncoder.encode(value.getValueString()); @@ -1535,11 +1682,10 @@ public QueryData addIncludeFilter(QueryData queryData, InclusionParameter inclus // > the specified version SHOULD be provided. /* SELECT R0.RESOURCE_ID, R0.LOGICAL_RESOURCE_ID, R0.VERSION_ID, R0.LAST_UPDATED, R0.IS_DELETED, R0.DATA, R0.RESOURCE_PAYLOAD_KEY, LR0.LOGICAL_ID - FROM fhirdata.ExplanationOfBenefit_TOKEN_VALUES_V AS P1 + FROM fhirdata.ExplanationOfBenefit_REF_VALUES AS P1 INNER JOIN fhirdata.Claim_LOGICAL_RESOURCES AS LR0 - ON LR0.LOGICAL_ID = P1.TOKEN_VALUE + ON LR0.LOGICAL_RESOURCE_ID = P1.REF_LOGICAL_RESOURCE_ID AND P1.PARAMETER_NAME_ID = 9263 - AND P1.CODE_SYSTEM_ID = 341729359 AND P1.LOGICAL_RESOURCE_ID IN (135010606,135010540,135010498,135010412,135010428) INNER JOIN fhirdata.Claim_RESOURCES AS R0 ON LR0.LOGICAL_RESOURCE_ID = R0.LOGICAL_RESOURCE_ID @@ -1597,12 +1743,11 @@ AND COALESCE(P1.REF_VERSION_ID,LR0.VERSION_ID) = R0.VERSION_ID .and(lrAlias, "VERSION_ID").eq(rAlias, "VERSION_ID") .and(rAlias, IS_DELETED).eq().literal("N")); } else { - final String tokenValues = joinResourceType + "_TOKEN_VALUES_V"; + final String tokenValues = joinResourceType + "_REF_VALUES"; select.from(tokenValues, alias(paramAlias)) .innerJoin(xxLogicalResources, alias(lrAlias), - on(lrAlias, "LOGICAL_ID").eq(paramAlias, "TOKEN_VALUE") + on(lrAlias, "LOGICAL_RESOURCE_ID").eq(paramAlias, "REF_LOGICAL_RESOURCE_ID") .and(paramAlias, "PARAMETER_NAME_ID").eq(getParameterNameId(inclusionParm.getSearchParameter())) - .and(paramAlias, "CODE_SYSTEM_ID").eq(getCodeSystemId(targetResourceType)) .and(paramAlias, "LOGICAL_RESOURCE_ID").inLiteralLong(logicalResourceIds)) .innerJoin(xxResources, alias(rAlias), on(lrAlias, "LOGICAL_RESOURCE_ID").eq(rAlias, "LOGICAL_RESOURCE_ID") @@ -1683,14 +1828,13 @@ public QueryData addRevIncludeFilter(QueryData queryData, InclusionParameter inc .or().col(nextPlus2ParamAlias, "STR_VALUE").eq().col(nextPlus1ParamAlias, "STR_VALUE") .rightParen()); } else { - final String tokenValues = joinResourceType + "_TOKEN_VALUES_V"; + final String tokenValues = joinResourceType + "_REF_VALUES"; query.from() .innerJoin(tokenValues, alias(paramAlias), on(parentLRAlias, "LOGICAL_RESOURCE_ID").eq(paramAlias, "LOGICAL_RESOURCE_ID") - .and(paramAlias, "PARAMETER_NAME_ID").eq(getParameterNameId(inclusionParm.getSearchParameter())) - .and(paramAlias, "CODE_SYSTEM_ID").eq(getCodeSystemId(targetResourceType))) + .and(paramAlias, "PARAMETER_NAME_ID").eq(getParameterNameId(inclusionParm.getSearchParameter()))) .innerJoin(targetLR, alias(lrAlias), - on(lrAlias, "LOGICAL_ID").eq(paramAlias, "TOKEN_VALUE") + on(lrAlias, "LOGICAL_RESOURCE_ID").eq(paramAlias, "REF_LOGICAL_RESOURCE_ID") .and().coalesce(col(paramAlias, "REF_VERSION_ID"), col(lrAlias, "VERSION_ID")).eq(lrAlias, "VERSION_ID") .and(lrAlias, "LOGICAL_RESOURCE_ID").inLiteralLong(logicalResourceIds)); } @@ -1988,6 +2132,7 @@ public QueryData addMissingParam(QueryData queryData, QueryParameter queryParm, // note that there's no filter here to look for a specific value. We simply want to know // whether or not the parameter exists for a given resource final String parameterName = queryParm.getCode(); + final int parameterNameId = getParameterNameId(parameterName); final int aliasIndex = getNextAliasIndex(); final String resourceType = queryData.getResourceType(); final String paramTableName = paramValuesTableName(resourceType, queryParm); @@ -2002,7 +2147,7 @@ public QueryData addMissingParam(QueryData queryData, QueryParameter queryParm, // their own tables. if (this.legacyWholeSystemSearchParamsEnabled || (!PROFILE.equals(parameterName) && !SECURITY.equals(parameterName) && !TAG.equals(parameterName))) { - exists.from().where().and(paramAlias, PARAMETER_NAME_ID).eq(getParameterNameId(parameterName)); + exists.from().where().and(paramAlias, PARAMETER_NAME_ID).eq(parameterNameId); } // Add the exists to the where clause of the main query which already has a predicate @@ -2024,14 +2169,15 @@ public QueryData addChained(QueryData queryData, QueryParameter currentParm) thr // In this variant, each chained element is added as join to the current statement. We still need // to add the EXISTS clause when depth == 0 (the first element in the chain) + // Because logical_resource_id is already unique across all resources, we don't need to constrain + // with resource_type_id. // AND EXISTS (SELECT 1 - // FROM fhirdata.Observation_TOKEN_VALUES_V AS P1 -- Observation references to - // INNER JOIN fhirdata.Device_LOGICAL_RESOURCES AS LR1 -- Device - // ON LR1.LOGICAL_ID = P1.TOKEN_VALUE -- Device.LOGICAL_ID = Observation.device - // AND P1.PARAMETER_NAME_ID = 1234 -- Observation.device reference param - // AND P1.CODE_SYSTEM_ID = 4321 -- code-system for Device - // AND LR1.IS_DELETED = 'N' -- referenced Device is not deleted - // WHERE P1.LOGICAL_RESOURCE_ID = LR0.LOGICAL_RESOURCE_ID -- correlate parameter to parent + // FROM fhirdata.Observation_REF_VALUES AS P1 -- Observation references to + // INNER JOIN fhirdata.Device_LOGICAL_RESOURCES AS LR1 -- Device + // ON LR1.LOGICAL_RESOURCE_ID = P1.REF_LOGICAL_RESOURCE_ID -- Device.LOGICAL_RESOURCE_ID = Observation.device + // AND P1.PARAMETER_NAME_ID = 1234 -- Observation.device reference param + // AND LR1.IS_DELETED = 'N' -- referenced Device is not deleted + // WHERE P1.LOGICAL_RESOURCE_ID = LR0.LOGICAL_RESOURCE_ID -- correlate parameter to parent final String sourceResourceType = queryData.getResourceType(); final SelectAdapter currentSubQuery = queryData.getQuery(); @@ -2083,14 +2229,13 @@ public QueryData addChained(QueryData queryData, QueryParameter currentParm) thr paramAlias = nextPlus2ParamAlias; } else { // Chain via the logical ID - final String tokenValues = sourceResourceType + "_TOKEN_VALUES_V"; // because we need TOKEN_VALUE + final String refValues = sourceResourceType + "_REF_VALUES"; currentSubQuery.from() - .innerJoin(tokenValues, alias(paramAlias), + .innerJoin(refValues, alias(paramAlias), on(paramAlias, "LOGICAL_RESOURCE_ID").eq(queryData.getLRAlias(), "LOGICAL_RESOURCE_ID")) .innerJoin(xxLogicalResources, alias(lrAlias), - on(lrAlias, "LOGICAL_ID").eq(paramAlias, "TOKEN_VALUE") + on(lrAlias, "LOGICAL_RESOURCE_ID").eq(paramAlias, "REF_LOGICAL_RESOURCE_ID") .and(paramAlias, "PARAMETER_NAME_ID").eq(getParameterNameId(currentParm.getCode())) - .and(paramAlias, "CODE_SYSTEM_ID").eq(nullCheck(getCodeSystemId(targetResourceType))) .and(lrAlias, "IS_DELETED").eq().literal("N")); } @@ -2136,7 +2281,21 @@ public void addFilter(QueryData queryData, String resourceType, QueryParameter c paramTable = paramValuesTableName(queryData.getResourceType(), currentParm); } - if (currentParm.getModifier() == Modifier.NOT) { + if (Type.REFERENCE.equals(currentParm.getType())) { + // V0027, reference filters now need to look at both xx_ref_values and xx_str_values + // so we use a full correlated sub-query for exists/not-exists + final String anchorAlias = "LR" + getNextAliasIndex(); + SelectAdapter exists = Select.select("1"); + exists.from("LOGICAL_RESOURCES", alias(anchorAlias)) + .where(anchorAlias, "LOGICAL_RESOURCE_ID").eq(lrAlias, "LOGICAL_RESOURCE_ID"); // correlate to parent query + QueryData subQuery = new QueryData(exists, anchorAlias, null, resourceType, 0); + addReferenceParam(subQuery, queryData.getResourceType(), currentParm); + if (currentParm.getModifier() == Modifier.NOT) { + currentSubQuery.from().where().and().notExists(exists.build()); + } else { + currentSubQuery.from().where().and().exists(exists.build()); + } + } else if (currentParm.getModifier() == Modifier.NOT) { // Needs to be handled as a NOT EXISTS correlated subquery SelectAdapter exists = Select.select("1"); exists.from(paramTable, alias(paramAlias)) @@ -2174,13 +2333,14 @@ public QueryData addReverseChained(QueryData queryData, QueryParameter currentPa // For reverse chaining, we connect the token-value (reference) // back to the parent query LOGICAL_ID and an xx_LOGICAL_RESOURCES // to provide the LOGICAL_ID as the target for future chain elements - // INNER JOIN fhirdata.Observation_TOKEN_VALUES_V AS P1 - // AND LR0.LOGICAL_ID = P1.TOKEN_VALUE -- 'Patient.LOGICAL_ID = Observation.patient' + + // INNER JOIN fhirdata.Observation_REF_VALUES AS P1 + // AND LR0.LOGICAL_RESOURCE_ID = P1.REF_LOGICAL_RESOURCE_ID -- 'Patient.LOGICAL_ID = Observation.patient' // AND LR0.VERSION_ID = COALESCE(P1.REF_VERSION_ID, LR0.VERSION_ID) // AND P1.PARAMETER_NAME_ID = 1246 -- 'Observation.patient' - // AND P1.CODE_SYSTEM_ID = 6 -- 'code system for Patient references' // INNER JOIN fhirdata.Observation_LOGICAL_RESOURCES LR1 // ON LR1.LOGICAL_RESOURCE_ID = P1.LOGICAL_RESOURCE_ID + final String refResourceType = queryData.getResourceType(); final SelectAdapter currentSubQuery = queryData.getQuery(); final int aliasIndex = getNextAliasIndex(); @@ -2225,13 +2385,12 @@ public QueryData addReverseChained(QueryData queryData, QueryParameter currentPa .rightParen()); paramAlias = nextPlus2ParamAlias; } else { - final String tokenValues = resourceTypeName + "_TOKEN_VALUES_V"; + final String refValues = resourceTypeName + "_REF_VALUES"; currentSubQuery.from() - .innerJoin(tokenValues, alias(paramAlias), - on(lrPrevAlias, "LOGICAL_ID").eq(paramAlias, "TOKEN_VALUE") // correlate with the main query + .innerJoin(refValues, alias(paramAlias), + on(lrPrevAlias, "LOGICAL_RESOURCE_ID").eq(paramAlias, "REF_LOGICAL_RESOURCE_ID") // correlate with the main query .and(lrPrevAlias, "VERSION_ID").eq().coalesce(col(paramAlias, "REF_VERSION_ID"), col(lrPrevAlias, "VERSION_ID")) - .and(paramAlias, "PARAMETER_NAME_ID").eq(getParameterNameId(currentParm.getCode())) - .and(paramAlias, "CODE_SYSTEM_ID").eq(nullCheck(getCodeSystemId(refResourceType)))); + .and(paramAlias, "PARAMETER_NAME_ID").eq(getParameterNameId(currentParm.getCode()))); } currentSubQuery.from() .innerJoin(xxLogicalResources, alias(lrAlias), @@ -2315,24 +2474,127 @@ public QueryData addLocationParam(QueryData queryData, String resourceType, Quer @Override public QueryData addReferenceParam(QueryData queryData, String resourceType, QueryParameter queryParm) throws FHIRPersistenceException { + + final int aliasIndex = getNextAliasIndex(); + final SelectAdapter query = queryData.getQuery(); + final String paramAlias = getParamAlias(aliasIndex); + final String lrAlias = queryData.getLRAlias(); + final boolean isIdentifier = Modifier.IDENTIFIER.equals(queryParm.getModifier()); + final ExpNode filter; + final String paramTableName; + if (isIdentifier) { + // Identifiers are tokens so we need to join with token_values. + // Grab the filter expression first. We can then inspect the expression to + // look for use of the TOKEN_VALUE column. If use of this column isn't found, + // we can apply an optimization by joining against the RESOURCE_TOKEN_REFS + // table directly. + filter = getIdentifierFilter(queryParm, paramAlias).getExpression(); + paramTableName = getTokenParamTable(filter, resourceType, paramAlias); + String queryParmCode = queryParm.getCode(); + queryParmCode += SearchConstants.IDENTIFIER_MODIFIER_SUFFIX; + query.from().innerJoin(paramTableName, alias(paramAlias), on(paramAlias, "LOGICAL_RESOURCE_ID").eq(lrAlias, "LOGICAL_RESOURCE_ID") + .and(paramAlias, "PARAMETER_NAME_ID").eq(getParameterNameId(queryParmCode)) + .and(filter)); + } else { + // For V0027 we need to handle parameters that may come from xx_ref_values or xx_str_values + return processRealReferenceParam(queryData, resourceType, queryParm); + } + + return queryData; + } + + /** + * FHIR Specification: + * A reference parameter refers to references between resources. For + * example, find all Conditions where the subject reference is a + * particular patient, where the patient is selected by name or + * identifier. The interpretation of a reference parameter is either: + * [1] [parameter]=[id] the logical [id] of a resource using a local reference (i.e. a relative reference) + * [2] [parameter]=[type]/[id] the logical [id] of a resource of a specified type using a local reference (i.e. a relative reference), for when the reference can point to different types of resources (e.g. Observation.subject) + * [3] [parameter]=[url] where the [url] is an absolute URL - a reference to a resource by its absolute location, or by its canonical URL + * + * For [1], the target resource type isn't known. This shouldn't matter, because + * we still look up the logical_resource_id by its logical_id. If there are + * multiple matches, they are by definition of different type, so this would be + * an error. Therefore the query still only needs to deal with a single logical_resource_id. + * For [2], we are guaranteed a single logical_resource_id because resourceType/logicalId + * is unique. + * For [3], we need to identify the value string as a url and not a local reference. + * + * @param queryData + * @param resourceType + * @param queryParm + * @return + * @throws FHIRPersistenceException + */ + private QueryData processRealReferenceParam(QueryData queryData, String resourceType, QueryParameter queryParm) throws FHIRPersistenceException { final int aliasIndex = getNextAliasIndex(); final SelectAdapter query = queryData.getQuery(); final String paramAlias = getParamAlias(aliasIndex); final String lrAlias = queryData.getLRAlias(); - // Grab the filter expression first. We can then inspect the expression to - // look for use of the TOKEN_VALUE column. If use of this column isn't found, - // we can apply an optimization by joining against the RESOURCE_TOKEN_REFS - // table directly. - ExpNode filter = getReferenceFilter(queryParm, paramAlias).getExpression(); - final String paramTableName = getTokenParamTable(filter, resourceType, paramAlias); + // For V0027 reference parameters are stored in xx_ref_values using the + // logical_id values stored in logical_resource_ident. Absolute references + // are stored using a resource_type of "Resource" (similar to the default + // code-system we used to use with common_token_values). - // Append the suffix for :identifier modifier - String queryParmCode = queryParm.getCode(); - if (Modifier.IDENTIFIER.equals(queryParm.getModifier())) { - queryParmCode += SearchConstants.IDENTIFIER_MODIFIER_SUFFIX; + // Firstly we need to split the query parm values into separate lists + List> resourceTypesAndIds = new ArrayList<>(queryParm.getValues().size()); + for (QueryParameterValue value : queryParm.getValues()) { + resourceTypesAndIds.add(getResourceTypeAndId(queryParm, value)); } + List logicalResourceIdList = new ArrayList<>(); + for (Pair resourceTypeAndId : resourceTypesAndIds) { + String targetResourceType = resourceTypeAndId.getLeft(); + String referenceValue = resourceTypeAndId.getRight(); + + if (targetResourceType != null) { + Integer resourceTypeId = identityCache.getResourceTypeId(targetResourceType); + if (resourceTypeId != null) { + // It's a valid resource type, so we treat as a local reference + logger.fine(() -> "reference search value: type[local] value[" + targetResourceType + "/" + referenceValue + "]"); + Long logicalResourceId = identityCache.getLogicalResourceId(targetResourceType, referenceValue); + logicalResourceIdList.add(logicalResourceId != null ? logicalResourceId : -1); + } else { + // Treat this as an error because it's not a valid local reference + throw new FHIRPersistenceException("Local reference specified with invalid resource type").withIssue( + Issue.builder() + .code(IssueType.INVALID) + .diagnostics("Local reference specified with invalid resource type") + .severity(IssueSeverity.ERROR) + .build()); + } + } else { + // Determine if the target value is an absolute or local reference + if (ReferenceUtil.isAbsolute(referenceValue)) { + logger.info(() -> "reference search value: type[absolute] value[" + referenceValue + "]"); + Long logicalResourceId = identityCache.getLogicalResourceId("Resource", referenceValue); + logicalResourceIdList.add(logicalResourceId != null ? logicalResourceId : -1); + } else { + // treat as a local reference where we don't know the type. + List localLogicalResourceIds = getLogicalResourceIdList(referenceValue); + if (localLogicalResourceIds.size() == 1) { + logger.info(() -> "reference search value: type[local] value[" + referenceValue + "]"); + logicalResourceIdList.add(localLogicalResourceIds.get(0)); + } else if (localLogicalResourceIds.size() == 0) { + logger.info(() -> "reference search value: type[local] value[" + referenceValue + "] notFound[true]"); + if (logicalResourceIdList.isEmpty()) { + logicalResourceIdList.add(-1L); // need at least one value + } + } else { + // We may match multiple resource types here, but it's only an error + // if we join with the xx_ref_value table and still get multiple rows + logicalResourceIdList.addAll(localLogicalResourceIds); + } + } + } + } + + // Only need to join with xx_ref_values + final String queryParmCode = queryParm.getCode(); + final ExpNode filter = getReferenceFilter(queryParm, paramAlias, logicalResourceIdList).getExpression(); + final String paramTableName = getRefParamTable(filter, resourceType, paramAlias); query.from().innerJoin(paramTableName, alias(paramAlias), on(paramAlias, "LOGICAL_RESOURCE_ID").eq(lrAlias, "LOGICAL_RESOURCE_ID") .and(paramAlias, "PARAMETER_NAME_ID").eq(getParameterNameId(queryParmCode)) .and(filter)); @@ -2610,6 +2872,8 @@ protected String getSortParameterTableName(String resourceType, String code, Typ sortParameterTableName.append("DATE_VALUES"); break; case REFERENCE: + sortParameterTableName.append("REF_VALUES_V"); + break; case TOKEN: if (!this.legacyWholeSystemSearchParamsEnabled && TAG.equals(code)) { sortParameterTableName.append("TAGS"); @@ -2704,7 +2968,7 @@ private List getValueAttributeNames(Type type) throws FHIRPersistenceExc attributeNames.add(STR_VALUE); break; case REFERENCE: - attributeNames.add(TOKEN_VALUE); + attributeNames.add(REF_VALUE); // V0027 using xx_REF_VALUES_V break; case DATE: attributeNames.add(DATE_START); @@ -2753,6 +3017,7 @@ private String getDataCol() { case DERBY: return "CAST(NULL AS BLOB) AS DATA"; case POSTGRESQL: + case CITUS: return "NULL::TEXT AS DATA"; default: throw new IllegalStateException("Database type not supported: " + translator.getType().name()); diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dto/Resource.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dto/Resource.java index 7cbd092be82..6602ad59dbd 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dto/Resource.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dto/Resource.java @@ -1,5 +1,5 @@ /* - * (C) Copyright IBM Corp. 2017,2021 + * (C) Copyright IBM Corp. 2017, 2022 * * SPDX-License-Identifier: Apache-2.0 */ @@ -17,14 +17,14 @@ public class Resource { /** - * This is the _RESOURCES.RESOURCE_ID column + * This is the _RESOURCES.RESOURCE_ID column. It is unique for a specific version + * of a resource. It is not used during create/update interactions. */ - private long id; + private long resourceId; /** - * This is the _LOGICAL_RESOURCES.LOGICAL_RESOURCE_ID column. It is only - * set when this DTO is used to read table data. It is not set when the DTO is - * used to insert/update. + * This is the _LOGICAL_RESOURCES.LOGICAL_RESOURCE_ID column. It is used during + * create/update interactions as well as read interactions */ private long logicalResourceId; @@ -114,18 +114,34 @@ public Integer getIfNoneMatchVersion() { return this.ifNoneMatchVersion; } - public long getId() { - return id; + /** + * Getter for the database xx_resources.resource_id value + * @return + */ + public long getResourceId() { + return resourceId; } - public void setId(long id) { - this.id = id; + /** + * Setter for the database xx_resources.resource_id value + * @param id + */ + public void setResourceId(long id) { + this.resourceId = id; } + /** + * Getter for the logical_resources.logical_resource_id value + * @return + */ public long getLogicalResourceId() { return logicalResourceId; } - + + /** + * Setter for the logical_resources.logical_resource_id value + * @param logicalResourceId + */ public void setLogicalResourceId(long logicalResourceId) { this.logicalResourceId = logicalResourceId; } @@ -156,7 +172,7 @@ public void setDeleted(boolean deleted) { @Override public String toString() { - return "Resource [id=" + id + ", logicalResourceId=" + logicalResourceId + ", logicalId=" + logicalId + + return "Resource [id=" + resourceId + ", logicalResourceId=" + logicalResourceId + ", logicalId=" + logicalId + ", versionId=" + versionId + ", resourceType=" + resourceType + ", lastUpdated=" + lastUpdated + ", deleted=" + deleted + "]"; } diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dto/ResourceReferenceValue.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dto/ResourceReferenceValue.java new file mode 100644 index 00000000000..88653e5ec7e --- /dev/null +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/dto/ResourceReferenceValue.java @@ -0,0 +1,100 @@ +/* + * (C) Copyright IBM Corp. 2022 + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.ibm.fhir.persistence.jdbc.dto; + +import java.util.Comparator; + +/** + * DTO representing a resource reference record. + */ +public class ResourceReferenceValue implements Comparable { + private static final Comparator NULL_SAFE_COMPARATOR = Comparator.nullsFirst(String::compareTo); + + private final String resourceType; + private final int resourceTypeId; + + // the target logicalId...can be null + private final String logicalId; + + /** + * Canonical constructor + * + * @param resourceType + * @param resourceTypeId + * @param logicalId + */ + public ResourceReferenceValue(String resourceType, int resourceTypeId, String logicalId) { + if (resourceTypeId < 0) { + throw new IllegalArgumentException("Invalid resourceTypeId argument"); + } + + this.resourceType = resourceType; + this.resourceTypeId = resourceTypeId; + this.logicalId = logicalId; + } + + @Override + public int hashCode() { + // We don't need to include codeSystem in the hash because codeSystemId is synonymous + // with codeSystem as far as identity is concerned + return Integer.hashCode(resourceTypeId) * 37 + (logicalId == null ? 7 : logicalId.hashCode()); + } + + @Override + public boolean equals(Object other) { + if (other instanceof ResourceReferenceValue) { + ResourceReferenceValue that = (ResourceReferenceValue)other; + return this.resourceTypeId == that.resourceTypeId + && ( this.logicalId == null && that.logicalId == null + || this.logicalId != null && this.logicalId.equals(that.logicalId) + ); + } else { + return false; + } + } + + @Override + public String toString() { + return "[resourceTypeId=" + resourceTypeId + ", logicalId=" + logicalId + "]"; + } + + @Override + public int compareTo(ResourceReferenceValue other) { + // allow ResourceReferenceValue objects to be sorted in a deterministic way. Note that + // we sort on resourceType not resourceTypeId. This is to help avoid deadlocks with + // Derby + int result = NULL_SAFE_COMPARATOR.compare(resourceType, other.resourceType); + if (result == 0) { + result = NULL_SAFE_COMPARATOR.compare(logicalId, other.logicalId); + } + return result; + } + + + /** + * @return the resourceType + */ + public String getResourceType() { + return resourceType; + } + + + /** + * @return the resourceTypeId + */ + public int getResourceTypeId() { + return resourceTypeId; + } + + + /** + * @return the logicalId + */ + public String getLogicalId() { + return logicalId; + } +} \ No newline at end of file diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/exception/FHIRPersistenceFKVException.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/exception/FHIRPersistenceFKVException.java index f2cfe4275a5..6dc29ec453f 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/exception/FHIRPersistenceFKVException.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/exception/FHIRPersistenceFKVException.java @@ -9,6 +9,7 @@ import java.util.Collection; import com.ibm.fhir.model.resource.OperationOutcome; +import com.ibm.fhir.persistence.exception.FHIRPersistenceDataAccessException; /** * This exception class is thrown when Foreign Key violations are encountered while attempting to access data in the FHIR DB. diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/impl/FHIRPersistenceJDBCImpl.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/impl/FHIRPersistenceJDBCImpl.java index 791ca4befbd..c3da80886fe 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/impl/FHIRPersistenceJDBCImpl.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/impl/FHIRPersistenceJDBCImpl.java @@ -55,6 +55,7 @@ import com.ibm.fhir.database.utils.api.DataAccessException; import com.ibm.fhir.database.utils.api.IConnectionProvider; import com.ibm.fhir.database.utils.api.IDatabaseTranslator; +import com.ibm.fhir.database.utils.api.SchemaType; import com.ibm.fhir.database.utils.api.UndefinedNameException; import com.ibm.fhir.database.utils.api.UniqueConstraintViolationException; import com.ibm.fhir.database.utils.model.DbType; @@ -101,8 +102,13 @@ import com.ibm.fhir.persistence.context.FHIRPersistenceContext; import com.ibm.fhir.persistence.context.FHIRPersistenceContextFactory; import com.ibm.fhir.persistence.erase.EraseDTO; +import com.ibm.fhir.persistence.exception.FHIRPersistenceDataAccessException; import com.ibm.fhir.persistence.exception.FHIRPersistenceException; import com.ibm.fhir.persistence.exception.FHIRPersistenceNotSupportedException; +import com.ibm.fhir.persistence.index.FHIRRemoteIndexService; +import com.ibm.fhir.persistence.index.IndexProviderResponse; +import com.ibm.fhir.persistence.index.RemoteIndexData; +import com.ibm.fhir.persistence.index.SearchParametersTransportAdapter; import com.ibm.fhir.persistence.jdbc.FHIRPersistenceJDBCCache; import com.ibm.fhir.persistence.jdbc.FHIRResourceDAOFactory; import com.ibm.fhir.persistence.jdbc.JDBCConstants; @@ -117,6 +123,7 @@ import com.ibm.fhir.persistence.jdbc.connection.SchemaNameFromProps; import com.ibm.fhir.persistence.jdbc.connection.SchemaNameImpl; import com.ibm.fhir.persistence.jdbc.connection.SchemaNameSupplier; +import com.ibm.fhir.persistence.jdbc.connection.SetMultiShardModifyModeAction; import com.ibm.fhir.persistence.jdbc.connection.SetTenantAction; import com.ibm.fhir.persistence.jdbc.dao.EraseResourceDAO; import com.ibm.fhir.persistence.jdbc.dao.ReindexResourceDAO; @@ -129,8 +136,10 @@ import com.ibm.fhir.persistence.jdbc.dao.impl.FetchResourcePayloadsDAO; import com.ibm.fhir.persistence.jdbc.dao.impl.JDBCIdentityCacheImpl; import com.ibm.fhir.persistence.jdbc.dao.impl.ParameterDAOImpl; +import com.ibm.fhir.persistence.jdbc.dao.impl.ParameterTransportVisitor; import com.ibm.fhir.persistence.jdbc.dao.impl.ResourceProfileRec; import com.ibm.fhir.persistence.jdbc.dao.impl.ResourceReferenceDAO; +import com.ibm.fhir.persistence.jdbc.dao.impl.ResourceReferenceValueRec; import com.ibm.fhir.persistence.jdbc.dao.impl.ResourceTokenValueRec; import com.ibm.fhir.persistence.jdbc.dao.impl.RetrieveIndexDAO; import com.ibm.fhir.persistence.jdbc.dao.impl.TransactionDataImpl; @@ -144,7 +153,6 @@ import com.ibm.fhir.persistence.jdbc.dto.StringParmVal; import com.ibm.fhir.persistence.jdbc.dto.TokenParmVal; import com.ibm.fhir.persistence.jdbc.exception.FHIRPersistenceDBConnectException; -import com.ibm.fhir.persistence.jdbc.exception.FHIRPersistenceDataAccessException; import com.ibm.fhir.persistence.jdbc.exception.FHIRPersistenceFKVException; import com.ibm.fhir.persistence.jdbc.util.ExtractedSearchParameters; import com.ibm.fhir.persistence.jdbc.util.JDBCParameterBuildingVisitor; @@ -233,6 +241,9 @@ public class FHIRPersistenceJDBCImpl implements FHIRPersistence, SchemaNameSuppl // A list of EraseResourceRec referencing offload resource records to erase if the current transaction commits private final List eraseResourceRecs = new ArrayList<>(); + // A list of the remote index messages we need to check we get ACKs for + private final List remoteIndexMessageList = new ArrayList<>(); + /** * Constructor for use when running as web application in WLP. * @throws Exception @@ -363,7 +374,9 @@ protected Action buildActionChain() { // reads/searches. result = new CreateTempTablesAction(result); - // For PostgreSQL + // For Citus SET LOCAL citus.multi_shard_modify_mode TO 'sequential' + result = new SetMultiShardModifyModeAction(result); + return result; } @@ -373,7 +386,7 @@ public SingleResourceResult create(FHIRPersistenceContex log.entering(CLASSNAME, METHODNAME); try (Connection connection = openConnection()) { - doCachePrefill(connection); + doCachePrefill(context, connection); if (context.getOffloadResponse() != null) { // Remember this payload offload response as part of the current transaction @@ -400,7 +413,7 @@ public SingleResourceResult create(FHIRPersistenceContex // The DAO objects are now created on-the-fly (not expensive to construct) and // given the connection to use while processing this request - ResourceDAO resourceDao = makeResourceDAO(connection); + ResourceDAO resourceDao = makeResourceDAO(context, connection); ParameterDAO parameterDao = makeParameterDAO(connection); // Persist the Resource DTO. @@ -408,10 +421,14 @@ public SingleResourceResult create(FHIRPersistenceContex ExtractedSearchParameters searchParameters = this.extractSearchParameters(updatedResource, resourceDTO); resourceDao.insert(resourceDTO, searchParameters.getParameters(), searchParameters.getParameterHashB64(), parameterDao, context.getIfNoneMatch()); if (log.isLoggable(Level.FINE)) { - log.fine("Persisted FHIR Resource '" + resourceDTO.getResourceType() + "/" + resourceDTO.getLogicalId() + "' id=" + resourceDTO.getId() + log.fine("Persisted FHIR Resource '" + resourceDTO.getResourceType() + "/" + resourceDTO.getLogicalId() + "' logicalResourceId=" + resourceDTO.getLogicalResourceId() + ", version=" + resourceDTO.getVersionId()); } + if (resourceDTO.getInteractionStatus() == InteractionStatus.MODIFIED) { + sendParametersToRemoteIndexService(resourceDTO.getResourceType(), resourceDTO.getLogicalId(), resourceDTO.getLogicalResourceId(), + resourceDTO.getVersionId(), resourceDTO.getLastUpdated().toInstant(), context.getRequestShard(), searchParameters); + } SingleResourceResult.Builder resultBuilder = new SingleResourceResult.Builder() .success(true) .interactionStatus(resourceDTO.getInteractionStatus()) @@ -445,6 +462,39 @@ public SingleResourceResult create(FHIRPersistenceContex } } + /** + * Convert the extracted parameters into a package we can send to a remote service + * for processing then send to that service (if so configured) + * @param resourceType + * @param logicalId + * @param logicalResourceId + * @param versionId + * @param lastUpdated + * @param requestShard + * @param searchParameters + */ + private void sendParametersToRemoteIndexService(String resourceType, String logicalId, long logicalResourceId, + int versionId, java.time.Instant lastUpdated, String requestShard, + ExtractedSearchParameters searchParameters) throws FHIRPersistenceException { + FHIRRemoteIndexService remoteIndexService = FHIRRemoteIndexService.getServiceInstance(); + if (remoteIndexService != null) { + // convert the parameters into a form that will be easy to ship to a remote service + SearchParametersTransportAdapter adapter = new SearchParametersTransportAdapter(resourceType, logicalId, logicalResourceId, + versionId, lastUpdated, requestShard, searchParameters.getParameterHashB64()); + ParameterTransportVisitor visitor = new ParameterTransportVisitor(adapter); + for (ExtractedParameterValue pv: searchParameters.getParameters()) { + pv.accept(visitor); + } + + // Note that the remote index service is supposed to be multi-tenant, using + // the tenantId from the request context on this thread, so we don't need + // to pass that here + final String kafkaPartitionKey = resourceType + "/" + logicalId; + IndexProviderResponse ipr = remoteIndexService.submit(new RemoteIndexData(kafkaPartitionKey, adapter.build())); + remoteIndexMessageList.add(ipr); // we'll check for an ACK just before we commit the transaction + } + } + /** * Prefill the cache if required * @throws FHIRPersistenceException @@ -452,7 +502,7 @@ public SingleResourceResult create(FHIRPersistenceContex private void doCachePrefill() throws FHIRPersistenceException { if (cache.needToPrefill()) { try (Connection connection = openConnection()) { - doCachePrefill(connection); + doCachePrefill(/*context=*/null, connection); } catch(FHIRPersistenceException e) { throw e; } catch(Throwable e) { @@ -508,26 +558,53 @@ private com.ibm.fhir.persistence.jdbc.dto.Resource createResourceDTO(Class SingleResourceResult update(FHIRPersistenceContex log.entering(CLASSNAME, METHODNAME); try (Connection connection = openConnection()) { - doCachePrefill(connection); + doCachePrefill(context, connection); if (context.getOffloadResponse() != null) { // Remember this payload offload response as part of the current transaction this.payloadPersistenceResponses.add(context.getOffloadResponse()); } - ResourceDAO resourceDao = makeResourceDAO(connection); + ResourceDAO resourceDao = makeResourceDAO(context, connection); ParameterDAO parameterDao = makeParameterDAO(connection); // Since 1869, the resource is already correctly configured so no need to modify it @@ -597,14 +674,20 @@ public SingleResourceResult update(FHIRPersistenceContex if (log.isLoggable(Level.FINE)) { if (resourceDTO.getInteractionStatus() == InteractionStatus.IF_NONE_MATCH_EXISTED) { - log.fine("If-None-Match: Existing FHIR Resource '" + resourceDTO.getResourceType() + "/" + resourceDTO.getLogicalId() + "' id=" + resourceDTO.getId() + log.fine("If-None-Match: Existing FHIR Resource '" + resourceDTO.getResourceType() + "/" + resourceDTO.getLogicalId() + "' logicalResourceId=" + resourceDTO.getLogicalResourceId() + ", version=" + resourceDTO.getVersionId()); } else { - log.fine("Persisted FHIR Resource '" + resourceDTO.getResourceType() + "/" + resourceDTO.getLogicalId() + "' id=" + resourceDTO.getId() + log.fine("Persisted FHIR Resource '" + resourceDTO.getResourceType() + "/" + resourceDTO.getLogicalId() + "' logicalResourceId=" + resourceDTO.getLogicalResourceId() + ", version=" + resourceDTO.getVersionId()); } } + // If configured, send the extracted parameters to the remote indexing service + if (resourceDTO.getInteractionStatus() == InteractionStatus.MODIFIED) { + sendParametersToRemoteIndexService(resourceDTO.getResourceType(), resourceDTO.getLogicalId(), resourceDTO.getLogicalResourceId(), + resourceDTO.getVersionId(), resourceDTO.getLastUpdated().toInstant(), context.getRequestShard(), searchParameters); + } + SingleResourceResult.Builder resultBuilder = new SingleResourceResult.Builder() .success(true) .interactionStatus(resourceDTO.getInteractionStatus()) @@ -658,10 +741,10 @@ public MultiResourceResult search(FHIRPersistenceContext context, Class newSearchForIncludeReso List allIncludeResources = new ArrayList<>(); // Used for de-duplication - Set allResourceIds = resourceDTOList.stream().map(r -> r.getId()).collect(Collectors.toSet()); + Set allResourceIds = resourceDTOList.stream().map(r -> r.getResourceId()).collect(Collectors.toSet()); // This is a map of iterations to query results. The query results is a map of // search resource type to returned logical resource IDs. The logical resource IDs @@ -826,7 +909,7 @@ private List newSearchForIncludeReso baseLogicalResourceIds, queryResultMap, resourceDao, 1, allResourceIds); // Add new ids to de-dup list - allResourceIds.addAll(includeResources.stream().map(r -> r.getId()).collect(Collectors.toSet())); + allResourceIds.addAll(includeResources.stream().map(r -> r.getResourceId()).collect(Collectors.toSet())); // Add resources to list allIncludeResources.addAll(includeResources); @@ -848,7 +931,7 @@ private List newSearchForIncludeReso baseLogicalResourceIds, queryResultMap, resourceDao, 1, allResourceIds); // Add new ids to de-dup list - allResourceIds.addAll(revincludeResources.stream().map(r -> r.getId()).collect(Collectors.toSet())); + allResourceIds.addAll(revincludeResources.stream().map(r -> r.getResourceId()).collect(Collectors.toSet())); // Add resources to list allIncludeResources.addAll(revincludeResources); @@ -892,7 +975,7 @@ private List newSearchForIncludeReso SearchConstants.INCLUDE, queryIds, queryResultMap, resourceDao, i+1, allResourceIds); // Add new ids to de-dup list - allResourceIds.addAll(includeResources.stream().map(r -> r.getId()).collect(Collectors.toSet())); + allResourceIds.addAll(includeResources.stream().map(r -> r.getResourceId()).collect(Collectors.toSet())); // Add resources to list allIncludeResources.addAll(includeResources); @@ -917,7 +1000,7 @@ private List newSearchForIncludeReso SearchConstants.REVINCLUDE, queryIds, queryResultMap, resourceDao, i+1, allResourceIds); // Add new ids to de-dup list - allResourceIds.addAll(revincludeResources.stream().map(r -> r.getId()).collect(Collectors.toSet())); + allResourceIds.addAll(revincludeResources.stream().map(r -> r.getResourceId()).collect(Collectors.toSet())); // Add resources to list allIncludeResources.addAll(revincludeResources); @@ -970,7 +1053,7 @@ private List runIncludeQuery(Class includeDTOs = - resourceDao.search(includeQuery).stream().filter(r -> !allResourceIds.contains(r.getId())).collect(Collectors.toList()); + resourceDao.search(includeQuery).stream().filter(r -> !allResourceIds.contains(r.getResourceId())).collect(Collectors.toList()); // Add query result to map. // The logical resource IDs are pulled from the returned DTOs and saved in a @@ -1043,14 +1126,14 @@ public void delete(FHIRPersistenceContext context, Class log.entering(CLASSNAME, METHODNAME); try (Connection connection = openConnection()) { - doCachePrefill(connection); + doCachePrefill(context, connection); if (context.getOffloadResponse() != null) { // Remember this payload offload response as part of the current transaction this.payloadPersistenceResponses.add(context.getOffloadResponse()); } - ResourceDAO resourceDao = makeResourceDAO(connection); + ResourceDAO resourceDao = makeResourceDAO(context, connection); // Create a new Resource DTO instance to represent the deletion marker. final int newVersionId = versionId + 1; @@ -1063,7 +1146,7 @@ public void delete(FHIRPersistenceContext context, Class resourceDao.insert(resourceDTO, null, null, null, IF_NONE_MATCH_NULL); if (log.isLoggable(Level.FINE)) { - log.fine("Deleted FHIR Resource '" + resourceDTO.getResourceType() + "/" + resourceDTO.getLogicalId() + "' id=" + resourceDTO.getId() + log.fine("Deleted FHIR Resource '" + resourceDTO.getResourceType() + "/" + resourceDTO.getLogicalId() + "' logicalResourceId=" + resourceDTO.getLogicalResourceId() + ", version=" + resourceDTO.getVersionId()); } } catch(FHIRPersistenceException e) { @@ -1117,8 +1200,8 @@ public SingleResourceResult read(FHIRPersistenceContext } try (Connection connection = openConnection()) { - doCachePrefill(connection); - ResourceDAO resourceDao = makeResourceDAO(connection); + doCachePrefill(context, connection); + ResourceDAO resourceDao = makeResourceDAO(context, connection); resourceDTO = resourceDao.read(logicalId, resourceType.getSimpleName()); boolean resourceIsDeleted = resourceDTO != null && resourceDTO.isDeleted(); @@ -1183,8 +1266,8 @@ public MultiResourceResult history(FHIRPersistenceContext context, Class SingleResourceResult vread(FHIRPersistenceContext } try (Connection connection = openConnection()) { - doCachePrefill(connection); - ResourceDAO resourceDao = makeResourceDAO(connection); + doCachePrefill(context, connection); + ResourceDAO resourceDao = makeResourceDAO(context, connection); version = Integer.parseInt(versionId); resourceDTO = resourceDao.versionRead(logicalId, resourceType.getSimpleName(), version); @@ -1401,7 +1484,7 @@ protected List buildSortedResourceDT // Store each ResourceDTO in its proper position in the returned sorted list. for (com.ibm.fhir.persistence.jdbc.dto.Resource resourceDTO : resourceDTOList) { - sortIndex = idPositionMap.get(resourceDTO.getId()); + sortIndex = idPositionMap.get(resourceDTO.getResourceId()); sortedResourceDTOs[sortIndex] = resourceDTO; } @@ -2498,14 +2581,14 @@ public String getSchemaForRequestContext(Connection connection) throws FHIRPersi /** * Prefill the caches */ - public void doCachePrefill(Connection connection) throws FHIRPersistenceException { + public void doCachePrefill(FHIRPersistenceContext context, Connection connection) throws FHIRPersistenceException { // Perform the cache prefill just once (for a given tenant). This isn't synchronous, so // there's a chance for other threads to slip in before the prefill completes. Those threads // just end up repeating the prefill - a little extra work one time to avoid unnecessary locking // Note - this is done as the first thing in a transaction so there's no concern about reading // uncommitted values. if (cache.needToPrefill()) { - ResourceDAO resourceDao = makeResourceDAO(connection); + ResourceDAO resourceDao = makeResourceDAO(context, connection); ParameterDAO parameterDao = makeParameterDAO(connection); FHIRPersistenceJDBCCacheUtil.prefill(resourceDao, parameterDao, cache); cache.clearNeedToPrefill(); @@ -2524,7 +2607,7 @@ public boolean isOffloadingSupported() { @Override public int reindex(FHIRPersistenceContext context, OperationOutcome.Builder operationOutcomeResult, java.time.Instant tstamp, List indexIds, - String resourceLogicalId) throws FHIRPersistenceException { + String resourceLogicalId, boolean force) throws FHIRPersistenceException { final String METHODNAME = "reindex"; log.entering(CLASSNAME, METHODNAME); @@ -2546,8 +2629,8 @@ public int reindex(FHIRPersistenceContext context, OperationOutcome.Builder oper } try (Connection connection = openConnection()) { - doCachePrefill(connection); - ResourceDAO resourceDao = makeResourceDAO(connection); + doCachePrefill(context, connection); + ResourceDAO resourceDao = makeResourceDAO(context, connection); ParameterDAO parameterDao = makeParameterDAO(connection); ReindexResourceDAO reindexDAO = FHIRResourceDAOFactory.getReindexResourceDAO(connection, FhirSchemaConstants.FHIR_ADMIN, schemaNameSupplier.getSchemaForRequestContext(connection), connectionStrategy.getFlavor(), this.trxSynchRegistry, this.cache, parameterDao); // Obtain a resource we will reindex in this request/transaction. The record is locked as part @@ -2596,7 +2679,7 @@ public int reindex(FHIRPersistenceContext context, OperationOutcome.Builder oper rir.setDeleted(false); // just to be clear Class resourceTypeClass = getResourceType(rir.getResourceType()); reindexDAO.setPersistenceContext(context); - updateParameters(rir, resourceTypeClass, existingResourceDTO, reindexDAO, operationOutcomeResult); + updateParameters(rir, resourceTypeClass, existingResourceDTO, reindexDAO, operationOutcomeResult, force); // result is only 0 if getResourceToReindex doesn't give us anything because this indicates // there's nothing left to do @@ -2655,10 +2738,11 @@ public int reindex(FHIRPersistenceContext context, OperationOutcome.Builder oper * @param existingResourceDTO the existing resource DTO * @param reindexDAO the reindex resource DAO * @param operationOutcomeResult the operation outcome result + * @param force * @throws Exception */ public void updateParameters(ResourceIndexRecord rir, Class resourceTypeClass, com.ibm.fhir.persistence.jdbc.dto.Resource existingResourceDTO, - ReindexResourceDAO reindexDAO, OperationOutcome.Builder operationOutcomeResult) throws Exception { + ReindexResourceDAO reindexDAO, OperationOutcome.Builder operationOutcomeResult, boolean force) throws Exception { if (existingResourceDTO != null && !existingResourceDTO.isDeleted()) { T existingResource = this.convertResourceDTO(existingResourceDTO, resourceTypeClass, null); @@ -2668,7 +2752,7 @@ public void updateParameters(ResourceIndexRecord rir, Class // Compare the hash of the extracted parameters with the hash in the index record. // If hash in the index record is not null and it matches the hash of the extracted parameters, then no need to replace the // extracted search parameters in the database tables for this resource, which helps with performance during reindex. - if (rir.getParameterHash() == null || !rir.getParameterHash().equals(searchParameters.getParameterHashB64())) { + if (force || rir.getParameterHash() == null || !rir.getParameterHash().equals(searchParameters.getParameterHashB64())) { reindexDAO.updateParameters(rir.getResourceType(), searchParameters.getParameters(), searchParameters.getParameterHashB64(), rir.getLogicalId(), rir.getLogicalResourceId()); } else { log.fine(() -> "Skipping update of unchanged parameters for FHIR Resource '" + rir.getResourceType() + "/" + rir.getLogicalId() + "'"); @@ -2770,6 +2854,7 @@ private void transactionCompleted(Boolean committed) { // important to clear this list after each transaction because batch bundles // use the same FHIRPersistenceJDBCImpl instance for each entry + remoteIndexMessageList.clear(); payloadPersistenceResponses.clear(); eraseResourceRecs.clear(); } @@ -2789,15 +2874,16 @@ private ParameterTransactionDataImpl createTransactionData(String datasourceId) * that have been accumulated during the transaction. This collection therefore * contains multiple resource types, which have to be processed separately. * @param records + * @param referenceRecords * @param profileRecs * @param tagRecs * @param securityRecs * @throws FHIRPersistenceException */ - public void onCommit(Collection records, Collection profileRecs, Collection tagRecs, Collection securityRecs) throws FHIRPersistenceException { + public void onCommit(Collection records, Collection referenceRecords, Collection profileRecs, Collection tagRecs, Collection securityRecs) throws FHIRPersistenceException { try (Connection connection = openConnection()) { IResourceReferenceDAO rrd = makeResourceReferenceDAO(connection); - rrd.persist(records, profileRecs, tagRecs, securityRecs); + rrd.persist(records, referenceRecords, profileRecs, tagRecs, securityRecs); } catch(FHIRPersistenceFKVException e) { log.log(Level.SEVERE, "FK violation", e); throw e; @@ -2828,6 +2914,22 @@ public void onCommit(Collection records, Collection records, Collection resourceType, java.time.Instant fromLastModified, java.time.Instant toLastModified, Function processor) throws FHIRPersistenceException { try (Connection connection = openConnection()) { - doCachePrefill(connection); + doCachePrefill(null, connection); // translator is required to handle some simple SQL syntax differences. This is easier // than creating separate DAO implementations for each database type IDatabaseTranslator translator = FHIRResourceDAOFactory.getTranslatorForFlavor(connectionStrategy.getFlavor()); @@ -2859,11 +2961,11 @@ public ResourcePayload fetchResourcePayloads(Class resourceT } @Override - public List changes(int resourceCount, java.time.Instant sinceLastModified, java.time.Instant beforeLastModified, + public List changes(FHIRPersistenceContext context, int resourceCount, java.time.Instant sinceLastModified, java.time.Instant beforeLastModified, Long changeIdMarker, List resourceTypeNames, boolean excludeTransactionTimeoutWindow, HistorySortOrder historySortOrder) throws FHIRPersistenceException { try (Connection connection = openConnection()) { - doCachePrefill(connection); + doCachePrefill(context, connection); // translator is required to handle some simple SQL syntax differences. This is easier // than creating separate DAO implementations for each database type final List resourceTypeIds; @@ -2896,13 +2998,13 @@ public List changes(int resourceCount, java.time.Instan } @Override - public ResourceEraseRecord erase(EraseDTO eraseDto) throws FHIRPersistenceException { + public ResourceEraseRecord erase(FHIRPersistenceContext context, EraseDTO eraseDto) throws FHIRPersistenceException { final String METHODNAME = "erase"; log.entering(CLASSNAME, METHODNAME); ResourceEraseRecord eraseRecord = new ResourceEraseRecord(); try (Connection connection = openConnection()) { - doCachePrefill(connection); + doCachePrefill(context, connection); IDatabaseTranslator translator = FHIRResourceDAOFactory.getTranslatorForFlavor(connectionStrategy.getFlavor()); IResourceReferenceDAO rrd = makeResourceReferenceDAO(connection); EraseResourceDAO eraseDao = new EraseResourceDAO(connection, FhirSchemaConstants.FHIR_ADMIN, translator, @@ -2994,12 +3096,12 @@ private boolean allSearchParmsAreGlobal(List queryParms) { } @Override - public List retrieveIndex(int count, java.time.Instant notModifiedAfter, Long afterIndexId, String resourceTypeName) throws FHIRPersistenceException { + public List retrieveIndex(FHIRPersistenceContext context, int count, java.time.Instant notModifiedAfter, Long afterIndexId, String resourceTypeName) throws FHIRPersistenceException { final String METHODNAME = "retrieveIndex"; log.entering(CLASSNAME, METHODNAME); try (Connection connection = openConnection()) { - doCachePrefill(connection); + doCachePrefill(context, connection); IDatabaseTranslator translator = FHIRResourceDAOFactory.getTranslatorForFlavor(connectionStrategy.getFlavor()); RetrieveIndexDAO dao = new RetrieveIndexDAO(translator, schemaNameSupplier.getSchemaForRequestContext(connection), resourceTypeName, count, notModifiedAfter, afterIndexId, this.cache); return dao.run(connection); diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/impl/ParameterTransactionDataImpl.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/impl/ParameterTransactionDataImpl.java index a337ffb3db9..4a6138554a4 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/impl/ParameterTransactionDataImpl.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/impl/ParameterTransactionDataImpl.java @@ -16,6 +16,7 @@ import com.ibm.fhir.persistence.jdbc.TransactionData; import com.ibm.fhir.persistence.jdbc.dao.impl.ResourceProfileRec; +import com.ibm.fhir.persistence.jdbc.dao.impl.ResourceReferenceValueRec; import com.ibm.fhir.persistence.jdbc.dao.impl.ResourceTokenValueRec; /** @@ -38,6 +39,9 @@ public class ParameterTransactionDataImpl implements TransactionData { // Collect all the token values so we can submit once per transaction private final List tokenValueRecs = new ArrayList<>(); + // Collect all the reference values so we can submit once per transaction + private final List referenceValueRecs = new ArrayList<>(); + // Collect all the profile values so we can submit once per transaction private final List profileRecs = new ArrayList<>(); @@ -63,7 +67,7 @@ public ParameterTransactionDataImpl(String datasourceId, FHIRPersistenceJDBCImpl public void persist() { try { - impl.onCommit(tokenValueRecs, profileRecs, tagRecs, securityRecs); + impl.onCommit(tokenValueRecs, referenceValueRecs, profileRecs, tagRecs, securityRecs); } catch (Throwable t) { logger.log(Level.SEVERE, "Failed persisting parameter transaction data. Marking transaction for rollback", t); try { @@ -82,6 +86,14 @@ public void addValue(ResourceTokenValueRec rec) { tokenValueRecs.add(rec); } + /** + * Add the record to the list of reference values being accumulated in this transaction + * @param rec + */ + public void addReferenceValue(ResourceReferenceValueRec rec) { + referenceValueRecs.add(rec); + } + /** * Add the given profile parameter record to the list of records being accumulated in * this transaction data. The records will be inserted to the database together at the diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/postgres/PostgresCodeSystemDAO.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/postgres/PostgresCodeSystemDAO.java index 3dfc8f7f7c0..01b4019f598 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/postgres/PostgresCodeSystemDAO.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/postgres/PostgresCodeSystemDAO.java @@ -12,8 +12,8 @@ import java.util.logging.Level; import java.util.logging.Logger; +import com.ibm.fhir.persistence.exception.FHIRPersistenceDataAccessException; import com.ibm.fhir.persistence.jdbc.dao.impl.CodeSystemDAOImpl; -import com.ibm.fhir.persistence.jdbc.exception.FHIRPersistenceDataAccessException; /** * PostgreSql variant DAO used to manage code_systems records. Uses diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/postgres/PostgresParameterNamesDAO.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/postgres/PostgresParameterNamesDAO.java index ffa9161402d..35d1d9a4916 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/postgres/PostgresParameterNamesDAO.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/postgres/PostgresParameterNamesDAO.java @@ -12,8 +12,8 @@ import java.util.logging.Level; import java.util.logging.Logger; +import com.ibm.fhir.persistence.exception.FHIRPersistenceDataAccessException; import com.ibm.fhir.persistence.jdbc.dao.impl.ParameterNameDAOImpl; -import com.ibm.fhir.persistence.jdbc.exception.FHIRPersistenceDataAccessException; public class PostgresParameterNamesDAO extends ParameterNameDAOImpl { private static final String CLASSNAME = PostgresParameterNamesDAO.class.getName(); diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/postgres/PostgresResourceDAO.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/postgres/PostgresResourceDAO.java index 8db642b5f25..8c1fa2700e8 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/postgres/PostgresResourceDAO.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/postgres/PostgresResourceDAO.java @@ -22,10 +22,13 @@ import javax.transaction.TransactionSynchronizationRegistry; +import com.ibm.fhir.database.utils.api.SchemaType; import com.ibm.fhir.database.utils.common.CalendarHelper; import com.ibm.fhir.persistence.InteractionStatus; +import com.ibm.fhir.persistence.exception.FHIRPersistenceDataAccessException; import com.ibm.fhir.persistence.exception.FHIRPersistenceException; import com.ibm.fhir.persistence.exception.FHIRPersistenceVersionIdMismatchException; +import com.ibm.fhir.persistence.index.FHIRRemoteIndexService; import com.ibm.fhir.persistence.jdbc.FHIRPersistenceJDBCCache; import com.ibm.fhir.persistence.jdbc.connection.FHIRDbFlavor; import com.ibm.fhir.persistence.jdbc.dao.api.FHIRDAOConstants; @@ -39,7 +42,6 @@ import com.ibm.fhir.persistence.jdbc.dto.ExtractedParameterValue; import com.ibm.fhir.persistence.jdbc.dto.Resource; import com.ibm.fhir.persistence.jdbc.exception.FHIRPersistenceDBConnectException; -import com.ibm.fhir.persistence.jdbc.exception.FHIRPersistenceDataAccessException; import com.ibm.fhir.persistence.jdbc.exception.FHIRPersistenceFKVException; import com.ibm.fhir.persistence.jdbc.impl.ParameterTransactionDataImpl; @@ -48,24 +50,53 @@ * the stored procedure (or function, in this case) */ public class PostgresResourceDAO extends ResourceDAOImpl { - private static final String CLASSNAME = PostgresResourceDAO.class.getSimpleName(); + private static final String CLASSNAME = PostgresResourceDAO.class.getName(); private static final Logger logger = Logger.getLogger(CLASSNAME); private static final String SQL_READ_RESOURCE_TYPE = "{CALL %s.add_resource_type(?, ?)}"; // 13 args (9 in, 4 out) private static final String SQL_INSERT_WITH_PARAMETERS = "{CALL %s.add_any_resource(?,?,?,?,?,?,?,?,?,?,?,?,?,?)}"; + // 14 args (10 in, 4 out) + private static final String SQL_SHARDED_INSERT_WITH_PARAMETERS = "{CALL %s.add_any_resource(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)}"; + + private static final String SQL_SHARDED_READ = "" + + "SELECT R.RESOURCE_ID, R.LOGICAL_RESOURCE_ID, R.VERSION_ID, R.LAST_UPDATED, R.IS_DELETED, " + + " R.DATA, LR.LOGICAL_ID, R.RESOURCE_PAYLOAD_KEY " + + " FROM %s_RESOURCES R, " + + " %s_LOGICAL_RESOURCES LR " + + " WHERE LR.SHARD_KEY = ? " + + " AND LR.LOGICAL_ID = ? " + + " AND R.RESOURCE_ID = LR.CURRENT_RESOURCE_ID " + + " AND R.SHARD_KEY = LR.SHARD_KEY "; + + // Read a specific version of the resource + private static final String SQL_SHARDED_VERSION_READ = "" + + "SELECT R.RESOURCE_ID, R.LOGICAL_RESOURCE_ID, R.VERSION_ID, R.LAST_UPDATED, R.IS_DELETED, " + + " R.DATA, LR.LOGICAL_ID, R.RESOURCE_PAYLOAD_KEY " + + " FROM %s_RESOURCES R, " + + " %s_LOGICAL_RESOURCES LR " + + " WHERE LR.SHARD_KEY = ? " + + " AND LR.LOGICAL_ID = ? " + + " AND R.VERSION_ID = ? " + + " AND R.LOGICAL_RESOURCE_ID = LR.LOGICAL_RESOURCE_ID " + + " AND R.SHARD_KEY = LR.SHARD_KEY "; // DAO used to obtain sequence values from FHIR_REF_SEQUENCE private FhirRefSequenceDAO fhirRefSequenceDAO; - public PostgresResourceDAO(Connection connection, String schemaName, FHIRDbFlavor flavor, FHIRPersistenceJDBCCache cache, IResourceReferenceDAO rrd) { + // The (optional) shard key used with sharded databases + private final Short shardKey; + + public PostgresResourceDAO(Connection connection, String schemaName, FHIRDbFlavor flavor, FHIRPersistenceJDBCCache cache, IResourceReferenceDAO rrd, Short shardKey) { super(connection, schemaName, flavor, cache, rrd); + this.shardKey = shardKey; } public PostgresResourceDAO(Connection connection, String schemaName, FHIRDbFlavor flavor, TransactionSynchronizationRegistry trxSynchRegistry, FHIRPersistenceJDBCCache cache, IResourceReferenceDAO rrd, - ParameterTransactionDataImpl ptdi) { + ParameterTransactionDataImpl ptdi, Short shardKey) { super(connection, schemaName, flavor, trxSynchRegistry, cache, rrd, ptdi); + this.shardKey = shardKey; } @Override @@ -76,7 +107,6 @@ public Resource insert(Resource resource, List paramete logger.entering(CLASSNAME, METHODNAME); final Connection connection = getConnection(); // do not close - CallableStatement stmt = null; String stmtString = null; Timestamp lastUpdated; long dbCallStartTime; @@ -87,63 +117,80 @@ public Resource insert(Resource resource, List paramete // hit the procedure Objects.requireNonNull(getResourceTypeId(resource.getResourceType())); - stmtString = String.format(SQL_INSERT_WITH_PARAMETERS, getSchemaName()); - stmt = connection.prepareCall(stmtString); - stmt.setString(1, resource.getResourceType()); - stmt.setString(2, resource.getLogicalId()); - - if (resource.getDataStream() != null) { - stmt.setBinaryStream(3, resource.getDataStream().inputStream()); + if (getFlavor().getSchemaType() == SchemaType.SHARDED) { + if (this.shardKey == null) { + throw new FHIRPersistenceException("Shard key value required when schema type is SHARDED"); + } + stmtString = String.format(SQL_SHARDED_INSERT_WITH_PARAMETERS, getSchemaName()); } else { - // payload was offloaded to another data store - stmt.setNull(3, Types.BINARY); + stmtString = String.format(SQL_INSERT_WITH_PARAMETERS, getSchemaName()); } - lastUpdated = resource.getLastUpdated(); - stmt.setTimestamp(4, lastUpdated, CalendarHelper.getCalendarForUTC()); - stmt.setString(5, resource.isDeleted() ? "Y": "N"); - stmt.setString(6, UUID.randomUUID().toString()); - stmt.setInt(7, resource.getVersionId()); - stmt.setString(8, parameterHashB64); - setInt(stmt, 9, ifNoneMatch); - setString(stmt, 10, resource.getResourcePayloadKey()); - stmt.registerOutParameter(11, Types.BIGINT); - stmt.registerOutParameter(12, Types.VARCHAR); // The old parameter_hash - stmt.registerOutParameter(13, Types.INTEGER); // o_interaction_status - stmt.registerOutParameter(14, Types.INTEGER); // o_if_none_match_version - - dbCallStartTime = System.nanoTime(); - stmt.execute(); - dbCallDuration = (System.nanoTime()-dbCallStartTime)/1e6; - - resource.setId(stmt.getLong(11)); - - if (stmt.getInt(13) == 1) { // interaction status - // no change, so skip parameter updates - resource.setInteractionStatus(InteractionStatus.IF_NONE_MATCH_EXISTED); - resource.setIfNoneMatchVersion(stmt.getInt(14)); // current version - } else { - resource.setInteractionStatus(InteractionStatus.MODIFIED); + try (CallableStatement stmt = connection.prepareCall(stmtString)) { + int arg = 1; + if (getFlavor().getSchemaType() == SchemaType.SHARDED) { + stmt.setShort(arg++, shardKey); + } + stmt.setString(arg++, resource.getResourceType()); + stmt.setString(arg++, resource.getLogicalId()); + + if (resource.getDataStream() != null) { + stmt.setBinaryStream(arg++, resource.getDataStream().inputStream()); + } else { + // payload was offloaded to another data store + stmt.setNull(arg++, Types.BINARY); + } + + lastUpdated = resource.getLastUpdated(); + stmt.setTimestamp(arg++, lastUpdated, CalendarHelper.getCalendarForUTC()); + stmt.setString(arg++, resource.isDeleted() ? "Y": "N"); + stmt.setString(arg++, UUID.randomUUID().toString()); + stmt.setInt(arg++, resource.getVersionId()); + stmt.setString(arg++, parameterHashB64); + setInt(stmt, arg++, ifNoneMatch); + setString(stmt, arg++, resource.getResourcePayloadKey()); + + // TODO use a helper function which can return the arg index to help clean up the syntax + stmt.registerOutParameter(arg, Types.BIGINT); final int logicalResourceIdIndex = arg++; + stmt.registerOutParameter(arg, Types.VARCHAR); final int oldParameterHashIndex = arg++; + stmt.registerOutParameter(arg, Types.INTEGER); final int interactionStatusIndex = arg++; + stmt.registerOutParameter(arg, Types.INTEGER); final int ifNoneMatchVersionIndex = arg++; - // Parameter time - // To keep things simple for the postgresql use-case, we just use a visitor to - // handle inserts of parameters directly in the resource parameter tables. - // Note we don't get any parameters for the resource soft-delete operation - final String currentParameterHash = stmt.getString(12); - if (parameters != null && (parameterHashB64 == null || parameterHashB64.isEmpty() - || !parameterHashB64.equals(currentParameterHash))) { - // postgresql doesn't support partitioned multi-tenancy, so we disable it on the DAO: - JDBCIdentityCache identityCache = new JDBCIdentityCacheImpl(getCache(), this, parameterDao, getResourceReferenceDAO()); - try (ParameterVisitorBatchDAO pvd = new ParameterVisitorBatchDAO(connection, null, resource.getResourceType(), false, resource.getId(), 100, - identityCache, getResourceReferenceDAO(), getTransactionData())) { - for (ExtractedParameterValue p: parameters) { - p.accept(pvd); + dbCallStartTime = System.nanoTime(); + stmt.execute(); + dbCallDuration = (System.nanoTime()-dbCallStartTime)/1e6; + + resource.setLogicalResourceId(stmt.getLong(logicalResourceIdIndex)); + if (stmt.getInt(interactionStatusIndex) == 1) { // interaction status + // no change, so skip parameter updates + resource.setInteractionStatus(InteractionStatus.IF_NONE_MATCH_EXISTED); + resource.setIfNoneMatchVersion(stmt.getInt(ifNoneMatchVersionIndex)); // current version + } else { + resource.setInteractionStatus(InteractionStatus.MODIFIED); + + // Parameter time + // To keep things simple for the postgresql use-case, we just use a visitor to + // handle inserts of parameters directly in the resource parameter tables. + // Note we don't get any parameters for the resource soft-delete operation + // Bypass the parameter insert here if we have the remoteIndexService configured + FHIRRemoteIndexService remoteIndexService = FHIRRemoteIndexService.getServiceInstance(); + final String currentParameterHash = stmt.getString(oldParameterHashIndex); + if (remoteIndexService == null + && parameters != null && (parameterHashB64 == null || parameterHashB64.isEmpty() + || !parameterHashB64.equals(currentParameterHash))) { + // postgresql doesn't support partitioned multi-tenancy, so we disable it on the DAO: + JDBCIdentityCache identityCache = new JDBCIdentityCacheImpl(getCache(), this, parameterDao, getResourceReferenceDAO()); + try (ParameterVisitorBatchDAO pvd = new ParameterVisitorBatchDAO(connection, null, resource.getResourceType(), false, resource.getLogicalResourceId(), 100, + identityCache, getResourceReferenceDAO(), getTransactionData())) { + for (ExtractedParameterValue p: parameters) { + p.accept(pvd); + } } } } - } - if (logger.isLoggable(Level.FINE)) { - logger.fine("Successfully inserted Resource. id=" + resource.getId() + " executionTime=" + dbCallDuration + "ms"); + if (logger.isLoggable(Level.FINE)) { + logger.fine("Successfully inserted Resource. logicalResourceId=" + resource.getLogicalResourceId() + " executionTime=" + dbCallDuration + "ms"); + } } } catch(FHIRPersistenceDBConnectException | FHIRPersistenceDataAccessException e) { throw e; @@ -168,21 +215,54 @@ identityCache, getResourceReferenceDAO(), getTransactionData())) { return resource; } - /** - * Delete all parameters for the given resourceId from the parameters table - * - * @param conn - * @param tableName - * @param logicalResourceId - * @throws SQLException - */ - protected void deleteFromParameterTable(Connection conn, String tableName, long logicalResourceId) throws SQLException { - final String delStrValues = "DELETE FROM " + tableName + " WHERE logical_resource_id = ?"; - try (PreparedStatement stmt = conn.prepareStatement(delStrValues)) { - // bind parameters - stmt.setLong(1, logicalResourceId); - stmt.executeUpdate(); + @Override + public Resource read(String logicalId, String resourceType) throws FHIRPersistenceDataAccessException, FHIRPersistenceDBConnectException { + final String METHODNAME = "read"; + logger.entering(CLASSNAME, METHODNAME); + + Resource resource = null; + if (getFlavor().getSchemaType() == SchemaType.SHARDED) { + List resources; + String stmtString = null; + + try { + stmtString = String.format(SQL_SHARDED_READ, resourceType, resourceType); + resources = this.runQuery(stmtString, shardKey, logicalId); + if (!resources.isEmpty()) { + resource = resources.get(0); + } + } finally { + logger.exiting(CLASSNAME, METHODNAME); + } + } else { + resource = super.read(logicalId, resourceType); + } + return resource; + } + + @Override + public Resource versionRead(String logicalId, String resourceType, int versionId) throws FHIRPersistenceDataAccessException, FHIRPersistenceDBConnectException { + final String METHODNAME = "versionRead"; + logger.entering(CLASSNAME, METHODNAME); + + Resource resource = null; + if (getFlavor().getSchemaType() == SchemaType.SHARDED) { + String stmtString = null; + + try { + stmtString = String.format(SQL_SHARDED_VERSION_READ, resourceType, resourceType); + List resources = this.runQuery(stmtString, shardKey, logicalId, versionId); + if (!resources.isEmpty()) { + resource = resources.get(0); + } + } finally { + logger.exiting(CLASSNAME, METHODNAME); + } + } else { + resource = super.versionRead(logicalId, resourceType, versionId); } + return resource; + } /** diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/postgres/PostgresResourceNoProcDAO.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/postgres/PostgresResourceNoProcDAO.java index 5e24825a8ef..bff777ad2ec 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/postgres/PostgresResourceNoProcDAO.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/postgres/PostgresResourceNoProcDAO.java @@ -27,6 +27,7 @@ import com.ibm.fhir.database.utils.common.CalendarHelper; import com.ibm.fhir.persistence.InteractionStatus; +import com.ibm.fhir.persistence.exception.FHIRPersistenceDataAccessException; import com.ibm.fhir.persistence.exception.FHIRPersistenceException; import com.ibm.fhir.persistence.exception.FHIRPersistenceVersionIdMismatchException; import com.ibm.fhir.persistence.jdbc.FHIRPersistenceJDBCCache; @@ -42,7 +43,6 @@ import com.ibm.fhir.persistence.jdbc.dto.ExtractedParameterValue; import com.ibm.fhir.persistence.jdbc.dto.Resource; import com.ibm.fhir.persistence.jdbc.exception.FHIRPersistenceDBConnectException; -import com.ibm.fhir.persistence.jdbc.exception.FHIRPersistenceDataAccessException; import com.ibm.fhir.persistence.jdbc.exception.FHIRPersistenceFKVException; import com.ibm.fhir.persistence.jdbc.impl.ParameterTransactionDataImpl; import com.ibm.fhir.persistence.jdbc.util.ParameterTableSupport; @@ -100,7 +100,7 @@ public Resource insert(Resource resource, List paramete AtomicInteger outInteractionStatus = new AtomicInteger(); AtomicInteger outIfNoneMatchVersion = new AtomicInteger(); - long resourceId = this.storeResource(resource.getResourceType(), + long logicalResourceId = this.storeResource(resource.getResourceType(), parameters, resource.getLogicalId(), resource.getDataStream().inputStream(), @@ -125,11 +125,11 @@ public Resource insert(Resource resource, List paramete resource.setIfNoneMatchVersion(outIfNoneMatchVersion.get()); } else { resource.setInteractionStatus(InteractionStatus.MODIFIED); - resource.setId(resourceId); + resource.setLogicalResourceId(logicalResourceId); } if (logger.isLoggable(Level.FINE)) { - logger.fine("Successfully inserted Resource. id=" + resource.getId() + " executionTime=" + dbCallDuration + "ms"); + logger.fine("Successfully inserted Resource. logicalResourceId=" + resource.getLogicalResourceId() + " executionTime=" + dbCallDuration + "ms"); } } catch(FHIRPersistenceDBConnectException | FHIRPersistenceDataAccessException e) { throw e; @@ -459,7 +459,7 @@ identityCache, getResourceReferenceDAO(), getTransactionData())) { } logger.exiting(CLASSNAME, METHODNAME); - return v_resource_id; + return v_logical_resource_id; } /** diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/postgres/PostgresResourceReferenceDAO.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/postgres/PostgresResourceReferenceDAO.java index 6a3ccf2746b..4a15aefd001 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/postgres/PostgresResourceReferenceDAO.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/postgres/PostgresResourceReferenceDAO.java @@ -10,17 +10,22 @@ import java.sql.PreparedStatement; import java.sql.SQLException; import java.util.Collection; +import java.util.List; import java.util.logging.Level; import java.util.logging.Logger; import com.ibm.fhir.database.utils.api.IDatabaseTranslator; +import com.ibm.fhir.persistence.exception.FHIRPersistenceDataAccessException; +import com.ibm.fhir.persistence.exception.FHIRPersistenceException; import com.ibm.fhir.persistence.jdbc.dao.api.ICommonTokenValuesCache; +import com.ibm.fhir.persistence.jdbc.dao.api.ILogicalResourceIdentCache; import com.ibm.fhir.persistence.jdbc.dao.api.INameIdCache; +import com.ibm.fhir.persistence.jdbc.dao.api.LogicalResourceIdentKey; +import com.ibm.fhir.persistence.jdbc.dao.api.LogicalResourceIdentValue; import com.ibm.fhir.persistence.jdbc.dao.api.ParameterNameDAO; import com.ibm.fhir.persistence.jdbc.dao.impl.ResourceReferenceDAO; import com.ibm.fhir.persistence.jdbc.dto.CommonTokenValue; import com.ibm.fhir.persistence.jdbc.exception.FHIRPersistenceDBConnectException; -import com.ibm.fhir.persistence.jdbc.exception.FHIRPersistenceDataAccessException; /** * Postgres-specific extension of the {@link ResourceReferenceDAO} to work around @@ -31,13 +36,17 @@ public class PostgresResourceReferenceDAO extends ResourceReferenceDAO { /** * Public constructor + * * @param t * @param c * @param schemaName * @param cache + * @param parameterNameCache + * @param logicalResourceIdentCache */ - public PostgresResourceReferenceDAO(IDatabaseTranslator t, Connection c, String schemaName, ICommonTokenValuesCache cache, INameIdCache parameterNameCache) { - super(t, c, schemaName, cache, parameterNameCache); + public PostgresResourceReferenceDAO(IDatabaseTranslator t, Connection c, String schemaName, ICommonTokenValuesCache cache, INameIdCache parameterNameCache, + ILogicalResourceIdentCache logicalResourceIdentCache) { + super(t, c, schemaName, cache, parameterNameCache, logicalResourceIdentCache); } @Override @@ -138,6 +147,40 @@ protected void doCommonTokenValuesUpsert(String paramList, Collection missing) throws FHIRPersistenceException { + // For PostgreSQL we can handle concurrency issues using ON CONFLICT DO NOTHING + // to skip inserts for records that already exist + final int batchSize = 256; + final String nextVal = getTranslator().nextValue(getSchemaName(), "fhir_sequence"); + StringBuilder insert = new StringBuilder(); + insert.append("INSERT INTO logical_resource_ident (resource_type_id, logical_id, logical_resource_id) VALUES (?,?,"); + insert.append(nextVal); // next sequence value + insert.append(") ON CONFLICT DO NOTHING"); + + logger.fine(() -> "ident insert: " + insert.toString()); + try (PreparedStatement ps = getConnection().prepareStatement(insert.toString())) { + int count = 0; + for (LogicalResourceIdentKey value: missing) { + ps.setInt(1, value.getResourceTypeId()); + ps.setString(2, value.getLogicalId()); + ps.addBatch(); + if (++count == batchSize) { + // not too many statements in a single batch + ps.executeBatch(); + count = 0; + } + } + if (count > 0) { + // final batch + ps.executeBatch(); + } + } catch (SQLException x) { + logger.log(Level.SEVERE, "logical_resource_ident insert failed: " + insert.toString(), x); + throw new FHIRPersistenceException("logical_resource_ident insert failed"); + } + } @Override protected int readOrAddParameterNameId(String parameterName) throws FHIRPersistenceDBConnectException, FHIRPersistenceDataAccessException { diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/util/ParameterTableSupport.java b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/util/ParameterTableSupport.java index 639c0ef15fd..5324a390628 100644 --- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/util/ParameterTableSupport.java +++ b/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/util/ParameterTableSupport.java @@ -33,6 +33,7 @@ public static void deleteFromParameterTables(Connection conn, String tablePrefix deleteFromParameterTable(conn, tablePrefix + "_profiles", v_logical_resource_id); deleteFromParameterTable(conn, tablePrefix + "_tags", v_logical_resource_id); deleteFromParameterTable(conn, tablePrefix + "_security", v_logical_resource_id); + deleteFromParameterTable(conn, tablePrefix + "_ref_values", v_logical_resource_id); // delete any system level parameters we have for this resource deleteFromParameterTable(conn, "str_values", v_logical_resource_id); diff --git a/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/search/test/JDBCSearchCompartmentTest.java b/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/search/test/JDBCSearchCompartmentTest.java index 975bce0b366..6eba549e9c6 100644 --- a/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/search/test/JDBCSearchCompartmentTest.java +++ b/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/search/test/JDBCSearchCompartmentTest.java @@ -6,21 +6,9 @@ package com.ibm.fhir.persistence.jdbc.search.test; -import java.util.Properties; - import com.ibm.fhir.config.FHIRConfigProvider; -import com.ibm.fhir.database.utils.api.IConnectionProvider; -import com.ibm.fhir.database.utils.pool.PoolConnectionProvider; -import com.ibm.fhir.model.test.TestUtil; import com.ibm.fhir.persistence.FHIRPersistence; -import com.ibm.fhir.persistence.jdbc.FHIRPersistenceJDBCCache; -import com.ibm.fhir.persistence.jdbc.cache.CommonTokenValuesCacheImpl; -import com.ibm.fhir.persistence.jdbc.cache.FHIRPersistenceJDBCCacheImpl; -import com.ibm.fhir.persistence.jdbc.cache.IdNameCache; -import com.ibm.fhir.persistence.jdbc.cache.NameIdCache; -import com.ibm.fhir.persistence.jdbc.dao.api.ICommonTokenValuesCache; -import com.ibm.fhir.persistence.jdbc.impl.FHIRPersistenceJDBCImpl; -import com.ibm.fhir.persistence.jdbc.test.util.DerbyInitializer; +import com.ibm.fhir.persistence.jdbc.test.util.PersistenceTestSupport; import com.ibm.fhir.persistence.search.test.AbstractSearchCompartmentTest; import com.ibm.fhir.search.util.SearchHelper; @@ -29,43 +17,23 @@ */ public class JDBCSearchCompartmentTest extends AbstractSearchCompartmentTest { - private Properties testProps; - - private PoolConnectionProvider connectionPool; - - private FHIRPersistenceJDBCCache cache; - - public JDBCSearchCompartmentTest() throws Exception { - this.testProps = TestUtil.readTestProperties("test.jdbc.properties"); - } + // Container to hide the instantiation of the persistence impl used for tests + private PersistenceTestSupport testSupport; @Override public void bootstrapDatabase() throws Exception { - DerbyInitializer derbyInit; - String dbDriverName = this.testProps.getProperty("dbDriverName"); - if (dbDriverName != null && dbDriverName.contains("derby")) { - derbyInit = new DerbyInitializer(this.testProps); - IConnectionProvider cp = derbyInit.getConnectionProvider(false); - this.connectionPool = new PoolConnectionProvider(cp, 1); - ICommonTokenValuesCache rrc = new CommonTokenValuesCacheImpl(100, 100, 100); - cache = new FHIRPersistenceJDBCCacheImpl(new NameIdCache(), new IdNameCache(), new NameIdCache(), rrc); - } + testSupport = new PersistenceTestSupport(); } @Override public FHIRPersistence getPersistenceImpl(FHIRConfigProvider configProvider, SearchHelper searchHelper) throws Exception { - if (this.connectionPool == null) { - throw new IllegalStateException("Database not bootstrapped"); - } - return new FHIRPersistenceJDBCImpl(this.testProps, this.connectionPool, configProvider, cache, searchHelper); + return testSupport.getPersistenceImpl(configProvider, searchHelper); } @Override protected void shutdownPools() throws Exception { - // Mark the pool as no longer in use. This allows the pool to check for - // lingering open connections/transactions. - if (this.connectionPool != null) { - this.connectionPool.close(); + if (testSupport != null) { + testSupport.shutdown(); } } } \ No newline at end of file diff --git a/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/search/test/JDBCSearchCompositeTest.java b/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/search/test/JDBCSearchCompositeTest.java index 7318e8f2b4b..821731a1d7e 100644 --- a/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/search/test/JDBCSearchCompositeTest.java +++ b/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/search/test/JDBCSearchCompositeTest.java @@ -6,65 +6,32 @@ package com.ibm.fhir.persistence.jdbc.search.test; -import java.util.Properties; - import com.ibm.fhir.config.FHIRConfigProvider; -import com.ibm.fhir.database.utils.api.IConnectionProvider; -import com.ibm.fhir.database.utils.pool.PoolConnectionProvider; -import com.ibm.fhir.model.test.TestUtil; import com.ibm.fhir.persistence.FHIRPersistence; -import com.ibm.fhir.persistence.jdbc.FHIRPersistenceJDBCCache; -import com.ibm.fhir.persistence.jdbc.cache.CommonTokenValuesCacheImpl; -import com.ibm.fhir.persistence.jdbc.cache.FHIRPersistenceJDBCCacheImpl; -import com.ibm.fhir.persistence.jdbc.cache.IdNameCache; -import com.ibm.fhir.persistence.jdbc.cache.NameIdCache; -import com.ibm.fhir.persistence.jdbc.dao.api.ICommonTokenValuesCache; -import com.ibm.fhir.persistence.jdbc.impl.FHIRPersistenceJDBCImpl; -import com.ibm.fhir.persistence.jdbc.test.util.DerbyInitializer; +import com.ibm.fhir.persistence.jdbc.test.util.PersistenceTestSupport; import com.ibm.fhir.persistence.search.test.AbstractSearchCompositeTest; import com.ibm.fhir.search.util.SearchHelper; public class JDBCSearchCompositeTest extends AbstractSearchCompositeTest { - private Properties testProps; - - private PoolConnectionProvider connectionPool; - - private FHIRPersistenceJDBCCache cache; - - public JDBCSearchCompositeTest() throws Exception { - this.testProps = TestUtil.readTestProperties("test.jdbc.properties"); - } + // Container to hide the instantiation of the persistence impl used for tests + private PersistenceTestSupport testSupport; @Override public void bootstrapDatabase() throws Exception { - DerbyInitializer derbyInit; - String dbDriverName = this.testProps.getProperty("dbDriverName"); - if (dbDriverName != null && dbDriverName.contains("derby")) { - derbyInit = new DerbyInitializer(this.testProps); - IConnectionProvider cp = derbyInit.getConnectionProvider(false); - this.connectionPool = new PoolConnectionProvider(cp, 1); - - ICommonTokenValuesCache rrc = new CommonTokenValuesCacheImpl(100, 100, 100); - cache = new FHIRPersistenceJDBCCacheImpl(new NameIdCache(), new IdNameCache(), new NameIdCache(), rrc); - } + testSupport = new PersistenceTestSupport(); } @Override public FHIRPersistence getPersistenceImpl(FHIRConfigProvider configProvider, SearchHelper searchHelper) throws Exception { - if (this.connectionPool == null) { - throw new IllegalStateException("Database not bootstrapped"); - } - return new FHIRPersistenceJDBCImpl(this.testProps, this.connectionPool, configProvider, cache, searchHelper); + return testSupport.getPersistenceImpl(configProvider, searchHelper); } @Override protected void shutdownPools() throws Exception { - // Mark the pool as no longer in use. This allows the pool to check for - // lingering open connections/transactions. - if (this.connectionPool != null) { - this.connectionPool.close(); + if (testSupport != null) { + testSupport.shutdown(); } } } \ No newline at end of file diff --git a/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/search/test/JDBCSearchDateTest.java b/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/search/test/JDBCSearchDateTest.java index 01fa14b21cb..2c25e61617e 100644 --- a/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/search/test/JDBCSearchDateTest.java +++ b/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/search/test/JDBCSearchDateTest.java @@ -6,62 +6,30 @@ package com.ibm.fhir.persistence.jdbc.search.test; -import java.util.Properties; - import com.ibm.fhir.config.FHIRConfigProvider; -import com.ibm.fhir.database.utils.api.IConnectionProvider; -import com.ibm.fhir.database.utils.pool.PoolConnectionProvider; -import com.ibm.fhir.model.test.TestUtil; import com.ibm.fhir.persistence.FHIRPersistence; -import com.ibm.fhir.persistence.jdbc.FHIRPersistenceJDBCCache; -import com.ibm.fhir.persistence.jdbc.cache.CommonTokenValuesCacheImpl; -import com.ibm.fhir.persistence.jdbc.cache.FHIRPersistenceJDBCCacheImpl; -import com.ibm.fhir.persistence.jdbc.cache.IdNameCache; -import com.ibm.fhir.persistence.jdbc.cache.NameIdCache; -import com.ibm.fhir.persistence.jdbc.dao.api.ICommonTokenValuesCache; -import com.ibm.fhir.persistence.jdbc.impl.FHIRPersistenceJDBCImpl; -import com.ibm.fhir.persistence.jdbc.test.util.DerbyInitializer; +import com.ibm.fhir.persistence.jdbc.test.util.PersistenceTestSupport; import com.ibm.fhir.persistence.search.test.AbstractSearchDateTest; import com.ibm.fhir.search.util.SearchHelper; public class JDBCSearchDateTest extends AbstractSearchDateTest { - private Properties testProps; - - private PoolConnectionProvider connectionPool; - - private FHIRPersistenceJDBCCache cache; - - public JDBCSearchDateTest() throws Exception { - this.testProps = TestUtil.readTestProperties("test.jdbc.properties"); - } + // Container to hide the instantiation of the persistence impl used for tests + private PersistenceTestSupport testSupport; @Override public void bootstrapDatabase() throws Exception { - DerbyInitializer derbyInit; - String dbDriverName = this.testProps.getProperty("dbDriverName"); - if (dbDriverName != null && dbDriverName.contains("derby")) { - derbyInit = new DerbyInitializer(this.testProps); - IConnectionProvider cp = derbyInit.getConnectionProvider(false); - this.connectionPool = new PoolConnectionProvider(cp, 1); - - ICommonTokenValuesCache rrc = new CommonTokenValuesCacheImpl(100, 100, 100); - cache = new FHIRPersistenceJDBCCacheImpl(new NameIdCache(), new IdNameCache(), new NameIdCache(), rrc); - } + testSupport = new PersistenceTestSupport(); } + @Override public FHIRPersistence getPersistenceImpl(FHIRConfigProvider configProvider, SearchHelper searchHelper) throws Exception { - if (this.connectionPool == null) { - throw new IllegalStateException("Database not bootstrapped"); - } - return new FHIRPersistenceJDBCImpl(this.testProps, this.connectionPool, configProvider, cache, searchHelper); + return testSupport.getPersistenceImpl(configProvider, searchHelper); } @Override protected void shutdownPools() throws Exception { - // Mark the pool as no longer in use. This allows the pool to check for - // lingering open connections/transactions. - if (this.connectionPool != null) { - this.connectionPool.close(); + if (testSupport != null) { + testSupport.shutdown(); } } } diff --git a/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/search/test/JDBCSearchIdLastUpdatedTest.java b/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/search/test/JDBCSearchIdLastUpdatedTest.java index abd0e22dd64..90c51acf3f5 100644 --- a/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/search/test/JDBCSearchIdLastUpdatedTest.java +++ b/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/search/test/JDBCSearchIdLastUpdatedTest.java @@ -6,62 +6,30 @@ package com.ibm.fhir.persistence.jdbc.search.test; -import java.util.Properties; - import com.ibm.fhir.config.FHIRConfigProvider; -import com.ibm.fhir.database.utils.api.IConnectionProvider; -import com.ibm.fhir.database.utils.pool.PoolConnectionProvider; -import com.ibm.fhir.model.test.TestUtil; import com.ibm.fhir.persistence.FHIRPersistence; -import com.ibm.fhir.persistence.jdbc.FHIRPersistenceJDBCCache; -import com.ibm.fhir.persistence.jdbc.cache.CommonTokenValuesCacheImpl; -import com.ibm.fhir.persistence.jdbc.cache.FHIRPersistenceJDBCCacheImpl; -import com.ibm.fhir.persistence.jdbc.cache.IdNameCache; -import com.ibm.fhir.persistence.jdbc.cache.NameIdCache; -import com.ibm.fhir.persistence.jdbc.dao.api.ICommonTokenValuesCache; -import com.ibm.fhir.persistence.jdbc.impl.FHIRPersistenceJDBCImpl; -import com.ibm.fhir.persistence.jdbc.test.util.DerbyInitializer; +import com.ibm.fhir.persistence.jdbc.test.util.PersistenceTestSupport; import com.ibm.fhir.persistence.search.test.AbstractSearchIdAndLastUpdatedTest; import com.ibm.fhir.search.util.SearchHelper; public class JDBCSearchIdLastUpdatedTest extends AbstractSearchIdAndLastUpdatedTest { - private Properties testProps; - - private PoolConnectionProvider connectionPool; - - private FHIRPersistenceJDBCCache cache; - - public JDBCSearchIdLastUpdatedTest() throws Exception { - this.testProps = TestUtil.readTestProperties("test.jdbc.properties"); - } + // Container to hide the instantiation of the persistence impl used for tests + private PersistenceTestSupport testSupport; @Override public void bootstrapDatabase() throws Exception { - DerbyInitializer derbyInit; - String dbDriverName = this.testProps.getProperty("dbDriverName"); - if (dbDriverName != null && dbDriverName.contains("derby")) { - derbyInit = new DerbyInitializer(this.testProps); - IConnectionProvider cp = derbyInit.getConnectionProvider(false); - this.connectionPool = new PoolConnectionProvider(cp, 1); - ICommonTokenValuesCache rrc = new CommonTokenValuesCacheImpl(100, 100, 100); - cache = new FHIRPersistenceJDBCCacheImpl(new NameIdCache(), new IdNameCache(), new NameIdCache(), rrc); - } + testSupport = new PersistenceTestSupport(); } @Override public FHIRPersistence getPersistenceImpl(FHIRConfigProvider configProvider, SearchHelper searchHelper) throws Exception { - if (this.connectionPool == null) { - throw new IllegalStateException("Database not bootstrapped"); - } - return new FHIRPersistenceJDBCImpl(this.testProps, this.connectionPool, configProvider, cache, searchHelper); + return testSupport.getPersistenceImpl(configProvider, searchHelper); } @Override protected void shutdownPools() throws Exception { - // Mark the pool as no longer in use. This allows the pool to check for - // lingering open connections/transactions. - if (this.connectionPool != null) { - this.connectionPool.close(); + if (testSupport != null) { + testSupport.shutdown(); } } } diff --git a/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/search/test/JDBCSearchNearTest.java b/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/search/test/JDBCSearchNearTest.java index 00ae1b1fbba..22858052857 100644 --- a/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/search/test/JDBCSearchNearTest.java +++ b/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/search/test/JDBCSearchNearTest.java @@ -20,7 +20,6 @@ import java.util.List; import java.util.Map; import java.util.Objects; -import java.util.Properties; import java.util.UUID; import java.util.logging.LogManager; @@ -32,8 +31,6 @@ import com.ibm.fhir.config.FHIRConfiguration; import com.ibm.fhir.config.FHIRRequestContext; -import com.ibm.fhir.database.utils.api.IConnectionProvider; -import com.ibm.fhir.database.utils.pool.PoolConnectionProvider; import com.ibm.fhir.model.resource.Location; import com.ibm.fhir.model.test.TestUtil; import com.ibm.fhir.model.type.Id; @@ -45,14 +42,7 @@ import com.ibm.fhir.persistence.context.FHIRPersistenceContext; import com.ibm.fhir.persistence.context.FHIRPersistenceContextFactory; import com.ibm.fhir.persistence.exception.FHIRPersistenceException; -import com.ibm.fhir.persistence.jdbc.FHIRPersistenceJDBCCache; -import com.ibm.fhir.persistence.jdbc.cache.CommonTokenValuesCacheImpl; -import com.ibm.fhir.persistence.jdbc.cache.FHIRPersistenceJDBCCacheImpl; -import com.ibm.fhir.persistence.jdbc.cache.IdNameCache; -import com.ibm.fhir.persistence.jdbc.cache.NameIdCache; -import com.ibm.fhir.persistence.jdbc.dao.api.ICommonTokenValuesCache; -import com.ibm.fhir.persistence.jdbc.impl.FHIRPersistenceJDBCImpl; -import com.ibm.fhir.persistence.jdbc.test.util.DerbyInitializer; +import com.ibm.fhir.persistence.jdbc.test.util.PersistenceTestSupport; import com.ibm.fhir.persistence.util.FHIRPersistenceUtil; import com.ibm.fhir.search.context.FHIRSearchContext; import com.ibm.fhir.search.util.SearchHelper; @@ -72,13 +62,15 @@ * */ public class JDBCSearchNearTest { - private Properties testProps; protected Location savedResource; protected static FHIRPersistence persistence; protected static SearchHelper searchHelper; + // Container to hide the instantiation of the persistence impl used for tests + private PersistenceTestSupport testSupport; + @BeforeClass public void startup() throws Exception { LogManager.getLogManager().readConfiguration( @@ -88,18 +80,7 @@ public void startup() throws Exception { searchHelper = new SearchHelper(); FHIRRequestContext.get().setTenantId("default"); - testProps = TestUtil.readTestProperties("test.jdbc.properties"); - - DerbyInitializer derbyInit; - PoolConnectionProvider connectionPool; - String dbDriverName = this.testProps.getProperty("dbDriverName"); - if (dbDriverName != null && dbDriverName.contains("derby")) { - derbyInit = new DerbyInitializer(this.testProps); - IConnectionProvider cp = derbyInit.getConnectionProvider(false); - connectionPool = new PoolConnectionProvider(cp, 1); - } else { - throw new IllegalStateException("dbDriverName must be set in test.jdbc.properties"); - } + testSupport = new PersistenceTestSupport(); savedResource = TestUtil.readExampleResource("json/spec/location-example.json"); savedResource = savedResource.toBuilder() @@ -110,9 +91,7 @@ public void startup() throws Exception { .build()) .build(); - ICommonTokenValuesCache rrc = new CommonTokenValuesCacheImpl(100, 100, 100); - FHIRPersistenceJDBCCache cache = new FHIRPersistenceJDBCCacheImpl(new NameIdCache(), new IdNameCache(), new NameIdCache(), rrc); - persistence = new FHIRPersistenceJDBCImpl(this.testProps, connectionPool, cache); + persistence = testSupport.getPersistenceImpl(); SingleResourceResult result = persistence.create(FHIRPersistenceContextFactory.createPersistenceContext(null), savedResource); @@ -132,7 +111,7 @@ public void teardown() throws Exception { FHIRSearchContext ctx = searchHelper.parseQueryParameters(Location.class, Collections.emptyMap(), true, true); FHIRPersistenceContext persistenceContext = - FHIRPersistenceContextFactory.createPersistenceContext(null, ctx); + FHIRPersistenceContextFactory.createPersistenceContext(null, ctx, null); com.ibm.fhir.model.type.Instant lastUpdated = FHIRPersistenceUtil.getUpdateTime(); persistence.delete(persistenceContext, savedResource.getClass(), savedResource.getId(), FHIRPersistenceSupport.getMetaVersionId(savedResource), lastUpdated); if (persistence.isTransactional()) { @@ -140,6 +119,9 @@ public void teardown() throws Exception { } } FHIRRequestContext.get().setTenantId("default"); + if (testSupport != null) { + testSupport.shutdown(); + } } public MultiResourceResult runQueryTest(String searchParamCode, String queryValue) throws Exception { @@ -164,7 +146,7 @@ public MultiResourceResult runQueryTestMultiples(String searchParamCode, String. public MultiResourceResult runQueryTest(Map> queryParms) throws Exception { FHIRSearchContext ctx = searchHelper.parseQueryParameters(Location.class, queryParms, true, true); - FHIRPersistenceContext persistenceContext = FHIRPersistenceContextFactory.createPersistenceContext(null, ctx); + FHIRPersistenceContext persistenceContext = FHIRPersistenceContextFactory.createPersistenceContext(null, ctx, null); MultiResourceResult result = persistence.search(persistenceContext, Location.class); return result; } diff --git a/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/search/test/JDBCSearchNumberTest.java b/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/search/test/JDBCSearchNumberTest.java index d551df53cdc..23038b1d14f 100644 --- a/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/search/test/JDBCSearchNumberTest.java +++ b/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/search/test/JDBCSearchNumberTest.java @@ -6,62 +6,30 @@ package com.ibm.fhir.persistence.jdbc.search.test; -import java.util.Properties; - import com.ibm.fhir.config.FHIRConfigProvider; -import com.ibm.fhir.database.utils.api.IConnectionProvider; -import com.ibm.fhir.database.utils.pool.PoolConnectionProvider; -import com.ibm.fhir.model.test.TestUtil; import com.ibm.fhir.persistence.FHIRPersistence; -import com.ibm.fhir.persistence.jdbc.FHIRPersistenceJDBCCache; -import com.ibm.fhir.persistence.jdbc.cache.CommonTokenValuesCacheImpl; -import com.ibm.fhir.persistence.jdbc.cache.FHIRPersistenceJDBCCacheImpl; -import com.ibm.fhir.persistence.jdbc.cache.IdNameCache; -import com.ibm.fhir.persistence.jdbc.cache.NameIdCache; -import com.ibm.fhir.persistence.jdbc.dao.api.ICommonTokenValuesCache; -import com.ibm.fhir.persistence.jdbc.impl.FHIRPersistenceJDBCImpl; -import com.ibm.fhir.persistence.jdbc.test.util.DerbyInitializer; +import com.ibm.fhir.persistence.jdbc.test.util.PersistenceTestSupport; import com.ibm.fhir.persistence.search.test.AbstractSearchNumberTest; import com.ibm.fhir.search.util.SearchHelper; public class JDBCSearchNumberTest extends AbstractSearchNumberTest { - private Properties testProps; - - private PoolConnectionProvider connectionPool; - - private FHIRPersistenceJDBCCache cache; - - public JDBCSearchNumberTest() throws Exception { - this.testProps = TestUtil.readTestProperties("test.jdbc.properties"); - } + // Container to hide the instantiation of the persistence impl used for tests + private PersistenceTestSupport testSupport; @Override public void bootstrapDatabase() throws Exception { - DerbyInitializer derbyInit; - String dbDriverName = this.testProps.getProperty("dbDriverName"); - if (dbDriverName != null && dbDriverName.contains("derby")) { - derbyInit = new DerbyInitializer(this.testProps); - IConnectionProvider cp = derbyInit.getConnectionProvider(false); - this.connectionPool = new PoolConnectionProvider(cp, 1); - ICommonTokenValuesCache rrc = new CommonTokenValuesCacheImpl(100, 100, 100); - cache = new FHIRPersistenceJDBCCacheImpl(new NameIdCache(), new IdNameCache(), new NameIdCache(), rrc); - } + testSupport = new PersistenceTestSupport(); } @Override public FHIRPersistence getPersistenceImpl(FHIRConfigProvider configProvider, SearchHelper searchHelper) throws Exception { - if (this.connectionPool == null) { - throw new IllegalStateException("Database not bootstrapped"); - } - return new FHIRPersistenceJDBCImpl(this.testProps, this.connectionPool, configProvider, cache, searchHelper); + return testSupport.getPersistenceImpl(configProvider, searchHelper); } @Override protected void shutdownPools() throws Exception { - // Mark the pool as no longer in use. This allows the pool to check for - // lingering open connections/transactions. - if (this.connectionPool != null) { - this.connectionPool.close(); + if (testSupport != null) { + testSupport.shutdown(); } } } diff --git a/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/search/test/JDBCSearchQuantityTest.java b/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/search/test/JDBCSearchQuantityTest.java index 0033a0a26be..56990df4af7 100644 --- a/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/search/test/JDBCSearchQuantityTest.java +++ b/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/search/test/JDBCSearchQuantityTest.java @@ -6,63 +6,30 @@ package com.ibm.fhir.persistence.jdbc.search.test; -import java.util.Properties; - import com.ibm.fhir.config.FHIRConfigProvider; -import com.ibm.fhir.database.utils.api.IConnectionProvider; -import com.ibm.fhir.database.utils.pool.PoolConnectionProvider; -import com.ibm.fhir.model.test.TestUtil; import com.ibm.fhir.persistence.FHIRPersistence; -import com.ibm.fhir.persistence.jdbc.FHIRPersistenceJDBCCache; -import com.ibm.fhir.persistence.jdbc.cache.CommonTokenValuesCacheImpl; -import com.ibm.fhir.persistence.jdbc.cache.FHIRPersistenceJDBCCacheImpl; -import com.ibm.fhir.persistence.jdbc.cache.IdNameCache; -import com.ibm.fhir.persistence.jdbc.cache.NameIdCache; -import com.ibm.fhir.persistence.jdbc.dao.api.ICommonTokenValuesCache; -import com.ibm.fhir.persistence.jdbc.impl.FHIRPersistenceJDBCImpl; -import com.ibm.fhir.persistence.jdbc.test.util.DerbyInitializer; +import com.ibm.fhir.persistence.jdbc.test.util.PersistenceTestSupport; import com.ibm.fhir.persistence.search.test.AbstractSearchQuantityTest; import com.ibm.fhir.search.util.SearchHelper; public class JDBCSearchQuantityTest extends AbstractSearchQuantityTest { - - private Properties testProps; - - private PoolConnectionProvider connectionPool; - - private FHIRPersistenceJDBCCache cache; - - public JDBCSearchQuantityTest() throws Exception { - this.testProps = TestUtil.readTestProperties("test.jdbc.properties"); - } + // Container to hide the instantiation of the persistence impl used for tests + private PersistenceTestSupport testSupport; @Override public void bootstrapDatabase() throws Exception { - DerbyInitializer derbyInit; - String dbDriverName = this.testProps.getProperty("dbDriverName"); - if (dbDriverName != null && dbDriverName.contains("derby")) { - derbyInit = new DerbyInitializer(this.testProps); - IConnectionProvider cp = derbyInit.getConnectionProvider(false); - this.connectionPool = new PoolConnectionProvider(cp, 1); - ICommonTokenValuesCache rrc = new CommonTokenValuesCacheImpl(100, 100, 100); - cache = new FHIRPersistenceJDBCCacheImpl(new NameIdCache(), new IdNameCache(), new NameIdCache(), rrc); - } + testSupport = new PersistenceTestSupport(); } @Override public FHIRPersistence getPersistenceImpl(FHIRConfigProvider configProvider, SearchHelper searchHelper) throws Exception { - if (this.connectionPool == null) { - throw new IllegalStateException("Database not bootstrapped"); - } - return new FHIRPersistenceJDBCImpl(this.testProps, this.connectionPool, configProvider, cache, searchHelper); + return testSupport.getPersistenceImpl(configProvider, searchHelper); } @Override protected void shutdownPools() throws Exception { - // Mark the pool as no longer in use. This allows the pool to check for - // lingering open connections/transactions. - if (this.connectionPool != null) { - this.connectionPool.close(); + if (testSupport != null) { + testSupport.shutdown(); } } -} +} \ No newline at end of file diff --git a/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/search/test/JDBCSearchReferenceTest.java b/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/search/test/JDBCSearchReferenceTest.java index 2315e81fd75..e39efbaca9c 100644 --- a/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/search/test/JDBCSearchReferenceTest.java +++ b/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/search/test/JDBCSearchReferenceTest.java @@ -6,63 +6,31 @@ package com.ibm.fhir.persistence.jdbc.search.test; -import java.util.Properties; - import com.ibm.fhir.config.FHIRConfigProvider; -import com.ibm.fhir.database.utils.api.IConnectionProvider; -import com.ibm.fhir.database.utils.pool.PoolConnectionProvider; -import com.ibm.fhir.model.test.TestUtil; import com.ibm.fhir.persistence.FHIRPersistence; -import com.ibm.fhir.persistence.jdbc.FHIRPersistenceJDBCCache; -import com.ibm.fhir.persistence.jdbc.cache.CommonTokenValuesCacheImpl; -import com.ibm.fhir.persistence.jdbc.cache.FHIRPersistenceJDBCCacheImpl; -import com.ibm.fhir.persistence.jdbc.cache.IdNameCache; -import com.ibm.fhir.persistence.jdbc.cache.NameIdCache; -import com.ibm.fhir.persistence.jdbc.dao.api.ICommonTokenValuesCache; -import com.ibm.fhir.persistence.jdbc.impl.FHIRPersistenceJDBCImpl; -import com.ibm.fhir.persistence.jdbc.test.util.DerbyInitializer; +import com.ibm.fhir.persistence.jdbc.test.util.PersistenceTestSupport; import com.ibm.fhir.persistence.search.test.AbstractSearchReferenceTest; import com.ibm.fhir.search.util.SearchHelper; public class JDBCSearchReferenceTest extends AbstractSearchReferenceTest { - private Properties testProps; - - private PoolConnectionProvider connectionPool; - - private FHIRPersistenceJDBCCache cache; - - public JDBCSearchReferenceTest() throws Exception { - this.testProps = TestUtil.readTestProperties("test.jdbc.properties"); - } + // Container to hide the instantiation of the persistence impl used for tests + private PersistenceTestSupport testSupport; @Override public void bootstrapDatabase() throws Exception { - DerbyInitializer derbyInit; - String dbDriverName = this.testProps.getProperty("dbDriverName"); - if (dbDriverName != null && dbDriverName.contains("derby")) { - derbyInit = new DerbyInitializer(this.testProps); - IConnectionProvider cp = derbyInit.getConnectionProvider(false); - this.connectionPool = new PoolConnectionProvider(cp, 1); - ICommonTokenValuesCache rrc = new CommonTokenValuesCacheImpl(100, 100, 100); - cache = new FHIRPersistenceJDBCCacheImpl(new NameIdCache(), new IdNameCache(), new NameIdCache(), rrc); - } + testSupport = new PersistenceTestSupport(); } @Override public FHIRPersistence getPersistenceImpl(FHIRConfigProvider configProvider, SearchHelper searchHelper) throws Exception { - if (this.connectionPool == null) { - throw new IllegalStateException("Database not bootstrapped"); - } - return new FHIRPersistenceJDBCImpl(this.testProps, this.connectionPool, configProvider, cache, searchHelper); + return testSupport.getPersistenceImpl(configProvider, searchHelper); } @Override protected void shutdownPools() throws Exception { - // Mark the pool as no longer in use. This allows the pool to check for - // lingering open connections/transactions. - if (this.connectionPool != null) { - this.connectionPool.close(); + if (testSupport != null) { + testSupport.shutdown(); } } } \ No newline at end of file diff --git a/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/search/test/JDBCSearchStringTest.java b/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/search/test/JDBCSearchStringTest.java index 9b0a8d62e89..f1f18367c70 100644 --- a/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/search/test/JDBCSearchStringTest.java +++ b/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/search/test/JDBCSearchStringTest.java @@ -6,64 +6,32 @@ package com.ibm.fhir.persistence.jdbc.search.test; -import java.util.Properties; - import com.ibm.fhir.config.FHIRConfigProvider; -import com.ibm.fhir.database.utils.api.IConnectionProvider; -import com.ibm.fhir.database.utils.pool.PoolConnectionProvider; -import com.ibm.fhir.model.test.TestUtil; import com.ibm.fhir.persistence.FHIRPersistence; -import com.ibm.fhir.persistence.jdbc.FHIRPersistenceJDBCCache; -import com.ibm.fhir.persistence.jdbc.cache.CommonTokenValuesCacheImpl; -import com.ibm.fhir.persistence.jdbc.cache.FHIRPersistenceJDBCCacheImpl; -import com.ibm.fhir.persistence.jdbc.cache.IdNameCache; -import com.ibm.fhir.persistence.jdbc.cache.NameIdCache; -import com.ibm.fhir.persistence.jdbc.dao.api.ICommonTokenValuesCache; -import com.ibm.fhir.persistence.jdbc.impl.FHIRPersistenceJDBCImpl; -import com.ibm.fhir.persistence.jdbc.test.util.DerbyInitializer; +import com.ibm.fhir.persistence.jdbc.test.util.PersistenceTestSupport; import com.ibm.fhir.persistence.search.test.AbstractSearchStringTest; import com.ibm.fhir.search.util.SearchHelper; public class JDBCSearchStringTest extends AbstractSearchStringTest { - private Properties testProps; - - private PoolConnectionProvider connectionPool; - - private FHIRPersistenceJDBCCache cache; - - public JDBCSearchStringTest() throws Exception { - this.testProps = TestUtil.readTestProperties("test.jdbc.properties"); - } + // Container to hide the instantiation of the persistence impl used for tests + private PersistenceTestSupport testSupport; @Override public void bootstrapDatabase() throws Exception { - DerbyInitializer derbyInit; - String dbDriverName = this.testProps.getProperty("dbDriverName"); - if (dbDriverName != null && dbDriverName.contains("derby")) { - derbyInit = new DerbyInitializer(this.testProps); - IConnectionProvider cp = derbyInit.getConnectionProvider(false); - this.connectionPool = new PoolConnectionProvider(cp, 1); - ICommonTokenValuesCache rrc = new CommonTokenValuesCacheImpl(100, 100, 100); - cache = new FHIRPersistenceJDBCCacheImpl(new NameIdCache(), new IdNameCache(), new NameIdCache(), rrc); - } + testSupport = new PersistenceTestSupport(); } @Override public FHIRPersistence getPersistenceImpl(FHIRConfigProvider configProvider, SearchHelper searchHelper) throws Exception { - if (this.connectionPool == null) { - throw new IllegalStateException("Database not bootstrapped"); - } - return new FHIRPersistenceJDBCImpl(this.testProps, this.connectionPool, configProvider, cache, searchHelper); + return testSupport.getPersistenceImpl(configProvider, searchHelper); } @Override protected void shutdownPools() throws Exception { - // Mark the pool as no longer in use. This allows the pool to check for - // lingering open connections/transactions. - if (this.connectionPool != null) { - this.connectionPool.close(); + if (testSupport != null) { + testSupport.shutdown(); } } } diff --git a/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/search/test/JDBCSearchTokenTest.java b/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/search/test/JDBCSearchTokenTest.java index 6c8cc150ad1..000cdb2616d 100644 --- a/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/search/test/JDBCSearchTokenTest.java +++ b/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/search/test/JDBCSearchTokenTest.java @@ -6,64 +6,32 @@ package com.ibm.fhir.persistence.jdbc.search.test; -import java.util.Properties; - import com.ibm.fhir.config.FHIRConfigProvider; -import com.ibm.fhir.database.utils.api.IConnectionProvider; -import com.ibm.fhir.database.utils.pool.PoolConnectionProvider; -import com.ibm.fhir.model.test.TestUtil; import com.ibm.fhir.persistence.FHIRPersistence; -import com.ibm.fhir.persistence.jdbc.FHIRPersistenceJDBCCache; -import com.ibm.fhir.persistence.jdbc.cache.CommonTokenValuesCacheImpl; -import com.ibm.fhir.persistence.jdbc.cache.FHIRPersistenceJDBCCacheImpl; -import com.ibm.fhir.persistence.jdbc.cache.IdNameCache; -import com.ibm.fhir.persistence.jdbc.cache.NameIdCache; -import com.ibm.fhir.persistence.jdbc.dao.api.ICommonTokenValuesCache; -import com.ibm.fhir.persistence.jdbc.impl.FHIRPersistenceJDBCImpl; -import com.ibm.fhir.persistence.jdbc.test.util.DerbyInitializer; +import com.ibm.fhir.persistence.jdbc.test.util.PersistenceTestSupport; import com.ibm.fhir.persistence.search.test.AbstractSearchTokenTest; import com.ibm.fhir.search.util.SearchHelper; public class JDBCSearchTokenTest extends AbstractSearchTokenTest { - private Properties testProps; - - private PoolConnectionProvider connectionPool; - - private FHIRPersistenceJDBCCache cache; - - public JDBCSearchTokenTest() throws Exception { - this.testProps = TestUtil.readTestProperties("test.jdbc.properties"); - } + // Container to hide the instantiation of the persistence impl used for tests + private PersistenceTestSupport testSupport; @Override public void bootstrapDatabase() throws Exception { - DerbyInitializer derbyInit; - String dbDriverName = this.testProps.getProperty("dbDriverName"); - if (dbDriverName != null && dbDriverName.contains("derby")) { - derbyInit = new DerbyInitializer(this.testProps); - IConnectionProvider cp = derbyInit.getConnectionProvider(false); - this.connectionPool = new PoolConnectionProvider(cp, 1); - ICommonTokenValuesCache rrc = new CommonTokenValuesCacheImpl(100, 100, 100); - cache = new FHIRPersistenceJDBCCacheImpl(new NameIdCache(), new IdNameCache(), new NameIdCache(), rrc); - } + testSupport = new PersistenceTestSupport(); } @Override public FHIRPersistence getPersistenceImpl(FHIRConfigProvider configProvider, SearchHelper searchHelper) throws Exception { - if (this.connectionPool == null) { - throw new IllegalStateException("Database not bootstrapped"); - } - return new FHIRPersistenceJDBCImpl(this.testProps, this.connectionPool, configProvider, cache, searchHelper); + return testSupport.getPersistenceImpl(configProvider, searchHelper); } @Override protected void shutdownPools() throws Exception { - // Mark the pool as no longer in use. This allows the pool to check for - // lingering open connections/transactions. - if (this.connectionPool != null) { - this.connectionPool.close(); + if (testSupport != null) { + testSupport.shutdown(); } } } diff --git a/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/search/test/JDBCSearchURITest.java b/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/search/test/JDBCSearchURITest.java index 4f8c27a5433..67c675094dd 100644 --- a/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/search/test/JDBCSearchURITest.java +++ b/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/search/test/JDBCSearchURITest.java @@ -6,62 +6,30 @@ package com.ibm.fhir.persistence.jdbc.search.test; -import java.util.Properties; - import com.ibm.fhir.config.FHIRConfigProvider; -import com.ibm.fhir.database.utils.api.IConnectionProvider; -import com.ibm.fhir.database.utils.pool.PoolConnectionProvider; -import com.ibm.fhir.model.test.TestUtil; import com.ibm.fhir.persistence.FHIRPersistence; -import com.ibm.fhir.persistence.jdbc.FHIRPersistenceJDBCCache; -import com.ibm.fhir.persistence.jdbc.cache.CommonTokenValuesCacheImpl; -import com.ibm.fhir.persistence.jdbc.cache.FHIRPersistenceJDBCCacheImpl; -import com.ibm.fhir.persistence.jdbc.cache.IdNameCache; -import com.ibm.fhir.persistence.jdbc.cache.NameIdCache; -import com.ibm.fhir.persistence.jdbc.dao.api.ICommonTokenValuesCache; -import com.ibm.fhir.persistence.jdbc.impl.FHIRPersistenceJDBCImpl; -import com.ibm.fhir.persistence.jdbc.test.util.DerbyInitializer; +import com.ibm.fhir.persistence.jdbc.test.util.PersistenceTestSupport; import com.ibm.fhir.persistence.search.test.AbstractSearchURITest; import com.ibm.fhir.search.util.SearchHelper; public class JDBCSearchURITest extends AbstractSearchURITest { - private Properties testProps; - - private PoolConnectionProvider connectionPool; - - private FHIRPersistenceJDBCCache cache; - - public JDBCSearchURITest() throws Exception { - this.testProps = TestUtil.readTestProperties("test.jdbc.properties"); - } + // Container to hide the instantiation of the persistence impl used for tests + private PersistenceTestSupport testSupport; @Override public void bootstrapDatabase() throws Exception { - DerbyInitializer derbyInit; - String dbDriverName = this.testProps.getProperty("dbDriverName"); - if (dbDriverName != null && dbDriverName.contains("derby")) { - derbyInit = new DerbyInitializer(this.testProps); - IConnectionProvider cp = derbyInit.getConnectionProvider(false); - this.connectionPool = new PoolConnectionProvider(cp, 1); - ICommonTokenValuesCache rrc = new CommonTokenValuesCacheImpl(100, 100, 100); - cache = new FHIRPersistenceJDBCCacheImpl(new NameIdCache(), new IdNameCache(), new NameIdCache(), rrc); - } + testSupport = new PersistenceTestSupport(); } @Override public FHIRPersistence getPersistenceImpl(FHIRConfigProvider configProvider, SearchHelper searchHelper) throws Exception { - if (this.connectionPool == null) { - throw new IllegalStateException("Database not bootstrapped"); - } - return new FHIRPersistenceJDBCImpl(this.testProps, this.connectionPool, configProvider, cache, searchHelper); + return testSupport.getPersistenceImpl(configProvider, searchHelper); } @Override protected void shutdownPools() throws Exception { - // Mark the pool as no longer in use. This allows the pool to check for - // lingering open connections/transactions. - if (this.connectionPool != null) { - this.connectionPool.close(); + if (testSupport != null) { + testSupport.shutdown(); } } } \ No newline at end of file diff --git a/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/search/test/JDBCWholeSystemSearchTest.java b/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/search/test/JDBCWholeSystemSearchTest.java index abb6523d653..b43ed2721c1 100644 --- a/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/search/test/JDBCWholeSystemSearchTest.java +++ b/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/search/test/JDBCWholeSystemSearchTest.java @@ -6,63 +6,31 @@ package com.ibm.fhir.persistence.jdbc.search.test; -import java.util.Properties; - import com.ibm.fhir.config.FHIRConfigProvider; -import com.ibm.fhir.database.utils.api.IConnectionProvider; -import com.ibm.fhir.database.utils.pool.PoolConnectionProvider; -import com.ibm.fhir.model.test.TestUtil; import com.ibm.fhir.persistence.FHIRPersistence; -import com.ibm.fhir.persistence.jdbc.FHIRPersistenceJDBCCache; -import com.ibm.fhir.persistence.jdbc.cache.CommonTokenValuesCacheImpl; -import com.ibm.fhir.persistence.jdbc.cache.FHIRPersistenceJDBCCacheImpl; -import com.ibm.fhir.persistence.jdbc.cache.IdNameCache; -import com.ibm.fhir.persistence.jdbc.cache.NameIdCache; -import com.ibm.fhir.persistence.jdbc.dao.api.ICommonTokenValuesCache; -import com.ibm.fhir.persistence.jdbc.impl.FHIRPersistenceJDBCImpl; -import com.ibm.fhir.persistence.jdbc.test.util.DerbyInitializer; +import com.ibm.fhir.persistence.jdbc.test.util.PersistenceTestSupport; import com.ibm.fhir.persistence.search.test.AbstractWholeSystemSearchTest; import com.ibm.fhir.search.util.SearchHelper; public class JDBCWholeSystemSearchTest extends AbstractWholeSystemSearchTest { - private Properties testProps; - - private PoolConnectionProvider connectionPool; - - private FHIRPersistenceJDBCCache cache; - - public JDBCWholeSystemSearchTest() throws Exception { - this.testProps = TestUtil.readTestProperties("test.jdbc.properties"); - } + // Container to hide the instantiation of the persistence impl used for tests + private PersistenceTestSupport testSupport; @Override public void bootstrapDatabase() throws Exception { - DerbyInitializer derbyInit; - String dbDriverName = this.testProps.getProperty("dbDriverName"); - if (dbDriverName != null && dbDriverName.contains("derby")) { - derbyInit = new DerbyInitializer(this.testProps); - IConnectionProvider cp = derbyInit.getConnectionProvider(false); - this.connectionPool = new PoolConnectionProvider(cp, 1); - ICommonTokenValuesCache rrc = new CommonTokenValuesCacheImpl(100, 100, 100); - cache = new FHIRPersistenceJDBCCacheImpl(new NameIdCache(), new IdNameCache(), new NameIdCache(), rrc); - } + testSupport = new PersistenceTestSupport(); } @Override public FHIRPersistence getPersistenceImpl(FHIRConfigProvider configProvider, SearchHelper searchHelper) throws Exception { - if (this.connectionPool == null) { - throw new IllegalStateException("Database not bootstrapped"); - } - return new FHIRPersistenceJDBCImpl(this.testProps, this.connectionPool, configProvider, cache, searchHelper); + return testSupport.getPersistenceImpl(configProvider, searchHelper); } @Override protected void shutdownPools() throws Exception { - // Mark the pool as no longer in use. This allows the pool to check for - // lingering open connections/transactions. - if (this.connectionPool != null) { - this.connectionPool.close(); + if (testSupport != null) { + testSupport.shutdown(); } } } \ No newline at end of file diff --git a/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/JDBCChangesTest.java b/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/JDBCChangesTest.java index cbad2e0d812..0c015efe4c7 100644 --- a/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/JDBCChangesTest.java +++ b/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/JDBCChangesTest.java @@ -6,24 +6,9 @@ package com.ibm.fhir.persistence.jdbc.test; -import java.sql.Connection; -import java.sql.SQLException; -import java.util.Properties; - import com.ibm.fhir.config.FHIRConfigProvider; -import com.ibm.fhir.database.utils.api.IConnectionProvider; -import com.ibm.fhir.database.utils.derby.DerbyMaster; -import com.ibm.fhir.database.utils.pool.PoolConnectionProvider; -import com.ibm.fhir.model.test.TestUtil; import com.ibm.fhir.persistence.FHIRPersistence; -import com.ibm.fhir.persistence.jdbc.FHIRPersistenceJDBCCache; -import com.ibm.fhir.persistence.jdbc.cache.CommonTokenValuesCacheImpl; -import com.ibm.fhir.persistence.jdbc.cache.FHIRPersistenceJDBCCacheImpl; -import com.ibm.fhir.persistence.jdbc.cache.IdNameCache; -import com.ibm.fhir.persistence.jdbc.cache.NameIdCache; -import com.ibm.fhir.persistence.jdbc.dao.api.ICommonTokenValuesCache; -import com.ibm.fhir.persistence.jdbc.impl.FHIRPersistenceJDBCImpl; -import com.ibm.fhir.persistence.jdbc.test.util.DerbyInitializer; +import com.ibm.fhir.persistence.jdbc.test.util.PersistenceTestSupport; import com.ibm.fhir.persistence.test.common.AbstractChangesTest; import com.ibm.fhir.search.util.SearchHelper; @@ -32,54 +17,28 @@ */ public class JDBCChangesTest extends AbstractChangesTest { - private Properties testProps; - - private PoolConnectionProvider connectionPool; - - private FHIRPersistenceJDBCCache cache; - - public JDBCChangesTest() throws Exception { - this.testProps = TestUtil.readTestProperties("test.jdbc.properties"); - } + // Container to hide the instantiation of the persistence impl used for tests + private PersistenceTestSupport testSupport; @Override public void bootstrapDatabase() throws Exception { - DerbyInitializer derbyInit; - String dbDriverName = this.testProps.getProperty("dbDriverName"); - if (dbDriverName != null && dbDriverName.contains("derby")) { - derbyInit = new DerbyInitializer(this.testProps); - IConnectionProvider cp = derbyInit.getConnectionProvider(false); - this.connectionPool = new PoolConnectionProvider(cp, 1); - ICommonTokenValuesCache rrc = new CommonTokenValuesCacheImpl(100, 100, 100); - cache = new FHIRPersistenceJDBCCacheImpl(new NameIdCache(), new IdNameCache(), new NameIdCache(), rrc); - } + testSupport = new PersistenceTestSupport(); } @Override public FHIRPersistence getPersistenceImpl(FHIRConfigProvider configProvider, SearchHelper searchHelper) throws Exception { - if (this.connectionPool == null) { - throw new IllegalStateException("Database not bootstrapped"); - } - return new FHIRPersistenceJDBCImpl(this.testProps, this.connectionPool, configProvider, cache, searchHelper); + return testSupport.getPersistenceImpl(configProvider, searchHelper); } @Override protected void shutdownPools() throws Exception { - // Mark the pool as no longer in use. This allows the pool to check for - // lingering open connections/transactions. - if (this.connectionPool != null) { - this.connectionPool.close(); + if (testSupport != null) { + testSupport.shutdown(); } } @Override protected void debugLocks() { - // Exception running a query. Let's dump the lock table - try (Connection c = connectionPool.getConnection()) { - DerbyMaster.dumpLockInfo(c); - } catch (SQLException x) { - // just log the error...things are already bad if this method has been called - logger.severe("dumpLockInfo - connection failure: " + x.getMessage()); - } + testSupport.debugLocks(); } } \ No newline at end of file diff --git a/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/JDBCCompartmentTest.java b/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/JDBCCompartmentTest.java index d5c304531e3..7cc38775792 100644 --- a/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/JDBCCompartmentTest.java +++ b/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/JDBCCompartmentTest.java @@ -6,64 +6,32 @@ package com.ibm.fhir.persistence.jdbc.test; -import java.util.Properties; - import com.ibm.fhir.config.FHIRConfigProvider; -import com.ibm.fhir.database.utils.api.IConnectionProvider; -import com.ibm.fhir.database.utils.pool.PoolConnectionProvider; -import com.ibm.fhir.model.test.TestUtil; import com.ibm.fhir.persistence.FHIRPersistence; -import com.ibm.fhir.persistence.jdbc.FHIRPersistenceJDBCCache; -import com.ibm.fhir.persistence.jdbc.cache.CommonTokenValuesCacheImpl; -import com.ibm.fhir.persistence.jdbc.cache.FHIRPersistenceJDBCCacheImpl; -import com.ibm.fhir.persistence.jdbc.cache.IdNameCache; -import com.ibm.fhir.persistence.jdbc.cache.NameIdCache; -import com.ibm.fhir.persistence.jdbc.dao.api.ICommonTokenValuesCache; -import com.ibm.fhir.persistence.jdbc.impl.FHIRPersistenceJDBCImpl; -import com.ibm.fhir.persistence.jdbc.test.util.DerbyInitializer; +import com.ibm.fhir.persistence.jdbc.test.util.PersistenceTestSupport; import com.ibm.fhir.persistence.test.common.AbstractCompartmentTest; import com.ibm.fhir.search.util.SearchHelper; public class JDBCCompartmentTest extends AbstractCompartmentTest { - private Properties testProps; - - private PoolConnectionProvider connectionPool; - - private FHIRPersistenceJDBCCache cache; - - public JDBCCompartmentTest() throws Exception { - this.testProps = TestUtil.readTestProperties("test.jdbc.properties"); - } + // Container to hide the instantiation of the persistence impl used for tests + private PersistenceTestSupport testSupport; @Override public void bootstrapDatabase() throws Exception { - DerbyInitializer derbyInit; - String dbDriverName = this.testProps.getProperty("dbDriverName"); - if (dbDriverName != null && dbDriverName.contains("derby")) { - derbyInit = new DerbyInitializer(this.testProps); - IConnectionProvider cp = derbyInit.getConnectionProvider(false); - this.connectionPool = new PoolConnectionProvider(cp, 1); - ICommonTokenValuesCache rrc = new CommonTokenValuesCacheImpl(100, 100, 100); - cache = new FHIRPersistenceJDBCCacheImpl(new NameIdCache(), new IdNameCache(), new NameIdCache(), rrc); - } + testSupport = new PersistenceTestSupport(); } @Override public FHIRPersistence getPersistenceImpl(FHIRConfigProvider configProvider, SearchHelper searchHelper) throws Exception { - if (this.connectionPool == null) { - throw new IllegalStateException("Database not bootstrapped"); - } - return new FHIRPersistenceJDBCImpl(this.testProps, this.connectionPool, configProvider, cache, searchHelper); + return testSupport.getPersistenceImpl(configProvider, searchHelper); } @Override protected void shutdownPools() throws Exception { - // Mark the pool as no longer in use. This allows the pool to check for - // lingering open connections/transactions. - if (this.connectionPool != null) { - this.connectionPool.close(); + if (testSupport != null) { + testSupport.shutdown(); } } } diff --git a/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/JDBCDeleteTest.java b/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/JDBCDeleteTest.java index db60c4eb331..2150208d371 100644 --- a/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/JDBCDeleteTest.java +++ b/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/JDBCDeleteTest.java @@ -6,21 +6,9 @@ package com.ibm.fhir.persistence.jdbc.test; -import java.util.Properties; - import com.ibm.fhir.config.FHIRConfigProvider; -import com.ibm.fhir.database.utils.api.IConnectionProvider; -import com.ibm.fhir.database.utils.pool.PoolConnectionProvider; -import com.ibm.fhir.model.test.TestUtil; import com.ibm.fhir.persistence.FHIRPersistence; -import com.ibm.fhir.persistence.jdbc.FHIRPersistenceJDBCCache; -import com.ibm.fhir.persistence.jdbc.cache.CommonTokenValuesCacheImpl; -import com.ibm.fhir.persistence.jdbc.cache.FHIRPersistenceJDBCCacheImpl; -import com.ibm.fhir.persistence.jdbc.cache.IdNameCache; -import com.ibm.fhir.persistence.jdbc.cache.NameIdCache; -import com.ibm.fhir.persistence.jdbc.dao.api.ICommonTokenValuesCache; -import com.ibm.fhir.persistence.jdbc.impl.FHIRPersistenceJDBCImpl; -import com.ibm.fhir.persistence.jdbc.test.util.DerbyInitializer; +import com.ibm.fhir.persistence.jdbc.test.util.PersistenceTestSupport; import com.ibm.fhir.persistence.test.common.AbstractDeleteTest; import com.ibm.fhir.search.util.SearchHelper; @@ -29,45 +17,23 @@ */ public class JDBCDeleteTest extends AbstractDeleteTest { - // test properties - private Properties testProps; - - // Connection pool used to provide connections for the FHIRPersistenceJDBCImpl - private PoolConnectionProvider connectionPool; - - private FHIRPersistenceJDBCCache cache; - - public JDBCDeleteTest() throws Exception { - this.testProps = TestUtil.readTestProperties("test.jdbc.properties"); - } + // Container to hide the instantiation of the persistence impl used for tests + private PersistenceTestSupport testSupport; @Override public void bootstrapDatabase() throws Exception { - DerbyInitializer derbyInit; - String dbDriverName = this.testProps.getProperty("dbDriverName"); - if (dbDriverName != null && dbDriverName.contains("derby")) { - derbyInit = new DerbyInitializer(this.testProps); - IConnectionProvider cp = derbyInit.getConnectionProvider(false); - this.connectionPool = new PoolConnectionProvider(cp, 1); - ICommonTokenValuesCache rrc = new CommonTokenValuesCacheImpl(100, 100, 100); - cache = new FHIRPersistenceJDBCCacheImpl(new NameIdCache(), new IdNameCache(), new NameIdCache(), rrc); - } + testSupport = new PersistenceTestSupport(); } @Override public FHIRPersistence getPersistenceImpl(FHIRConfigProvider configProvider, SearchHelper searchHelper) throws Exception { - if (this.connectionPool == null) { - throw new IllegalStateException("Database not bootstrapped"); - } - return new FHIRPersistenceJDBCImpl(this.testProps, this.connectionPool, configProvider, cache, searchHelper); + return testSupport.getPersistenceImpl(configProvider, searchHelper); } @Override protected void shutdownPools() throws Exception { - // Mark the pool as no longer in use. This allows the pool to check for - // lingering open connections/transactions. - if (this.connectionPool != null) { - this.connectionPool.close(); + if (testSupport != null) { + testSupport.shutdown(); } } } diff --git a/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/JDBCEraseTest.java b/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/JDBCEraseTest.java index b2fe7143eac..2e0e779b9b1 100644 --- a/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/JDBCEraseTest.java +++ b/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/JDBCEraseTest.java @@ -6,24 +6,9 @@ package com.ibm.fhir.persistence.jdbc.test; -import java.sql.Connection; -import java.sql.SQLException; -import java.util.Properties; - import com.ibm.fhir.config.FHIRConfigProvider; -import com.ibm.fhir.database.utils.api.IConnectionProvider; -import com.ibm.fhir.database.utils.derby.DerbyMaster; -import com.ibm.fhir.database.utils.pool.PoolConnectionProvider; -import com.ibm.fhir.model.test.TestUtil; import com.ibm.fhir.persistence.FHIRPersistence; -import com.ibm.fhir.persistence.jdbc.FHIRPersistenceJDBCCache; -import com.ibm.fhir.persistence.jdbc.cache.CommonTokenValuesCacheImpl; -import com.ibm.fhir.persistence.jdbc.cache.FHIRPersistenceJDBCCacheImpl; -import com.ibm.fhir.persistence.jdbc.cache.IdNameCache; -import com.ibm.fhir.persistence.jdbc.cache.NameIdCache; -import com.ibm.fhir.persistence.jdbc.dao.api.ICommonTokenValuesCache; -import com.ibm.fhir.persistence.jdbc.impl.FHIRPersistenceJDBCImpl; -import com.ibm.fhir.persistence.jdbc.test.util.DerbyInitializer; +import com.ibm.fhir.persistence.jdbc.test.util.PersistenceTestSupport; import com.ibm.fhir.persistence.test.common.AbstractEraseTest; import com.ibm.fhir.search.util.SearchHelper; @@ -32,54 +17,28 @@ */ public class JDBCEraseTest extends AbstractEraseTest { - private Properties testProps; - - private PoolConnectionProvider connectionPool; - - private FHIRPersistenceJDBCCache cache; - - public JDBCEraseTest() throws Exception { - this.testProps = TestUtil.readTestProperties("test.jdbc.properties"); - } + // Container to hide the instantiation of the persistence impl used for tests + private PersistenceTestSupport testSupport; @Override public void bootstrapDatabase() throws Exception { - DerbyInitializer derbyInit; - String dbDriverName = this.testProps.getProperty("dbDriverName"); - if (dbDriverName != null && dbDriverName.contains("derby")) { - derbyInit = new DerbyInitializer(this.testProps); - IConnectionProvider cp = derbyInit.getConnectionProvider(false); - this.connectionPool = new PoolConnectionProvider(cp, 1); - ICommonTokenValuesCache rrc = new CommonTokenValuesCacheImpl(100, 100, 100); - cache = new FHIRPersistenceJDBCCacheImpl(new NameIdCache(), new IdNameCache(), new NameIdCache(), rrc); - } + testSupport = new PersistenceTestSupport(); } @Override public FHIRPersistence getPersistenceImpl(FHIRConfigProvider configProvider, SearchHelper searchHelper) throws Exception { - if (this.connectionPool == null) { - throw new IllegalStateException("Database not bootstrapped"); - } - return new FHIRPersistenceJDBCImpl(this.testProps, this.connectionPool, configProvider, cache, searchHelper); + return testSupport.getPersistenceImpl(configProvider, searchHelper); } @Override protected void shutdownPools() throws Exception { - // Mark the pool as no longer in use. This allows the pool to check for - // lingering open connections/transactions. - if (this.connectionPool != null) { - this.connectionPool.close(); + if (testSupport != null) { + testSupport.shutdown(); } } @Override protected void debugLocks() { - // Exception running a query. Let's dump the lock table - try (Connection c = connectionPool.getConnection()) { - DerbyMaster.dumpLockInfo(c); - } catch (SQLException x) { - // just log the error...things are already bad if this method has been called - logger.severe("dumpLockInfo - connection failure: " + x.getMessage()); - } + testSupport.debugLocks(); } } \ No newline at end of file diff --git a/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/JDBCExportTest.java b/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/JDBCExportTest.java index 9d1f4c08781..0ee7369f06f 100644 --- a/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/JDBCExportTest.java +++ b/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/JDBCExportTest.java @@ -6,24 +6,9 @@ package com.ibm.fhir.persistence.jdbc.test; -import java.sql.Connection; -import java.sql.SQLException; -import java.util.Properties; - import com.ibm.fhir.config.FHIRConfigProvider; -import com.ibm.fhir.database.utils.api.IConnectionProvider; -import com.ibm.fhir.database.utils.derby.DerbyMaster; -import com.ibm.fhir.database.utils.pool.PoolConnectionProvider; -import com.ibm.fhir.model.test.TestUtil; import com.ibm.fhir.persistence.FHIRPersistence; -import com.ibm.fhir.persistence.jdbc.FHIRPersistenceJDBCCache; -import com.ibm.fhir.persistence.jdbc.cache.CommonTokenValuesCacheImpl; -import com.ibm.fhir.persistence.jdbc.cache.FHIRPersistenceJDBCCacheImpl; -import com.ibm.fhir.persistence.jdbc.cache.IdNameCache; -import com.ibm.fhir.persistence.jdbc.cache.NameIdCache; -import com.ibm.fhir.persistence.jdbc.dao.api.ICommonTokenValuesCache; -import com.ibm.fhir.persistence.jdbc.impl.FHIRPersistenceJDBCImpl; -import com.ibm.fhir.persistence.jdbc.test.util.DerbyInitializer; +import com.ibm.fhir.persistence.jdbc.test.util.PersistenceTestSupport; import com.ibm.fhir.persistence.test.common.AbstractExportTest; import com.ibm.fhir.search.util.SearchHelper; @@ -32,54 +17,28 @@ */ public class JDBCExportTest extends AbstractExportTest { - private Properties testProps; - - private PoolConnectionProvider connectionPool; - - private FHIRPersistenceJDBCCache cache; - - public JDBCExportTest() throws Exception { - this.testProps = TestUtil.readTestProperties("test.jdbc.properties"); - } + // Container to hide the instantiation of the persistence impl used for tests + private PersistenceTestSupport testSupport; @Override public void bootstrapDatabase() throws Exception { - DerbyInitializer derbyInit; - String dbDriverName = this.testProps.getProperty("dbDriverName"); - if (dbDriverName != null && dbDriverName.contains("derby")) { - derbyInit = new DerbyInitializer(this.testProps); - IConnectionProvider cp = derbyInit.getConnectionProvider(false); - this.connectionPool = new PoolConnectionProvider(cp, 1); - ICommonTokenValuesCache rrc = new CommonTokenValuesCacheImpl(100, 100, 100); - cache = new FHIRPersistenceJDBCCacheImpl(new NameIdCache(), new IdNameCache(), new NameIdCache(), rrc); - } + testSupport = new PersistenceTestSupport(); } @Override public FHIRPersistence getPersistenceImpl(FHIRConfigProvider configProvider, SearchHelper searchHelper) throws Exception { - if (this.connectionPool == null) { - throw new IllegalStateException("Database not bootstrapped"); - } - return new FHIRPersistenceJDBCImpl(this.testProps, this.connectionPool, configProvider, cache, searchHelper); + return testSupport.getPersistenceImpl(configProvider, searchHelper); } @Override protected void shutdownPools() throws Exception { - // Mark the pool as no longer in use. This allows the pool to check for - // lingering open connections/transactions. - if (this.connectionPool != null) { - this.connectionPool.close(); + if (testSupport != null) { + testSupport.shutdown(); } } @Override protected void debugLocks() { - // Exception running a query. Let's dump the lock table - try (Connection c = connectionPool.getConnection()) { - DerbyMaster.dumpLockInfo(c); - } catch (SQLException x) { - // just log the error...things are already bad if this method has been called - logger.severe("dumpLockInfo - connection failure: " + x.getMessage()); - } + testSupport.debugLocks(); } } \ No newline at end of file diff --git a/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/JDBCIfNoneMatchTest.java b/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/JDBCIfNoneMatchTest.java index 2ea33af86f9..665602a98b5 100644 --- a/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/JDBCIfNoneMatchTest.java +++ b/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/JDBCIfNoneMatchTest.java @@ -6,21 +6,9 @@ package com.ibm.fhir.persistence.jdbc.test; -import java.util.Properties; - import com.ibm.fhir.config.FHIRConfigProvider; -import com.ibm.fhir.database.utils.api.IConnectionProvider; -import com.ibm.fhir.database.utils.pool.PoolConnectionProvider; -import com.ibm.fhir.model.test.TestUtil; import com.ibm.fhir.persistence.FHIRPersistence; -import com.ibm.fhir.persistence.jdbc.FHIRPersistenceJDBCCache; -import com.ibm.fhir.persistence.jdbc.cache.CommonTokenValuesCacheImpl; -import com.ibm.fhir.persistence.jdbc.cache.FHIRPersistenceJDBCCacheImpl; -import com.ibm.fhir.persistence.jdbc.cache.IdNameCache; -import com.ibm.fhir.persistence.jdbc.cache.NameIdCache; -import com.ibm.fhir.persistence.jdbc.dao.api.ICommonTokenValuesCache; -import com.ibm.fhir.persistence.jdbc.impl.FHIRPersistenceJDBCImpl; -import com.ibm.fhir.persistence.jdbc.test.util.DerbyInitializer; +import com.ibm.fhir.persistence.jdbc.test.util.PersistenceTestSupport; import com.ibm.fhir.persistence.test.common.AbstractIfNoneMatchTest; import com.ibm.fhir.search.util.SearchHelper; @@ -29,45 +17,23 @@ */ public class JDBCIfNoneMatchTest extends AbstractIfNoneMatchTest { - // test properties - private Properties testProps; - - // Connection pool used to provide connections for the FHIRPersistenceJDBCImpl - private PoolConnectionProvider connectionPool; - - private FHIRPersistenceJDBCCache cache; - - public JDBCIfNoneMatchTest() throws Exception { - this.testProps = TestUtil.readTestProperties("test.jdbc.properties"); - } + // Container to hide the instantiation of the persistence impl used for tests + private PersistenceTestSupport testSupport; @Override public void bootstrapDatabase() throws Exception { - DerbyInitializer derbyInit; - String dbDriverName = this.testProps.getProperty("dbDriverName"); - if (dbDriverName != null && dbDriverName.contains("derby")) { - derbyInit = new DerbyInitializer(this.testProps); - IConnectionProvider cp = derbyInit.getConnectionProvider(false); - this.connectionPool = new PoolConnectionProvider(cp, 1); - ICommonTokenValuesCache rrc = new CommonTokenValuesCacheImpl(100, 100, 100); - cache = new FHIRPersistenceJDBCCacheImpl(new NameIdCache(), new IdNameCache(), new NameIdCache(), rrc); - } + testSupport = new PersistenceTestSupport(); } @Override public FHIRPersistence getPersistenceImpl(FHIRConfigProvider configProvider, SearchHelper searchHelper) throws Exception { - if (this.connectionPool == null) { - throw new IllegalStateException("Database not bootstrapped"); - } - return new FHIRPersistenceJDBCImpl(this.testProps, this.connectionPool, configProvider, cache, searchHelper); + return testSupport.getPersistenceImpl(configProvider, searchHelper); } @Override protected void shutdownPools() throws Exception { - // Mark the pool as no longer in use. This allows the pool to check for - // lingering open connections/transactions. - if (this.connectionPool != null) { - this.connectionPool.close(); + if (testSupport != null) { + testSupport.shutdown(); } } } diff --git a/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/JDBCIncludeRevincludeTest.java b/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/JDBCIncludeRevincludeTest.java index ab58238a985..ee9d3894134 100644 --- a/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/JDBCIncludeRevincludeTest.java +++ b/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/JDBCIncludeRevincludeTest.java @@ -6,63 +6,30 @@ package com.ibm.fhir.persistence.jdbc.test; -import java.util.Properties; - import com.ibm.fhir.config.FHIRConfigProvider; -import com.ibm.fhir.database.utils.api.IConnectionProvider; -import com.ibm.fhir.database.utils.pool.PoolConnectionProvider; -import com.ibm.fhir.model.test.TestUtil; import com.ibm.fhir.persistence.FHIRPersistence; -import com.ibm.fhir.persistence.jdbc.FHIRPersistenceJDBCCache; -import com.ibm.fhir.persistence.jdbc.cache.CommonTokenValuesCacheImpl; -import com.ibm.fhir.persistence.jdbc.cache.FHIRPersistenceJDBCCacheImpl; -import com.ibm.fhir.persistence.jdbc.cache.IdNameCache; -import com.ibm.fhir.persistence.jdbc.cache.NameIdCache; -import com.ibm.fhir.persistence.jdbc.dao.api.ICommonTokenValuesCache; -import com.ibm.fhir.persistence.jdbc.impl.FHIRPersistenceJDBCImpl; -import com.ibm.fhir.persistence.jdbc.test.util.DerbyInitializer; +import com.ibm.fhir.persistence.jdbc.test.util.PersistenceTestSupport; import com.ibm.fhir.persistence.test.common.AbstractIncludeRevincludeTest; import com.ibm.fhir.search.util.SearchHelper; public class JDBCIncludeRevincludeTest extends AbstractIncludeRevincludeTest { - private Properties testProps; - - // The connection pool wrapping the Derby test database - private PoolConnectionProvider connectionPool; - - private FHIRPersistenceJDBCCache cache; - - public JDBCIncludeRevincludeTest() throws Exception { - this.testProps = TestUtil.readTestProperties("test.jdbc.properties"); - } + // Container to hide the instantiation of the persistence impl used for tests + private PersistenceTestSupport testSupport; @Override public void bootstrapDatabase() throws Exception { - DerbyInitializer derbyInit; - String dbDriverName = this.testProps.getProperty("dbDriverName"); - if (dbDriverName != null && dbDriverName.contains("derby")) { - derbyInit = new DerbyInitializer(this.testProps); - IConnectionProvider cp = derbyInit.getConnectionProvider(false); - this.connectionPool = new PoolConnectionProvider(cp, 1); - ICommonTokenValuesCache rrc = new CommonTokenValuesCacheImpl(100, 100, 100); - cache = new FHIRPersistenceJDBCCacheImpl(new NameIdCache(), new IdNameCache(), new NameIdCache(), rrc); - } + testSupport = new PersistenceTestSupport(); } @Override public FHIRPersistence getPersistenceImpl(FHIRConfigProvider configProvider, SearchHelper searchHelper) throws Exception { - if (this.connectionPool == null) { - throw new IllegalStateException("Database not bootstrapped"); - } - return new FHIRPersistenceJDBCImpl(this.testProps, this.connectionPool, configProvider, cache, searchHelper); + return testSupport.getPersistenceImpl(configProvider, searchHelper); } @Override protected void shutdownPools() throws Exception { - // Mark the pool as no longer in use. This allows the pool to check for - // lingering open connections/transactions. - if (this.connectionPool != null) { - this.connectionPool.close(); + if (testSupport != null) { + testSupport.shutdown(); } } diff --git a/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/JDBCMultiResourceTest.java b/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/JDBCMultiResourceTest.java index 2a21f761678..d9397486fa9 100644 --- a/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/JDBCMultiResourceTest.java +++ b/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/JDBCMultiResourceTest.java @@ -6,65 +6,32 @@ package com.ibm.fhir.persistence.jdbc.test; -import java.util.Properties; - import com.ibm.fhir.config.FHIRConfigProvider; -import com.ibm.fhir.database.utils.api.IConnectionProvider; -import com.ibm.fhir.database.utils.pool.PoolConnectionProvider; -import com.ibm.fhir.model.test.TestUtil; import com.ibm.fhir.persistence.FHIRPersistence; -import com.ibm.fhir.persistence.jdbc.FHIRPersistenceJDBCCache; -import com.ibm.fhir.persistence.jdbc.cache.CommonTokenValuesCacheImpl; -import com.ibm.fhir.persistence.jdbc.cache.FHIRPersistenceJDBCCacheImpl; -import com.ibm.fhir.persistence.jdbc.cache.IdNameCache; -import com.ibm.fhir.persistence.jdbc.cache.NameIdCache; -import com.ibm.fhir.persistence.jdbc.dao.api.ICommonTokenValuesCache; -import com.ibm.fhir.persistence.jdbc.impl.FHIRPersistenceJDBCImpl; -import com.ibm.fhir.persistence.jdbc.test.util.DerbyInitializer; +import com.ibm.fhir.persistence.jdbc.test.util.PersistenceTestSupport; import com.ibm.fhir.persistence.test.common.AbstractMultiResourceTest; import com.ibm.fhir.search.util.SearchHelper; public class JDBCMultiResourceTest extends AbstractMultiResourceTest { - private Properties testProps; - - // The connection pool wrapping the Derby test database - private PoolConnectionProvider connectionPool; - - private FHIRPersistenceJDBCCache cache; - - public JDBCMultiResourceTest() throws Exception { - this.testProps = TestUtil.readTestProperties("test.jdbc.properties"); - } + // Container to hide the instantiation of the persistence impl used for tests + private PersistenceTestSupport testSupport; @Override public void bootstrapDatabase() throws Exception { - DerbyInitializer derbyInit; - String dbDriverName = this.testProps.getProperty("dbDriverName"); - if (dbDriverName != null && dbDriverName.contains("derby")) { - derbyInit = new DerbyInitializer(this.testProps); - IConnectionProvider cp = derbyInit.getConnectionProvider(false); - this.connectionPool = new PoolConnectionProvider(cp, 1); - ICommonTokenValuesCache rrc = new CommonTokenValuesCacheImpl(100, 100, 100); - cache = new FHIRPersistenceJDBCCacheImpl(new NameIdCache(), new IdNameCache(), new NameIdCache(), rrc); - } + testSupport = new PersistenceTestSupport(); } @Override public FHIRPersistence getPersistenceImpl(FHIRConfigProvider configProvider, SearchHelper searchHelper) throws Exception { - if (this.connectionPool == null) { - throw new IllegalStateException("Database not bootstrapped"); - } - return new FHIRPersistenceJDBCImpl(this.testProps, this.connectionPool, configProvider, cache, searchHelper); + return testSupport.getPersistenceImpl(configProvider, searchHelper); } @Override protected void shutdownPools() throws Exception { - // Mark the pool as no longer in use. This allows the pool to check for - // lingering open connections/transactions. - if (this.connectionPool != null) { - this.connectionPool.close(); + if (testSupport != null) { + testSupport.shutdown(); } } } diff --git a/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/JDBCPagingTest.java b/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/JDBCPagingTest.java index 1d783983bed..81647415f89 100644 --- a/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/JDBCPagingTest.java +++ b/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/JDBCPagingTest.java @@ -6,78 +6,37 @@ package com.ibm.fhir.persistence.jdbc.test; -import java.sql.Connection; -import java.sql.SQLException; -import java.util.Properties; - import com.ibm.fhir.config.FHIRConfigProvider; -import com.ibm.fhir.database.utils.api.IConnectionProvider; -import com.ibm.fhir.database.utils.derby.DerbyMaster; -import com.ibm.fhir.database.utils.pool.PoolConnectionProvider; -import com.ibm.fhir.model.test.TestUtil; import com.ibm.fhir.persistence.FHIRPersistence; -import com.ibm.fhir.persistence.jdbc.FHIRPersistenceJDBCCache; -import com.ibm.fhir.persistence.jdbc.cache.CommonTokenValuesCacheImpl; -import com.ibm.fhir.persistence.jdbc.cache.FHIRPersistenceJDBCCacheImpl; -import com.ibm.fhir.persistence.jdbc.cache.IdNameCache; -import com.ibm.fhir.persistence.jdbc.cache.NameIdCache; -import com.ibm.fhir.persistence.jdbc.dao.api.ICommonTokenValuesCache; -import com.ibm.fhir.persistence.jdbc.impl.FHIRPersistenceJDBCImpl; -import com.ibm.fhir.persistence.jdbc.test.util.DerbyInitializer; +import com.ibm.fhir.persistence.jdbc.test.util.PersistenceTestSupport; import com.ibm.fhir.persistence.test.common.AbstractPagingTest; import com.ibm.fhir.search.util.SearchHelper; public class JDBCPagingTest extends AbstractPagingTest { - private Properties testProps; - - private PoolConnectionProvider connectionPool; - - private FHIRPersistenceJDBCCache cache; - - public JDBCPagingTest() throws Exception { - this.testProps = TestUtil.readTestProperties("test.jdbc.properties"); - } + // Container to hide the instantiation of the persistence impl used for tests + private PersistenceTestSupport testSupport; @Override public void bootstrapDatabase() throws Exception { - DerbyInitializer derbyInit; - String dbDriverName = this.testProps.getProperty("dbDriverName"); - if (dbDriverName != null && dbDriverName.contains("derby")) { - derbyInit = new DerbyInitializer(this.testProps); - IConnectionProvider cp = derbyInit.getConnectionProvider(false); - this.connectionPool = new PoolConnectionProvider(cp, 1); - ICommonTokenValuesCache rrc = new CommonTokenValuesCacheImpl(100, 100, 100); - cache = new FHIRPersistenceJDBCCacheImpl(new NameIdCache(), new IdNameCache(), new NameIdCache(), rrc); - } + testSupport = new PersistenceTestSupport(); } @Override public FHIRPersistence getPersistenceImpl(FHIRConfigProvider configProvider, SearchHelper searchHelper) throws Exception { - if (this.connectionPool == null) { - throw new IllegalStateException("Database not bootstrapped"); - } - return new FHIRPersistenceJDBCImpl(this.testProps, this.connectionPool, configProvider, cache, searchHelper); + return testSupport.getPersistenceImpl(configProvider, searchHelper); } @Override protected void shutdownPools() throws Exception { - // Mark the pool as no longer in use. This allows the pool to check for - // lingering open connections/transactions. - if (this.connectionPool != null) { - this.connectionPool.close(); + if (testSupport != null) { + testSupport.shutdown(); } } @Override protected void debugLocks() { - // Exception running a query. Let's dump the lock table - try (Connection c = connectionPool.getConnection()) { - DerbyMaster.dumpLockInfo(c); - } catch (SQLException x) { - // just log the error...things are already bad if this method has been called - logger.severe("dumpLockInfo - connection failure: " + x.getMessage()); - } + testSupport.debugLocks(); } -} +} \ No newline at end of file diff --git a/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/JDBCReverseChainTest.java b/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/JDBCReverseChainTest.java index 7ae0232ba7a..41209e9d1a8 100644 --- a/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/JDBCReverseChainTest.java +++ b/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/JDBCReverseChainTest.java @@ -6,63 +6,30 @@ package com.ibm.fhir.persistence.jdbc.test; -import java.util.Properties; - import com.ibm.fhir.config.FHIRConfigProvider; -import com.ibm.fhir.database.utils.api.IConnectionProvider; -import com.ibm.fhir.database.utils.pool.PoolConnectionProvider; -import com.ibm.fhir.model.test.TestUtil; import com.ibm.fhir.persistence.FHIRPersistence; -import com.ibm.fhir.persistence.jdbc.FHIRPersistenceJDBCCache; -import com.ibm.fhir.persistence.jdbc.cache.CommonTokenValuesCacheImpl; -import com.ibm.fhir.persistence.jdbc.cache.FHIRPersistenceJDBCCacheImpl; -import com.ibm.fhir.persistence.jdbc.cache.IdNameCache; -import com.ibm.fhir.persistence.jdbc.cache.NameIdCache; -import com.ibm.fhir.persistence.jdbc.dao.api.ICommonTokenValuesCache; -import com.ibm.fhir.persistence.jdbc.impl.FHIRPersistenceJDBCImpl; -import com.ibm.fhir.persistence.jdbc.test.util.DerbyInitializer; +import com.ibm.fhir.persistence.jdbc.test.util.PersistenceTestSupport; import com.ibm.fhir.persistence.test.common.AbstractReverseChainTest; import com.ibm.fhir.search.util.SearchHelper; public class JDBCReverseChainTest extends AbstractReverseChainTest { - private Properties testProps; - - // The connection pool wrapping the Derby test database - private PoolConnectionProvider connectionPool; - - private FHIRPersistenceJDBCCache cache; - - public JDBCReverseChainTest() throws Exception { - this.testProps = TestUtil.readTestProperties("test.jdbc.properties"); - } + // Container to hide the instantiation of the persistence impl used for tests + private PersistenceTestSupport testSupport; @Override public void bootstrapDatabase() throws Exception { - DerbyInitializer derbyInit; - String dbDriverName = this.testProps.getProperty("dbDriverName"); - if (dbDriverName != null && dbDriverName.contains("derby")) { - derbyInit = new DerbyInitializer(this.testProps); - IConnectionProvider cp = derbyInit.getConnectionProvider(false); - this.connectionPool = new PoolConnectionProvider(cp, 1); - ICommonTokenValuesCache rrc = new CommonTokenValuesCacheImpl(100, 100, 100); - cache = new FHIRPersistenceJDBCCacheImpl(new NameIdCache(), new IdNameCache(), new NameIdCache(), rrc); - } + testSupport = new PersistenceTestSupport(); } @Override public FHIRPersistence getPersistenceImpl(FHIRConfigProvider configProvider, SearchHelper searchHelper) throws Exception { - if (this.connectionPool == null) { - throw new IllegalStateException("Database not bootstrapped"); - } - return new FHIRPersistenceJDBCImpl(this.testProps, this.connectionPool, configProvider, cache, searchHelper); + return testSupport.getPersistenceImpl(configProvider, searchHelper); } @Override protected void shutdownPools() throws Exception { - // Mark the pool as no longer in use. This allows the pool to check for - // lingering open connections/transactions. - if (this.connectionPool != null) { - this.connectionPool.close(); + if (testSupport != null) { + testSupport.shutdown(); } } diff --git a/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/JDBCSortTest.java b/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/JDBCSortTest.java index 716796d6b60..476ed5ba36a 100644 --- a/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/JDBCSortTest.java +++ b/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/JDBCSortTest.java @@ -6,64 +6,33 @@ package com.ibm.fhir.persistence.jdbc.test; -import java.util.Properties; - import com.ibm.fhir.config.FHIRConfigProvider; -import com.ibm.fhir.database.utils.api.IConnectionProvider; -import com.ibm.fhir.database.utils.pool.PoolConnectionProvider; -import com.ibm.fhir.model.test.TestUtil; +import com.ibm.fhir.config.FHIRConfigProvider; import com.ibm.fhir.persistence.FHIRPersistence; -import com.ibm.fhir.persistence.jdbc.FHIRPersistenceJDBCCache; -import com.ibm.fhir.persistence.jdbc.cache.CommonTokenValuesCacheImpl; -import com.ibm.fhir.persistence.jdbc.cache.FHIRPersistenceJDBCCacheImpl; -import com.ibm.fhir.persistence.jdbc.cache.IdNameCache; -import com.ibm.fhir.persistence.jdbc.cache.NameIdCache; -import com.ibm.fhir.persistence.jdbc.dao.api.ICommonTokenValuesCache; -import com.ibm.fhir.persistence.jdbc.impl.FHIRPersistenceJDBCImpl; -import com.ibm.fhir.persistence.jdbc.test.util.DerbyInitializer; +import com.ibm.fhir.persistence.jdbc.test.util.PersistenceTestSupport; import com.ibm.fhir.persistence.test.common.AbstractSortTest; import com.ibm.fhir.search.util.SearchHelper; public class JDBCSortTest extends AbstractSortTest { - private Properties testProps; - - private PoolConnectionProvider connectionPool; - - private FHIRPersistenceJDBCCache cache; - - public JDBCSortTest() throws Exception { - this.testProps = TestUtil.readTestProperties("test.jdbc.properties"); - } + // Container to hide the instantiation of the persistence impl used for tests + private PersistenceTestSupport testSupport; @Override public void bootstrapDatabase() throws Exception { - DerbyInitializer derbyInit; - String dbDriverName = this.testProps.getProperty("dbDriverName"); - if (dbDriverName != null && dbDriverName.contains("derby")) { - derbyInit = new DerbyInitializer(this.testProps); - IConnectionProvider cp = derbyInit.getConnectionProvider(false); - this.connectionPool = new PoolConnectionProvider(cp, 1); - ICommonTokenValuesCache rrc = new CommonTokenValuesCacheImpl(100, 100, 100); - cache = new FHIRPersistenceJDBCCacheImpl(new NameIdCache(), new IdNameCache(), new NameIdCache(), rrc); - } + testSupport = new PersistenceTestSupport(); } @Override public FHIRPersistence getPersistenceImpl(FHIRConfigProvider configProvider, SearchHelper searchHelper) throws Exception { - if (this.connectionPool == null) { - throw new IllegalStateException("Database not bootstrapped"); - } - return new FHIRPersistenceJDBCImpl(this.testProps, this.connectionPool, configProvider, cache, searchHelper); + return testSupport.getPersistenceImpl(configProvider, searchHelper); } @Override protected void shutdownPools() throws Exception { - // Mark the pool as no longer in use. This allows the pool to check for - // lingering open connections/transactions. - if (this.connectionPool != null) { - this.connectionPool.close(); + if (testSupport != null) { + testSupport.shutdown(); } } } \ No newline at end of file diff --git a/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/JDBCanonicalTest.java b/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/JDBCanonicalTest.java index 4d593c30f5c..e77766477ee 100644 --- a/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/JDBCanonicalTest.java +++ b/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/JDBCanonicalTest.java @@ -1,69 +1,35 @@ /* - * (C) Copyright IBM Corp. 2021, 2022 + * (C) Copyright IBM Corp. 2022 * * SPDX-License-Identifier: Apache-2.0 */ package com.ibm.fhir.persistence.jdbc.test; -import java.util.Properties; - import com.ibm.fhir.config.FHIRConfigProvider; -import com.ibm.fhir.database.utils.api.IConnectionProvider; -import com.ibm.fhir.database.utils.pool.PoolConnectionProvider; -import com.ibm.fhir.model.test.TestUtil; import com.ibm.fhir.persistence.FHIRPersistence; -import com.ibm.fhir.persistence.jdbc.FHIRPersistenceJDBCCache; -import com.ibm.fhir.persistence.jdbc.cache.CommonTokenValuesCacheImpl; -import com.ibm.fhir.persistence.jdbc.cache.FHIRPersistenceJDBCCacheImpl; -import com.ibm.fhir.persistence.jdbc.cache.IdNameCache; -import com.ibm.fhir.persistence.jdbc.cache.NameIdCache; -import com.ibm.fhir.persistence.jdbc.dao.api.ICommonTokenValuesCache; -import com.ibm.fhir.persistence.jdbc.impl.FHIRPersistenceJDBCImpl; -import com.ibm.fhir.persistence.jdbc.test.util.DerbyInitializer; +import com.ibm.fhir.persistence.jdbc.test.util.PersistenceTestSupport; import com.ibm.fhir.persistence.test.common.AbstractCanonicalTest; import com.ibm.fhir.search.util.SearchHelper; public class JDBCanonicalTest extends AbstractCanonicalTest { - private Properties testProps; - - // The connection pool wrapping the Derby test database - private PoolConnectionProvider connectionPool; - - private FHIRPersistenceJDBCCache cache; - - public JDBCanonicalTest() throws Exception { - this.testProps = TestUtil.readTestProperties("test.jdbc.properties"); - } + // Container to hide the instantiation of the persistence impl used for tests + private PersistenceTestSupport testSupport; @Override public void bootstrapDatabase() throws Exception { - DerbyInitializer derbyInit; - String dbDriverName = this.testProps.getProperty("dbDriverName"); - if (dbDriverName != null && dbDriverName.contains("derby")) { - derbyInit = new DerbyInitializer(this.testProps); - IConnectionProvider cp = derbyInit.getConnectionProvider(false); - this.connectionPool = new PoolConnectionProvider(cp, 1); - ICommonTokenValuesCache rrc = new CommonTokenValuesCacheImpl(100, 100, 100); - cache = new FHIRPersistenceJDBCCacheImpl(new NameIdCache(), new IdNameCache(), new NameIdCache(), rrc); - } + testSupport = new PersistenceTestSupport(); } @Override public FHIRPersistence getPersistenceImpl(FHIRConfigProvider configProvider, SearchHelper searchHelper) throws Exception { - if (this.connectionPool == null) { - throw new IllegalStateException("Database not bootstrapped"); - } - return new FHIRPersistenceJDBCImpl(this.testProps, this.connectionPool, configProvider, cache, searchHelper); + return testSupport.getPersistenceImpl(configProvider, searchHelper); } @Override protected void shutdownPools() throws Exception { - // Mark the pool as no longer in use. This allows the pool to check for - // lingering open connections/transactions. - if (this.connectionPool != null) { - this.connectionPool.close(); + if (testSupport != null) { + testSupport.shutdown(); } } - } \ No newline at end of file diff --git a/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/erase/EraseTestMain.java b/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/erase/EraseTestMain.java index 141979182ab..61d783509fe 100644 --- a/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/erase/EraseTestMain.java +++ b/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/erase/EraseTestMain.java @@ -18,6 +18,7 @@ import java.util.logging.Logger; import com.ibm.fhir.database.utils.api.IDatabaseTranslator; +import com.ibm.fhir.database.utils.api.SchemaType; import com.ibm.fhir.database.utils.common.JdbcPropertyAdapter; import com.ibm.fhir.database.utils.model.DbType; import com.ibm.fhir.database.utils.postgres.PostgresTranslator; @@ -31,6 +32,7 @@ import com.ibm.fhir.persistence.jdbc.dao.EraseResourceDAO; import com.ibm.fhir.persistence.jdbc.dao.api.ICommonTokenValuesCache; import com.ibm.fhir.persistence.jdbc.dao.api.IIdNameCache; +import com.ibm.fhir.persistence.jdbc.dao.api.ILogicalResourceIdentCache; import com.ibm.fhir.persistence.jdbc.dao.api.INameIdCache; import com.ibm.fhir.schema.app.util.CommonUtil; import com.ibm.fhir.schema.control.FhirSchemaConstants; @@ -87,7 +89,7 @@ protected void erase() throws Exception { try (Connection c = createConnection()) { System.out.println("Got a Connection"); try { - FHIRDbFlavor flavor = new FHIRDbFlavorImpl(dbType, true); + FHIRDbFlavor flavor = new FHIRDbFlavorImpl(dbType, SchemaType.PLAIN); EraseResourceDAO dao = new EraseResourceDAO(c, FhirSchemaConstants.FHIR_ADMIN, translator, schemaName, flavor, new MockLocalCache(), null); ResourceEraseRecord record = new ResourceEraseRecord(); @@ -255,6 +257,12 @@ public void transactionCommitted() { public void transactionRolledBack() { // No Operation } + + @Override + public ILogicalResourceIdentCache getLogicalResourceIdentCache() { + // TODO Auto-generated method stub + return null; + } } /** diff --git a/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/spec/Main.java b/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/spec/Main.java index b6d10daeac0..4bfc0914a0a 100644 --- a/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/spec/Main.java +++ b/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/spec/Main.java @@ -49,8 +49,10 @@ import com.ibm.fhir.persistence.jdbc.cache.CommonTokenValuesCacheImpl; import com.ibm.fhir.persistence.jdbc.cache.FHIRPersistenceJDBCCacheImpl; import com.ibm.fhir.persistence.jdbc.cache.IdNameCache; +import com.ibm.fhir.persistence.jdbc.cache.LogicalResourceIdentCacheImpl; import com.ibm.fhir.persistence.jdbc.cache.NameIdCache; import com.ibm.fhir.persistence.jdbc.dao.api.ICommonTokenValuesCache; +import com.ibm.fhir.persistence.jdbc.dao.api.ILogicalResourceIdentCache; import com.ibm.fhir.persistence.jdbc.impl.FHIRPersistenceJDBCImpl; import com.ibm.fhir.schema.derby.DerbyFhirDatabase; import com.ibm.fhir.validation.test.ValidationProcessor; @@ -124,7 +126,7 @@ public class Main { // mode of operation private static enum Operation { - DB2, DERBY, DERBYNETWORK, POSTGRESQL, PARSE + DB2, DERBY, DERBYNETWORK, POSTGRESQL, CITUS, PARSE } private Operation mode = Operation.DB2; @@ -226,6 +228,9 @@ protected void parseArgs(String[] args) { case "--postgresql": this.mode = Operation.POSTGRESQL; break; + case "--citus": + this.mode = Operation.CITUS; + break; case "--parse": this.mode = Operation.PARSE; break; @@ -303,6 +308,7 @@ protected void process() throws Exception { processDerbyNetwork(); break; case POSTGRESQL: + case CITUS: processPostgreSql(); break; case PARSE: @@ -352,7 +358,8 @@ protected void processDB2() throws Exception { TestFHIRConfigProvider configProvider = new TestFHIRConfigProvider(new DefaultFHIRConfigProvider()); configure(configProvider); ICommonTokenValuesCache rrc = new CommonTokenValuesCacheImpl(100, 100, 100); - FHIRPersistenceJDBCCache cache = new FHIRPersistenceJDBCCacheImpl(new NameIdCache(), new IdNameCache(), new NameIdCache(), rrc); + ILogicalResourceIdentCache lric = new LogicalResourceIdentCacheImpl(100); + FHIRPersistenceJDBCCache cache = new FHIRPersistenceJDBCCacheImpl(new NameIdCache(), new IdNameCache(), new NameIdCache(), rrc, lric); // Provide the credentials we need for accessing a multi-tenant schema (if enabled) // Must set this BEFORE we create our persistence object @@ -434,8 +441,9 @@ protected void processDerby() throws Exception { // layer to obtain connections. try (DerbyFhirDatabase database = new DerbyFhirDatabase()) { ICommonTokenValuesCache rrc = new CommonTokenValuesCacheImpl(100, 100, 100); - FHIRPersistenceJDBCCache cache = new FHIRPersistenceJDBCCacheImpl(new NameIdCache(), - new IdNameCache(), new NameIdCache(), rrc); + ILogicalResourceIdentCache lric = new LogicalResourceIdentCacheImpl(100); + FHIRPersistenceJDBCCache cache = new FHIRPersistenceJDBCCacheImpl(new NameIdCache(), + new IdNameCache(), new NameIdCache(), rrc, lric); persistence = new FHIRPersistenceJDBCImpl(this.configProps, database, cache); // create a custom list of operations to apply in order to each resource @@ -487,7 +495,8 @@ protected void processDerbyNetwork() throws Exception { ITransactionProvider transactionProvider = new SimpleTransactionProvider(connectionPool); FHIRConfigProvider configProvider = new DefaultFHIRConfigProvider(); ICommonTokenValuesCache rrc = new CommonTokenValuesCacheImpl(100, 100, 100); - FHIRPersistenceJDBCCache cache = new FHIRPersistenceJDBCCacheImpl(new NameIdCache(), new IdNameCache(), new NameIdCache(), rrc); + ILogicalResourceIdentCache lric = new LogicalResourceIdentCacheImpl(100); + FHIRPersistenceJDBCCache cache = new FHIRPersistenceJDBCCacheImpl(new NameIdCache(), new IdNameCache(), new NameIdCache(), rrc, lric); // create a custom list of operations to apply in order to each resource DriverMetrics dm = new DriverMetrics(); @@ -542,7 +551,8 @@ protected void processPostgreSql() throws Exception { ITransactionProvider transactionProvider = new SimpleTransactionProvider(connectionPool); FHIRConfigProvider configProvider = new DefaultFHIRConfigProvider(); ICommonTokenValuesCache rrc = new CommonTokenValuesCacheImpl(100, 100, 100); - FHIRPersistenceJDBCCache cache = new FHIRPersistenceJDBCCacheImpl(new NameIdCache(), new IdNameCache(), new NameIdCache(), rrc); + ILogicalResourceIdentCache lric = new LogicalResourceIdentCacheImpl(100); + FHIRPersistenceJDBCCache cache = new FHIRPersistenceJDBCCacheImpl(new NameIdCache(), new IdNameCache(), new NameIdCache(), rrc, lric); // create a custom list of operations to apply in order to each resource diff --git a/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/spec/R4JDBCExamplesTest.java b/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/spec/R4JDBCExamplesTest.java index 2553478f240..a64ffd46164 100644 --- a/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/spec/R4JDBCExamplesTest.java +++ b/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/spec/R4JDBCExamplesTest.java @@ -29,8 +29,10 @@ import com.ibm.fhir.persistence.jdbc.cache.CommonTokenValuesCacheImpl; import com.ibm.fhir.persistence.jdbc.cache.FHIRPersistenceJDBCCacheImpl; import com.ibm.fhir.persistence.jdbc.cache.IdNameCache; +import com.ibm.fhir.persistence.jdbc.cache.LogicalResourceIdentCacheImpl; import com.ibm.fhir.persistence.jdbc.cache.NameIdCache; import com.ibm.fhir.persistence.jdbc.dao.api.ICommonTokenValuesCache; +import com.ibm.fhir.persistence.jdbc.dao.api.ILogicalResourceIdentCache; import com.ibm.fhir.persistence.jdbc.test.util.DerbyInitializer; import com.ibm.fhir.persistence.test.common.AbstractPersistenceTest; import com.ibm.fhir.search.util.SearchHelper; @@ -61,7 +63,8 @@ public void perform() throws Exception { ITransactionProvider transactionProvider = new SimpleTransactionProvider(connectionPool); FHIRConfigProvider configProvider = new DefaultFHIRConfigProvider(); ICommonTokenValuesCache rrc = new CommonTokenValuesCacheImpl(100, 100, 100); - FHIRPersistenceJDBCCache cache = new FHIRPersistenceJDBCCacheImpl(new NameIdCache(), new IdNameCache(), new NameIdCache(), rrc); + ILogicalResourceIdentCache lric = new LogicalResourceIdentCacheImpl(100); + FHIRPersistenceJDBCCache cache = new FHIRPersistenceJDBCCacheImpl(new NameIdCache(), new IdNameCache(), new NameIdCache(), rrc, lric); List operations = new ArrayList<>(); operations.add(new CreateOperation()); diff --git a/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/util/ParameterCounter.java b/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/util/ParameterCounter.java index 838e6c9c3d4..edad2c86ea5 100644 --- a/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/util/ParameterCounter.java +++ b/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/util/ParameterCounter.java @@ -25,6 +25,7 @@ import com.ibm.fhir.core.util.ResourceTypeHelper; import com.ibm.fhir.database.utils.api.IDatabaseAdapter; import com.ibm.fhir.database.utils.api.IDatabaseTranslator; +import com.ibm.fhir.database.utils.citus.CitusTranslator; import com.ibm.fhir.database.utils.common.JdbcPropertyAdapter; import com.ibm.fhir.database.utils.common.JdbcTarget; import com.ibm.fhir.database.utils.db2.Db2Translator; @@ -92,6 +93,12 @@ public void configure() { schemaName = "FHIRDATA"; } break; + case CITUS: + translator = new CitusTranslator(); + if (schemaName == null) { + schemaName = "FHIRDATA"; + } + break; case DB2: default: translator = new Db2Translator(); diff --git a/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/util/PersistenceTestSupport.java b/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/util/PersistenceTestSupport.java new file mode 100644 index 00000000000..9ee0795ed63 --- /dev/null +++ b/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/util/PersistenceTestSupport.java @@ -0,0 +1,110 @@ +/* + * (C) Copyright IBM Corp. 2022 + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.ibm.fhir.persistence.jdbc.test.util; + +import java.sql.Connection; +import java.sql.SQLException; +import java.util.Properties; +import java.util.logging.Logger; + +import com.ibm.fhir.config.FHIRConfigProvider; +import com.ibm.fhir.database.utils.api.IConnectionProvider; +import com.ibm.fhir.database.utils.derby.DerbyMaster; +import com.ibm.fhir.database.utils.pool.PoolConnectionProvider; +import com.ibm.fhir.model.test.TestUtil; +import com.ibm.fhir.persistence.FHIRPersistence; +import com.ibm.fhir.persistence.jdbc.FHIRPersistenceJDBCCache; +import com.ibm.fhir.persistence.jdbc.cache.CommonTokenValuesCacheImpl; +import com.ibm.fhir.persistence.jdbc.cache.FHIRPersistenceJDBCCacheImpl; +import com.ibm.fhir.persistence.jdbc.cache.IdNameCache; +import com.ibm.fhir.persistence.jdbc.cache.LogicalResourceIdentCacheImpl; +import com.ibm.fhir.persistence.jdbc.cache.NameIdCache; +import com.ibm.fhir.persistence.jdbc.dao.api.ICommonTokenValuesCache; +import com.ibm.fhir.persistence.jdbc.dao.api.ILogicalResourceIdentCache; +import com.ibm.fhir.persistence.jdbc.impl.FHIRPersistenceJDBCImpl; +import com.ibm.fhir.search.util.SearchHelper; + +/** + * Encapsulates the instantiation of objects needed to support the JDBC persistence tests. + * If the constructors for these objects change, we only need to modify thir instantiation + * here instead of every for every concrete test class + */ +public class PersistenceTestSupport { + private static final Logger logger = Logger.getLogger(PersistenceTestSupport.class.getName()); + private Properties testProps; + + private PoolConnectionProvider connectionPool; + + private FHIRPersistenceJDBCCache cache; + + /** + * Public constructor + * @throws Exception + */ + public PersistenceTestSupport() throws Exception { + this.testProps = TestUtil.readTestProperties("test.jdbc.properties"); + DerbyInitializer derbyInit; + String dbDriverName = this.testProps.getProperty("dbDriverName"); + if (dbDriverName != null && dbDriverName.contains("derby")) { + derbyInit = new DerbyInitializer(this.testProps); + IConnectionProvider cp = derbyInit.getConnectionProvider(false); + this.connectionPool = new PoolConnectionProvider(cp, 1); + ICommonTokenValuesCache rrc = new CommonTokenValuesCacheImpl(100, 100, 100); + ILogicalResourceIdentCache lric = new LogicalResourceIdentCacheImpl(100); + cache = new FHIRPersistenceJDBCCacheImpl(new NameIdCache(), new IdNameCache(), new NameIdCache(), rrc, lric); + } + } + + /** + * Return a new FHIRPersistence implementation configured using the connection pool + * and cache from this object and the given configProvider and searchHelper. + * @return + * @throws Exception + */ + public FHIRPersistence getPersistenceImpl(FHIRConfigProvider configProvider, SearchHelper searchHelper) throws Exception { + if (this.connectionPool == null) { + throw new IllegalStateException("Database not bootstrapped"); + } + return new FHIRPersistenceJDBCImpl(this.testProps, this.connectionPool, configProvider, cache, searchHelper); + } + + /** + * Return a new FHIRPersistence implementation configured using the connection pool + * and cache from this object + * @return + * @throws Exception + */ + public FHIRPersistence getPersistenceImpl() throws Exception { + if (this.connectionPool == null) { + throw new IllegalStateException("Database not bootstrapped"); + } + return new FHIRPersistenceJDBCImpl(this.testProps, this.connectionPool, cache); + } + + /** + * Close any resources we may still have open + */ + public void shutdown() { + if (this.connectionPool != null) { + this.connectionPool.close(); + } + } + + /** + * Debug locks in the Derby database we're using + */ + public void debugLocks() { + // Exception running a query. Let's dump the lock table + try (Connection c = connectionPool.getConnection()) { + DerbyMaster.dumpLockInfo(c); + } catch (SQLException x) { + // just log the error...things are already bad if this method has been called + logger.severe("dumpLockInfo - connection failure: " + x.getMessage()); + } + + } +} \ No newline at end of file diff --git a/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/util/QuantityParmBehaviorUtilTest.java b/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/util/QuantityParmBehaviorUtilTest.java index a2e0b8dc31e..f1a4f5f3450 100644 --- a/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/util/QuantityParmBehaviorUtilTest.java +++ b/fhir-persistence-jdbc/src/test/java/com/ibm/fhir/persistence/jdbc/test/util/QuantityParmBehaviorUtilTest.java @@ -1,5 +1,5 @@ /* - * (C) Copyright IBM Corp. 2019, 2021 + * (C) Copyright IBM Corp. 2019, 2022 * * SPDX-License-Identifier: Apache-2.0 */ @@ -25,6 +25,7 @@ import com.ibm.fhir.persistence.jdbc.JDBCConstants; import com.ibm.fhir.persistence.jdbc.dao.api.JDBCIdentityCache; import com.ibm.fhir.persistence.jdbc.dto.CommonTokenValue; +import com.ibm.fhir.persistence.jdbc.dto.ResourceReferenceValue; import com.ibm.fhir.persistence.jdbc.util.type.NewNumberParmBehaviorUtil; import com.ibm.fhir.persistence.jdbc.util.type.NewQuantityParmBehaviorUtil; import com.ibm.fhir.search.SearchConstants; @@ -169,6 +170,21 @@ public List getResourceTypeNames() throws FHIRPersistenceException { public List getResourceTypeIds() throws FHIRPersistenceException { return null; } + + @Override + public Long getLogicalResourceId(String resourceType, String logicalId) throws FHIRPersistenceException { + return null; + } + + @Override + public Set getLogicalResourceIds(Collection referenceValues) throws FHIRPersistenceException { + return null; + } + + @Override + public List getLogicalResourceIdList(String logicalId) throws FHIRPersistenceException { + return null; + } }; } //--------------------------------------------------------------------------------------------------------- diff --git a/fhir-persistence-schema/docs/SchemaDesign.md b/fhir-persistence-schema/docs/SchemaDesign.md index 814fd11afae..6b138646f94 100644 --- a/fhir-persistence-schema/docs/SchemaDesign.md +++ b/fhir-persistence-schema/docs/SchemaDesign.md @@ -39,6 +39,7 @@ The following table highlights the main differences among the database implement | PostgreSQL | Uses a function for the resource persistence logic | | PostgreSQL | TEXT type used instead of CLOB for large data values | | Derby | Resource persistence is implemented at the DAO layer as a sequence of individual statements instead of one procedure call. At a functional level, the process is identical. Simplifies debugging and supports easier unit-test construction. | +| Citus | Experimental. Distributed tables for large scale deployments. Most tables are distributed by logical_resource_id, except for `logical_resource_ident` which is distributed by `logical_id`. Search interactions using chaining or include/revinclude not currently supported. | ---------------------------------------------------------------- # Schema Design - Physical Data Model @@ -60,7 +61,7 @@ These sequences should not require updating and should never be altered as this By convention, tables are named using the plural form of the data they represent. The following diagram shows the main relationships among tables in the schema. Note that only the search parameter tables for the Patient resource are shown. Each resource type gets its own set of search parameter tables. -![Physical Schema](physical_schema_V0024.png) +![Physical Schema](physical_schema_V0027.png) ## TABLESPACES @@ -86,6 +87,8 @@ These table definitions are more completely described in [DB2MultiTenancy.md](DB Each logical resource instance such as a Patient, Device or Observation is stored as a row in the `LOGICAL_RESOURCES` table. A corresponding row is also stored in a resource-specific logical resources tables for example: `PATIENT_LOGICAL_RESOURCES`, `DEVICE_LOGICAL_RESOURCES` or `OBSERVATION_LOGICAL_RESOURCES` etc. +Each logical resource also has record stored in `LOGICAL_RESOURCE_IDENT`. This table is used to hold all identifiers which could be the target of a relation. When a reference is external, the logical_id value will be the url target of the resource. In all other cases, the logical_id value will be the resource id value. As of release 5.0.0, the `LOGICAL_RESOURCE_IDENT` table is used for the `SELECT ... FOR UPDATE` locking used inside the `add_any_resource` stored procedures instead of logical_resources. + Each time a logical resource is updated a new version is created. Each new version is stored in the resource-type-specific table `xx_RESOURCES` where xx is the resource type name for example: `PATIENT_RESOURCES`, `DEVICE_RESOURCES` or `OBSERVATION_RESOURCES` etc. Unless payload offloading has been configured, the resource payload is rendered in JSON, compressed, and stored in the `DATA` column of this table. If payload offloading is configured, the `DATA` column will be null instead. Each version is allocated an integer `VERSION_ID` number starting from 1 which increments by 1 for each new version. Row locking (SELECT FOR UPDATE) guarantees there will be no gaps in the version numbers unless a version-specific `$erase` custom operation has been invoked. The `$erase` operation is the only time rows from the `xx_RESOURCES` table are ever deleted. HTTP `DELETE` interactions are implemented as _soft_ deletes and create a new version of the resource with the `xx_RESOURCES.IS_DELETED` column value equal to 'Y'. This column is repeated (denormalized) in the `xx_LOGICAL_RESOURCES` table as a performance optimization. Most queries need only non-deleted resources and the query is much faster if the join to the wide `xx_RESOURCES` table can be avoided. See the [Finding and Reading a Resource](#finding-and-reading-a-resource) section for practical examples on how data is accessed in the schema. @@ -106,6 +109,7 @@ The following table describes the purpose of each table or group of tables in th | parameter_names | Normalized data | Search parameter names. | | common_token_values | Normalized data | Normalized token and code system values. | | common_canonical_values | Normalized data | Normalized canonical values. | +| logical_resource_ident | Whole system | One row for each logical resource or a reference to a logical resource. When the reference is external, the associated resource type will be `Resource`. | | logical_resources | Whole system | One row for each logical resource, regardless of type. | | resource_change_log | Whole system | One row for each CREATE, UPDATE or DELETE interaction. | | xx_logical_resources | Resource | Resource-type specific entry for each logical resource. | @@ -115,6 +119,7 @@ The following table describes the purpose of each table or group of tables in th | xx_quantity_values | Resource parameters | Quantity search parameter values. | | xx_date_values | Resource parameters | Date search parameter values. | | xx_latlng_values | Resource parameters | Latitude/longitude search parameter values. | +| xx_ref_values | Resource parameters | Reference search parameter values. The reference values are normalized, with the target values stored as records in logical_resource_ident. | | xx_resource_token_refs | Resource parameters | Token search parameter values. The token values are normalized to save space from repeating long strings. This table acts as a mapping table between the normalized values stored in COMMON_TOKEN_VALUES and the logical resource. | | xx_tags | Resource parameters | Tag search parameters. Tag is a search parameter (rather than a type of search parameter) so no reference to PARAMETER_NAMES is needed. Tags are token values which are normalized in the COMMON_TOKEN_VALUES table. This table acts as a mapping between the normalized value and the logical resource. | | xx_profiles | Resource parameters | Profile search parameters. Profile is a search parameter (rather than a type of search parameter) so no reference to PARAMETER_NAMES is needed. | @@ -134,6 +139,8 @@ Search parameters which are part of a composite value include a value in their ` ## Finding and Reading a Resource +Note: see the section describing Citus for a more efficient way to read a resource when using the `DISTRIBUTED` variant of the schema. + The name of the FHIR resource type is normalized and stored in the `RESOURCE_TYPES` table. The `RESOURCE_TYPE_ID` is then used as a foreign key to reference the resource type throughout the schema. ``` @@ -484,8 +491,17 @@ The DDL for most objects (like tables) is specified once. Changes to the table s The schema update utility first reads the VERSION_HISTORY table, loading all records for the target schema (e.g. FHIRDATA). The utility only applies changes which have a version number greater than the currently recorded version. Once the DDL has been applied successfully, the version number is updated in VERSION_HISTORY. This makes the processed idempotent. Subsequent runs of the schema update utility only apply changes which have a greater version id value than the most recently stored value for each object. +### Schema Version V0027 + +A new `logical_resource_ident` table has been added. This table is now the primary owner of the `(resource_type_id, logical_id) to (logical_resource_id)` mapping. During ingestion, the SELECT FOR UPDATE lock is now obtained on this record instead of the `logical_resources` record. This change supports more data efficient distribution when using Citus, and more efficient handling of reference search parameter values in general. +Schema version V0027 changes the way reference search parameter values are stored. Prior to V0027, reference parameters were treated as tokens and stored in the `common_token_values` table, with the `xx_resource_token_refs` providing the many-to-many mapping between the logical resource record in `xx_logical_resources` and `common_token_values`. +As of schema version V0027, the reference mapping is now stored in `xx_ref_values` with the normalized referenced value stored in the `logical_resource_ident` table. This is useful, because it reduces the size of the `common_token_values` table, allowing it to be treated as a REFERENCE table when using the distributed schema in Citus. This arrangement also makes it more efficient to store local references (references between resources both stored within the same IBM FHIR Server database). Each logical_resource_ident record includes a foreign key referencing the resource type. Where the reference is an external reference (perhaps a URL pointing to another FHIR server), the target of the resource may not be known, in which case the `logical_resource_ident` record is assigned the `resource_type_id` for the `Resource` resource type. + +During schema migration, the schema tool will check to see if the `logical_resource_ident` table is empty, and if so, will populate the table with each record from the `logical_resources` table. + +Following migration, these changes require the search parameter data to be reindexed before any FHIR searches are run. ## Managing Resource Tables @@ -510,6 +526,7 @@ VISIONPRESCRIPTION_DATE_VALUES VISIONPRESCRIPTION_STR_VALUES VISIONPRESCRIPTION_PROFILES VISIONPRESCRIPTION_RESOURCE_TOKEN_REFS +VISIONPRESCRIPTION_REF_VALUES VISIONPRESCRIPTION_TAGS VISIONPRESCRIPTION_SECURITY VISIONPRESCRIPTION_QUANTITY_VALUES @@ -535,6 +552,7 @@ For example, for the `STRING_VALUES` table: // Parameters are tied to the logical resource Table tbl = Table.builder(schemaName, tableName) .setVersion(2) + .setDistributionType(DistributionType.DISTRIBUTED) // V0027 support for sharding .addTag(FhirSchemaTags.RESOURCE_TYPE, prefix) .setTenantColumnName(MT_ID) .addBigIntColumn( ROW_ID, false) @@ -648,9 +666,117 @@ This was necessary because version 4.0.1 of the fhir-persistence-schema cli does The `fhir-install` module contains scripts for building Docker containers of the IBM FHIR Server and IBM Db2 and, optionally, bringing them up via `docker-compose`. When releasing new versions of the IBM FHIR Server, the `SCHEMA_VERSION` variable should be updated within `fhir-install/docker/copy-dependencies-db2-migration.sh` in order to test migrations from the previously released version of the `fhir-persistencne-schema` module. -## References +## Distributed Schema Support for Citus + +With Citus, we can distribute data across multiple nodes for increased scalability. The cost is some search functionality is restricted and some queries need to be executed in parallel across multiple nodes which can increase latency. + +The tables are distributed as follows: + +| Tables | Distribution Type | Distribution Column | +| --------------- | ----------------- | ------------------- | +| whole_schema_version | NONE | N/A | +| resource_change_log | NONE | N/A | +| logical_resource_ident | DISTRIBUTED | logical_id | +| resource_types | REFERENCE | N/A | +| parameter_names | REFERENCE | N/A | +| code_systems | REFERENCE | N/A | +| common_token_values | REFERENCE | N/A | +| common_canonical_values | REFERENCE | N/A | +| *all other data tables* | DISTRIBUTED | logical_resource_id | + +For Citus, all joins between distributed tables must include the distribution column which in this case will always be `logical_resource_id`. The logical_resource_ident table is now used to manage the identity of all logical resources, including those that may not yet exist but are the target of a reference. There may therefore be entries in logical_resource_ident which do not currently have a corresponding record in logical_resources. For local references, the resource_type_id will refer to the actual resource type of the target resource. Where the reference is an external reference we may not know the type, so the resource_type_id refers to the resource_types record where resource_type is `Resource`. + +No support is provided to migrate from the PLAIN to DISTRIBUTED flavors of the schema. Therefore, to use Citus, the schema must be newly created. After initial creation, migrations will continue to work as usual. + +Because the logical_resources table is distributed by logical_resource_id, if a component only has the {resource_type, logical_id} tuple, it is more efficient to read the logical_resource_id from the logical_resource_ident table first (this value can also be cached by the application if desired): + +``` +fhirdb=> \d fhirdata.logical_resource_ident + Table "fhirdata.logical_resource_ident" + Column | Type | Collation | Nullable | Default +---------------------+-------------------------+-----------+----------+--------- + resource_type_id | integer | | not null | + logical_id | character varying(1024) | | not null | + logical_resource_id | bigint | | not null | +Indexes: + "logical_resource_ident_pk" PRIMARY KEY, btree (logical_id, resource_type_id) + "idx_logical_resource_ident_lrid" btree (logical_resource_id) +Foreign-key constraints: + "fk_logical_resource_ident_rtid" FOREIGN KEY (resource_type_id) REFERENCES fhirdata.resource_types(resource_type_id) +``` + +Note that the `logical_id` type is defined as `VARCHAR(1024)` which is much larger than the 64 characters required for a FHIR `Resource.id`. This is because this column must also accommodate external reference values, which are typically full URLS and therefore much longer. + +The query to obtain the logical_resource_id is: + +``` + SELECT logical_resource_id + FROM logical_resource_ident + WHERE resource_type_id = ? + AND logical_id = ?; +``` + +Because the table is distributed by `logical_id` and the value is given in the query, Citus can route the query to a single target node. + +The `resource_type_id` value comes from the `resource_types` table and is fixed when the schema is first installed. This value is not guaranteed to be same across databases and schemas, so it must always be read from the `resource_types` table for a given schema. For Citus, the `resource_types` table is distributed as a REFERENCE table which means there is a complete copy of the records on every node. It is therefore possible to join to a reference table without needing a common distribution key. For example: + +``` + SELECT lri.logical_resource_id + FROM logical_resource_ident lri + JOIN resource_types rt ON (lri.resource_type_id = rt.resource_type_id) + WHERE rt.resource_type = 'Patient' + AND lri.logical_id = ?; +``` + + +### Schema Differences + +In the standard `PLAIN` schema variant, tables may use IDENTITY columns to automate the generation of primary key values. IDENTITY columns are not supported in Citus so when the schema type is `DISTRIBUTED` (which is required for Citus), the primary key values are obtained from the sequence `fhir_sequence` instead. This impacts the common_token_values table, whose definition includes the following: +``` + .setIdentityColumn( COMMON_TOKEN_VALUE_ID, Generated.ALWAYS) +``` + +Also, if an index on a DISTRIBUTED table does not include the distribution column (typically `logical_resource_id`), the index cannot be declared UNIQUE. Instead, a non-unique index is defined. For example, the `logical_resources` definition includes: +``` + .addUniqueIndex("UNQ_" + LOGICAL_RESOURCES, RESOURCE_TYPE_ID, LOGICAL_ID) +``` + +For the `DISTRIBUTED` schema type variant, uniqueness of the {resource_type_id, logical_id} tuple can no longer be enforced by the above unique index on `logical_resources`, because `logical_resources` is distributed by `logical_resource_id` and this column is not part of the index definition. Instead, the IBM FHIR Server relies on the new `logical_resource_ident` table to manage uniqueness of this tuple, and the `resource_type_id` and `logical_id` columns in `logical_resources` are just denormalized copies of the data. + +### Distributed Procedures/Functions + +The following description uses the term stored procedure for simplicity even though some implementations use stored functions. + +For the `PLAIN` schema type variant, the IBM FHIR Server uses a stored procedure called `add_any_resource` to create or update the database `logical_resources`, `xx_logical_resources` and `xx_resources` records (where `xx` represents the resource type name). Using a stored procedure improves ingestion performance by reducing the number of database round-trips required to execute the required logic. + +For the `DISTRIBUTED` schema type variant, the logic has been split into two procedures as follows: + +1. `add_logical_resource_ident(resource_type_id, logical_id)` - contains the logic to create a new logical_resource_ident record, or if one exists already, obtain a lock on the row by executing a SELECT FOR UPDATE. This procedure is therefore now responsible for allocating a new `logical_resource_id` value for all new logical resources; +2. `add_any_resource(logical_resource_id, ...)` - contains the remaining logic to create/update the `logical_resources`, `xx_logical_resources` and `xx_resources` records. + +Because the `logical_resource_id` value is no longer generated inside `add_any_resource`, it is now passed as a parameter to this procedure. This approach has some significant benefits with Citus, which allows stored procedures and functions to be distributed by one of their parameters. The `add_logical_resource_ident` includes SQL and DML statements involving only the`logical_resource_ident` table, and all statements use `logical_id` which is the distribution column for that table. This allows us to also distribute the procedure by the `logical_id` parameter value, allowing the database to optimize how the procedure is executed. + +Similarly, all SQL and DML statements within the `add_any_resource` procedure use `logical_resource_id`, so the `add_any_resource` procedure is distributed by the `logical_resource_id` parameter to provide the same benefit at runtime. + +### Schema Tool Changes to Support Citus + +When the database type is given as citus (`--db-type citus`), the schema-tool applies changes in this order: + +1. Create Tables (without any foreign key constraints) +2. Apply table distribution rules for all REFERENCE tables +3. Apply table distribution rules for all DISTRIBUTED tables +4. Add foreign key constraints +5. Add/replace stored functions +6. Apply stored function distribution rules + +The foreign key constraints have to be added after the tables are distributed. Attempting to distribute tables after the foreign key constraints have been applied leads to errors. + +The distribution step can take some time due to the amount of DDL Citus must execute for each table. + +## References and Additional Reading - [Git Issue: Document the schema migration process on the project wiki #270](https://github.com/IBM/FHIR/issues/270) - [Db2 11.5: Extent sizes in table spaces](https://www.ibm.com/support/knowledgecenter/SSEPGG_11.5.0/com.ibm.db2.luw.admin.dbobj.doc/doc/c0004964.html) - [Db2 11.5: Altering table spaces](https://www.ibm.com/support/producthub/db2/docs/content/SSEPGG_11.5.0/com.ibm.db2.luw.admin.dbobj.doc/doc/t0005096.html) +- [Citus Data Modeling](https://docs.citusdata.com/en/v11.0-beta/sharding/data_modeling.html) FHIR® is the registered trademark of HL7 and is used with the permission of HL7. \ No newline at end of file diff --git a/fhir-persistence-schema/docs/SchemaToolUsageGuide.md b/fhir-persistence-schema/docs/SchemaToolUsageGuide.md index 53b3df9c1f7..29826f56c93 100644 --- a/fhir-persistence-schema/docs/SchemaToolUsageGuide.md +++ b/fhir-persistence-schema/docs/SchemaToolUsageGuide.md @@ -21,11 +21,12 @@ For details on the schema design, refer to the [Schema Design](https://github.co ## Database Support -| Database | Version | Support | -|------------|-----------|-----------------------------------| -| DB2 | 11.5+ | Supports multi-tenancy. | -| PostgreSQL | 12+ | Single tenant per database. | -| Derby | 10.14.2.0 | Development only. Single tenant per database. | +| Database | Version | Support | +|------------|----------------|-----------------------------------| +| DB2 | 11.5+ | Supports multi-tenancy. | +| PostgreSQL | 12+ | Single tenant per database. | +| Derby | 10.14.2.0 | Development only. Single tenant per database. | +| Citus | PostgreSQL 12+ | Experimental. | ---------------------------------------------------------------- ## Getting started @@ -112,24 +113,27 @@ java -jar ./fhir-persistence-schema-${VERSION}-cli.jar [OPTIONS] Note: Replace `${VERSION}` with the version of the jar you're using or use the wildcard `*` to match any version. +Note: Prior to IBM FHIR Server Release 5.0.0, the default value for `--db-type` was `db2`. As of IBM FHIR Server Release 5.0.0, there is no longer a default value and `--db-type` must be specified every time. + The following sections include common values for `OPTIONS`. ### Create new schema For Db2: ``` ---prop-file db2.properties ---schema-name FHIRDATA +--db-type db2 \ +--prop-file db2.properties \ +--schema-name FHIRDATA \ --create-schemas ``` For PostgreSQL: ``` ---prop-file postgresql.properties ---schema-name fhirdata +--db-type postgresql \ +--prop-file postgresql.properties \ +--schema-name fhirdata \ --create-schemas ---db-type postgresql ``` ### Deploy new schema or update an existing schema @@ -141,9 +145,10 @@ for the IBM FHIR Server to operate. The FHIRADMIN user should only be used for schema updates, not for IBM FHIR Server access. ``` ---prop-file db2.properties ---schema-name FHIRDATA ---update-schema +--db-type db2 \ +--prop-file db2.properties \ +--schema-name FHIRDATA \ +--update-schema \ --grant-to FHIRSERVER ``` @@ -161,12 +166,13 @@ for the IBM FHIR Server to operate. The FHIRADMIN user should only be used for schema updates, not for IBM FHIR Server access. ``` ---prop-file postgresql.properties ---schema-name FHIRDATA ---update-schema +--db-type postgresql \ +--prop-file postgresql.properties \ +--schema-name FHIRDATA \ +--update-schema \ --grant-to FHIRSERVER ---db-type postgresql ``` + If the --grant-to is provided, the grants will be processed after the schema objects have been created for a particular schema. No grant changes will be applied if the schema is already at the latest version according to the @@ -178,16 +184,18 @@ When updating the postgres schema, the autovacuum settings are configured. ### Grant privileges to another data access user ``` ---prop-file db2.properties ---schema-name FHIRDATA +--db-type db2 \ +--prop-file db2.properties \ +--schema-name FHIRDATA \ --grant-to FHIRSERVER ``` ### Add a new tenant (e.g. default) (Db2 only) ``` ---prop-file db2.properties ---schema-name FHIRDATA +--db-type db2 \ +--prop-file db2.properties \ +--schema-name FHIRDATA \ --allocate-tenant default ``` @@ -209,6 +217,7 @@ After a schema update you must run the refresh-tenants command to ensure that an ``` java -jar schema/fhir-persistence-schema-*-cli.jar \ + --db-type db2 \ --prop-file db2.properties --refresh-tenants ``` @@ -243,9 +252,10 @@ Edit `wlp/usr/servers/fhir-server/config/default/fhir-server-config.json` and ad ### Test a tenant (Db2 only) ``` ---prop-file db2.properties ---schema-name FHIRDATA ---test-tenant default +--db-type db2 \ +--prop-file db2.properties \ +--schema-name FHIRDATA \ +--test-tenant default \ --tenant-key "" ``` @@ -255,8 +265,9 @@ Use `--tenant-key-file tenant.key` to read the tenant-key to a file. You do not To add a tenant key for an existing tenant, replace FHIRDATA with your client schema, and change default to your tenant's name. ``` ---prop-file db2.properties ---schema-name FHIRDATA +--db-type db2 \ +--prop-file db2.properties \ +--schema-name FHIRDATA \ --add-tenant-key default ``` @@ -277,9 +288,9 @@ Use `--tenant-key-file tenant.key.file` to direct the action to read the tenant- To remove all tenant keys for an existing tenant, replace FHIRDATA with your client schema, and change default to your tenant's name. ``` ---prop-file db2.properties ---schema-name FHIRDATA ---db-type db2 +--db-type db2 \ +--prop-file db2.properties \ +--schema-name FHIRDATA \ --revoke-all-tenant-keys default ``` @@ -295,10 +306,10 @@ To remove all tenant keys for an existing tenant, replace FHIRDATA with your cli To remove a tenant key for an existing tenant, replace FHIRDATA with your client schema, and change default to your tenant's name. ``` ---prop-file db2.properties ---schema-name FHIRDATA ---db-type db2 ---revoke-tenant-key default +--db-type db2 \ +--prop-file db2.properties \ +--schema-name FHIRDATA \ +--revoke-tenant-key default \ --tenant-key rZ59TLyEpjU+FAKEtgVk8J44J0= ``` @@ -317,63 +328,66 @@ Use `--tenant-key-file tenant.key.file` to direct the action to read the tenant- For Db2: ``` ---prop-file db2.properties ---schema-name FHIRDATA +--db-type db2 \ +--prop-file db2.properties \ +--schema-name FHIRDATA \ --update-proc ``` For PostgreSQL: ``` ---prop-file postgresql.properties ---schema-name fhirdata +--db-type postgresql \ +--prop-file postgresql.properties \ +--schema-name fhirdata \ --update-proc ---db-type postgresql ``` ### Drop the FHIR schema specified by `schema-name` (e.g. FHIRDATA) For Db2: ``` ---prop-file db2.properties ---schema-name FHIRDATA ---drop-schema-fhir +--db-type db2 \ +--prop-file db2.properties \ +--schema-name FHIRDATA \ +--drop-schema-fhir \ --confirm-drop ``` For PostgreSQL: ``` ---prop-file postgresql.properties ---schema-name FHIRDATA ---drop-schema-fhir +--db-type postgresql \ +--prop-file postgresql.properties \ +--schema-name FHIRDATA \ +--drop-schema-fhir \ --confirm-drop ---db-type postgresql ``` ### Drop all tables created by `--create-schemas` (including the FHIR-ADMIN schema) For Db2: ``` ---prop-file db2.properties ---schema-name FHIRDATA ---drop-schema-fhir ---drop-schema-batch ---drop-schema-oauth ---drop-admin +--db-type db2 \ +--prop-file db2.properties \ +--schema-name FHIRDATA \ +--drop-schema-fhir \ +--drop-schema-batch \ +--drop-schema-oauth \ +--drop-admin \ --confirm-drop ``` For PostgreSQL: ``` ---prop-file postgresql.properties ---schema-name FHIRDATA ---drop-schema-fhir ---drop-schema-batch ---drop-schema-oauth +--db-type postgresql \ +--prop-file postgresql.properties \ +--schema-name FHIRDATA \ +--drop-schema-fhir \ +--drop-schema-batch \ +--drop-schema-oauth \ --drop-admin ---db-type postgresql ``` Alternatively, you can drop specific schemas with `--drop-schema-batch schema-name-to-drop` and @@ -387,10 +401,11 @@ For those using multiple schemas for each customer, for instance, customer 2 nee ``` java -jar ./fhir-persistence-schema-${VERSION}-cli.jar \ ---prop-file db2.properties ---create-schemas ---create-schema-batch FHIR_JBATCH_2ND ---create-schema-oauth FHIR_OAUTH_2ND +--db-type db2 \ +--prop-file db2.properties \ +--create-schemas \ +--create-schema-batch FHIR_JBATCH_2ND \ +--create-schema-oauth FHIR_OAUTH_2ND \ --create-schema-fhir FHIRDATA_2ND ``` @@ -398,32 +413,36 @@ java -jar ./fhir-persistence-schema-${VERSION}-cli.jar \ ``` java -jar ./fhir-persistence-schema-${VERSION}-cli.jar \ ---prop-file db2.properties ---schema-name FHIRDATA ---update-schema-batch FHIR_JBATCH_2ND ---update-schema-oauth FHIR_OAUTH_2ND +--db-type db2 \ +--prop-file db2.properties \ +--schema-name FHIRDATA \ +--update-schema-batch FHIR_JBATCH_2ND \ +--update-schema-oauth FHIR_OAUTH_2ND \ --update-schema-fhir FHIRDATA_2ND ``` ### Grant privileges to data access user ``` java -jar ./fhir-persistence-schema-${VERSION}-cli.jar \ ---prop-file db2.properties ---grant-to FHIRSERVER +--db-type db2 \ +--prop-file db2.properties \ +--grant-to FHIRSERVER \ --target BATCH FHIR_JBATCH_2ND ``` ``` java -jar ./fhir-persistence-schema-${VERSION}-cli.jar \ ---prop-file db2.properties ---grant-to FHIRSERVER +--db-type db2 \ +--prop-file db2.properties \ +--grant-to FHIRSERVER \ --target OAUTH FHIR_OAUTH_2ND ``` ``` java -jar ./fhir-persistence-schema-${VERSION}-cli.jar \ ---prop-file db2.properties ---grant-to FHIRSERVER +--db-type db2 \ +--prop-file db2.properties \ +--grant-to FHIRSERVER \ --target DATA FHIRDATA_2ND ``` @@ -532,6 +551,23 @@ Note: the jar file is stored locally in `fhir-persistence-schema/target` or in t If there is data in the DOMAINRESOURCE and RESOURCE table groups, which is unexpected, the administrator may run the tool with `--force-unused-table-removal` to force the removal of the unused tables. +---------------------------------------------------------------- +# Distributed Database Support for Citus + +IBM FHIR Server Release 5.0.0 includes experimental support for Citus. Configuration is mostly identical to PostgreSQL, except that the `-db-type` argument should be given as `citus`, for example: + +``` +java -jar ./fhir-persistence-schema-${VERSION}-cli.jar \ +--prop-file citus.properties \ +--schema-name fhirdata \ +--update-schema \ +--db-type citus +``` + +When `--db-type citus` is specified, the schema tool builds a slightly modified DISTRIBUTED version of the schema which introduces different behavior for some indexes and foreign key constraints. For details on the DISTRIBUTED schema design, refer to the [Schema Design](https://github.com/IBM/FHIR/tree/main/fhir-persistence-schema/docs/SchemaDesign.md) document. + +Note that the datasource must also be identified as type `citus` in the fhir-server-config.json file. See the [IBM FHIR Server Users Guide](https://ibm.github.io/FHIR/guides/FHIRServerUsersGuide) for more details. + ---------------------------------------------------------------- # Database Size Report (Db2, PostgreSQL) @@ -539,8 +575,8 @@ Run this command to show a summary of the space used by IBM FHIR Server resource ``` shell java -jar ./fhir-persistence-schema-${VERSION}-cli.jar \ ---prop-file fhiradmin.properties \ --db-type postgresql \ +--prop-file fhiradmin.properties \ --schema-name FHIRDATA \ --show-db-size ``` @@ -549,8 +585,8 @@ To include per-table and per-index in the output, add the `--show-db-size-detail ``` shell java -jar ./fhir-persistence-schema-${VERSION}-cli.jar \ ---prop-file fhiradmin.properties \ --db-type postgresql \ +--prop-file fhiradmin.properties \ --schema-name FHIRDATA \ --show-db-size \ --show-db-size-detail @@ -560,8 +596,8 @@ java -jar ./fhir-persistence-schema-${VERSION}-cli.jar \ ``` shell java -jar ./fhir-persistence-schema-${VERSION}-cli.jar \ ---prop-file fhiradmin.properties \ --db-type db2 \ +--prop-file fhiradmin.properties \ --schema-name FHIRDATA \ --tenant-name MY_TENANT_NAME \ --show-db-size @@ -576,8 +612,6 @@ The detail rows are tab-separated, making it easy to load the data into a spread ---------------------------------------------------------------- # List of IBM FHIR Server Persistence Schema Tool Flags -|Flag|Variable|Description| -|----------------|----------------|----------------| |Flag|Variable|Description| |----------------|----------------|----------------| |--help||This menu| @@ -597,7 +631,8 @@ and grants permission to the username| |--tenant-key|tenantKey|the tenant-key in the queries| |--tenant-key-file|tenant-key-file-location|sets the tenant key file location| |--list-tenants||fetches list of tenants and current status| -|--db-type|dbType|Either derby, postgresql, db2| +|--db-type|dbType|Either derby, postgresql, db2, citus. Required.| +|--schema-type|schemaType|Override the default schema type created for the configured database type. PostgresSQL->PLAIN, Derby->PLAIN, Db2->MULTITENANT, Citus->DISTRIBUTED | |--delete-tenant-meta|tenantName|deletes tenant metadata given the tenantName| |--drop-detached|tenantName|(phase 2) drops the detached tenant partition tables given the tenantName| |--freeze-tenant||Changes the tenant state to frozen, and subsequently (Db2 only)| diff --git a/fhir-persistence-schema/docs/physical_schema_V0027.png b/fhir-persistence-schema/docs/physical_schema_V0027.png new file mode 100644 index 00000000000..9b6770cbc66 Binary files /dev/null and b/fhir-persistence-schema/docs/physical_schema_V0027.png differ diff --git a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/app/Main.java b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/app/Main.java index 397bbd5269e..e443fecd44d 100644 --- a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/app/Main.java +++ b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/app/Main.java @@ -22,6 +22,7 @@ import static com.ibm.fhir.schema.app.menu.Menu.DROP_SCHEMA_BATCH; import static com.ibm.fhir.schema.app.menu.Menu.DROP_SCHEMA_FHIR; import static com.ibm.fhir.schema.app.menu.Menu.DROP_SCHEMA_OAUTH; +import static com.ibm.fhir.schema.app.menu.Menu.DROP_SPLIT_TRANSACTION; import static com.ibm.fhir.schema.app.menu.Menu.DROP_TENANT; import static com.ibm.fhir.schema.app.menu.Menu.FORCE; import static com.ibm.fhir.schema.app.menu.Menu.FORCE_UNUSED_TABLE_REMOVAL; @@ -36,6 +37,7 @@ import static com.ibm.fhir.schema.app.menu.Menu.REVOKE_ALL_TENANT_KEYS; import static com.ibm.fhir.schema.app.menu.Menu.REVOKE_TENANT_KEY; import static com.ibm.fhir.schema.app.menu.Menu.SCHEMA_NAME; +import static com.ibm.fhir.schema.app.menu.Menu.SCHEMA_TYPE; import static com.ibm.fhir.schema.app.menu.Menu.SHOW_DB_SIZE; import static com.ibm.fhir.schema.app.menu.Menu.SHOW_DB_SIZE_DETAIL; import static com.ibm.fhir.schema.app.menu.Menu.SKIP_ALLOCATE_IF_TENANT_EXISTS; @@ -59,6 +61,7 @@ import static com.ibm.fhir.schema.app.util.CommonUtil.getDbAdapter; import static com.ibm.fhir.schema.app.util.CommonUtil.getPropertyAdapter; import static com.ibm.fhir.schema.app.util.CommonUtil.getRandomKey; +import static com.ibm.fhir.schema.app.util.CommonUtil.getSchemaAdapter; import static com.ibm.fhir.schema.app.util.CommonUtil.loadDriver; import static com.ibm.fhir.schema.app.util.CommonUtil.logClasspath; @@ -93,12 +96,17 @@ import com.ibm.fhir.database.utils.api.IDatabaseAdapter; import com.ibm.fhir.database.utils.api.IDatabaseTranslator; import com.ibm.fhir.database.utils.api.ILeaseManagerConfig; +import com.ibm.fhir.database.utils.api.ISchemaAdapter; import com.ibm.fhir.database.utils.api.ITransaction; import com.ibm.fhir.database.utils.api.ITransactionProvider; +import com.ibm.fhir.database.utils.api.SchemaApplyContext; +import com.ibm.fhir.database.utils.api.SchemaType; import com.ibm.fhir.database.utils.api.TableSpaceRemovalException; import com.ibm.fhir.database.utils.api.TenantStatus; import com.ibm.fhir.database.utils.api.UndefinedNameException; import com.ibm.fhir.database.utils.api.UniqueConstraintViolationException; +import com.ibm.fhir.database.utils.citus.CitusTranslator; +import com.ibm.fhir.database.utils.citus.ConfigureConnectionDAO; import com.ibm.fhir.database.utils.common.DataDefinitionUtil; import com.ibm.fhir.database.utils.common.JdbcConnectionProvider; import com.ibm.fhir.database.utils.common.JdbcPropertyAdapter; @@ -135,6 +143,7 @@ import com.ibm.fhir.model.util.ModelSupport; import com.ibm.fhir.schema.app.menu.Menu; import com.ibm.fhir.schema.app.util.TenantKeyFileUtil; +import com.ibm.fhir.schema.control.AddForeignKey; import com.ibm.fhir.schema.control.BackfillResourceChangeLog; import com.ibm.fhir.schema.control.BackfillResourceChangeLogDb2; import com.ibm.fhir.schema.control.DisableForeignKey; @@ -144,6 +153,7 @@ import com.ibm.fhir.schema.control.FhirSchemaGenerator; import com.ibm.fhir.schema.control.FhirSchemaVersion; import com.ibm.fhir.schema.control.GetLogicalResourceNeedsV0014Migration; +import com.ibm.fhir.schema.control.GetLogicalResourceNeedsV0027Migration; import com.ibm.fhir.schema.control.GetResourceChangeLogEmpty; import com.ibm.fhir.schema.control.GetResourceTypeList; import com.ibm.fhir.schema.control.GetTenantInfo; @@ -153,6 +163,8 @@ import com.ibm.fhir.schema.control.JavaBatchSchemaGenerator; import com.ibm.fhir.schema.control.MigrateV0014LogicalResourceIsDeletedLastUpdated; import com.ibm.fhir.schema.control.MigrateV0021AbstractTypeRemoval; +import com.ibm.fhir.schema.control.MigrateV0027LogicalResourceIdent; +import com.ibm.fhir.schema.control.MigrateV0027LogicalResourceIdentMT; import com.ibm.fhir.schema.control.OAuthSchemaGenerator; import com.ibm.fhir.schema.control.PopulateParameterNames; import com.ibm.fhir.schema.control.PopulateResourceTypes; @@ -194,8 +206,8 @@ public class Main { // Indicates if the feature is enabled for the DbType public List MULTITENANT_FEATURE_ENABLED = Arrays.asList(DbType.DB2); - public List STORED_PROCEDURE_ENABLED = Arrays.asList(DbType.DB2, DbType.POSTGRESQL); - public List PRIVILEGES_FEATURE_ENABLED = Arrays.asList(DbType.DB2, DbType.POSTGRESQL); + public List STORED_PROCEDURE_ENABLED = Arrays.asList(DbType.DB2, DbType.POSTGRESQL, DbType.CITUS); + public List PRIVILEGES_FEATURE_ENABLED = Arrays.asList(DbType.DB2, DbType.POSTGRESQL, DbType.CITUS); // Properties accumulated as we parse args and read configuration files private final Properties properties = new Properties(); @@ -249,9 +261,9 @@ public class Main { // How many seconds to wait to obtain the update lease private int waitForUpdateLeaseSeconds = 10; - // The database type being populated (default: Db2) - private DbType dbType = DbType.DB2; - private IDatabaseTranslator translator = new Db2Translator(); + // The database type being populated. Now a required parameter. + private DbType dbType; + private IDatabaseTranslator translator; // Optional subset of resource types (for faster schema builds when testing) private Set resourceTypeSubset; @@ -282,6 +294,9 @@ public class Main { // Include detail output in the report (default is no) private boolean showDbSizeDetail = false; + // Split drops into multiple transactions? + private boolean dropSplitTransaction = false; + // Tenant Key Output or Input File private String tenantKeyFileName; private TenantKeyFileUtil tenantKeyFileUtil = new TenantKeyFileUtil(); @@ -303,6 +318,9 @@ public class Main { // Configuration to control how the LeaseManager operates private ILeaseManagerConfig leaseManagerConfig; + // Which flavor of the FHIR data schema should we build? + private SchemaType dataSchemaType = SchemaType.PLAIN; + // ----------------------------------------------------------------------------------------------------------------- // The following method is related to the common methods and functions /** @@ -334,6 +352,10 @@ protected void configureConnectionPool() { JdbcConnectionProvider cp = new JdbcConnectionProvider(this.translator, adapter); this.connectionPool = new PoolConnectionProvider(cp, this.maxConnectionPoolSize); this.transactionProvider = new SimpleTransactionProvider(this.connectionPool); + + if (this.dbType == DbType.CITUS) { + connectionPool.setNewConnectionHandler(Main::configureCitusConnection); + } } /** @@ -343,7 +365,8 @@ protected void configureConnectionPool() { */ protected void buildAdminSchemaModel(PhysicalDataModel pdm) { // Add the tenant and tenant_keys tables and any other admin schema stuff - FhirSchemaGenerator gen = new FhirSchemaGenerator(schema.getAdminSchemaName(), schema.getSchemaName(), isMultitenant()); + SchemaType adminSchemaType = isMultitenant() ? SchemaType.MULTITENANT : SchemaType.PLAIN; + FhirSchemaGenerator gen = new FhirSchemaGenerator(schema.getAdminSchemaName(), schema.getSchemaName(), adminSchemaType); gen.buildAdminSchema(pdm); } @@ -377,9 +400,12 @@ protected void buildJavaBatchSchemaModel(PhysicalDataModel pdm) { * @param collector * @param vhs */ - protected void applyModel(PhysicalDataModel pdm, IDatabaseAdapter adapter, ITaskCollector collector, VersionHistoryService vhs) { + protected void applyModel(PhysicalDataModel pdm, ISchemaAdapter adapter, ITaskCollector collector, VersionHistoryService vhs, SchemaType schemaType) { logger.info("Collecting model update tasks"); - pdm.collect(collector, adapter, this.transactionProvider, vhs); + // If using a distributed RDBMS (like Citus) then skip the initial FK creation + final boolean includeForeignKeys = schemaType != SchemaType.DISTRIBUTED; + SchemaApplyContext context = SchemaApplyContext.builder().setIncludeForeignKeys(includeForeignKeys).build(); + pdm.collect(collector, adapter, context, this.transactionProvider, vhs); // FHIR in the hole! logger.info("Starting model updates"); @@ -462,9 +488,9 @@ protected void updateSchemas() { // Make sure that we have the CONTROL table created before we try any // schema update work - IDatabaseAdapter adapter = getDbAdapter(dbType, connectionPool); + ISchemaAdapter schemaAdapter = getSchemaAdapter(SchemaType.PLAIN, dbType, connectionPool); try (ITransaction tx = transactionProvider.getTransaction()) { - CreateControl.createTableIfNeeded(schema.getAdminSchemaName(), adapter); + CreateControl.createTableIfNeeded(schema.getAdminSchemaName(), schemaAdapter); } catch (UniqueConstraintViolationException x) { // Race condition - two or more instances trying to create the CONTROL table throw new ConcurrentUpdateException("Concurrent update - create control table"); @@ -490,9 +516,9 @@ protected void updateSchemas() { protected void buildFhirDataSchemaModel(PhysicalDataModel pdm) { FhirSchemaGenerator gen; if (resourceTypeSubset == null || resourceTypeSubset.isEmpty()) { - gen = new FhirSchemaGenerator(schema.getAdminSchemaName(), schema.getSchemaName(), isMultitenant()); + gen = new FhirSchemaGenerator(schema.getAdminSchemaName(), schema.getSchemaName(), getDataSchemaType()); } else { - gen = new FhirSchemaGenerator(schema.getAdminSchemaName(), schema.getSchemaName(), isMultitenant(), resourceTypeSubset); + gen = new FhirSchemaGenerator(schema.getAdminSchemaName(), schema.getSchemaName(), getDataSchemaType(), resourceTypeSubset); } gen.buildSchema(pdm); @@ -506,6 +532,9 @@ protected void buildFhirDataSchemaModel(PhysicalDataModel pdm) { case POSTGRESQL: gen.buildDatabaseSpecificArtifactsPostgres(pdm); break; + case CITUS: + gen.buildDatabaseSpecificArtifactsCitus(pdm); + break; default: throw new IllegalStateException("Unsupported db type: " + dbType); } @@ -524,19 +553,28 @@ protected void updateFhirSchema() { } final String targetSchemaName = schema.getSchemaName(); - IDatabaseAdapter adapter = getDbAdapter(dbType, connectionPool); + ISchemaAdapter schemaAdapter = getSchemaAdapter(SchemaType.PLAIN, dbType, connectionPool); try (ITransaction tx = transactionProvider.getTransaction()) { - CreateWholeSchemaVersion.createTableIfNeeded(targetSchemaName, adapter); + CreateWholeSchemaVersion.createTableIfNeeded(targetSchemaName, schemaAdapter); } // If our schema is already at the latest version, we can skip a lot of processing SchemaVersionsManager svm = new SchemaVersionsManager(translator, connectionPool, transactionProvider, targetSchemaName, FhirSchemaVersion.getLatestFhirSchemaVersion().vid()); if (svm.isSchemaOld() || this.force && svm.isSchemaVersionMatch()) { + if (this.dbType == DbType.CITUS) { + // First version with Citus support is V0027 and we can't upgrade + // from before that + int currentSchemaVersion = svm.getVersionForSchema(); + if (currentSchemaVersion >= 0 && currentSchemaVersion < FhirSchemaVersion.V0027.vid()) { + throw new IllegalStateException("Cannot upgrade Citus databases with schema version < V0027"); + } + } + // Build/update the FHIR-related tables as well as the stored procedures - PhysicalDataModel pdm = new PhysicalDataModel(); + PhysicalDataModel pdm = new PhysicalDataModel(isDistributed()); buildFhirDataSchemaModel(pdm); - updateSchema(pdm); + boolean isNewDb = updateSchema(pdm, getDataSchemaType()); if (this.exitStatus == EXIT_OK) { // If the db is multi-tenant, we populate the resource types and parameter names in allocate-tenant. @@ -562,6 +600,9 @@ protected void updateFhirSchema() { // V0021 removes Abstract Type tables which are unused. applyTableRemovalForV0021(); + // V0027 populate the new LOGICAL_RESOURCE_IDENT table + applyDataMigrationForV0027(); + // Apply privileges if asked if (grantTo != null) { grantPrivilegesForFhirData(); @@ -600,18 +641,18 @@ protected void updateOauthSchema() { } final String targetSchemaName = schema.getOauthSchemaName(); - IDatabaseAdapter adapter = getDbAdapter(dbType, connectionPool); + ISchemaAdapter schemaAdapter = getSchemaAdapter(SchemaType.PLAIN, dbType, connectionPool); try (ITransaction tx = transactionProvider.getTransaction()) { - CreateWholeSchemaVersion.createTableIfNeeded(targetSchemaName, adapter); + CreateWholeSchemaVersion.createTableIfNeeded(targetSchemaName, schemaAdapter); } // If our schema is already at the latest version, we can skip a lot of processing SchemaVersionsManager svm = new SchemaVersionsManager(translator, connectionPool, transactionProvider, targetSchemaName, FhirSchemaVersion.getLatestFhirSchemaVersion().vid()); if (svm.isSchemaOld() || this.force && svm.isSchemaVersionMatch()) { - PhysicalDataModel pdm = new PhysicalDataModel(); + PhysicalDataModel pdm = new PhysicalDataModel(false); buildOAuthSchemaModel(pdm); - updateSchema(pdm); + updateSchema(pdm, SchemaType.PLAIN); if (this.exitStatus == EXIT_OK) { // Apply privileges if asked @@ -646,18 +687,18 @@ protected void updateJavaBatchSchema() { } final String targetSchemaName = schema.getJavaBatchSchemaName(); - IDatabaseAdapter adapter = getDbAdapter(dbType, connectionPool); + ISchemaAdapter schemaAdapter = getSchemaAdapter(SchemaType.PLAIN, dbType, connectionPool); try (ITransaction tx = transactionProvider.getTransaction()) { - CreateWholeSchemaVersion.createTableIfNeeded(targetSchemaName, adapter); + CreateWholeSchemaVersion.createTableIfNeeded(targetSchemaName, schemaAdapter); } // If our schema is already at the latest version, we can skip a lot of processing SchemaVersionsManager svm = new SchemaVersionsManager(translator, connectionPool, transactionProvider, targetSchemaName, FhirSchemaVersion.getLatestFhirSchemaVersion().vid()); if (svm.isSchemaOld() || this.force && svm.isSchemaVersionMatch()) { - PhysicalDataModel pdm = new PhysicalDataModel(); + PhysicalDataModel pdm = new PhysicalDataModel(false); buildJavaBatchSchemaModel(pdm); - updateSchema(pdm); + updateSchema(pdm, SchemaType.PLAIN); if (this.exitStatus == EXIT_OK) { // Apply privileges if asked @@ -683,7 +724,7 @@ protected void updateJavaBatchSchema() { * Update the schema associated with the given {@link PhysicalDataModel} * @return true if the database is new */ - protected boolean updateSchema(PhysicalDataModel pdm) { + protected boolean updateSchema(PhysicalDataModel pdm, SchemaType schemaType) { // The objects are applied in parallel, which relies on each object // expressing its dependencies correctly. Changes are only applied @@ -694,12 +735,13 @@ protected boolean updateSchema(PhysicalDataModel pdm) { ExecutorService pool = Executors.newFixedThreadPool(this.threadPoolSize); ITaskCollector collector = taskService.makeTaskCollector(pool); IDatabaseAdapter adapter = getDbAdapter(dbType, connectionPool); + ISchemaAdapter schemaAdapter = getSchemaAdapter(schemaType, dbType, connectionPool); // Before we start anything, we need to make sure our schema history // and control tables are in place. These tables are used to manage // all FHIR data, oauth and JavaBatch schemas we build try (ITransaction tx = transactionProvider.getTransaction()) { - CreateVersionHistory.createTableIfNeeded(schema.getAdminSchemaName(), adapter); + CreateVersionHistory.createTableIfNeeded(schema.getAdminSchemaName(), schemaAdapter); } // Current version history for the data schema @@ -713,12 +755,50 @@ protected boolean updateSchema(PhysicalDataModel pdm) { boolean isNewDb = vhs.getVersion(schema.getSchemaName(), DatabaseObjectType.TABLE.name(), "PARAMETER_NAMES") == null || vhs.getVersion(schema.getSchemaName(), DatabaseObjectType.TABLE.name(), "PARAMETER_NAMES") == 0; - applyModel(pdm, adapter, collector, vhs); + applyModel(pdm, schemaAdapter, collector, vhs, schemaType); + applyDistributionRules(pdm, schemaType); + // The physical database objects should now match what was defined in the PhysicalDataModel return isNewDb; } + /** + * Apply any table distribution rules then add all the + * FK constraints that are needed. Applying all the distribution rules + * in one transaction causes issues with Citus/PostgreSQL (out of shared memory + * errors) so instead we provide a function to allow the visitor to break + * things up into smaller transactions. + * @param pdm + */ + private void applyDistributionRules(PhysicalDataModel pdm, SchemaType schemaType) { + if (dbType == DbType.CITUS) { + ISchemaAdapter schemaAdapter = getSchemaAdapter(getDataSchemaType(), dbType, connectionPool); + pdm.applyDistributionRules(schemaAdapter, () -> TransactionFactory.openTransaction(connectionPool)); + } + + final boolean includedForeignKeys = schemaType != SchemaType.DISTRIBUTED; + if (!includedForeignKeys) { + // Now that all the tables have been distributed, it should be safe + // to apply the FK constraints + final String tenantColumnName = isMultitenant() ? "mt_id" : null; + ISchemaAdapter adapter = getSchemaAdapter(getDataSchemaType(), dbType, connectionPool); + AddForeignKey adder = new AddForeignKey(adapter, tenantColumnName); + pdm.visit(adder, FhirSchemaGenerator.SCHEMA_GROUP_TAG, FhirSchemaGenerator.FHIRDATA_GROUP, () -> TransactionFactory.openTransaction(connectionPool)); + } + } + + /** + * Special case to initialize new connections created by the connection pool + * for Citus databases + * @param c + */ + private static void configureCitusConnection(Connection c) { + logger.info("Citus: Configuring new database connection"); + ConfigureConnectionDAO dao = new ConfigureConnectionDAO(); + dao.run(new CitusTranslator(), c); + } + /** * populates for the given tenantId the RESOURCE_TYPE table. * @@ -759,7 +839,7 @@ protected void populateResourceTypeAndParameterNameTableEntries(Integer tenantId * Typically used during development. */ protected void dropSchema() { - PhysicalDataModel pdm = new PhysicalDataModel(); + PhysicalDataModel pdm = new PhysicalDataModel(isDistributed()); buildCommonModel(pdm, dropFhirSchema, dropOauthSchema, dropJavaBatchSchema); // Dropping the schema in PostgreSQL can fail with an out of shared memory error @@ -792,6 +872,8 @@ protected void dropSchema() { try { JdbcTarget target = new JdbcTarget(c); IDatabaseAdapter adapter = getDbAdapter(dbType, target); + ISchemaAdapter schemaAdapter = getSchemaAdapter(getDataSchemaType(), adapter); + ISchemaAdapter plainSchemaAdapter = getSchemaAdapter(SchemaType.PLAIN, adapter); VersionHistoryService vhs = new VersionHistoryService(schema.getAdminSchemaName(), schema.getSchemaName(), schema.getOauthSchemaName(), schema.getJavaBatchSchemaName()); vhs.setTransactionProvider(transactionProvider); @@ -800,8 +882,18 @@ protected void dropSchema() { if (dropFhirSchema) { // Just drop the objects associated with the FHIRDATA schema group final String schemaName = schema.getSchemaName(); - pdm.drop(adapter, FhirSchemaGenerator.SCHEMA_GROUP_TAG, FhirSchemaGenerator.FHIRDATA_GROUP); - CreateWholeSchemaVersion.dropTable(schemaName, adapter); + if (this.dropSplitTransaction) { + // important that we use an adapter connected with the connection pool + // (which is connected to the transaction provider) + ISchemaAdapter poolSchemaAdapter = getSchemaAdapter(getDataSchemaType(), dbType, connectionPool); + pdm.dropSplitTransaction(poolSchemaAdapter, this.transactionProvider, FhirSchemaGenerator.SCHEMA_GROUP_TAG, FhirSchemaGenerator.FHIRDATA_GROUP); + } else { + // old fashioned drop where we do everything in one (big) transaction + pdm.drop(schemaAdapter, FhirSchemaGenerator.SCHEMA_GROUP_TAG, FhirSchemaGenerator.FHIRDATA_GROUP); + } + + // Drop the whole-schema-version table + CreateWholeSchemaVersion.dropTable(schemaName, plainSchemaAdapter); if (!checkSchemaIsEmpty(adapter, schemaName)) { throw new DataAccessException("Schema '" + schemaName + "' not empty after drop"); } @@ -811,8 +903,8 @@ protected void dropSchema() { if (dropOauthSchema) { // Just drop the objects associated with the OAUTH schema group final String schemaName = schema.getOauthSchemaName(); - pdm.drop(adapter, FhirSchemaGenerator.SCHEMA_GROUP_TAG, OAuthSchemaGenerator.OAUTH_GROUP); - CreateWholeSchemaVersion.dropTable(schemaName, adapter); + pdm.drop(schemaAdapter, FhirSchemaGenerator.SCHEMA_GROUP_TAG, OAuthSchemaGenerator.OAUTH_GROUP); + CreateWholeSchemaVersion.dropTable(schemaName, plainSchemaAdapter); if (!checkSchemaIsEmpty(adapter, schemaName)) { throw new DataAccessException("Schema '" + schemaName + "' not empty after drop"); } @@ -822,8 +914,8 @@ protected void dropSchema() { if (dropJavaBatchSchema) { // Just drop the objects associated with the BATCH schema group final String schemaName = schema.getJavaBatchSchemaName(); - pdm.drop(adapter, FhirSchemaGenerator.SCHEMA_GROUP_TAG, JavaBatchSchemaGenerator.BATCH_GROUP); - CreateWholeSchemaVersion.dropTable(schemaName, adapter); + pdm.drop(plainSchemaAdapter, FhirSchemaGenerator.SCHEMA_GROUP_TAG, JavaBatchSchemaGenerator.BATCH_GROUP); + CreateWholeSchemaVersion.dropTable(schemaName, plainSchemaAdapter); if (!checkSchemaIsEmpty(adapter, schemaName)) { throw new DataAccessException("Schema '" + schemaName + "' not empty after drop"); } @@ -834,7 +926,7 @@ protected void dropSchema() { // Just drop the objects associated with the ADMIN schema group CreateVersionHistory.generateTable(pdm, ADMIN_SCHEMANAME, true); CreateControl.buildTableDef(pdm, ADMIN_SCHEMANAME, true); - pdm.drop(adapter, FhirSchemaGenerator.SCHEMA_GROUP_TAG, FhirSchemaGenerator.ADMIN_GROUP); + pdm.drop(plainSchemaAdapter, FhirSchemaGenerator.SCHEMA_GROUP_TAG, FhirSchemaGenerator.ADMIN_GROUP); if (!checkSchemaIsEmpty(adapter, schema.getAdminSchemaName())) { throw new DataAccessException("Schema '" + schema.getAdminSchemaName() + "' not empty after drop"); } @@ -865,7 +957,7 @@ private void dropForeignKeyConstraints(PhysicalDataModel pdm, String tagGroup, S Set
referencedTables = new HashSet<>(); DropForeignKey dropper = new DropForeignKey(adapter, referencedTables); - pdm.visit(dropper, tagGroup, tag); + pdm.visit(dropper, tagGroup, tag, null); } catch (Exception x) { c.rollback(); throw x; @@ -888,7 +980,7 @@ protected void updateProcedures() { } // Build/update the tables as well as the stored procedures - PhysicalDataModel pdm = new PhysicalDataModel(); + PhysicalDataModel pdm = new PhysicalDataModel(isDistributed()); // Since this is a stored procedure, we need the model. // We must pass in true to flag to the underlying layer that the // Procedures need to be generated. @@ -899,14 +991,15 @@ protected void updateProcedures() { try (ITransaction tx = TransactionFactory.openTransaction(connectionPool)) { try (Connection c = connectionPool.getConnection();) { try { - IDatabaseAdapter adapter = getDbAdapter(dbType, connectionPool); - pdm.applyProcedures(adapter); - pdm.applyFunctions(adapter); + ISchemaAdapter schemaAdapter = getSchemaAdapter(getDataSchemaType(), dbType, connectionPool); + SchemaApplyContext context = SchemaApplyContext.getDefault(); + pdm.applyProcedures(schemaAdapter, context); + pdm.applyFunctions(schemaAdapter, context); // Because we're replacing the procedures, we should also check if // we need to apply the associated privileges if (this.grantTo != null) { - pdm.applyProcedureAndFunctionGrants(adapter, FhirSchemaConstants.FHIR_USER_GRANT_GROUP, grantTo); + pdm.applyProcedureAndFunctionGrants(schemaAdapter, FhirSchemaConstants.FHIR_USER_GRANT_GROUP, grantTo); } } catch (DataAccessException x) { // Something went wrong, so mark the transaction as failed @@ -949,16 +1042,16 @@ protected void buildCommonModel(PhysicalDataModel pdm, boolean addFhirDataSchema */ protected void grantPrivilegesForFhirData() { - final IDatabaseAdapter adapter = getDbAdapter(dbType, connectionPool); + final ISchemaAdapter schemaAdapter = getSchemaAdapter(getDataSchemaType(), dbType, connectionPool); try (ITransaction tx = TransactionFactory.openTransaction(connectionPool)) { try { - PhysicalDataModel pdm = new PhysicalDataModel(); + PhysicalDataModel pdm = new PhysicalDataModel(isDistributed()); buildFhirDataSchemaModel(pdm); - pdm.applyGrants(adapter, FhirSchemaConstants.FHIR_USER_GRANT_GROUP, grantTo); + pdm.applyGrants(schemaAdapter, FhirSchemaConstants.FHIR_USER_GRANT_GROUP, grantTo); // Grant SELECT on WHOLE_SCHEMA_VERSION to the FHIR server user // Note the constant comes from SchemaConstants on purpose - CreateWholeSchemaVersion.grantPrivilegesTo(adapter, schema.getSchemaName(), SchemaConstants.FHIR_USER_GRANT_GROUP, grantTo); + CreateWholeSchemaVersion.grantPrivilegesTo(schemaAdapter, schema.getSchemaName(), SchemaConstants.FHIR_USER_GRANT_GROUP, grantTo); } catch (DataAccessException x) { // Something went wrong, so mark the transaction as failed tx.setRollbackOnly(); @@ -971,12 +1064,12 @@ protected void grantPrivilegesForFhirData() { * Apply grants to the OAuth schema objects */ protected void grantPrivilegesForOAuth() { - final IDatabaseAdapter adapter = getDbAdapter(dbType, connectionPool); + final ISchemaAdapter schemaAdapter = getSchemaAdapter(SchemaType.PLAIN, dbType, connectionPool); try (ITransaction tx = TransactionFactory.openTransaction(connectionPool)) { try { - PhysicalDataModel pdm = new PhysicalDataModel(); + PhysicalDataModel pdm = new PhysicalDataModel(false); buildOAuthSchemaModel(pdm); - pdm.applyGrants(adapter, FhirSchemaConstants.FHIR_OAUTH_GRANT_GROUP, grantTo); + pdm.applyGrants(schemaAdapter, FhirSchemaConstants.FHIR_OAUTH_GRANT_GROUP, grantTo); } catch (DataAccessException x) { // Something went wrong, so mark the transaction as failed @@ -991,15 +1084,15 @@ protected void grantPrivilegesForOAuth() { * Apply grants to the JavaBatch schema objects */ protected void grantPrivilegesForBatch() { - final IDatabaseAdapter adapter = getDbAdapter(dbType, connectionPool); + final ISchemaAdapter schemaAdapter = getSchemaAdapter(SchemaType.PLAIN, dbType, connectionPool); try (ITransaction tx = TransactionFactory.openTransaction(connectionPool)) { try { - PhysicalDataModel pdm = new PhysicalDataModel(); + PhysicalDataModel pdm = new PhysicalDataModel(false); buildJavaBatchSchemaModel(pdm); - pdm.applyGrants(adapter, FhirSchemaConstants.FHIR_BATCH_GRANT_GROUP, grantTo); + pdm.applyGrants(schemaAdapter, FhirSchemaConstants.FHIR_BATCH_GRANT_GROUP, grantTo); // special case for the JavaBatch schema in PostgreSQL - adapter.grantAllSequenceUsage(schema.getJavaBatchSchemaName(), grantTo); + schemaAdapter.grantAllSequenceUsage(schema.getJavaBatchSchemaName(), grantTo); } catch (DataAccessException x) { // Something went wrong, so mark the transaction as failed @@ -1047,7 +1140,15 @@ protected void grantPrivileges() { * @return */ protected boolean isMultitenant() { - return MULTITENANT_FEATURE_ENABLED.contains(this.dbType); + return dataSchemaType == SchemaType.MULTITENANT; + } + + /** + * What type of FHIR data schema do we want to build? + * @return + */ + protected SchemaType getDataSchemaType() { + return this.dataSchemaType; } // ----------------------------------------------------------------------------------------------------------------- @@ -1231,8 +1332,8 @@ protected void allocateTenant() { } // Build/update the tables as well as the stored procedures - FhirSchemaGenerator gen = new FhirSchemaGenerator(schema.getAdminSchemaName(), schema.getSchemaName(), isMultitenant()); - PhysicalDataModel pdm = new PhysicalDataModel(); + FhirSchemaGenerator gen = new FhirSchemaGenerator(schema.getAdminSchemaName(), schema.getSchemaName(), getDataSchemaType()); + PhysicalDataModel pdm = new PhysicalDataModel(isDistributed()); gen.buildSchema(pdm); // Get the data model to create the table partitions. This is threaded, so transactions are @@ -1364,8 +1465,8 @@ protected void refreshTenants() { if (ti.getTenantSchema() != null && (!schema.isOverrideDataSchema() || schema.matchesDataSchema(ti.getTenantSchema()))) { // It's crucial we use the correct schema for each particular tenant, which // is why we have to build the PhysicalDataModel separately for each tenant - FhirSchemaGenerator gen = new FhirSchemaGenerator(schema.getAdminSchemaName(), ti.getTenantSchema(), isMultitenant()); - PhysicalDataModel pdm = new PhysicalDataModel(); + FhirSchemaGenerator gen = new FhirSchemaGenerator(schema.getAdminSchemaName(), ti.getTenantSchema(), getDataSchemaType()); + PhysicalDataModel pdm = new PhysicalDataModel(isDistributed()); gen.buildSchema(pdm); Db2Adapter adapter = new Db2Adapter(connectionPool); @@ -1582,8 +1683,8 @@ protected void dropTenant() { TenantInfo tenantInfo = freezeTenant(); // Build the model of the data (FHIRDATA) schema which is then used to drive the drop - FhirSchemaGenerator gen = new FhirSchemaGenerator(schema.getAdminSchemaName(), tenantInfo.getTenantSchema(), isMultitenant()); - PhysicalDataModel pdm = new PhysicalDataModel(); + FhirSchemaGenerator gen = new FhirSchemaGenerator(schema.getAdminSchemaName(), tenantInfo.getTenantSchema(), getDataSchemaType()); + PhysicalDataModel pdm = new PhysicalDataModel(isDistributed()); gen.buildSchema(pdm); // Detach the tenant partition from each of the data tables @@ -1601,8 +1702,8 @@ protected void dropTenant() { protected void dropDetachedPartitionTables() { TenantInfo tenantInfo = getTenantInfo(); - FhirSchemaGenerator gen = new FhirSchemaGenerator(schema.getAdminSchemaName(), tenantInfo.getTenantSchema(), isMultitenant()); - PhysicalDataModel pdm = new PhysicalDataModel(); + FhirSchemaGenerator gen = new FhirSchemaGenerator(schema.getAdminSchemaName(), tenantInfo.getTenantSchema(), getDataSchemaType()); + PhysicalDataModel pdm = new PhysicalDataModel(isDistributed()); gen.buildSchema(pdm); dropDetachedPartitionTables(pdm, tenantInfo); @@ -1995,6 +2096,9 @@ protected void parseArgs(String[] args) { i++; } break; + case DROP_SPLIT_TRANSACTION: + this.dropSplitTransaction = true; + break; case POOL_SIZE: if (++i < args.length) { this.maxConnectionPoolSize = Integer.parseInt(args[i]); @@ -2051,6 +2155,13 @@ protected void parseArgs(String[] args) { case CONFIRM_DROP: this.confirmDrop = true; break; + case SCHEMA_TYPE: + if (++i < args.length) { + this.dataSchemaType = SchemaType.valueOf(args[i]); + } else { + throw new IllegalArgumentException("Missing value for argument at posn: " + i); + } + break; case ALLOCATE_TENANT: if (++i < args.length) { this.tenantName = args[i]; @@ -2115,7 +2226,14 @@ protected void parseArgs(String[] args) { case POSTGRESQL: translator = new PostgresTranslator(); break; + case CITUS: + translator = new CitusTranslator(); + dataSchemaType = SchemaType.DISTRIBUTED; // by default + break; case DB2: + translator = new Db2Translator(); + dataSchemaType = SchemaType.MULTITENANT; + break; default: break; } @@ -2153,6 +2271,10 @@ protected void parseArgs(String[] args) { this.maxConnectionPoolSize = threadPoolSize + FhirSchemaConstants.CONNECTION_POOL_HEADROOM; logger.warning("Connection pool size below minimum headroom. Setting it to " + this.maxConnectionPoolSize); } + + if (this.dbType == null) { + throw new IllegalArgumentException(DB_TYPE + " is required"); + } } /** @@ -2238,6 +2360,23 @@ protected void applyDataMigrationForV0014() { } } + protected void applyDataMigrationForV0027() { + if (MULTITENANT_FEATURE_ENABLED.contains(dbType)) { + // Process each tenant one-by-one + List tenants = getTenantList(); + for (TenantInfo ti : tenants) { + + // If no --schema-name override was specified, we process all tenants, otherwise we + // process only tenants which belong to the override schema name + if (!schema.isOverrideDataSchema() || schema.matchesDataSchema(ti.getTenantSchema())) { + dataMigrationForV0027(ti); + } + } + } else { + dataMigrationForV0027(); + } + } + /** * Get the list of resource types to drive resource-by-resource operations * @@ -2395,6 +2534,44 @@ private void dataMigrationForV0014() { } } + private void dataMigrationForV0027(TenantInfo ti) { + // Multi-tenant schema so we know this is Db2: + Db2Adapter adapter = new Db2Adapter(connectionPool); + + try (ITransaction tx = TransactionFactory.openTransaction(connectionPool)) { + try { + SetTenantIdDb2 setTenantId = new SetTenantIdDb2(schema.getAdminSchemaName(), ti.getTenantId()); + adapter.runStatement(setTenantId); + + logger.info("V0027 Migration: Populating LOGICAL_RESOURCE_IDENT for tenant['" + + ti.getTenantName() + "' in schema '" + ti.getTenantSchema() + "']"); + + dataMigrationForV0027(adapter, ti.getTenantSchema()); + } catch (DataAccessException x) { + // Something went wrong, so mark the transaction as failed + tx.setRollbackOnly(); + throw x; + } + } + } + + /** + * V0027 migration. Populate LOGICAL_RESOURCE_IDENT from LOGICAL_RESOURCES + */ + private void dataMigrationForV0027() { + IDatabaseAdapter adapter = getDbAdapter(dbType, connectionPool); + + try (ITransaction tx = TransactionFactory.openTransaction(connectionPool)) { + try { + dataMigrationForV0027(adapter, schema.getSchemaName()); + } catch (DataAccessException x) { + // Something went wrong, so mark the transaction as failed + tx.setRollbackOnly(); + throw x; + } + } + } + /** * only process tables which have not yet had their data migrated. The migration can't be * done as part of the schema change because some tables need a REORG which @@ -2415,6 +2592,25 @@ private void dataMigrationForV0014(IDatabaseAdapter adapter, String schemaName, } } + /** + * If the LOGICAL_RESOURCE_IDENT table is empty, fill it using values from + * LOGICAL_RESOURCES + * @param adapter + * @param schemaName + */ + private void dataMigrationForV0027(IDatabaseAdapter adapter, String schemaName) { + GetLogicalResourceNeedsV0027Migration needsMigrating = new GetLogicalResourceNeedsV0027Migration(schemaName); + if (adapter.runStatement(needsMigrating)) { + if (this.dataSchemaType == SchemaType.MULTITENANT) { + MigrateV0027LogicalResourceIdentMT cmd = new MigrateV0027LogicalResourceIdentMT(schemaName); + adapter.runStatement(cmd); + } else { + MigrateV0027LogicalResourceIdent cmd = new MigrateV0027LogicalResourceIdent(schemaName); + adapter.runStatement(cmd); + } + } + } + /** * Backfill the RESOURCE_CHANGE_LOG table if it is empty */ @@ -2503,16 +2699,16 @@ private void doBackfill(IDatabaseAdapter adapter) { } /** - * updates the vacuum settings for postgres. + * updates the vacuum settings for postgres/citus. */ public void updateVacuumSettings() { - if (dbType != DbType.POSTGRESQL) { + if (dbType != DbType.POSTGRESQL && dbType != DbType.CITUS) { logger.severe("Updating the vacuum settings is only supported on postgres and the setting is for '" + dbType + "'"); return; } // Create the Physical Data Model - PhysicalDataModel pdm = new PhysicalDataModel(); + PhysicalDataModel pdm = new PhysicalDataModel(isDistributed()); buildCommonModel(pdm, true, false, false); // Setup the Connection Pool @@ -2533,6 +2729,16 @@ public void updateVacuumSettings() { } } + /** + * Should we build the distributed variant of the FHIR data schema. This + * changes how we need to handle certain unique indexes and foreign key + * constraints. + * @return + */ + private boolean isDistributed() { + return dataSchemaType == SchemaType.DISTRIBUTED; + } + /** * runs the vacuum update inside a single connection and single transaction. * @@ -2588,6 +2794,7 @@ private void generateDbSizeReport() { final ISizeCollector collector; switch (dbType) { case POSTGRESQL: + case CITUS: // assume for now this works for Citus collector = new PostgresSizeCollector(model); break; case DB2: diff --git a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/app/SchemaPrinter.java b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/app/SchemaPrinter.java index 01b45b8579e..d3323afe176 100644 --- a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/app/SchemaPrinter.java +++ b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/app/SchemaPrinter.java @@ -1,5 +1,5 @@ /* - * (C) Copyright IBM Corp. 2019, 2020 + * (C) Copyright IBM Corp. 2019, 2022 * * SPDX-License-Identifier: Apache-2.0 */ @@ -46,8 +46,12 @@ import java.util.Properties; import java.util.concurrent.Executor; +import com.ibm.fhir.database.utils.api.ISchemaAdapter; +import com.ibm.fhir.database.utils.api.SchemaApplyContext; +import com.ibm.fhir.database.utils.api.SchemaType; import com.ibm.fhir.database.utils.common.JdbcTarget; import com.ibm.fhir.database.utils.db2.Db2Adapter; +import com.ibm.fhir.database.utils.derby.DerbyMaster; import com.ibm.fhir.database.utils.model.PhysicalDataModel; import com.ibm.fhir.database.utils.version.CreateVersionHistory; import com.ibm.fhir.database.utils.version.VersionHistoryService; @@ -89,7 +93,7 @@ public class SchemaPrinter { private static final String STORED_PROCEDURE_DELIMITER = "@"; private final boolean toFile; - private final boolean multitenant; + private final SchemaType schemaType; private File schemaFile = new File("schema.sql"); private File spFile = new File("stored-procedures.sql"); private File grantFile = new File("grants.sql"); @@ -104,11 +108,15 @@ public class SchemaPrinter { /** * constructor that switches behavior toFile our output stream. + * + * @param toFile + * @param schemaType + * @throws FileNotFoundException */ - public SchemaPrinter(boolean toFile, boolean multitenant) throws FileNotFoundException { + public SchemaPrinter(boolean toFile, SchemaType schemaType) throws FileNotFoundException { this.toFile = toFile; - this.multitenant = multitenant; + this.schemaType = schemaType; if (this.toFile) { out = new PrintStream(new FileOutputStream(schemaFile)); @@ -153,9 +161,10 @@ public void process() { PrintConnection connection = new PrintConnection(); JdbcTarget target = new JdbcTarget(connection); Db2Adapter adapter = new Db2Adapter(target); + ISchemaAdapter schemaAdapter = DerbyMaster.wrap(adapter); // Set up the version history service first if it doesn't yet exist - CreateVersionHistory.createTableIfNeeded(Main.ADMIN_SCHEMANAME, adapter); + CreateVersionHistory.createTableIfNeeded(Main.ADMIN_SCHEMANAME, schemaAdapter); // Current version history for the database. This is used by applyWithHistory // to determine which updates to apply and to record the new changes as they @@ -165,7 +174,7 @@ public void process() { // Create an instance of the service and use it to test creation // of the FHIR schema - FhirSchemaGenerator gen = new FhirSchemaGenerator(Main.ADMIN_SCHEMANAME, Main.DATA_SCHEMANAME, multitenant); + FhirSchemaGenerator gen = new FhirSchemaGenerator(Main.ADMIN_SCHEMANAME, Main.DATA_SCHEMANAME, this.schemaType); PhysicalDataModel model = new PhysicalDataModel(); gen.buildSchema(model); @@ -174,7 +183,8 @@ public void process() { JavaBatchSchemaGenerator javaBatchSchemaGenerator = new JavaBatchSchemaGenerator(Main.BATCH_SCHEMANAME); javaBatchSchemaGenerator.buildJavaBatchSchema(model); - model.apply(adapter); + SchemaApplyContext context = SchemaApplyContext.getDefault(); + model.apply(schemaAdapter, context); } public void processApplyGrants() { @@ -182,9 +192,10 @@ public void processApplyGrants() { PrintConnection connection = new PrintConnection(); JdbcTarget target = new JdbcTarget(connection); Db2Adapter adapter = new Db2Adapter(target); + ISchemaAdapter schemaAdapter = DerbyMaster.wrap(adapter); // Set up the version history service first if it doesn't yet exist - CreateVersionHistory.createTableIfNeeded(Main.ADMIN_SCHEMANAME, adapter); + CreateVersionHistory.createTableIfNeeded(Main.ADMIN_SCHEMANAME, schemaAdapter); // Current version history for the database. This is used by applyWithHistory // to determine which updates to apply and to record the new changes as they @@ -194,7 +205,7 @@ public void processApplyGrants() { // Create an instance of the service and use it to test creation // of the FHIR schema - FhirSchemaGenerator gen = new FhirSchemaGenerator(Main.ADMIN_SCHEMANAME, Main.DATA_SCHEMANAME, multitenant); + FhirSchemaGenerator gen = new FhirSchemaGenerator(Main.ADMIN_SCHEMANAME, Main.DATA_SCHEMANAME, schemaType); PhysicalDataModel model = new PhysicalDataModel(); gen.buildSchema(model); @@ -206,7 +217,7 @@ public void processApplyGrants() { // clear it out. commands.clear(); - model.applyGrants(adapter, FhirSchemaConstants.FHIR_USER_GRANT_GROUP, "FHIRUSER"); + model.applyGrants(schemaAdapter, FhirSchemaConstants.FHIR_USER_GRANT_GROUP, "FHIRUSER"); } /** @@ -260,6 +271,7 @@ public void print() { public static void main(String[] args) { boolean outputToFile = false; boolean multitenant = false; + boolean distributed = false; String outputFile = ""; // If there are files @@ -272,13 +284,22 @@ public static void main(String[] args) { case "--multitenant": multitenant = true; break; + case "--distributed": + distributed = true; + break; default: throw new IllegalArgumentException("Invalid argument: " + arg); } } + if (multitenant && distributed) { + throw new IllegalArgumentException("--multitenant and --distributed are mutually exclusive"); + } + + SchemaType schemaType = multitenant ? SchemaType.MULTITENANT : distributed ? SchemaType.DISTRIBUTED : SchemaType.PLAIN; + try { - SchemaPrinter printer = new SchemaPrinter(outputToFile, multitenant); + SchemaPrinter printer = new SchemaPrinter(outputToFile, schemaType); printer.process(); printer.print(); printer.processApplyGrants(); diff --git a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/app/menu/Menu.java b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/app/menu/Menu.java index aba8adfab71..4dbd5029a71 100644 --- a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/app/menu/Menu.java +++ b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/app/menu/Menu.java @@ -32,6 +32,7 @@ public class Menu { public static final String DROP_DETACHED = "--drop-detached"; public static final String FREEZE_TENANT = "--freeze-tenant"; public static final String DROP_TENANT = "--drop-tenant"; + public static final String DROP_SPLIT_TRANSACTION = "--drop-split-transaction"; public static final String REFRESH_TENANTS = "--refresh-tenants"; public static final String ALLOCATE_TENANT = "--allocate-tenant"; public static final String CONFIRM_DROP = "--confirm-drop"; @@ -61,6 +62,7 @@ public class Menu { public static final String HELP = "--help"; public static final String SHOW_DB_SIZE = "--show-db-size"; public static final String SHOW_DB_SIZE_DETAIL = "--show-db-size-detail"; + public static final String SCHEMA_TYPE = "--schema-type"; public Menu() { // NOP @@ -83,7 +85,7 @@ public enum HelpMenu { MI_TENANT_KEY(TENANT_KEY, "tenantKey", "the tenant-key in the queries"), MI_TENANT_KEY_FILE(TENANT_KEY_FILE, "tenant-key-file-location", "sets the tenant key file location"), MI_LIST_TENANTS(LIST_TENANTS, "", "fetches list of tenants and current status"), - MI_DB_TYPE(DB_TYPE, "dbType" , "Either derby, postgresql, db2"), + MI_DB_TYPE(DB_TYPE, "dbType" , "Either derby, postgresql, db2, citus"), MI_DELETE_TENANT_META(DELETE_TENANT_META, "tenantName", "deletes tenant metadata given the tenantName"), MI_DROP_DETACHED(DROP_DETACHED, "tenantName", "(phase 2) drops the detached tenant partition tables given the tenantName"), MI_FREEZE_TENANT(FREEZE_TENANT, "", "Changes the tenant state to frozen, and subsequently (Db2 only)"), @@ -115,6 +117,7 @@ public enum HelpMenu { MI_CREATE_SCHEMA_FHIR(CREATE_SCHEMA_FHIR, "schemaName", "Create the FHIR Data Schema"), MI_CREATE_SCHEMA_BATCH(CREATE_SCHEMA_BATCH, "schemaName", "Create the Batch Schema"), MI_CREATE_SCHEMA_OAUTH(CREATE_SCHEMA_OAUTH, "schemaName", "Create the OAuth Schema"), + MI_SCHEMA_TYPE(SCHEMA_TYPE, "", "Which variant of the FHIR data schema to use"), MI_SHOW_DB_SIZE(SHOW_DB_SIZE, "", "Generate report with a breakdown of database size"), MI_SHOW_DB_SIZE_DETAIL(SHOW_DB_SIZE_DETAIL, "", "Include detailed table and index info in size report"); diff --git a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/app/util/CommonUtil.java b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/app/util/CommonUtil.java index 035556a525e..73eb2332f7e 100644 --- a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/app/util/CommonUtil.java +++ b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/app/util/CommonUtil.java @@ -1,5 +1,5 @@ /* - * (C) Copyright IBM Corp. 2020 + * (C) Copyright IBM Corp. 2020, 2022 * * SPDX-License-Identifier: Apache-2.0 */ @@ -19,6 +19,9 @@ import com.ibm.fhir.database.utils.api.IConnectionProvider; import com.ibm.fhir.database.utils.api.IDatabaseAdapter; import com.ibm.fhir.database.utils.api.IDatabaseTranslator; +import com.ibm.fhir.database.utils.api.ISchemaAdapter; +import com.ibm.fhir.database.utils.api.SchemaType; +import com.ibm.fhir.database.utils.citus.CitusAdapter; import com.ibm.fhir.database.utils.common.JdbcPropertyAdapter; import com.ibm.fhir.database.utils.common.JdbcTarget; import com.ibm.fhir.database.utils.common.LogFormatter; @@ -29,6 +32,10 @@ import com.ibm.fhir.database.utils.model.DbType; import com.ibm.fhir.database.utils.postgres.PostgresAdapter; import com.ibm.fhir.database.utils.postgres.PostgresPropertyAdapter; +import com.ibm.fhir.schema.build.DistributedSchemaAdapter; +import com.ibm.fhir.schema.build.FhirSchemaAdapter; +import com.ibm.fhir.schema.build.ShardedSchemaAdapter; +import com.ibm.fhir.schema.control.FhirSchemaConstants; /** * @@ -36,6 +43,7 @@ public final class CommonUtil { // Random generator for new tenant keys and salts private static final SecureRandom random = new SecureRandom(); + private static final String DEFAULT_DISTRIBUTION_COLUMN = "LOGICAL_RESOURCE_ID"; /** * Set up the logger using the log.dir system property @@ -105,6 +113,7 @@ public static JdbcPropertyAdapter getPropertyAdapter(DbType dbType, Properties p case DERBY: return new DerbyPropertyAdapter(props); case POSTGRESQL: + case CITUS: return new PostgresPropertyAdapter(props); default: throw new IllegalStateException("Unsupported db type: " + dbType); @@ -119,10 +128,57 @@ public static IDatabaseAdapter getDbAdapter(DbType dbType, JdbcTarget target) { return new DerbyAdapter(target); case POSTGRESQL: return new PostgresAdapter(target); + case CITUS: + return new CitusAdapter(target); default: throw new IllegalStateException("Unsupported db type: " + dbType); } } + /** + * Get the schema adapter which will build the schema variant described by + * the given schemaType + * @param schemaType + * @param dbType + * @param connectionProvider + * @return + */ + public static ISchemaAdapter getSchemaAdapter(SchemaType schemaType, DbType dbType, IConnectionProvider connectionProvider) { + IDatabaseAdapter dbAdapter = getDbAdapter(dbType, connectionProvider); + switch (schemaType) { + case PLAIN: + return new FhirSchemaAdapter(dbAdapter); + case MULTITENANT: + return new FhirSchemaAdapter(dbAdapter); + case DISTRIBUTED: + return new DistributedSchemaAdapter(dbAdapter, DEFAULT_DISTRIBUTION_COLUMN); + case SHARDED: + return new ShardedSchemaAdapter(dbAdapter, FhirSchemaConstants.SHARD_KEY); + default: + throw new IllegalArgumentException("Unsupported schema type: " + schemaType); + } + } + + /** + * Wrap the given databaseAdapter in an ISchemaAdapter implementation selected + * by the given schemaType + * @param schemaType + * @param dbAdapter + * @return + */ + public static ISchemaAdapter getSchemaAdapter(SchemaType schemaType, IDatabaseAdapter dbAdapter) { + switch (schemaType) { + case PLAIN: + return new FhirSchemaAdapter(dbAdapter); + case MULTITENANT: + return new FhirSchemaAdapter(dbAdapter); + case DISTRIBUTED: + return new DistributedSchemaAdapter(dbAdapter, DEFAULT_DISTRIBUTION_COLUMN); + case SHARDED: + return new ShardedSchemaAdapter(dbAdapter, FhirSchemaConstants.SHARD_KEY); + default: + throw new IllegalArgumentException("Unsupported schema type: " + schemaType); + } + } public static IDatabaseAdapter getDbAdapter(DbType dbType, IConnectionProvider connectionProvider) { switch (dbType) { @@ -132,6 +188,8 @@ public static IDatabaseAdapter getDbAdapter(DbType dbType, IConnectionProvider c return new DerbyAdapter(connectionProvider); case POSTGRESQL: return new PostgresAdapter(connectionProvider); + case CITUS: + return new CitusAdapter(connectionProvider); default: throw new IllegalStateException("Unsupported db type: " + dbType); } diff --git a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/build/DistributedSchemaAdapter.java b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/build/DistributedSchemaAdapter.java new file mode 100644 index 00000000000..237fc96a4e1 --- /dev/null +++ b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/build/DistributedSchemaAdapter.java @@ -0,0 +1,82 @@ +/* + * (C) Copyright IBM Corp. 2022 + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.ibm.fhir.schema.build; + +import java.util.List; + +import com.ibm.fhir.database.utils.api.DistributionContext; +import com.ibm.fhir.database.utils.api.DistributionType; +import com.ibm.fhir.database.utils.api.IDatabaseAdapter; +import com.ibm.fhir.database.utils.common.PlainSchemaAdapter; +import com.ibm.fhir.database.utils.model.CheckConstraint; +import com.ibm.fhir.database.utils.model.ColumnBase; +import com.ibm.fhir.database.utils.model.IdentityDef; +import com.ibm.fhir.database.utils.model.OrderedColumnDef; +import com.ibm.fhir.database.utils.model.PrimaryKeyDef; +import com.ibm.fhir.database.utils.model.With; + +/** + * Represents an adapter used to build the FHIR schema when + * used with a distributed databse like Citus + */ +public class DistributedSchemaAdapter extends PlainSchemaAdapter { + // The distribution column to use by default for DISTRIBUTED tables + private final String defaultDistributionColumnName; + + /** + * Public constructor + * + * @param databaseAdapter + */ + public DistributedSchemaAdapter(IDatabaseAdapter databaseAdapter, String defaultDistributionColumnName) { + super(databaseAdapter); + this.defaultDistributionColumnName = defaultDistributionColumnName; + } + + /** + * Create a DistributionContext using the given type and column. If the distribution column + * is null, then the default distribution column is used. + * @param distributionType + * @param distributionColumnName + * @return + */ + private DistributionContext createContext(DistributionType distributionType, String distributionColumnName) { + if (distributionType == DistributionType.DISTRIBUTED && distributionColumnName == null) { + distributionColumnName = defaultDistributionColumnName; + } + return new DistributionContext(distributionType, distributionColumnName); + } + + @Override + public void createTable(String schemaName, String name, String tenantColumnName, List columns, PrimaryKeyDef primaryKey, IdentityDef identity, + String tablespaceName, List withs, List checkConstraints, DistributionType distributionType, String distributionColumnName) { + + DistributionContext dc = createContext(distributionType, distributionColumnName); + databaseAdapter.createTable(schemaName, name, tenantColumnName, columns, primaryKey, identity, tablespaceName, withs, checkConstraints, dc); + } + + @Override + public void createUniqueIndex(String schemaName, String tableName, String indexName, String tenantColumnName, List indexColumns, + List includeColumns, DistributionType distributionType, String distributionColumnName) { + DistributionContext dc = createContext(distributionType, distributionColumnName); + databaseAdapter.createUniqueIndex(schemaName, tableName, indexName, tenantColumnName, indexColumns, includeColumns, dc); + } + + @Override + public void createUniqueIndex(String schemaName, String tableName, String indexName, String tenantColumnName, List indexColumns, + DistributionType distributionType, String distributionColumnName) { + + DistributionContext dc = createContext(distributionType, distributionColumnName); + databaseAdapter.createUniqueIndex(schemaName, tableName, indexName, tenantColumnName, indexColumns, dc); + } + + @Override + public void applyDistributionRules(String schemaName, String tableName, DistributionType distributionType, String distributionColumnName) { + DistributionContext dc = createContext(distributionType, distributionColumnName); + databaseAdapter.applyDistributionRules(schemaName, tableName, dc); + } +} diff --git a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/build/FhirSchemaAdapter.java b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/build/FhirSchemaAdapter.java new file mode 100644 index 00000000000..386b435327b --- /dev/null +++ b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/build/FhirSchemaAdapter.java @@ -0,0 +1,25 @@ +/* + * (C) Copyright IBM Corp. 2022 + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.ibm.fhir.schema.build; + +import com.ibm.fhir.database.utils.api.IDatabaseAdapter; +import com.ibm.fhir.database.utils.common.PlainSchemaAdapter; + +/** + * Represents an adapter used to build the standard FHIR schema + */ +public class FhirSchemaAdapter extends PlainSchemaAdapter { + + /** + * Public constructor + * + * @param databaseAdapter + */ + public FhirSchemaAdapter(IDatabaseAdapter databaseAdapter) { + super(databaseAdapter); + } +} diff --git a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/build/ShardedSchemaAdapter.java b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/build/ShardedSchemaAdapter.java new file mode 100644 index 00000000000..57c1a187da1 --- /dev/null +++ b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/build/ShardedSchemaAdapter.java @@ -0,0 +1,118 @@ +/* + * (C) Copyright IBM Corp. 2022 + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.ibm.fhir.schema.build; + +import java.util.ArrayList; +import java.util.List; + +import com.ibm.fhir.database.utils.api.DistributionContext; +import com.ibm.fhir.database.utils.api.DistributionType; +import com.ibm.fhir.database.utils.api.IDatabaseAdapter; +import com.ibm.fhir.database.utils.model.CheckConstraint; +import com.ibm.fhir.database.utils.model.ColumnBase; +import com.ibm.fhir.database.utils.model.IdentityDef; +import com.ibm.fhir.database.utils.model.OrderedColumnDef; +import com.ibm.fhir.database.utils.model.PrimaryKeyDef; +import com.ibm.fhir.database.utils.model.SmallIntColumn; +import com.ibm.fhir.database.utils.model.With; + +/** + * Adapter implementation used to build the distributed variant of + * the IBM FHIR Server RDBMS schema. + * + * This schema adds a distribution key column to every table identified as + * distributed. This column is also added to every index and FK relationship + * as needed. We use a smallint (2 bytes) which represents a signed integer + * holding values in the range [-32768, 32767]. This provides sufficient spread, + * assuming we won't be using a database with thousands of nodes. + */ +public class ShardedSchemaAdapter extends FhirSchemaAdapter { + + // The distribution column to add to each table marked as distributed + final String shardColumnName; + + /** + * @param databaseAdapter + */ + public ShardedSchemaAdapter(IDatabaseAdapter databaseAdapter, String shardColumnName) { + super(databaseAdapter); + this.shardColumnName = shardColumnName; + } + + @Override + public void createTable(String schemaName, String name, String tenantColumnName, List columns, PrimaryKeyDef primaryKey, IdentityDef identity, + String tablespaceName, List withs, List checkConstraints, DistributionType distributionType, String distributionColumnName) { + // If the table is distributed, we need to inject the distribution column into the columns list. This same + // column will need to be injected into each of the index definitions + List actualColumns = new ArrayList<>(); + if (distributionType == DistributionType.DISTRIBUTED) { + ColumnBase distributionColumn = new SmallIntColumn(shardColumnName, false, null); + actualColumns.add(distributionColumn); + if (primaryKey != null) { + // we need to alter the primary so it includes the distribution column + // as the last member + List newCols = new ArrayList<>(primaryKey.getColumns()); + newCols.add(distributionColumnName); + primaryKey = new PrimaryKeyDef(primaryKey.getConstraintName(), newCols); + } + } + + actualColumns.addAll(columns); + DistributionContext dc = new DistributionContext(distributionType, shardColumnName); + databaseAdapter.createTable(schemaName, name, tenantColumnName, actualColumns, primaryKey, identity, tablespaceName, withs, checkConstraints, dc); + } + + @Override + public void createUniqueIndex(String schemaName, String tableName, String indexName, String tenantColumnName, List indexColumns, + List includeColumns, DistributionType distributionType, String distributionColumnName) { + + List actualColumns = new ArrayList<>(indexColumns); + if (distributionType == DistributionType.DISTRIBUTED) { + // inject the distribution column into the index definition + actualColumns.add(new OrderedColumnDef(this.shardColumnName, null, null)); + } + + // Create the index using the modified set of index columns + DistributionContext dc = new DistributionContext(distributionType, shardColumnName); + databaseAdapter.createUniqueIndex(schemaName, tableName, indexName, tenantColumnName, actualColumns, includeColumns, dc); + } + + @Override + public void createUniqueIndex(String schemaName, String tableName, String indexName, String tenantColumnName, List indexColumns, + DistributionType distributionType, String distributionColumnName) { + + List actualColumns = new ArrayList<>(indexColumns); + if (distributionType == DistributionType.DISTRIBUTED) { + // inject the distribution column into the index definition + actualColumns.add(new OrderedColumnDef(this.shardColumnName, null, null)); + } + + // Create the index using the modified set of index columns + DistributionContext dc = new DistributionContext(distributionType, shardColumnName); + databaseAdapter.createUniqueIndex(schemaName, tableName, indexName, tenantColumnName, actualColumns, dc); + } + + @Override + public void createIndex(String schemaName, String tableName, String indexName, String tenantColumnName, List indexColumns, DistributionType distributionType) { + // Create the index using the modified set of index columns + databaseAdapter.createIndex(schemaName, tableName, indexName, tenantColumnName, indexColumns); + } + + @Override + public void createForeignKeyConstraint(String constraintName, String schemaName, String name, String targetSchema, String targetTable, + String targetColumnName, String tenantColumnName, List columns, boolean enforced, DistributionType distributionType, boolean targetIsReference) { + // If both this and the target table are distributed, we need to add the distributionColumnName + // to the FK relationship definition. If the target is a reference, it won't have the shard_key + // column because the table is fully replicated across all nodes and therefore any FK relationship + // can be based on the original PK definition without the extra sharding column. + List newCols = new ArrayList<>(columns); + if (distributionType == DistributionType.DISTRIBUTED && !targetIsReference) { + newCols.add(shardColumnName); + } + databaseAdapter.createForeignKeyConstraint(constraintName, schemaName, name, targetSchema, targetTable, targetColumnName, tenantColumnName, newCols, enforced); + } +} \ No newline at end of file diff --git a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/AddForeignKey.java b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/AddForeignKey.java new file mode 100644 index 00000000000..07a2b038d40 --- /dev/null +++ b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/AddForeignKey.java @@ -0,0 +1,42 @@ +/* + * (C) Copyright IBM Corp. 2021 + * + * SPDX-License-Identifier: Apache-2.0 + */ +package com.ibm.fhir.schema.control; + +import java.util.logging.Logger; + +import com.ibm.fhir.database.utils.api.ISchemaAdapter; +import com.ibm.fhir.database.utils.model.DataModelVisitorBase; +import com.ibm.fhir.database.utils.model.ForeignKeyConstraint; +import com.ibm.fhir.database.utils.model.Table; + +/** + * Visitor adapter used to add all the foreign key constraints + * associated with tables in the schema. + * + * Expects any transaction handling to be performed outside this class. + */ +public class AddForeignKey extends DataModelVisitorBase { + private static final Logger logger = Logger.getLogger(DropForeignKey.class.getName()); + + // The database adapter used to issue changes to the database + private final ISchemaAdapter adapter; + private final String tenantColumnName; + /** + * Public constructor + * @param adapter + */ + public AddForeignKey(ISchemaAdapter adapter, String tenantColumnName) { + this.adapter = adapter; + this.tenantColumnName = tenantColumnName; + } + + @Override + public void visited(Table fromChildTable, ForeignKeyConstraint fk) { + // Enable (add) the FK constraint + logger.info(String.format("Adding foreign key: %s.%s[%s]", fromChildTable.getSchemaName(), fromChildTable.getObjectName(), fk.getConstraintName())); + fk.apply(fromChildTable.getSchemaName(), fromChildTable.getObjectName(), this.tenantColumnName, adapter, fromChildTable.getDistributionType()); + } +} \ No newline at end of file diff --git a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/FhirResourceTableGroup.java b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/FhirResourceTableGroup.java index 1fd848c4e1b..55565fc9212 100644 --- a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/FhirResourceTableGroup.java +++ b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/FhirResourceTableGroup.java @@ -36,6 +36,7 @@ import static com.ibm.fhir.schema.control.FhirSchemaConstants.LOGICAL_ID_BYTES; import static com.ibm.fhir.schema.control.FhirSchemaConstants.LOGICAL_RESOURCES; import static com.ibm.fhir.schema.control.FhirSchemaConstants.LOGICAL_RESOURCE_ID; +import static com.ibm.fhir.schema.control.FhirSchemaConstants.LOGICAL_RESOURCE_IDENT; import static com.ibm.fhir.schema.control.FhirSchemaConstants.LONGITUDE_VALUE; import static com.ibm.fhir.schema.control.FhirSchemaConstants.MAX_SEARCH_STRING_BYTES; import static com.ibm.fhir.schema.control.FhirSchemaConstants.MT_ID; @@ -51,6 +52,9 @@ import static com.ibm.fhir.schema.control.FhirSchemaConstants.QUANTITY_VALUE; import static com.ibm.fhir.schema.control.FhirSchemaConstants.QUANTITY_VALUE_HIGH; import static com.ibm.fhir.schema.control.FhirSchemaConstants.QUANTITY_VALUE_LOW; +import static com.ibm.fhir.schema.control.FhirSchemaConstants.REF_LOGICAL_RESOURCE_ID; +import static com.ibm.fhir.schema.control.FhirSchemaConstants.REF_VALUES; +import static com.ibm.fhir.schema.control.FhirSchemaConstants.REF_VALUES_V; import static com.ibm.fhir.schema.control.FhirSchemaConstants.REF_VERSION_ID; import static com.ibm.fhir.schema.control.FhirSchemaConstants.RESOURCE_ID; import static com.ibm.fhir.schema.control.FhirSchemaConstants.RESOURCE_PAYLOAD_KEY; @@ -73,6 +77,7 @@ import java.util.List; import java.util.Set; +import com.ibm.fhir.database.utils.api.DistributionType; import com.ibm.fhir.database.utils.api.IDatabaseStatement; import com.ibm.fhir.database.utils.common.AddColumn; import com.ibm.fhir.database.utils.common.CreateIndexStatement; @@ -80,6 +85,7 @@ import com.ibm.fhir.database.utils.common.DropIndex; import com.ibm.fhir.database.utils.common.DropPrimaryKey; import com.ibm.fhir.database.utils.common.DropTable; +import com.ibm.fhir.database.utils.common.DropView; import com.ibm.fhir.database.utils.common.ReorgTable; import com.ibm.fhir.database.utils.model.ColumnBase; import com.ibm.fhir.database.utils.model.ColumnDefBuilder; @@ -176,7 +182,9 @@ public ObjectGroup addResourceType(String resourceTypeName) { addQuantityValues(group, tablePrefix); // composites table removed by issue-1683 addResourceTokenRefs(group, tablePrefix); + addRefValues(group, tablePrefix); addTokenValuesView(group, tablePrefix); + addRefValuesView(group, tablePrefix); addProfiles(group, tablePrefix); addTags(group, tablePrefix); addSecurity(group, tablePrefix); @@ -200,8 +208,9 @@ public void addLogicalResources(List group, String prefix) { // We also have a FK constraint pointing back to that table to try and keep // things sensible. Table.Builder builder = Table.builder(schemaName, tableName) + .setVersion(FhirSchemaVersion.V0027.vid()) // V0027: add support for distribution/sharding .setTenantColumnName(MT_ID) - .setVersion(FhirSchemaVersion.V0020.vid()) // V0020: Updated to support Postgres fillfactor changes + .setDistributionType(DistributionType.DISTRIBUTED) // V0027 support for sharding .addTag(FhirSchemaTags.RESOURCE_TYPE, prefix) .addBigIntColumn(LOGICAL_RESOURCE_ID, false) .addVarcharColumn(LOGICAL_ID, LOGICAL_ID_BYTES, false) @@ -336,8 +345,9 @@ public void addResources(List group, String prefix) { final String tableName = prefix + _RESOURCES; Table tbl = Table.builder(schemaName, tableName) - .setVersion(FhirSchemaVersion.V0024.vid()) + .setVersion(FhirSchemaVersion.V0027.vid()) // V0027: add support for distribution/sharding .setTenantColumnName(MT_ID) + .setDistributionType(DistributionType.DISTRIBUTED) // V0027 support for sharding .addTag(FhirSchemaTags.RESOURCE_TYPE, prefix) .addBigIntColumn( RESOURCE_ID, false) .addBigIntColumn( LOGICAL_RESOURCE_ID, false) @@ -420,8 +430,9 @@ public void addStrValues(List group, String prefix) { // Parameters are tied to the logical resource Table tbl = Table.builder(schemaName, tableName) .addTag(FhirSchemaTags.RESOURCE_TYPE, prefix) - .setVersion(FhirSchemaVersion.V0020.vid()) // V0020: Updated to support Postgres fillfactor changes + .setVersion(FhirSchemaVersion.V0027.vid()) // V0027: add support for distribution/sharding .setTenantColumnName(MT_ID) + .setDistributionType(DistributionType.DISTRIBUTED) // V0027 support for sharding // .addBigIntColumn( ROW_ID, false) // Removed by issue-1683 - composites refactor .addIntColumn( PARAMETER_NAME_ID, false) .addVarcharColumn( STR_VALUE, msb, true) @@ -509,12 +520,12 @@ public Table addResourceTokenRefs(List group, String prefix) { // logical_resources (1) ---- (*) patient_resource_token_refs (*) ---- (0|1) common_token_values Table tbl = Table.builder(schemaName, tableName) - .setVersion(FhirSchemaVersion.V0020.vid()) // V0020: Updated to support Postgres fillfactor changes + .setVersion(FhirSchemaVersion.V0028.vid()) // V0028: ref_version_id removed because refs are now stored in xx_ref_values .setTenantColumnName(MT_ID) + .setDistributionType(DistributionType.DISTRIBUTED) // V0027 support for sharding .addIntColumn( PARAMETER_NAME_ID, false) .addBigIntColumn(COMMON_TOKEN_VALUE_ID, true) .addBigIntColumn( LOGICAL_RESOURCE_ID, false) - .addIntColumn( REF_VERSION_ID, true) // for when the referenced value is a logical resource with a version .addIntColumn(COMPOSITE_ID, true) // V0009 .addIndex(IDX + tableName + "_TPLR", COMMON_TOKEN_VALUE_ID, PARAMETER_NAME_ID, LOGICAL_RESOURCE_ID) // V0008 change .addIndex(IDX + tableName + "_LRPT", LOGICAL_RESOURCE_ID, PARAMETER_NAME_ID, COMMON_TOKEN_VALUE_ID) // V0008 change @@ -552,6 +563,7 @@ public Table addResourceTokenRefs(List group, String prefix) { statements.add(new CreateIndexStatement(schemaName, IDX + tableName + "_LRPT", tableName, mtId, lrpt)); } + boolean needReorg = false; if (priorVersion < FhirSchemaVersion.V0009.vid()) { addCompositeMigrationStepsV0009(statements, tableName); } @@ -563,6 +575,67 @@ public Table addResourceTokenRefs(List group, String prefix) { if (priorVersion < FhirSchemaVersion.V0020.vid()) { statements.add(new PostgresFillfactorSettingDAO(schemaName, tableName, FhirSchemaConstants.PG_FILLFACTOR_VALUE)); } + + if (priorVersion < FhirSchemaVersion.V0028.vid()) { + // For PostgreSQL we can't drop the column because a view depends on it. Just drop the view + // because the definition has been updated and it will be added again later + final String viewName = prefix + "_" + TOKEN_VALUES_V; + statements.add(new DropView(schemaName, viewName)); + statements.add(new DropColumn(schemaName, tableName, REF_VERSION_ID)); + needReorg = true; + } + + if (needReorg) { + // Required for Db2, ignored otherwise + statements.add(new ReorgTable(schemaName, tableName)); + } + + return statements; + }) + .build(model); + + tbl.addTag(FhirSchemaGenerator.SCHEMA_GROUP_TAG, FhirSchemaGenerator.FHIRDATA_GROUP); + + group.add(tbl); + model.addTable(tbl); + + return tbl; + } + + /** + * Schema V0027 adds a dedicated table for supporting reference values instead of using + * token values. + * @param pdm + * @return + */ + public Table addRefValues(List group, String prefix) { + + final String tableName = prefix + "_REF_VALUES"; + + // logical_resources (1) ---- (*) patient_ref_values (*) ---- (0|1) logical_resource_ident + Table tbl = Table.builder(schemaName, tableName) + .setVersion(FhirSchemaVersion.V0028.vid()) // V0028: tweak vacuum and fillfactor + .setTenantColumnName(MT_ID) + .setDistributionType(DistributionType.DISTRIBUTED) // V0027 support for sharding + .addIntColumn( PARAMETER_NAME_ID, false) + .addBigIntColumn( LOGICAL_RESOURCE_ID, false) + .addBigIntColumn( REF_LOGICAL_RESOURCE_ID, true) + .addIntColumn( REF_VERSION_ID, true) // for when the referenced value is a logical resource with a version + .addIntColumn( COMPOSITE_ID, true) + .addIndex(IDX + tableName + "_RFPN", REF_LOGICAL_RESOURCE_ID, PARAMETER_NAME_ID) + .addIndex(IDX + tableName + "_LRPN", LOGICAL_RESOURCE_ID, PARAMETER_NAME_ID) + .addForeignKeyConstraint(FK + tableName + "_PNID", schemaName, PARAMETER_NAMES, PARAMETER_NAME_ID) + .addForeignKeyConstraint(FK + tableName + "_LR", schemaName, LOGICAL_RESOURCES, LOGICAL_RESOURCE_ID) + .setTablespace(fhirTablespace) + .addPrivileges(resourceTablePrivileges) + .enableAccessControl(this.sessionVariable) + .addWiths(withs) + .addMigration(priorVersion -> { + List statements = new ArrayList<>(); + if (priorVersion < FhirSchemaVersion.V0028.vid()) { + statements.add(new PostgresVacuumSettingDAO(schemaName, tableName, 2000, null, 1000)); + statements.add(new PostgresFillfactorSettingDAO(schemaName, tableName, FhirSchemaConstants.PG_FILLFACTOR_VALUE)); + } return statements; }) .build(model); @@ -588,8 +661,9 @@ public Table addProfiles(List group, String prefix) { // logical_resources (1) ---- (*) patient_profiles (*) ---- (0|1) common_canonical_values Table tbl = Table.builder(schemaName, tableName) - .setVersion(FhirSchemaVersion.V0020.vid()) // V0020: Updated to support Postgres fillfactor changes + .setVersion(FhirSchemaVersion.V0027.vid()) // V0027: add support for distribution/sharding .setTenantColumnName(MT_ID) + .setDistributionType(DistributionType.DISTRIBUTED) // V0027 support for sharding .addBigIntColumn( CANONICAL_ID, false) .addBigIntColumn( LOGICAL_RESOURCE_ID, false) .addVarcharColumn( VERSION, VERSION_BYTES, true) @@ -636,8 +710,9 @@ public Table addTags(List group, String prefix) { // logical_resources (1) ---- (*) patient_tags (*) ---- (0|1) common_token_values Table tbl = Table.builder(schemaName, tableName) - .setVersion(FhirSchemaVersion.V0020.vid()) // V0020: Updated to support Postgres fillfactor changes + .setVersion(FhirSchemaVersion.V0027.vid()) // V0027: add support for distribution/sharding .setTenantColumnName(MT_ID) + .setDistributionType(DistributionType.DISTRIBUTED) // V0027 support for sharding .addBigIntColumn(COMMON_TOKEN_VALUE_ID, false) .addBigIntColumn( LOGICAL_RESOURCE_ID, false) .addIndex(IDX + tableName + "_TPLR", COMMON_TOKEN_VALUE_ID, LOGICAL_RESOURCE_ID) @@ -679,8 +754,9 @@ public Table addSecurity(List group, String prefix) { // logical_resources (1) ---- (*) patient_security (*) ---- (0|1) common_token_values Table tbl = Table.builder(schemaName, tableName) - .setVersion(FhirSchemaVersion.V0020.vid()) // V0020: Updated to support Postgres fillfactor changes + .setVersion(FhirSchemaVersion.V0027.vid()) // V0027: add support for distribution/sharding .setTenantColumnName(MT_ID) + .setDistributionType(DistributionType.DISTRIBUTED) // V0027 support for sharding .addBigIntColumn(COMMON_TOKEN_VALUE_ID, false) .addBigIntColumn( LOGICAL_RESOURCE_ID, false) .addIndex(IDX + tableName + "_TPLR", COMMON_TOKEN_VALUE_ID, LOGICAL_RESOURCE_ID) @@ -736,20 +812,20 @@ public void addTokenValuesView(List group, String prefix) { // in the join condition to give the optimizer the best chance at finding a good nested // loop strategy select.append("SELECT ref.").append(MT_ID); - select.append(", ref.parameter_name_id, ctv.code_system_id, ctv.token_value, ref.logical_resource_id, ref.ref_version_id, ref.common_token_value_id, ref." + COMPOSITE_ID); + select.append(", ref.parameter_name_id, ctv.code_system_id, ctv.token_value, ref.logical_resource_id, ref.common_token_value_id, ref." + COMPOSITE_ID); select.append(" FROM ").append(commonTokenValues.getName()).append(" AS ctv, "); select.append(resourceTokenRefs.getName()).append(" AS ref "); select.append(" WHERE ctv.common_token_value_id = ref.common_token_value_id "); select.append(" AND ctv.").append(MT_ID).append(" = ").append("ref.").append(MT_ID); } else { - select.append("SELECT ref.parameter_name_id, ctv.code_system_id, ctv.token_value, ref.logical_resource_id, ref.ref_version_id, ref.common_token_value_id, ref." + COMPOSITE_ID); + select.append("SELECT ref.parameter_name_id, ctv.code_system_id, ctv.token_value, ref.logical_resource_id, ref.common_token_value_id, ref." + COMPOSITE_ID); select.append(" FROM ").append(commonTokenValues.getName()).append(" AS ctv, "); select.append(resourceTokenRefs.getName()).append(" AS ref "); select.append(" WHERE ctv.common_token_value_id = ref.common_token_value_id "); } View view = View.builder(schemaName, viewName) - .setVersion(FhirSchemaVersion.V0009.vid()) + .setVersion(FhirSchemaVersion.V0028.vid()) .setSelectClause(select.toString()) .addPrivileges(resourceTablePrivileges) .addDependency(commonTokenValues) @@ -759,6 +835,54 @@ public void addTokenValuesView(List group, String prefix) { group.add(view); } + /** + * View to encapsulate the join between xx_ref_values and logical_resource_ident + * tables, which makes it easier for the search query builder to compose search + * queries using reference parameters. + * @param group + * @param prefix + */ + public void addRefValuesView(List group, String prefix) { + + final String viewName = prefix + "_" + REF_VALUES_V; + + // Find the two dependencies we need for this view + IDatabaseObject logicalResourceIdent = model.findTable(schemaName, LOGICAL_RESOURCE_IDENT); + IDatabaseObject refValues = model.findTable(schemaName, prefix + "_" + REF_VALUES); + + StringBuilder select = new StringBuilder(); + if (this.multitenant) { + // Make sure we include MT_ID in both the select list and join condition. It's needed + // in the join condition to give the optimizer the best chance at finding a good nested + // loop strategy + select.append("SELECT ref.").append(MT_ID).append(", "); + select.append(" ref.parameter_name_id, lri.resource_type_id, lri.logical_id, ref.logical_resource_id, "); + select.append(" ref.ref_version_id, ref.ref_logical_resource_id, ref.composite_id, "); + select.append(" lri.logical_id AS ref_value "); + select.append(" FROM ").append(logicalResourceIdent.getName()).append(" AS lri, "); + select.append(refValues.getName()).append(" AS ref "); + select.append(" WHERE lri.logical_resource_id = ref.ref_logical_resource_id "); + select.append(" AND lri.").append(MT_ID).append(" = ").append("ref.").append(MT_ID); + } else { + select.append("SELECT ref.parameter_name_id, lri.resource_type_id, lri.logical_id, ref.logical_resource_id, "); + select.append(" ref.ref_version_id, ref.ref_logical_resource_id, ref.composite_id, "); + select.append(" lri.logical_id AS ref_value "); + select.append(" FROM ").append(logicalResourceIdent.getName()).append(" AS lri, "); + select.append(refValues.getName()).append(" AS ref "); + select.append(" WHERE lri.logical_resource_id = ref.ref_logical_resource_id "); + } + + View view = View.builder(schemaName, viewName) + .setVersion(FhirSchemaVersion.V0027.vid()) + .setSelectClause(select.toString()) + .addPrivileges(resourceTablePrivileges) + .addDependency(logicalResourceIdent) + .addDependency(refValues) + .build(); + + group.add(view); + } + /** *
 CREATE TABLE device_date_values  (
@@ -784,9 +908,10 @@ public void addDateValues(List group, String prefix) {
         final String logicalResourcesTable = prefix + _LOGICAL_RESOURCES;
 
         Table tbl = Table.builder(schemaName, tableName)
-                .setVersion(FhirSchemaVersion.V0020.vid()) // V0020: Updated to support Postgres fillfactor changes
-                .addTag(FhirSchemaTags.RESOURCE_TYPE, prefix)
+                .setVersion(FhirSchemaVersion.V0027.vid()) // V0027: add support for distribution/sharding
                 .setTenantColumnName(MT_ID)
+                .setDistributionType(DistributionType.DISTRIBUTED) // V0027 support for sharding
+                .addTag(FhirSchemaTags.RESOURCE_TYPE, prefix)
                 .addIntColumn(     PARAMETER_NAME_ID,      false)
                 .addTimestampColumn(      DATE_START,      true)
                 .addTimestampColumn(        DATE_END,      true)
@@ -850,9 +975,10 @@ public void addNumberValues(List group, String prefix) {
         final String logicalResourcesTable = prefix + _LOGICAL_RESOURCES;
 
         Table tbl = Table.builder(schemaName, tableName)
-                .setVersion(FhirSchemaVersion.V0020.vid()) // V0020: Updated to support Postgres fillfactor changes
-                .addTag(FhirSchemaTags.RESOURCE_TYPE, prefix)
+                .setVersion(FhirSchemaVersion.V0027.vid()) // V0027: add support for distribution/sharding
                 .setTenantColumnName(MT_ID)
+                .setDistributionType(DistributionType.DISTRIBUTED) // V0027 support for sharding
+                .addTag(FhirSchemaTags.RESOURCE_TYPE, prefix)
                 .addIntColumn(     PARAMETER_NAME_ID,      false)
                 .addDoubleColumn(       NUMBER_VALUE,       true)
                 .addBigIntColumn(LOGICAL_RESOURCE_ID,      false)
@@ -923,9 +1049,10 @@ public void addLatLngValues(List group, String prefix) {
         final String logicalResourcesTable = prefix + _LOGICAL_RESOURCES;
 
         Table tbl = Table.builder(schemaName, tableName)
-                .addTag(FhirSchemaTags.RESOURCE_TYPE, prefix)
-                .setVersion(FhirSchemaVersion.V0020.vid()) // V0020: Updated to support Postgres fillfactor changes
+                .setVersion(FhirSchemaVersion.V0027.vid()) // V0027: add support for distribution/sharding
                 .setTenantColumnName(MT_ID)
+                .setDistributionType(DistributionType.DISTRIBUTED) // V0027 support for sharding
+                .addTag(FhirSchemaTags.RESOURCE_TYPE, prefix)
                 .addIntColumn(     PARAMETER_NAME_ID,      false)
                 .addDoubleColumn(     LATITUDE_VALUE,       true)
                 .addDoubleColumn(    LONGITUDE_VALUE,       true)
@@ -995,9 +1122,10 @@ public void addQuantityValues(List group, String prefix) {
         final String logicalResourcesTable = prefix + _LOGICAL_RESOURCES;
 
         Table tbl = Table.builder(schemaName, tableName)
-                .addTag(FhirSchemaTags.RESOURCE_TYPE, prefix)
-                .setVersion(FhirSchemaVersion.V0020.vid()) // V0020: Updated to support Postgres fillfactor changes
+                .setVersion(FhirSchemaVersion.V0027.vid()) // V0027: add support for distribution/sharding
                 .setTenantColumnName(MT_ID)
+                .setDistributionType(DistributionType.DISTRIBUTED) // V0027 support for sharding
+                .addTag(FhirSchemaTags.RESOURCE_TYPE, prefix)
                 .addIntColumn(     PARAMETER_NAME_ID,      false)
                 .addVarcharColumn(              CODE, 255, false)
                 .addDoubleColumn(     QUANTITY_VALUE,      true)
@@ -1053,9 +1181,10 @@ public void addListLogicalResourceItems(List group, String pref
         final int lib = LOGICAL_ID_BYTES;
 
         Table tbl = Table.builder(schemaName, LIST_LOGICAL_RESOURCE_ITEMS)
-                .setVersion(FhirSchemaVersion.V0020.vid()) // V0020: Updated to support Postgres vacuum changes
-                .addTag(FhirSchemaTags.RESOURCE_TYPE, prefix)
+                .setVersion(FhirSchemaVersion.V0027.vid()) // V0027: add support for distribution/sharding
                 .setTenantColumnName(MT_ID)
+                .setDistributionType(DistributionType.DISTRIBUTED) // V0027 support for sharding
+                .addTag(FhirSchemaTags.RESOURCE_TYPE, prefix)
                 .addBigIntColumn( LOGICAL_RESOURCE_ID,      false)
                 .addIntColumn(       RESOURCE_TYPE_ID,      false)
                 .addVarcharColumn(    ITEM_LOGICAL_ID, lib,  true)
@@ -1096,9 +1225,10 @@ public void addPatientCurrentRefs(List group, String prefix) {
         // model with a foreign key to avoid order of insertion issues
 
         Table tbl = Table.builder(schemaName, PATIENT_CURRENT_REFS)
-                .setVersion(FhirSchemaVersion.V0020.vid()) // V0020: Updated to support Postgres fillfactor changes
-                .addTag(FhirSchemaTags.RESOURCE_TYPE, prefix)
+                .setVersion(FhirSchemaVersion.V0027.vid()) // V0027: add support for distribution/sharding
                 .setTenantColumnName(MT_ID)
+                .setDistributionType(DistributionType.DISTRIBUTED) // V0027 support for sharding
+                .addTag(FhirSchemaTags.RESOURCE_TYPE, prefix)
                 .addBigIntColumn(         LOGICAL_RESOURCE_ID,      false)
                 .addVarcharColumn(      CURRENT_PROBLEMS_LIST, lib,  true)
                 .addVarcharColumn(   CURRENT_MEDICATIONS_LIST, lib,  true)
diff --git a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/FhirSchemaConstants.java b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/FhirSchemaConstants.java
index 7705f563f04..16fe9c22ee4 100644
--- a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/FhirSchemaConstants.java
+++ b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/FhirSchemaConstants.java
@@ -39,6 +39,7 @@ public class FhirSchemaConstants {
 
     public static final String FHIR_SEQUENCE = "FHIR_SEQUENCE";
     public static final String FHIR_REF_SEQUENCE = "FHIR_REF_SEQUENCE";
+    public static final String FHIR_CHANGE_SEQUENCE = "FHIR_CHANGE_SEQUENCE";
     public static final String TENANT_SEQUENCE = "TENANT_SEQUENCE";
     public static final long FHIR_REF_SEQUENCE_START = 20000;
     public static final int FHIR_REF_SEQUENCE_CACHE = 1000;
@@ -68,6 +69,7 @@ public class FhirSchemaConstants {
     public static final int FRAGMENT_BYTES = 16;
 
     // R4 Logical Resources
+    public static final String LOGICAL_RESOURCE_IDENT = "LOGICAL_RESOURCE_IDENT";
     public static final String LOGICAL_RESOURCES = "LOGICAL_RESOURCES";
     public static final String REINDEX_TSTAMP = "REINDEX_TSTAMP";
     public static final String REINDEX_TXID = "REINDEX_TXID";
@@ -161,7 +163,8 @@ public class FhirSchemaConstants {
 
     // Table for Normalization of References (Internal and External)
     public static final String LOCAL_REFERENCES = "LOCAL_REFERENCES";
-    public static final String REF_LOGICAL_RESOURCE_ID = "REF_LOGICAL_RESOURCE_ID";
+    public static final String COMMON_REFERENCE_VALUES = "COMMON_REFERENCE_VALUES";
+    public static final String COMMON_REFERENCE_VALUE_ID = "COMMON_REFERENCE_VALUE_ID";
 //    public static final String EXTERNAL_SYSTEMS = "EXTERNAL_SYSTEMS";
 //    public static final String EXTERNAL_SYSTEM_ID = "EXTERNAL_SYSTEM_ID";
 //    public static final String EXTERNAL_SYSTEM_NAME = "EXTERNAL_SYSTEM_NAME";
@@ -181,8 +184,13 @@ public class FhirSchemaConstants {
     public static final String REF_RESOURCE_TYPE_ID = "REF_RESOURCE_TYPE_ID";
     public static final String REF_VERSION_ID = "REF_VERSION_ID";
 
+    // logical_resource_id value used to point to a local reference
+    public static final String REF_LOGICAL_RESOURCE_ID = "REF_LOGICAL_RESOURCE_ID";
+
     // View suffix to overlay the new common_token_values and resource_token_refs tables
     public static final String TOKEN_VALUES_V = "TOKEN_VALUES_V";
+    public static final String REF_VALUES = "REF_VALUES";
+    public static final String REF_VALUES_V = "REF_VALUES_V";
 
     public static final String LOGICAL_RESOURCE_COMPARTMENTS = "LOGICAL_RESOURCE_COMPARTMENTS";
     public static final String COMPARTMENT_LOGICAL_RESOURCE_ID = "COMPARTMENT_LOGICAL_RESOURCE_ID";
@@ -195,4 +203,7 @@ public class FhirSchemaConstants {
     public static final String ERASED_RESOURCES = "ERASED_RESOURCES";
     public static final String ERASED_RESOURCE_ID = "ERASED_RESOURCE_ID";
     public static final String ERASED_RESOURCE_GROUP_ID = "ERASED_RESOURCE_GROUP_ID";
+
+    // Data Distribution/Sharding Constants
+    public static final String SHARD_KEY = "SHARD_KEY";
 }
\ No newline at end of file
diff --git a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/FhirSchemaGenerator.java b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/FhirSchemaGenerator.java
index 66bafd79742..2dd8904f59b 100644
--- a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/FhirSchemaGenerator.java
+++ b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/FhirSchemaGenerator.java
@@ -24,7 +24,7 @@
 import static com.ibm.fhir.schema.control.FhirSchemaConstants.DATE_VALUE_DROPPED_COLUMN;
 import static com.ibm.fhir.schema.control.FhirSchemaConstants.ERASED_RESOURCES;
 import static com.ibm.fhir.schema.control.FhirSchemaConstants.ERASED_RESOURCE_GROUP_ID;
-import static com.ibm.fhir.schema.control.FhirSchemaConstants.ERASED_RESOURCE_ID;
+import static com.ibm.fhir.schema.control.FhirSchemaConstants.FHIR_CHANGE_SEQUENCE;
 import static com.ibm.fhir.schema.control.FhirSchemaConstants.FHIR_REF_SEQUENCE;
 import static com.ibm.fhir.schema.control.FhirSchemaConstants.FHIR_SEQUENCE;
 import static com.ibm.fhir.schema.control.FhirSchemaConstants.FK;
@@ -38,6 +38,7 @@
 import static com.ibm.fhir.schema.control.FhirSchemaConstants.LOGICAL_RESOURCES;
 import static com.ibm.fhir.schema.control.FhirSchemaConstants.LOGICAL_RESOURCE_COMPARTMENTS;
 import static com.ibm.fhir.schema.control.FhirSchemaConstants.LOGICAL_RESOURCE_ID;
+import static com.ibm.fhir.schema.control.FhirSchemaConstants.LOGICAL_RESOURCE_IDENT;
 import static com.ibm.fhir.schema.control.FhirSchemaConstants.LOGICAL_RESOURCE_PROFILES;
 import static com.ibm.fhir.schema.control.FhirSchemaConstants.LOGICAL_RESOURCE_SECURITY;
 import static com.ibm.fhir.schema.control.FhirSchemaConstants.LOGICAL_RESOURCE_TAGS;
@@ -87,12 +88,15 @@
 import java.util.logging.Logger;
 import java.util.stream.Collectors;
 
+import com.ibm.fhir.database.utils.api.DistributionType;
 import com.ibm.fhir.database.utils.api.IDatabaseStatement;
+import com.ibm.fhir.database.utils.api.SchemaType;
 import com.ibm.fhir.database.utils.common.AddColumn;
 import com.ibm.fhir.database.utils.common.CreateIndexStatement;
 import com.ibm.fhir.database.utils.common.DropColumn;
 import com.ibm.fhir.database.utils.common.DropIndex;
 import com.ibm.fhir.database.utils.common.DropTable;
+import com.ibm.fhir.database.utils.common.ReorgTable;
 import com.ibm.fhir.database.utils.model.AlterSequenceStartWith;
 import com.ibm.fhir.database.utils.model.BaseObject;
 import com.ibm.fhir.database.utils.model.CharColumn;
@@ -128,8 +132,8 @@ public class FhirSchemaGenerator {
     // The schema used for administration objects like the tenants table, variable etc
     private final String adminSchemaName;
 
-    // Build the multitenant variant of the schema
-    private final boolean multitenant;
+    // Which variant of the schema do we want to build
+    private final SchemaType schemaType;
 
     // No abstract types
     private static final Set ALL_RESOURCE_TYPES = getAllResourceTypes();
@@ -138,6 +142,10 @@ public class FhirSchemaGenerator {
     private static final String ADD_PARAMETER_NAME = "ADD_PARAMETER_NAME";
     private static final String ADD_RESOURCE_TYPE = "ADD_RESOURCE_TYPE";
     private static final String ADD_ANY_RESOURCE = "ADD_ANY_RESOURCE";
+    private static final String ADD_LOGICAL_RESOURCE_IDENT = "ADD_LOGICAL_RESOURCE_IDENT";
+    
+    // Special procedure for Citus database support
+    private static final String ADD_LOGICAL_RESOURCE = "ADD_LOGICAL_RESOURCE";
     private static final String DELETE_RESOURCE_PARAMETERS = "DELETE_RESOURCE_PARAMETERS";
     private static final String ERASE_RESOURCE = "ERASE_RESOURCE";
 
@@ -175,6 +183,9 @@ public class FhirSchemaGenerator {
     // The common sequence used for allocated resource ids
     private Sequence fhirSequence;
 
+    // The sequence for tracking change history in distributed schemas like Citus
+    private Sequence fhirChangeSequence;
+
     // The sequence used for the reference tables (parameter_names, code_systems etc)
     private Sequence fhirRefSequence;
 
@@ -211,8 +222,8 @@ public class FhirSchemaGenerator {
      * @param adminSchemaName
      * @param schemaName
      */
-    public FhirSchemaGenerator(String adminSchemaName, String schemaName, boolean multitenant) {
-        this(adminSchemaName, schemaName, multitenant, ALL_RESOURCE_TYPES);
+    public FhirSchemaGenerator(String adminSchemaName, String schemaName, SchemaType schemaType) {
+        this(adminSchemaName, schemaName, schemaType, ALL_RESOURCE_TYPES);
     }
 
     /**
@@ -221,10 +232,10 @@ public FhirSchemaGenerator(String adminSchemaName, String schemaName, boolean mu
      * @param adminSchemaName
      * @param schemaName
      */
-    public FhirSchemaGenerator(String adminSchemaName, String schemaName, boolean multitenant, Set resourceTypes) {
+    public FhirSchemaGenerator(String adminSchemaName, String schemaName, SchemaType schemaType, Set resourceTypes) {
         this.adminSchemaName = adminSchemaName;
         this.schemaName = schemaName;
-        this.multitenant = multitenant;
+        this.schemaType = schemaType;
 
         // The FHIR user (e.g. "FHIRSERVER") will need these privileges to be granted to it. Note that
         // we use the group identified by FHIR_USER_GRANT_GROUP here - these privileges can be applied
@@ -389,6 +400,7 @@ public void buildSchema(PhysicalDataModel model) {
         addCodeSystems(model);
         addCommonTokenValues(model);
         addResourceTypes(model);
+        addLogicalResourceIdent(model);
         addLogicalResources(model); // for system-level parameter search
         addReferencesSequence(model);
         addLogicalResourceCompartments(model);
@@ -475,7 +487,7 @@ public void buildDatabaseSpecificArtifactsDb2(PhysicalDataModel model) {
     }
 
     public void buildDatabaseSpecificArtifactsPostgres(PhysicalDataModel model) {
-        // Add stored procedures/functions for postgresql.
+        // Add stored procedures/functions for PostgreSQL
         // Have to use different object names from DB2, because the group processing doesn't support 2 objects with the same name.
         final String ROOT_DIR = "postgres/";
         FunctionDef fd = model.addFunction(this.schemaName,
@@ -503,10 +515,18 @@ public void buildDatabaseSpecificArtifactsPostgres(PhysicalDataModel model) {
         fd.addTag(SCHEMA_GROUP_TAG, FHIRDATA_GROUP);
 
         // We currently only support functions with PostgreSQL, although this is really just a procedure
+        final String deleteResourceParametersScript;
+        final String addAnyResourceScript;
+        final String eraseResourceScript;
+        final String schemaTypeSuffix = getSchemaTypeSuffix();
+        addAnyResourceScript = ROOT_DIR + ADD_ANY_RESOURCE.toLowerCase() + schemaTypeSuffix;
+        deleteResourceParametersScript = ROOT_DIR + DELETE_RESOURCE_PARAMETERS.toLowerCase() + ".sql";
+        eraseResourceScript = ROOT_DIR + ERASE_RESOURCE.toLowerCase() + ".sql";
+
         FunctionDef deleteResourceParameters = model.addFunction(this.schemaName,
             DELETE_RESOURCE_PARAMETERS,
             FhirSchemaVersion.V0020.vid(),
-            () -> SchemaGeneratorUtil.readTemplate(adminSchemaName, schemaName, ROOT_DIR + DELETE_RESOURCE_PARAMETERS.toLowerCase() + ".sql", null),
+            () -> SchemaGeneratorUtil.readTemplate(adminSchemaName, schemaName, deleteResourceParametersScript, null),
             Arrays.asList(fhirSequence, resourceTypesTable, allTablesComplete),
             procedurePrivileges);
         deleteResourceParameters.addTag(SCHEMA_GROUP_TAG, FHIRDATA_GROUP);
@@ -514,19 +534,119 @@ public void buildDatabaseSpecificArtifactsPostgres(PhysicalDataModel model) {
         fd = model.addFunction(this.schemaName,
                 ADD_ANY_RESOURCE,
                 FhirSchemaVersion.V0001.vid(),
-                () -> SchemaGeneratorUtil.readTemplate(adminSchemaName, schemaName, ROOT_DIR + ADD_ANY_RESOURCE.toLowerCase()
-                        + ".sql", null),
+                () -> SchemaGeneratorUtil.readTemplate(adminSchemaName, schemaName, addAnyResourceScript, null),
                 Arrays.asList(fhirSequence, resourceTypesTable, deleteResourceParameters, allTablesComplete), procedurePrivileges);
         fd.addTag(SCHEMA_GROUP_TAG, FHIRDATA_GROUP);
 
         fd = model.addFunction(this.schemaName,
             ERASE_RESOURCE,
             FhirSchemaVersion.V0013.vid(),
-            () -> SchemaGeneratorUtil.readTemplate(adminSchemaName, schemaName, ROOT_DIR + ERASE_RESOURCE.toLowerCase() + ".sql", null),
+            () -> SchemaGeneratorUtil.readTemplate(adminSchemaName, schemaName, eraseResourceScript, null),
+            Arrays.asList(fhirSequence, resourceTypesTable, deleteResourceParameters, allTablesComplete), procedurePrivileges);
+        fd.addTag(SCHEMA_GROUP_TAG, FHIRDATA_GROUP);
+    }
+
+    /**
+     * Add stored procedures/functions for Citus (largely based on PostgreSQL, but some functions are distributed
+     * based on a parameter to make them more efficient.
+     * @implNote https://docs.microsoft.com/en-us/azure/postgresql/hyperscale/reference-functions#create_distributed_function
+     * @param model
+     */
+    public void buildDatabaseSpecificArtifactsCitus(PhysicalDataModel model) {
+        // Have to use different object names from DB2, because the group processing doesn't support 2 objects with the same name.
+        final String ROOT_DIR = "postgres/";
+        final String CITUS_ROOT_DIR = "citus/";
+        FunctionDef fd = model.addFunction(this.schemaName,
+                ADD_CODE_SYSTEM,
+                FhirSchemaVersion.V0001.vid(),
+                () -> SchemaGeneratorUtil.readTemplate(adminSchemaName, schemaName, ROOT_DIR + ADD_CODE_SYSTEM.toLowerCase() + ".sql", null),
+                Arrays.asList(fhirSequence, codeSystemsTable, allTablesComplete),
+                procedurePrivileges);
+        fd.addTag(SCHEMA_GROUP_TAG, FHIRDATA_GROUP);
+
+        fd = model.addFunction(this.schemaName,
+                ADD_PARAMETER_NAME,
+                FhirSchemaVersion.V0001.vid(),
+                () -> SchemaGeneratorUtil.readTemplate(adminSchemaName, schemaName, ROOT_DIR + ADD_PARAMETER_NAME.toLowerCase()
+                        + ".sql", null),
+                Arrays.asList(fhirSequence, parameterNamesTable, allTablesComplete), procedurePrivileges);
+        fd.addTag(SCHEMA_GROUP_TAG, FHIRDATA_GROUP);
+
+        fd = model.addFunction(this.schemaName,
+                ADD_RESOURCE_TYPE,
+                FhirSchemaVersion.V0001.vid(),
+                () -> SchemaGeneratorUtil.readTemplate(adminSchemaName, schemaName, ROOT_DIR + ADD_RESOURCE_TYPE.toLowerCase()
+                        + ".sql", null),
+                Arrays.asList(fhirSequence, resourceTypesTable, allTablesComplete), procedurePrivileges);
+        fd.addTag(SCHEMA_GROUP_TAG, FHIRDATA_GROUP);
+
+        // We currently only support functions with PostgreSQL, although this is really just a procedure
+        final String deleteResourceParametersScript;
+        final String addAnyResourceScript;
+        final String eraseResourceScript;
+        final String schemaTypeSuffix = getSchemaTypeSuffix();
+        addAnyResourceScript = CITUS_ROOT_DIR + ADD_ANY_RESOURCE.toLowerCase() + schemaTypeSuffix;
+        deleteResourceParametersScript = ROOT_DIR + DELETE_RESOURCE_PARAMETERS.toLowerCase() + ".sql";
+        eraseResourceScript = ROOT_DIR + ERASE_RESOURCE.toLowerCase() + ".sql";
+        final String addLogicalResourceIdentScript = CITUS_ROOT_DIR + ADD_LOGICAL_RESOURCE_IDENT.toLowerCase() + ".sql";
+        
+        FunctionDef deleteResourceParameters = model.addFunction(this.schemaName,
+            DELETE_RESOURCE_PARAMETERS,
+            FhirSchemaVersion.V0020.vid(),
+            () -> SchemaGeneratorUtil.readTemplate(adminSchemaName, schemaName, deleteResourceParametersScript, null),
+            Arrays.asList(fhirSequence, resourceTypesTable, allTablesComplete),
+            procedurePrivileges, 2); // distributed by p_logical_resource_id
+        deleteResourceParameters.addTag(SCHEMA_GROUP_TAG, FHIRDATA_GROUP);
+
+        // For Citus, we use an additional function to create/lock the logical_resource_ident record
+        // Function is distributed by p_logical_id (parameter 2)
+        fd = model.addFunction(this.schemaName,
+            ADD_LOGICAL_RESOURCE_IDENT,
+            FhirSchemaVersion.V0001.vid(),
+            () -> SchemaGeneratorUtil.readTemplate(adminSchemaName, schemaName, addLogicalResourceIdentScript, null),
+            Arrays.asList(fhirSequence, resourceTypesTable, deleteResourceParameters, allTablesComplete), procedurePrivileges, 2);
+        fd.addTag(SCHEMA_GROUP_TAG, FHIRDATA_GROUP);
+
+        // Function is distributed by p_logical_resource_id (parameter 1)
+        fd = model.addFunction(this.schemaName,
+                ADD_ANY_RESOURCE,
+                FhirSchemaVersion.V0001.vid(),
+                () -> SchemaGeneratorUtil.readTemplate(adminSchemaName, schemaName, addAnyResourceScript, null),
+                Arrays.asList(fhirSequence, resourceTypesTable, deleteResourceParameters, allTablesComplete), procedurePrivileges, 1);
+        fd.addTag(SCHEMA_GROUP_TAG, FHIRDATA_GROUP);
+
+        fd = model.addFunction(this.schemaName,
+            ERASE_RESOURCE,
+            FhirSchemaVersion.V0013.vid(),
+            () -> SchemaGeneratorUtil.readTemplate(adminSchemaName, schemaName, eraseResourceScript, null),
             Arrays.asList(fhirSequence, resourceTypesTable, deleteResourceParameters, allTablesComplete), procedurePrivileges);
         fd.addTag(SCHEMA_GROUP_TAG, FHIRDATA_GROUP);
     }
 
+    /**
+     * Get the suffix to select the appropriate procedure/function script 
+     * for the schema type
+     * @return
+     */
+    private String getSchemaTypeSuffix() {
+        switch (this.schemaType) {
+        case DISTRIBUTED:
+            return ".sql";
+        case SHARDED:
+            return "_sharded.sql";
+        default:
+            return ".sql";
+        }
+    }
+
+    /**
+     * Are we building the Db2-specific multitenant schema variant
+     * @return
+     */
+    private boolean isMultitenant() {
+        return this.schemaType == SchemaType.MULTITENANT;
+    }
+
     /**
      * Add the system-wide logical_resources table. Note that LOGICAL_ID is
      * denormalized, stored in both LOGICAL_RESOURCES and _LOGICAL_RESOURCES.
@@ -536,14 +656,15 @@ public void buildDatabaseSpecificArtifactsPostgres(PhysicalDataModel model) {
      */
     public void addLogicalResources(PhysicalDataModel pdm) {
         final String tableName = LOGICAL_RESOURCES;
-        final String mtId = this.multitenant ? MT_ID : null;
+        final String mtId = isMultitenant() ? MT_ID : null;
 
         final String IDX_LOGICAL_RESOURCES_RITS = "IDX_" + LOGICAL_RESOURCES + "_RITS";
         final String IDX_LOGICAL_RESOURCES_LUPD = "IDX_" + LOGICAL_RESOURCES + "_LUPD";
 
         Table tbl = Table.builder(schemaName, tableName)
-                .setVersion(FhirSchemaVersion.V0020.vid()) // V0020: Updated to support Postgres fillfactor changes
+                .setVersion(FhirSchemaVersion.V0027.vid()) // V0027: add support for distribution/sharding
                 .setTenantColumnName(MT_ID)
+                .setDistributionType(DistributionType.DISTRIBUTED) // V0027 support for sharding
                 .addBigIntColumn(LOGICAL_RESOURCE_ID, false)
                 .addIntColumn(RESOURCE_TYPE_ID, false)
                 .addVarcharColumn(LOGICAL_ID, LOGICAL_ID_BYTES, false)
@@ -628,6 +749,70 @@ public void addLogicalResources(PhysicalDataModel pdm) {
         pdm.addObject(tbl);
     }
 
+    /**
+     * Adds a table to support logical identity management of resources
+     * when using a distributed RDBMS such as Citus. It represents the
+     * mapping:
+     * 
+     * (RESOURCE_TYPE_ID, LOGICAL_ID) -> (LOGICAL_RESOURCE_ID)
+     * 
+     * LOGICAL_RESOURCE_ID values are assigned from the sequence FHIR_SEQUENCE.
+     * 
+     * When using Citus (or similar), this table is distributed by LOGICAL_ID,
+     * which means we can use a primary key of {RESOURCE_TYPE_ID, LOGICAL_ID}. 
+     * This is required to ensure that we can lock the logical resource to 
+     * avoid any concurrency issues.
+     * 
+     * LOGICAL_RESOURCE_IDENT records are also generated when the tuple 
+     * (RESOURCE_TYPE_ID. LOGICAL_ID) is used as a local resource reference 
+     * value. For example:
+     *   "reference": "Patient/aPatientId"
+     * will create a new LOGICAL_RESOURCE_IDENT record if the Patient resource
+     * "aPatientId" has not yet been created. The LOGICAL_RESOURCES record is 
+     * not created until the actual resource is created.
+     * 
+     * The index IDX_LOGICAL_RESOURCE_IDENT_LRID is specified as non-unique
+     * because in Citus, this table is distributed by logical_id, which isn't
+     * part of the index. This is intentional. We have to rely on the logic
+     * in the add_any_resource procedures to guarantee that logical_resource_id
+     * values are assigned correctly. The logical_resource_id column is the
+     * primary key for the logical_resources table, so we are guaranteed
+     * uniqueness there.
+     * @param pdm
+     */
+    private void addLogicalResourceIdent(PhysicalDataModel pdm) {
+        final String tableName = LOGICAL_RESOURCE_IDENT;
+        final String mtId = isMultitenant() ? MT_ID : null;
+
+        Table tbl = Table.builder(schemaName, tableName)
+                .setVersion(FhirSchemaVersion.V0027.vid()) // add support for distribution/sharding
+                .setTenantColumnName(mtId)
+                .setDistributionType(DistributionType.DISTRIBUTED)
+                .setDistributionColumnName(LOGICAL_ID)             // override distribution column for this table
+                .addIntColumn(RESOURCE_TYPE_ID, false)
+                .addVarcharColumn(LOGICAL_ID, MAX_SEARCH_STRING_BYTES, false) // used to also store absolute reference values
+                .addBigIntColumn(LOGICAL_RESOURCE_ID, false)
+                .addPrimaryKey(tableName + "_PK", LOGICAL_ID, RESOURCE_TYPE_ID) // do not change this order
+                .addIndex("IDX_" + LOGICAL_RESOURCE_IDENT + "_LRID", LOGICAL_RESOURCE_ID) // non-unique to allow easy distribution
+                .setTablespace(fhirTablespace)
+                .addPrivileges(resourceTablePrivileges)
+                .addForeignKeyConstraint(FK + tableName + "_RTID", schemaName, RESOURCE_TYPES, RESOURCE_TYPE_ID)
+                .enableAccessControl(this.sessionVariable)
+                .addWiths(addWiths()) // add table tuning
+                .addMigration(priorVersion -> {
+                    List statements = new ArrayList<>();
+                    // NOP for now
+                    return statements;
+                })
+                .build(pdm);
+
+        // TODO should not need to add as a table and an object. Get the table to add itself?
+        tbl.addTag(SCHEMA_GROUP_TAG, FHIRDATA_GROUP);
+        this.procedureDependencies.add(tbl);
+        pdm.addTable(tbl);
+        pdm.addObject(tbl);
+    }
+
     /**
      * Create the COMMON_CANONICAL_VALUES table. Used from schema V0014 to normalize
      * meta.profile search parameters (similar to common_token_values). Only the url
@@ -643,8 +828,9 @@ public void addCommonCanonicalValues(PhysicalDataModel pdm) {
         final String tableName = COMMON_CANONICAL_VALUES;
         final String unqCanonicalUrl = "UNQ_" + tableName + "_URL";
         Table tbl = Table.builder(schemaName, tableName)
-                .setVersion(FhirSchemaVersion.V0014.vid())
+                .setVersion(FhirSchemaVersion.V0027.vid()) // V0027: add support for distribution/sharding
                 .setTenantColumnName(MT_ID)
+                .setDistributionType(DistributionType.REFERENCE) // V0027 support for sharding
                 .addBigIntColumn(CANONICAL_ID, false)
                 .addVarcharColumn(URL, CANONICAL_URL_BYTES, false)
                 .addPrimaryKey(tableName + "_PK", CANONICAL_ID)
@@ -681,8 +867,9 @@ public Table addLogicalResourceProfiles(PhysicalDataModel pdm) {
 
         // logical_resources (1) ---- (*) logical_resource_profiles (*) ---- (1) common_canonical_values
         Table tbl = Table.builder(schemaName, tableName)
-                .setVersion(FhirSchemaVersion.V0020.vid()) // V0020: Updated to support Postgres fillfactor changes (Original Table at V0014)
+                .setVersion(FhirSchemaVersion.V0027.vid()) // V0027: add support for distribution/sharding
                 .setTenantColumnName(MT_ID)
+                .setDistributionType(DistributionType.DISTRIBUTED) // V0027 support for sharding
                 .addBigIntColumn(         CANONICAL_ID,     false) // FK referencing COMMON_CANONICAL_VALUES
                 .addBigIntColumn(  LOGICAL_RESOURCE_ID,     false) // FK referencing LOGICAL_RESOURCES
                 .addVarcharColumn(             VERSION,  VERSION_BYTES, true)
@@ -729,8 +916,9 @@ public Table addLogicalResourceTags(PhysicalDataModel pdm) {
 
         // logical_resources (1) ---- (*) logical_resource_tags (*) ---- (1) common_token_values
         Table tbl = Table.builder(schemaName, tableName)
-                .setVersion(FhirSchemaVersion.V0020.vid()) // V0019: Updated to support Postgres vacuum changes, original table created at version V0014
+                .setVersion(FhirSchemaVersion.V0027.vid()) // V0027: add support for distribution/sharding
                 .setTenantColumnName(MT_ID)
+                .setDistributionType(DistributionType.DISTRIBUTED) // V0027 support for sharding
                 .addBigIntColumn(COMMON_TOKEN_VALUE_ID,    false) // FK referencing COMMON_CANONICAL_VALUES
                 .addBigIntColumn(  LOGICAL_RESOURCE_ID,    false) // FK referencing LOGICAL_RESOURCES
                 .addIndex(IDX + tableName + "_CCVLR", COMMON_TOKEN_VALUE_ID, LOGICAL_RESOURCE_ID)
@@ -772,8 +960,9 @@ public Table addLogicalResourceSecurity(PhysicalDataModel pdm) {
 
         // logical_resources (1) ---- (*) logical_resource_security (*) ---- (1) common_token_values
         Table tbl = Table.builder(schemaName, tableName)
-                .setVersion(FhirSchemaVersion.V0020.vid()) // V0019: Updated to support Postgres vacuum changes, original table created at version V0016
+                .setVersion(FhirSchemaVersion.V0027.vid()) // V0027: add support for distribution/sharding
                 .setTenantColumnName(MT_ID)
+                .setDistributionType(DistributionType.DISTRIBUTED) // V0027 support for sharding
                 .addBigIntColumn(COMMON_TOKEN_VALUE_ID,    false) // FK referencing COMMON_CANONICAL_VALUES
                 .addBigIntColumn(  LOGICAL_RESOURCE_ID,    false) // FK referencing LOGICAL_RESOURCES
                 .addIndex(IDX + tableName + "_CCVLR", COMMON_TOKEN_VALUE_ID, LOGICAL_RESOURCE_ID)
@@ -821,9 +1010,11 @@ public void addResourceChangeLog(PhysicalDataModel pdm) {
             With.with("autovacuum_vacuum_cost_limit", "2000")    // V0019
             );
 
+        // Each shard gets its own history
         Table tbl = Table.builder(schemaName, tableName)
                 .setTenantColumnName(MT_ID)
                 .setVersion(FhirSchemaVersion.V0019.vid()) // V0019: Updated to support Postgres vacuum changes
+                .setDistributionType(DistributionType.NONE) // don't distribute the history log
                 .addBigIntColumn(RESOURCE_ID, false)
                 .addIntColumn(RESOURCE_TYPE_ID, false)
                 .addBigIntColumn(LOGICAL_RESOURCE_ID, false)
@@ -869,8 +1060,10 @@ public Table addLogicalResourceCompartments(PhysicalDataModel pdm) {
         // because it makes it very easy to find the most recent changes to resources associated with
         // a given patient (for example).
         Table tbl = Table.builder(schemaName, tableName)
-                .setVersion(FhirSchemaVersion.V0020.vid()) // V0020: Updated to support Postgres fillfactor changes
+                .setCreate(false) // V0027 no longer used
+                .setVersion(FhirSchemaVersion.V0027.vid()) // V0027: add support for distribution/sharding
                 .setTenantColumnName(MT_ID)
+                .setDistributionType(DistributionType.DISTRIBUTED) // V0027 support for sharding
                 .addIntColumn(     COMPARTMENT_NAME_ID,      false)
                 .addBigIntColumn(LOGICAL_RESOURCE_ID,      false)
                 .addTimestampColumn(LAST_UPDATED, false)
@@ -891,6 +1084,11 @@ public Table addLogicalResourceCompartments(PhysicalDataModel pdm) {
                     if (priorVersion < FhirSchemaVersion.V0020.vid()) {
                         statements.add(new PostgresFillfactorSettingDAO(schemaName, tableName, FhirSchemaConstants.PG_FILLFACTOR_VALUE));
                     }
+                    if (priorVersion < FhirSchemaVersion.V0027.vid()) {
+                        // This table is never used and the FK_LOGICAL_RESOURCE_COMPARTMENTS_COMP FK
+                        // causes issues with Citus distribution, so it's time for it to go
+                        statements.add(new DropTable(schemaName, tableName));
+                    }
                     return statements;
                 })
                 .build(pdm);
@@ -915,7 +1113,7 @@ public Table addResourceStrValues(PhysicalDataModel pdm) {
         final int msb = MAX_SEARCH_STRING_BYTES;
 
         Table tbl = Table.builder(schemaName, STR_VALUES)
-                .setVersion(FhirSchemaVersion.V0020.vid()) // V0019: Updated to support Postgres vacuum changes
+                .setVersion(FhirSchemaVersion.V0027.vid()) // V0027: add support for distribution/sharding
                 .setTenantColumnName(MT_ID)
                 .addIntColumn(     PARAMETER_NAME_ID,      false)
                 .addVarcharColumn(         STR_VALUE, msb,  true)
@@ -931,6 +1129,7 @@ public Table addResourceStrValues(PhysicalDataModel pdm) {
                 .addPrivileges(resourceTablePrivileges)
                 .enableAccessControl(this.sessionVariable)
                 .addWiths(addWiths()) // New Column for V0017
+                .setDistributionType(DistributionType.DISTRIBUTED) // V0027 support for sharding
                 .addMigration(priorVersion -> {
                     List statements = new ArrayList<>();
                     if (priorVersion < FhirSchemaVersion.V0019.vid()) {
@@ -961,7 +1160,7 @@ public Table addResourceDateValues(PhysicalDataModel model) {
         final String logicalResourcesTable = LOGICAL_RESOURCES;
 
         Table tbl = Table.builder(schemaName, tableName)
-                .setVersion(FhirSchemaVersion.V0020.vid()) // V0019: Updated to support Postgres vacuum changes
+                .setVersion(FhirSchemaVersion.V0027.vid()) // V0027: add support for distribution/sharding
                 .setTenantColumnName(MT_ID)
                 .addIntColumn(     PARAMETER_NAME_ID,      false)
                 .addTimestampColumn(      DATE_START,6,    true)
@@ -976,6 +1175,7 @@ public Table addResourceDateValues(PhysicalDataModel model) {
                 .addPrivileges(resourceTablePrivileges)
                 .enableAccessControl(this.sessionVariable)
                 .addWiths(addWiths()) // New Column for V0017
+                .setDistributionType(DistributionType.DISTRIBUTED) // V0027 support for sharding
                 .addMigration(priorVersion -> {
                     List statements = new ArrayList<>();
                     if (priorVersion == 1) {
@@ -1017,7 +1217,7 @@ resource_type   VARCHAR(64) NOT NULL
      */
     protected void addResourceTypes(PhysicalDataModel model) {
         resourceTypesTable = Table.builder(schemaName, RESOURCE_TYPES)
-                .setVersion(FhirSchemaVersion.V0025.vid())
+                .setVersion(FhirSchemaVersion.V0027.vid()) // V0027: add support for distribution/sharding
                 .setTenantColumnName(MT_ID)
                 .addIntColumn(    RESOURCE_TYPE_ID,      false)
                 .addVarcharColumn(   RESOURCE_TYPE,  64, false)
@@ -1027,6 +1227,7 @@ protected void addResourceTypes(PhysicalDataModel model) {
                 .setTablespace(fhirTablespace)
                 .addPrivileges(resourceTablePrivileges)
                 .enableAccessControl(this.sessionVariable)
+                .setDistributionType(DistributionType.REFERENCE) // V0027 supporting for sharding
                 .addMigration(priorVersion -> {
                     List statements = new ArrayList<>();
 
@@ -1058,7 +1259,7 @@ protected void addResourceTables(PhysicalDataModel model, IDatabaseObject... dep
 
         // The sessionVariable is used to enable access control on every table, so we
         // provide it as a dependency
-        FhirResourceTableGroup frg = new FhirResourceTableGroup(model, this.schemaName, this.multitenant, sessionVariable,
+        FhirResourceTableGroup frg = new FhirResourceTableGroup(model, this.schemaName, isMultitenant(), sessionVariable,
                 this.procedureDependencies, this.fhirTablespace, this.resourceTablePrivileges, addWiths());
         for (String resourceType: this.resourceTypes) {
 
@@ -1101,6 +1302,7 @@ protected void addParameterNames(PhysicalDataModel model) {
         String[] prfIncludeCols = {PARAMETER_NAME_ID};
 
         parameterNamesTable = Table.builder(schemaName, PARAMETER_NAMES)
+                .setVersion(FhirSchemaVersion.V0027.vid()) // V0027: add support for distribution/sharding
                 .setTenantColumnName(MT_ID)
                 .addIntColumn(     PARAMETER_NAME_ID,              false)
                 .addVarcharColumn(    PARAMETER_NAME,         255, false)
@@ -1109,6 +1311,7 @@ protected void addParameterNames(PhysicalDataModel model) {
                 .setTablespace(fhirTablespace)
                 .addPrivileges(resourceTablePrivileges)
                 .enableAccessControl(this.sessionVariable)
+                .setDistributionType(DistributionType.REFERENCE) // V0027 supporting for sharding
                 .addMigration(priorVersion -> {
                     List statements = new ArrayList<>();
                     // Intentionally a NOP
@@ -1137,7 +1340,7 @@ code_system_name       VARCHAR(255 OCTETS) NOT NULL
      */
     protected void addCodeSystems(PhysicalDataModel model) {
         codeSystemsTable = Table.builder(schemaName, CODE_SYSTEMS)
-                .setVersion(FhirSchemaVersion.V0019.vid()) // V0019: Updated to support Postgres vacuum changes
+                .setVersion(FhirSchemaVersion.V0027.vid()) // V0027: add support for distribution/sharding
                 .setTenantColumnName(MT_ID)
                 .addIntColumn(      CODE_SYSTEM_ID,         false)
                 .addVarcharColumn(CODE_SYSTEM_NAME,    255, false)
@@ -1146,6 +1349,7 @@ protected void addCodeSystems(PhysicalDataModel model) {
                 .setTablespace(fhirTablespace)
                 .addPrivileges(resourceTablePrivileges)
                 .enableAccessControl(this.sessionVariable)
+                .setDistributionType(DistributionType.REFERENCE) // V0027 supporting for sharding
                 .addMigration(priorVersion -> {
                     List statements = new ArrayList<>();
                     if (priorVersion < FhirSchemaVersion.V0019.vid()) {
@@ -1183,13 +1387,16 @@ protected void addCodeSystems(PhysicalDataModel model) {
      * the token_value represents its logical_id. This approach simplifies query writing when
      * following references.
      *
+     * When using a distributed database (Citus), this table is distributed as a REFERENCE
+     * table, meaning that all records will exist on all nodes.
+     * 
      * @param pdm
      * @return the table definition
      */
     public void addCommonTokenValues(PhysicalDataModel pdm) {
         final String tableName = COMMON_TOKEN_VALUES;
         commonTokenValuesTable = Table.builder(schemaName, tableName)
-                .setVersion(FhirSchemaVersion.V0006.vid())
+                .setVersion(FhirSchemaVersion.V0027.vid()) // V0027: add support for distribution/sharding
                 .setTenantColumnName(MT_ID)
                 .addBigIntColumn(     COMMON_TOKEN_VALUE_ID,                          false)
                 .setIdentityColumn(   COMMON_TOKEN_VALUE_ID, Generated.ALWAYS)
@@ -1201,6 +1408,7 @@ public void addCommonTokenValues(PhysicalDataModel pdm) {
                 .setTablespace(fhirTablespace)
                 .addPrivileges(resourceTablePrivileges)
                 .enableAccessControl(this.sessionVariable)
+                .setDistributionType(DistributionType.REFERENCE) // V0027 shard using token_value
                 .addMigration(priorVersion -> {
                     List statements = new ArrayList<>();
                     // Intentionally a NOP
@@ -1231,12 +1439,12 @@ public Table addResourceTokenRefs(PhysicalDataModel pdm) {
 
         // logical_resources (0|1) ---- (*) resource_token_refs
         Table tbl = Table.builder(schemaName, tableName)
-                .setVersion(FhirSchemaVersion.V0020.vid()) // V0020: Updated to support Postgres fillfactor changes
+                .setVersion(FhirSchemaVersion.V0028.vid()) // V0028: drop column ref_version_id
                 .setTenantColumnName(MT_ID)
+                .setDistributionType(DistributionType.DISTRIBUTED) // V0027 support for sharding
                 .addIntColumn(       PARAMETER_NAME_ID,    false)
                 .addBigIntColumn(COMMON_TOKEN_VALUE_ID,     true) // support for null token value entries
                 .addBigIntColumn(  LOGICAL_RESOURCE_ID,    false)
-                .addIntColumn(          REF_VERSION_ID,     true) // for when the referenced value is a logical resource with a version
                 .addIndex(IDX + tableName + "_TPLR", COMMON_TOKEN_VALUE_ID, PARAMETER_NAME_ID, LOGICAL_RESOURCE_ID) // V0009 change
                 .addIndex(IDX + tableName + "_LRPT", LOGICAL_RESOURCE_ID, PARAMETER_NAME_ID, COMMON_TOKEN_VALUE_ID) // V0009 change
                 .addForeignKeyConstraint(FK + tableName + "_CTV", schemaName, COMMON_TOKEN_VALUES, COMMON_TOKEN_VALUE_ID)
@@ -1249,6 +1457,7 @@ public Table addResourceTokenRefs(PhysicalDataModel pdm) {
                 .addMigration(priorVersion -> {
                     // Replace the indexes initially defined in the V0006 version with better ones
                     List statements = new ArrayList<>();
+                    boolean needReorg = false;
                     if (priorVersion == FhirSchemaVersion.V0006.vid()) {
                         // Migrate the index definitions as part of the V0008 version of the schema
                         // This table was originally introduced as part of the V0006 schema, which
@@ -1256,7 +1465,7 @@ public Table addResourceTokenRefs(PhysicalDataModel pdm) {
                         statements.add(new DropIndex(schemaName, IDX + tableName + "_TVLR"));
                         statements.add(new DropIndex(schemaName, IDX + tableName + "_LRTV"));
 
-                        final String mtId = multitenant ? MT_ID : null;
+                        final String mtId = isMultitenant() ? MT_ID : null;
                         // Replace the original TVLR index on (common_token_value_id, parameter_name_id, logical_resource_id)
                         List tplr = Arrays.asList(
                             new OrderedColumnDef(COMMON_TOKEN_VALUE_ID, OrderedColumnDef.Direction.ASC, null),
@@ -1279,6 +1488,15 @@ public Table addResourceTokenRefs(PhysicalDataModel pdm) {
                     if (priorVersion < FhirSchemaVersion.V0020.vid()) {
                         statements.add(new PostgresFillfactorSettingDAO(schemaName, tableName, FhirSchemaConstants.PG_FILLFACTOR_VALUE));
                     }
+                    if (priorVersion < FhirSchemaVersion.V0028.vid()) {
+                        statements.add(new DropColumn(schemaName,  tableName, REF_VERSION_ID));
+                        needReorg = true;
+                    }
+
+                    if (needReorg) {
+                        // Required for Db2, ignored otherwise
+                        statements.add(new ReorgTable(schemaName, tableName));
+                    }
                     return statements;
                 })
                 .build(pdm);
@@ -1302,7 +1520,7 @@ public Table addResourceTokenRefs(PhysicalDataModel pdm) {
      */
     public void addErasedResources(PhysicalDataModel pdm) {
         final String tableName = ERASED_RESOURCES;
-        final String mtId = this.multitenant ? MT_ID : null;
+        final String mtId = isMultitenant() ? MT_ID : null;
 
         // Each erase operation is allocated an ERASED_RESOURCE_GROUP_ID
         // value which can be used to retrieve the resource and/or
@@ -1312,15 +1530,12 @@ public void addErasedResources(PhysicalDataModel pdm) {
         // or resource_id values here, because those records may have
         // already been deleted by $erase.
         Table tbl = Table.builder(schemaName, tableName)
-                .setVersion(FhirSchemaVersion.V0023.vid())
+                .setVersion(FhirSchemaVersion.V0027.vid())
                 .setTenantColumnName(mtId)
-                .addBigIntColumn(ERASED_RESOURCE_ID, false)
-                .setIdentityColumn(ERASED_RESOURCE_ID, Generated.ALWAYS)
                 .addBigIntColumn(ERASED_RESOURCE_GROUP_ID, false)
                 .addIntColumn(RESOURCE_TYPE_ID, false)
                 .addVarcharColumn(LOGICAL_ID, LOGICAL_ID_BYTES, false)
                 .addIntColumn(VERSION_ID, true)
-                .addPrimaryKey(tableName + "_PK", ERASED_RESOURCE_ID)
                 .addIndex(IDX + tableName + "_GID", ERASED_RESOURCE_GROUP_ID)
                 .setTablespace(fhirTablespace)
                 .addPrivileges(resourceTablePrivileges)
@@ -1330,6 +1545,8 @@ public void addErasedResources(PhysicalDataModel pdm) {
                 .addMigration(priorVersion -> {
                     List statements = new ArrayList<>();
                     // Nothing yet
+                    
+                    // TODO migrate to simplified design (no PK, FK)
                     return statements;
                 })
                 .build(pdm);
@@ -1361,6 +1578,19 @@ protected void addFhirSequence(PhysicalDataModel pdm) {
         pdm.addObject(fhirSequence);
     }
 
+    /**
+     * Adds a new sequence required for distributed databases like Citus
+     * @param pdm
+     */
+    protected void addFhirChangeSequence(PhysicalDataModel pdm) {
+        this.fhirChangeSequence = new Sequence(schemaName, FHIR_CHANGE_SEQUENCE, FhirSchemaVersion.V0027.vid(), 1, 1000);
+        this.fhirChangeSequence.addTag(SCHEMA_GROUP_TAG, FHIRDATA_GROUP);
+        procedureDependencies.add(fhirChangeSequence);
+        sequencePrivileges.forEach(p -> p.addToObject(fhirChangeSequence));
+
+        pdm.addObject(fhirChangeSequence);
+    }
+
     protected void addFhirRefSequence(PhysicalDataModel pdm) {
         this.fhirRefSequence = new Sequence(schemaName, FHIR_REF_SEQUENCE, FhirSchemaVersion.V0001.vid(), FhirSchemaConstants.FHIR_REF_SEQUENCE_START, FhirSchemaConstants.FHIR_REF_SEQUENCE_CACHE);
         this.fhirRefSequence.addTag(SCHEMA_GROUP_TAG, FHIRDATA_GROUP);
diff --git a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/FhirSchemaVersion.java b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/FhirSchemaVersion.java
index 2d96417ad39..6d3738471f0 100644
--- a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/FhirSchemaVersion.java
+++ b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/FhirSchemaVersion.java
@@ -45,7 +45,9 @@ public enum FhirSchemaVersion {
     ,V0023(23, "issue-2900 erased_resources to support $erase when offloading payloads", false)
     ,V0024(24, "issue-2900 for offloading add resource_payload_key to xx_resources", false)
     ,V0025(25, "issue-3158 stored proc updates to prevent deleting currently deleted resources", false)
-    ,V0026(26, "Add new resource types for FHIR R4B", false)
+    ,V0026(26, "issue-nnnn Add new resource types for FHIR R4B", false)
+    ,V0027(27, "issue-3437 extensions to support distribution/sharding", true)
+    ,V0028(28, "issue-3437 remove ref_version_id from xx_resource_token_refs", false) // parameter storage updated by V0027
     ;
 
     // The version number recorded in the VERSION_HISTORY
diff --git a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/GetLogicalResourceNeedsV0027Migration.java b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/GetLogicalResourceNeedsV0027Migration.java
new file mode 100644
index 00000000000..3a3faa8e266
--- /dev/null
+++ b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/GetLogicalResourceNeedsV0027Migration.java
@@ -0,0 +1,54 @@
+/*
+ * (C) Copyright IBM Corp. 2021
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package com.ibm.fhir.schema.control;
+
+import java.sql.Connection;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
+
+import com.ibm.fhir.database.utils.api.IDatabaseSupplier;
+import com.ibm.fhir.database.utils.api.IDatabaseTranslator;
+import com.ibm.fhir.database.utils.common.DataDefinitionUtil;
+
+/**
+ * Check to see if we have any data in LOGICAL_RESOURCE_IDENT. If it is empty,
+ * we assume we need to perform a migration
+ */
+public class GetLogicalResourceNeedsV0027Migration implements IDatabaseSupplier {
+
+    // The FHIR data schema
+    private final String schemaName;
+
+    /**
+     * Public constructor
+     * 
+     * @param schemaName
+     */
+    public GetLogicalResourceNeedsV0027Migration(String schemaName) {
+        this.schemaName = schemaName;
+    }
+
+    @Override
+    public Boolean run(IDatabaseTranslator translator, Connection c) {
+        Boolean result = true;
+        final String tableName = DataDefinitionUtil.getQualifiedName(schemaName, "LOGICAL_RESOURCE_IDENT");
+        final String SQL = "SELECT 1 FROM " + tableName + " " + translator.limit("1");
+
+        try (Statement s = c.createStatement()) {
+            ResultSet rs = s.executeQuery(SQL);
+            if (rs.next()) {
+                // logical_resource_ident already contains data, so no need to migrate
+                result = false;
+            }
+        } catch (SQLException x) {
+            throw translator.translate(x);
+        }
+
+        return result;
+    }
+}
\ No newline at end of file
diff --git a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/JavaBatchSchemaGenerator.java b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/JavaBatchSchemaGenerator.java
index 8427b8a45aa..4a50338d39a 100644
--- a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/JavaBatchSchemaGenerator.java
+++ b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/JavaBatchSchemaGenerator.java
@@ -245,7 +245,7 @@ public void addStepThreadExecutionTable(PhysicalDataModel model) {
                 .addBigIntColumn(M_WRITESKIP, NOT_NULL) // M_WRITESKIP BIGINT NOT NULL
                 .addBigIntColumn(FK_JOBEXECID, NOT_NULL) // FK_JOBEXECID BIGINT NOT NULL
                 .addBigIntColumn(FK_TOPLVL_STEPEXECID, NULL) // FK_TOPLVL_STEPEXECID BIGINT
-                .addSmallIntColumn(ISPARTITIONEDSTEP, 0, NULL) // ISPARTITIONEDSTEP SMALLINT DEFAULT 0
+                .addSmallIntBooleanColumn(ISPARTITIONEDSTEP, 0, NULL) // ISPARTITIONEDSTEP SMALLINT DEFAULT 0
                 .addPrimaryKey(PK + STEPTHREADEXECUTION_TABLE, STEPEXECID) // PRIMARY KEY (STEPEXECID)
                 .addIndex("STE_FKJOBEXECID_IX", FK_JOBEXECID) // STE_FKJOBEXECID_IX (FK_JOBEXECID)
                 .addIndex("STE_FKTLSTEPEID_IX", FK_TOPLVL_STEPEXECID) // STE_FKTLSTEPEID_IX (FK_TOPLVL_STEPEXECID)
@@ -391,7 +391,7 @@ public void addStepThreadInstanceTable(PhysicalDataModel model) {
                 .addBlobColumn(CHECKPOINTDATA, 2147483647, 10240, NULL) // CHECKPOINTDATA BLOB(2147483647)
                 .addBigIntColumn(FK_JOBINSTANCEID, NOT_NULL) // FK_JOBINSTANCEID BIGINT NOT NULL
                 .addBigIntColumn(FK_LATEST_STEPEXECID, NOT_NULL) // FK_LATEST_STEPEXECID BIGINT NOT NULL
-                .addSmallIntColumn(PARTITIONED, 0, NULL) //PARTITIONED SMALLINT DEFAULT 0 NOT NULL
+                .addSmallIntBooleanColumn(PARTITIONED, 0, NULL) //PARTITIONED SMALLINT DEFAULT 0 NOT NULL
                 .addIntColumn(PARTITIONPLANSIZE, NULL) // PARTITIONPLANSIZE INTEGER
                 .addIntColumn(STARTCOUNT,NULL) // STARTCOUNT INTEGER
                 .addIndex("STI_FKINSTANCEID_IX", FK_JOBINSTANCEID)
diff --git a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/MigrateV0021AbstractTypeRemoval.java b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/MigrateV0021AbstractTypeRemoval.java
index 711bdf61b90..243eefc413f 100644
--- a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/MigrateV0021AbstractTypeRemoval.java
+++ b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/MigrateV0021AbstractTypeRemoval.java
@@ -84,6 +84,7 @@ public MigrateV0021AbstractTypeRemoval(IDatabaseAdapter adapter, String adminSch
     public void run(IDatabaseTranslator translator, Connection c) {
         switch (translator.getType()) {
         case POSTGRESQL:
+        case CITUS:
         case DERBY:
             checkDataTables(translator, c);
             checkShouldThrowException();
@@ -172,7 +173,7 @@ private void checkDataTables(IDatabaseTranslator translator, Connection c) {
         for (String deprecatedTable : UnusedTableRemovalNeedsV0021Migration.DEPRECATED_TABLES) {
             if (adapter.doesTableExist(schemaName, deprecatedTable)) {
                 String table = schemaName + "." + deprecatedTable;
-                if (translator.getType() == DbType.POSTGRESQL) {
+                if (translator.isFamilyPostgreSQL()) {
                     table = schemaName.toLowerCase() + "." + deprecatedTable;
                 }
 
@@ -210,7 +211,7 @@ private void removeBaseArtifacts(IDatabaseTranslator translator, Connection c) {
         // Run across both tables
         for (String tablePrefix : tables) {
             // Drop the View for the Table
-            if (translator.getType() == DbType.POSTGRESQL) {
+            if (translator.isFamilyPostgreSQL()) {
                 runDropTableResourceGroup(translator, c, schemaName.toLowerCase(), tablePrefix.toLowerCase(), VALUE_TYPES_LOWER);
             } else {
                 runDropTableResourceGroup(translator, c, schemaName, tablePrefix, VALUE_TYPES);
@@ -227,7 +228,7 @@ private void removeBaseArtifacts(IDatabaseTranslator translator, Connection c) {
         // and logs print warnings saying the tables don't exist. That's OK.
         for (String deprecatedTable : UnusedTableRemovalNeedsV0021Migration.DEPRECATED_TABLES) {
             String table = prefix + deprecatedTable;
-            if (translator.getType() == DbType.POSTGRESQL) {
+            if (translator.isFamilyPostgreSQL()) {
                 adapter.dropTable(schemaName.toLowerCase(), table.toLowerCase());
             } else {
                 adapter.dropTable(schemaName, table);
diff --git a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/MigrateV0027LogicalResourceIdent.java b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/MigrateV0027LogicalResourceIdent.java
new file mode 100644
index 00000000000..fa87521167d
--- /dev/null
+++ b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/MigrateV0027LogicalResourceIdent.java
@@ -0,0 +1,48 @@
+/*
+ * (C) Copyright IBM Corp. 2021
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package com.ibm.fhir.schema.control;
+
+import java.sql.Connection;
+import java.sql.PreparedStatement;
+import java.sql.SQLException;
+
+import com.ibm.fhir.database.utils.api.IDatabaseStatement;
+import com.ibm.fhir.database.utils.api.IDatabaseTranslator;
+import com.ibm.fhir.database.utils.common.DataDefinitionUtil;
+
+/**
+ * Populate LOGICAL_RESOURCE_IDENT with records from LOGICAL_RESOURCES
+ */
+public class MigrateV0027LogicalResourceIdent implements IDatabaseStatement {
+
+    // The FHIR data schema
+    private final String schemaName;
+
+    /**
+     * Public constructor
+     * @param schemaName
+     */
+    public MigrateV0027LogicalResourceIdent(String schemaName) {
+        this.schemaName = schemaName;
+    }
+
+    @Override
+    public void run(IDatabaseTranslator translator, Connection c) {
+        final String logicalResources = DataDefinitionUtil.getQualifiedName(schemaName, "LOGICAL_RESOURCES");
+        final String logicalResourceIdent = DataDefinitionUtil.getQualifiedName(schemaName, "LOGICAL_RESOURCE_IDENT");
+        final String DML = ""
+                + "INSERT INTO " + logicalResourceIdent +"(resource_type_id, logical_id, logical_resource_id) "
+                + "     SELECT resource_type_id, logical_id, logical_resource_id "
+                + "       FROM " + logicalResources;
+
+        try (PreparedStatement ps = c.prepareStatement(DML)) {
+                ps.executeUpdate();
+        } catch (SQLException x) {
+            throw translator.translate(x);
+        }
+    }
+}
\ No newline at end of file
diff --git a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/MigrateV0027LogicalResourceIdentMT.java b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/MigrateV0027LogicalResourceIdentMT.java
new file mode 100644
index 00000000000..d73f265e01c
--- /dev/null
+++ b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/MigrateV0027LogicalResourceIdentMT.java
@@ -0,0 +1,49 @@
+/*
+ * (C) Copyright IBM Corp. 2021
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package com.ibm.fhir.schema.control;
+
+import java.sql.Connection;
+import java.sql.PreparedStatement;
+import java.sql.SQLException;
+
+import com.ibm.fhir.database.utils.api.IDatabaseStatement;
+import com.ibm.fhir.database.utils.api.IDatabaseTranslator;
+import com.ibm.fhir.database.utils.common.DataDefinitionUtil;
+
+/**
+ * Populate LOGICAL_RESOURCE_IDENT with records from LOGICAL_RESOURCES.
+ * Variant for use with the Db2 multitenant schema.
+ */
+public class MigrateV0027LogicalResourceIdentMT implements IDatabaseStatement {
+
+    // The FHIR data schema
+    private final String schemaName;
+
+    /**
+     * Public constructor
+     * @param schemaName
+     */
+    public MigrateV0027LogicalResourceIdentMT(String schemaName) {
+        this.schemaName = schemaName;
+    }
+
+    @Override
+    public void run(IDatabaseTranslator translator, Connection c) {
+        final String logicalResources = DataDefinitionUtil.getQualifiedName(schemaName, "LOGICAL_RESOURCES");
+        final String logicalResourceIdent = DataDefinitionUtil.getQualifiedName(schemaName, "LOGICAL_RESOURCE_IDENT");
+        final String DML = ""
+                + "INSERT INTO " + logicalResourceIdent +"(mt_id, resource_type_id, logical_id, logical_resource_id) "
+                + "     SELECT mt_id, resource_type_id, logical_id, logical_resource_id "
+                + "       FROM " + logicalResources;
+
+        try (PreparedStatement ps = c.prepareStatement(DML)) {
+                ps.executeUpdate();
+        } catch (SQLException x) {
+            throw translator.translate(x);
+        }
+    }
+}
\ No newline at end of file
diff --git a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/UnusedTableRemovalNeedsV0021Migration.java b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/UnusedTableRemovalNeedsV0021Migration.java
index 4cb3a690874..ec39b39b4bc 100644
--- a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/UnusedTableRemovalNeedsV0021Migration.java
+++ b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/control/UnusedTableRemovalNeedsV0021Migration.java
@@ -15,7 +15,6 @@
 
 import com.ibm.fhir.database.utils.api.IDatabaseSupplier;
 import com.ibm.fhir.database.utils.api.IDatabaseTranslator;
-import com.ibm.fhir.database.utils.model.DbType;
 
 /**
  * Checks to see if any of the tables exist in the target database.
@@ -73,6 +72,7 @@ public UnusedTableRemovalNeedsV0021Migration(String schemaName) {
     public Boolean run(IDatabaseTranslator translator, Connection c) {
         switch (translator.getType()) {
         case POSTGRESQL:
+        case CITUS:
             return checkPostgres(translator, c);
         case DB2:
             return checkDb2(translator, c);
@@ -163,7 +163,7 @@ private boolean hasTables(IDatabaseTranslator translator, Connection c, final St
             ps.setString(i++, schemaName);
 
             for (String deprecatedTable : DEPRECATED_TABLES) {
-                if (translator.getType() == DbType.POSTGRESQL) {
+                if (translator.isFamilyPostgreSQL()) {
                     ps.setString(i++, deprecatedTable.toLowerCase());
                 } else {
                     ps.setString(i++, deprecatedTable);
diff --git a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/derby/DerbyFhirDatabase.java b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/derby/DerbyFhirDatabase.java
index 78583215f48..14122855eed 100644
--- a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/derby/DerbyFhirDatabase.java
+++ b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/derby/DerbyFhirDatabase.java
@@ -7,7 +7,6 @@
 package com.ibm.fhir.schema.derby;
 
 import java.sql.Connection;
-import java.sql.PreparedStatement;
 import java.sql.ResultSet;
 import java.sql.SQLException;
 import java.sql.Statement;
@@ -19,6 +18,7 @@
 import com.ibm.fhir.database.utils.api.IDatabaseTranslator;
 import com.ibm.fhir.database.utils.api.ITransaction;
 import com.ibm.fhir.database.utils.api.ITransactionProvider;
+import com.ibm.fhir.database.utils.api.SchemaType;
 import com.ibm.fhir.database.utils.common.JdbcTarget;
 import com.ibm.fhir.database.utils.derby.DerbyAdapter;
 import com.ibm.fhir.database.utils.derby.DerbyConnectionProvider;
@@ -87,11 +87,11 @@ public DerbyFhirDatabase(String dbPath,  Set resourceTypeNames) throws S
         // Database objects for the admin schema (shared across multiple tenants in the same DB)
         PhysicalDataModel pdm = new PhysicalDataModel();
         if (resourceTypeNames == null) {
-            FhirSchemaGenerator gen = new FhirSchemaGenerator(ADMIN_SCHEMA_NAME, SCHEMA_NAME, false);
+            FhirSchemaGenerator gen = new FhirSchemaGenerator(ADMIN_SCHEMA_NAME, SCHEMA_NAME, SchemaType.PLAIN);
             gen.buildSchema(pdm);
         } else {
             // just build out a subset of tables
-            FhirSchemaGenerator gen = new FhirSchemaGenerator(ADMIN_SCHEMA_NAME, SCHEMA_NAME, false, resourceTypeNames);
+            FhirSchemaGenerator gen = new FhirSchemaGenerator(ADMIN_SCHEMA_NAME, SCHEMA_NAME, SchemaType.PLAIN, resourceTypeNames);
             gen.buildSchema(pdm);
         }
 
@@ -177,7 +177,7 @@ public VersionHistoryService createVersionHistoryService() throws SQLException {
             try {
                 JdbcTarget target = new JdbcTarget(c);
                 DerbyAdapter derbyAdapter = new DerbyAdapter(target);
-                CreateVersionHistory.createTableIfNeeded(ADMIN_SCHEMA_NAME, derbyAdapter);
+                CreateVersionHistory.createTableIfNeeded(ADMIN_SCHEMA_NAME, DerbyMaster.wrap(derbyAdapter));
                 c.commit();
             } catch (SQLException x) {
                 logger.log(Level.SEVERE, "failed to create version history table", x);
diff --git a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/patch/Main.java b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/patch/Main.java
index 7b4e4b87316..aff460dfc17 100644
--- a/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/patch/Main.java
+++ b/fhir-persistence-schema/src/main/java/com/ibm/fhir/schema/patch/Main.java
@@ -31,6 +31,7 @@
 import com.ibm.fhir.database.utils.api.DatabaseNotReadyException;
 import com.ibm.fhir.database.utils.api.IDatabaseAdapter;
 import com.ibm.fhir.database.utils.api.IDatabaseTranslator;
+import com.ibm.fhir.database.utils.citus.CitusTranslator;
 import com.ibm.fhir.database.utils.common.DataDefinitionUtil;
 import com.ibm.fhir.database.utils.common.DropForeignKeyConstraint;
 import com.ibm.fhir.database.utils.common.JdbcPropertyAdapter;
@@ -219,6 +220,9 @@ protected void parseArgs(String[] args) {
                 case POSTGRESQL:
                     translator = new PostgresTranslator();
                     break;
+                case CITUS:
+                    translator = new CitusTranslator();
+                    break;
                 case DB2:
                 default:
                     break;
diff --git a/fhir-persistence-schema/src/main/resources/citus/add_any_resource.sql b/fhir-persistence-schema/src/main/resources/citus/add_any_resource.sql
new file mode 100644
index 00000000000..5c719b387cf
--- /dev/null
+++ b/fhir-persistence-schema/src/main/resources/citus/add_any_resource.sql
@@ -0,0 +1,190 @@
+-------------------------------------------------------------------------------
+-- (C) Copyright IBM Corp. 2020, 2022
+--
+-- SPDX-License-Identifier: Apache-2.0
+-------------------------------------------------------------------------------
+
+-- ----------------------------------------------------------------------------
+-- Procedure to add a resource version and its associated parameters. These
+-- parameters only ever point to the latest version of a resource, never to
+-- previous versions, which are kept to support history queries.
+-- From V0027, we now use a logical_resource_ident table for locking. Records
+-- can be created in this table either by this procedure, or as part of
+-- reference parameter processing.
+--
+-- This variant is for use with the Citus distributed variant of the schema.
+-- This function is distributed by logical_resource_id (the first parameter)
+-- because all SQL/DML it executes uses logical_resource_id. The
+-- logical_resource_ident record must already exist and be locked for update
+-- before this function is called.
+--
+-- Because this function is distributed all object names must be fully
+-- qualified.
+--
+-- implNote - Conventions:
+--           p_... prefix used to represent input parameters
+--           v_... prefix used to represent declared variables
+--           t_... prefix used to represent temp variables
+--           o_... prefix used to represent output parameters
+-- Parameters:
+--   p_logical_resource_id: the logical_resource_ident primary key value for the resource
+--   p_resource_type_id: the resource_type_id from resource_types
+--   p_resource_type: the resource type name
+--   p_logical_id: the logical id given to the resource by the FHIR server
+--   p_payload:    the BLOB (of JSON) which is the resource content
+--   p_last_updated the last_updated time given by the FHIR server
+--   p_is_deleted: the soft delete flag
+--   p_version_id: the intended new version id of the resource (matching the JSON payload)
+--   p_parameter_hash_b64 the Base64 encoded hash of parameter values
+--   p_if_none_match the encoded If-None-Match value
+--   o_current_parameter_hash: Base64 current parameter hash if existing resource
+--   o_interaction_status: output indicating whether a change was made or IfNoneMatch hit
+--   o_if_none_match_version: output revealing the version found when o_interaction_status is 1 (IfNoneMatch)
+-- Exceptions:
+--   SQLSTATE 99001: on version conflict (concurrency)
+--   SQLSTATE 99002: missing expected row (data integrity)
+--   SQLSTATE 99004: delete a currently deleted resource (data integrity)
+-- ----------------------------------------------------------------------------
+    ( IN p_logical_resource_id            BIGINT,
+      IN p_resource_type_id                  INT,
+      IN p_resource_type                 VARCHAR( 36),
+      IN p_logical_id                    VARCHAR(255), 
+      IN p_payload                         BYTEA,
+      IN p_last_updated                TIMESTAMP,
+      IN p_is_deleted                       CHAR(  1),
+      IN p_source_key                    VARCHAR( 64),
+      IN p_version                           INT,
+      IN p_parameter_hash_b64            VARCHAR( 44),
+      IN p_if_none_match                     INT,
+      IN p_resource_payload_key          VARCHAR( 36),
+      OUT o_current_parameter_hash       VARCHAR( 44),
+      OUT o_interaction_status               INT,
+      OUT o_if_none_match_version            INT)
+    LANGUAGE plpgsql
+     AS $$
+
+  DECLARE 
+  v_schema_name         VARCHAR(128);
+  t_logical_resource_id  BIGINT := NULL;
+  v_current_resource_id  BIGINT := NULL;
+  v_resource_id          BIGINT := NULL;
+  v_currently_deleted      CHAR(1) := NULL;
+  v_new_resource            INT := 0;
+  v_duplicate               INT := 0;
+  v_current_version         INT := 0;
+  v_ghost_resource          INT := 0;
+  v_change_type            CHAR(1) := NULL;
+
+BEGIN
+  -- default value unless we hit If-None-Match
+  o_interaction_status := 0;
+
+  -- LOADED ON: {{DATE}}
+  v_schema_name := '{{SCHEMA_NAME}}';
+
+  -- Grab the new resource_id so that we can use it right away (and skip an update to xx_logical_resources later)
+  SELECT NEXTVAL('{{SCHEMA_NAME}}.fhir_sequence') INTO v_resource_id;
+
+  -- Read the record from logical_resources to see if this is an existing resource
+  SELECT logical_resource_id, parameter_hash, is_deleted 
+    INTO t_logical_resource_id, o_current_parameter_hash, v_currently_deleted
+    FROM {{SCHEMA_NAME}}.logical_resources 
+   WHERE logical_resource_id = p_logical_resource_id;
+  IF (t_logical_resource_id IS NULL)
+  THEN
+     v_new_resource := 1;
+    -- we already own the lock on the ident record, so we can safely create
+    -- the corresponding records in the logical_resources and resource-type-specific 
+    -- xx_logical_resources tables
+    INSERT INTO {{SCHEMA_NAME}}.logical_resources (logical_resource_id, resource_type_id, logical_id, reindex_tstamp, is_deleted, last_updated, parameter_hash)
+         VALUES (p_logical_resource_id, p_resource_type_id, p_logical_id, '1970-01-01', p_is_deleted, p_last_updated, p_parameter_hash_b64) ON CONFLICT DO NOTHING;
+
+    EXECUTE 'INSERT INTO ' || v_schema_name || '.' || p_resource_type || '_logical_resources (logical_resource_id, logical_id, is_deleted, last_updated, version_id, current_resource_id) '
+      || '     VALUES ($1, $2, $3, $4, $5, $6)' USING p_logical_resource_id, p_logical_id, p_is_deleted, p_last_updated, p_version, v_resource_id;
+
+    -- Since the resource did not previously exist, make sure o_current_parameter_hash is null
+    o_current_parameter_hash := NULL;
+  ELSE
+    -- as this is an existing resource, we need to know the current resource id.
+    -- This is only available at the resource-specific logical_resources level
+    EXECUTE
+         'SELECT current_resource_id, version_id FROM ' || v_schema_name || '.' || p_resource_type || '_logical_resources '
+      || ' WHERE logical_resource_id = $1 '
+    INTO v_current_resource_id, v_current_version USING p_logical_resource_id;
+    
+    IF v_current_resource_id IS NULL OR v_current_version IS NULL
+    THEN
+        -- our concurrency protection means that this shouldn't happen
+        RAISE 'Schema data corruption - missing logical resource' USING ERRCODE = '99002';
+    END IF;
+
+    -- If-None-Match does not apply if the resource is currently deleted
+    IF v_currently_deleted = 'N' AND p_if_none_match = 0
+    THEN
+        -- If-None-Match hit. Raising an exception here causes PostgreSQL to mark the
+        -- connection with a fatal error, so instead we use an out parameter to
+        -- indicate the match
+        o_interaction_status := 1;
+        o_if_none_match_version := v_current_version;
+        RETURN;
+    END IF;
+
+    -- Concurrency check:
+    --   the version parameter we've been given (which is also embedded in the JSON payload) must be 
+    --   one greater than the current version, otherwise we've hit a concurrent update race condition
+    IF p_version != v_current_version + 1
+    THEN
+      RAISE 'Concurrent update - mismatch of version in JSON' USING ERRCODE = '99001';
+    END IF;
+
+    -- Prevent creating a new deletion marker if the resource is currently deleted
+    IF v_currently_deleted = 'Y' AND p_is_deleted = 'Y'
+    THEN
+      RAISE 'Unexpected attempt to delete a Resource which is currently deleted' USING ERRCODE = '99004';
+    END IF;
+
+    IF o_current_parameter_hash IS NULL OR p_parameter_hash_b64 != o_current_parameter_hash
+    THEN
+        -- existing resource, so need to delete all its parameters (select because it's a function, not a procedure)
+        -- TODO patch parameter sets instead of all delete/all insert.
+        EXECUTE 'SELECT {{SCHEMA_NAME}}.delete_resource_parameters($1, $2)'
+        USING p_resource_type, p_logical_resource_id;
+    END IF; -- end if check parameter hash
+  END IF; -- end if existing resource
+
+  -- create the new resource version entry in xx_resources
+  EXECUTE
+         'INSERT INTO ' || v_schema_name || '.' || p_resource_type || '_resources (resource_id, logical_resource_id, version_id, data, last_updated, is_deleted, resource_payload_key) '
+      || ' VALUES ($1, $2, $3, $4, $5, $6, $7)'
+    USING v_resource_id, p_logical_resource_id, p_version, p_payload, p_last_updated, p_is_deleted, p_resource_payload_key;
+
+  IF v_new_resource = 0 THEN
+    -- As this is an existing logical resource, we need to update the xx_logical_resource values to match
+    -- the values of the current resource. For new resources, these are added by the insert so we don't
+    -- need to update them here.
+    EXECUTE 'UPDATE ' || v_schema_name || '.' || p_resource_type || '_logical_resources SET current_resource_id = $1, is_deleted = $2, last_updated = $3, version_id = $4 WHERE logical_resource_id = $5'
+      USING v_resource_id, p_is_deleted, p_last_updated, p_version, p_logical_resource_id;
+
+    -- For V0014 we now also store is_deleted and last_updated values at the whole-system logical_resources level
+    EXECUTE 'UPDATE ' || v_schema_name || '.logical_resources SET is_deleted = $1, last_updated = $2, parameter_hash = $3 WHERE logical_resource_id = $4'
+      USING p_is_deleted, p_last_updated, p_parameter_hash_b64, p_logical_resource_id;
+  END IF;
+
+  -- Finally, write a record to RESOURCE_CHANGE_LOG which records each event
+  -- related to resources changes (issue-1955)
+  IF p_is_deleted = 'Y'
+  THEN
+    v_change_type := 'D';
+  ELSE 
+    IF v_new_resource = 0 AND v_currently_deleted = 'N'
+    THEN
+      v_change_type := 'U';
+    ELSE
+      v_change_type := 'C';
+    END IF;
+  END IF;
+
+  INSERT INTO {{SCHEMA_NAME}}.resource_change_log(resource_id, change_tstamp, resource_type_id, logical_resource_id, version_id, change_type)
+       VALUES (v_resource_id, p_last_updated, p_resource_type_id, p_logical_resource_id, p_version, v_change_type);
+  
+END $$;
diff --git a/fhir-persistence-schema/src/main/resources/citus/add_logical_resource_ident.sql b/fhir-persistence-schema/src/main/resources/citus/add_logical_resource_ident.sql
new file mode 100644
index 00000000000..4ea33574c96
--- /dev/null
+++ b/fhir-persistence-schema/src/main/resources/citus/add_logical_resource_ident.sql
@@ -0,0 +1,80 @@
+-------------------------------------------------------------------------------
+-- (C) Copyright IBM Corp. 2022
+--
+-- SPDX-License-Identifier: Apache-2.0
+-------------------------------------------------------------------------------
+
+-- ----------------------------------------------------------------------------
+-- Procedure to either create or select for update a logical_resource_ident
+-- record. For Citus, this part of the "add_any_resource" logic is split
+-- off into its own function here which allows us to distribute the function
+-- on logical_id, which is used in all the SQL/DML executed by this
+-- function. This allows Citus to push execution of the entire function
+-- down to the worker node.
+--
+-- implNote - Conventions:
+--           p_... prefix used to represent input parameters
+--           v_... prefix used to represent declared variables
+--           t_... prefix used to represent temp variables
+--           o_... prefix used to represent output parameters
+-- Parameters:
+--   p_resource_type_id: the resource type id from resource_types
+--   p_logical_id: the logical id given to the resource by the FHIR server
+--   o_logical_resource_id: output field returning the newly assigned logical_resource_id value
+--
+-- ----------------------------------------------------------------------------
+    (  IN p_resource_type_id                 INT,
+       IN p_logical_id                   VARCHAR(64), 
+      OUT o_logical_resource_id           BIGINT)
+    LANGUAGE plpgsql
+     AS $$
+
+  DECLARE 
+  v_schema_name         VARCHAR(128);
+  v_logical_resource_id  BIGINT := NULL;
+  t_logical_resource_id  BIGINT := NULL;
+  
+  -- Because we don't really update any existing key, so use NO KEY UPDATE to achieve better concurrence performance. 
+  lock_cur CURSOR (t_resource_type_id INT, t_logical_id VARCHAR(1024)) FOR SELECT logical_resource_id FROM {{SCHEMA_NAME}}.logical_resource_ident WHERE resource_type_id = t_resource_type_id AND logical_id = t_logical_id FOR NO KEY UPDATE;
+
+BEGIN
+
+  -- LOADED ON: {{DATE}}
+  v_schema_name := '{{SCHEMA_NAME}}';
+
+  -- Get a lock on the logical resource identity record
+  OPEN lock_cur(t_resource_type_id := p_resource_type_id, t_logical_id := p_logical_id);
+  FETCH lock_cur INTO v_logical_resource_id;
+  CLOSE lock_cur;
+  
+  -- Create the resource ident record if we don't have it already
+  IF v_logical_resource_id IS NULL
+  THEN
+    -- allocate the new logical_resource_id value
+    SELECT nextval('{{SCHEMA_NAME}}.fhir_sequence') INTO v_logical_resource_id;
+
+    -- remember that we have a concurrent system...so there is a possibility
+    -- that another thread snuck in before us and created the ident record. To
+    -- handle this in PostgreSQL, we INSERT...ON CONFLICT DO NOTHING, then turn
+    -- around and read again to check that the logical_resource_id in the table
+    -- matches the value we tried to insert.
+    INSERT INTO {{SCHEMA_NAME}}.logical_resource_ident (resource_type_id, logical_id, logical_resource_id)
+         VALUES (p_resource_type_id, p_logical_id, v_logical_resource_id) ON CONFLICT DO NOTHING;
+
+    -- Do a read so that we can verify that *we* did the insert
+    OPEN lock_cur(t_resource_type_id := p_resource_type_id, t_logical_id := p_logical_id);
+    FETCH lock_cur INTO t_logical_resource_id;
+    CLOSE lock_cur;
+
+    IF v_logical_resource_id != t_logical_resource_id
+    THEN
+      -- logical_resource_ident record was created by another thread...so use that id instead
+      v_logical_resource_id := t_logical_resource_id;
+    END IF;
+  END IF;
+
+  -- Hand back the id of the logical resource we created earlier. In the new R4 schema
+  -- only the logical_resource_id is the target of any FK, so there's no need to return
+  -- the resource_id (which is now private to the _resources tables).
+  o_logical_resource_id := v_logical_resource_id;
+END $$;
diff --git a/fhir-persistence-schema/src/main/resources/db2/add_any_resource.sql b/fhir-persistence-schema/src/main/resources/db2/add_any_resource.sql
index 7f15ea4fa8f..a09673ea2ac 100644
--- a/fhir-persistence-schema/src/main/resources/db2/add_any_resource.sql
+++ b/fhir-persistence-schema/src/main/resources/db2/add_any_resource.sql
@@ -54,6 +54,7 @@ BEGIN
 
   DECLARE v_schema_name         VARCHAR(128 OCTETS);
   DECLARE v_logical_resource_id  BIGINT     DEFAULT NULL;
+  DECLARE t_logical_resource_id  BIGINT     DEFAULT NULL;
   DECLARE v_current_resource_id  BIGINT     DEFAULT NULL;
   DECLARE v_resource_id          BIGINT     DEFAULT NULL;
   DECLARE v_resource_type_id        INT     DEFAULT NULL;
@@ -83,9 +84,11 @@ BEGIN
     FROM {{SCHEMA_NAME}}.resource_types WHERE resource_type = p_resource_type;
   
   -- FOR UPDATE WITH RS does not appear to work using a prepared statement and
-  -- cursor, so we have to run this directly against the logical_resources table.
-  SELECT logical_resource_id, parameter_hash, is_deleted INTO v_logical_resource_id, o_current_parameter_hash, v_currently_deleted
-    FROM {{SCHEMA_NAME}}.logical_resources
+  -- cursor, so we have to run as compiled SQL. For V0027 we now use
+  -- logical_resource_ident for managing the identity of a resource and the
+  -- associated locking
+  SELECT logical_resource_id INTO v_logical_resource_id
+    FROM {{SCHEMA_NAME}}.logical_resource_ident
    WHERE resource_type_id = v_resource_type_id AND logical_id = p_logical_id
      FOR UPDATE WITH RS
    ;
@@ -94,40 +97,73 @@ BEGIN
   IF v_logical_resource_id IS NULL
   THEN
     VALUES NEXT VALUE FOR {{SCHEMA_NAME}}.fhir_sequence INTO v_logical_resource_id;
-    PREPARE stmt FROM
-       'INSERT INTO ' || v_schema_name || '.logical_resources (mt_id, logical_resource_id, resource_type_id, logical_id, reindex_tstamp, is_deleted, last_updated, parameter_hash) '
-    || '     VALUES (?, ?, ?, ?, ?, ?, ?, ?)';
-    EXECUTE stmt USING {{ADMIN_SCHEMA_NAME}}.sv_tenant_id, v_logical_resource_id, v_resource_type_id, p_logical_id, '1970-01-01-00.00.00.0', p_is_deleted, p_last_updated, p_parameter_hash_b64;
 
+    PREPARE stmt FROM
+       'INSERT INTO ' || v_schema_name || '.logical_resource_ident (mt_id, resource_type_id, logical_id, logical_resource_id) '
+    || '     VALUES (?, ?, ?, ?)';
+    EXECUTE stmt USING {{ADMIN_SCHEMA_NAME}}.sv_tenant_id, v_resource_type_id, p_logical_id, v_logical_resource_id;
     -- remember that we have a concurrent system...so there is a possibility
-    -- that another thread snuck in before us and created the logical resource. This
+    -- that another thread snuck in before us and created the logical resource ident. This
     -- is easy to handle, just turn around and read it
     IF v_duplicate = 1
     THEN
-      -- row exists, so we just need to obtain a lock on it. Because logical resource records are
-      -- never deleted, we don't need to worry about it disappearing again before we grab the row lock
-      SELECT logical_resource_id, parameter_hash, is_deleted INTO v_logical_resource_id, o_current_parameter_hash, v_currently_deleted
-        FROM {{SCHEMA_NAME}}.logical_resources
+      SELECT logical_resource_id INTO v_logical_resource_id
+        FROM {{SCHEMA_NAME}}.logical_resource_ident
        WHERE resource_type_id = v_resource_type_id AND logical_id = p_logical_id
          FOR UPDATE WITH RS
        ;
+
+       -- Because someone else created the logical_resoure_ident record, we need to see if
+       -- they also created the corresponding logical_resources record
+       SELECT logical_resource_id, parameter_hash, is_deleted 
+         INTO t_logical_resource_id, o_current_parameter_hash, v_currently_deleted
+         FROM {{SCHEMA_NAME}}.logical_resources 
+        WHERE logical_resource_id = v_logical_resource_id;
+       
+       IF (t_logical_resource_id IS NULL)
+       THEN
+         -- other thread only created the ident record, so we still need to treat
+         -- this as a new resource
+         SET v_new_resource = 1;
+       END IF;
+     ELSE
+       -- we created the logical_resource_ident, so we know this is a new resource
+       SET v_new_resource = 1;
+     END IF;
+   ELSE
+     -- the logical_resource_ident record exists, so now we need to find out
+     -- if the corresponding logical_resources record exists
+     SELECT logical_resource_id, parameter_hash, is_deleted 
+       INTO t_logical_resource_id, o_current_parameter_hash, v_currently_deleted
+       FROM {{SCHEMA_NAME}}.logical_resources 
+      WHERE logical_resource_id = v_logical_resource_id;
        
-      -- Since the resource did not previously exist, set o_current_parameter_hash back to NULL
-      SET o_current_parameter_hash = NULL;
+     IF (t_logical_resource_id IS NULL)
+     THEN
+       -- the ident record was created as a reference, but because there's no logical_resources
+       -- record, we treat this as a new resource
+       SET v_new_resource = 1;
+     END IF;
+  END IF;
 
-    ELSE
-      -- we created the logical resource and therefore we already own the lock. So now we can
-      -- safely create the corresponding record in the resource-type-specific logical_resources table
-      PREPARE stmt FROM
+  IF v_new_resource = 1
+  THEN
+    -- create the logical_resources record
+    PREPARE stmt FROM
+       'INSERT INTO ' || v_schema_name || '.logical_resources (mt_id, logical_resource_id, resource_type_id, logical_id, reindex_tstamp, is_deleted, last_updated, parameter_hash) '
+    || '     VALUES (?, ?, ?, ?, ?, ?, ?, ?)';
+    EXECUTE stmt USING {{ADMIN_SCHEMA_NAME}}.sv_tenant_id, v_logical_resource_id, v_resource_type_id, p_logical_id, '1970-01-01-00.00.00.0', p_is_deleted, p_last_updated, p_parameter_hash_b64;
+
+    -- create the xx_logical_resources record
+    PREPARE stmt FROM
          'INSERT INTO ' || v_schema_name || '.' || p_resource_type || '_logical_resources (mt_id, logical_resource_id, logical_id, is_deleted, last_updated, version_id, current_resource_id) '
       || '     VALUES (?, ?, ?, ?, ?, ?, ?)';
-      EXECUTE stmt USING {{ADMIN_SCHEMA_NAME}}.sv_tenant_id, v_logical_resource_id, p_logical_id, p_is_deleted, p_last_updated, p_version, v_resource_id;
-      SET v_new_resource = 1;
-    END IF;
-  END IF;
+    EXECUTE stmt USING {{ADMIN_SCHEMA_NAME}}.sv_tenant_id, v_logical_resource_id, p_logical_id, p_is_deleted, p_last_updated, p_version, v_resource_id;
 
-  -- Remember everying is locked at the logical resource level, so we are thread-safe here
-  IF v_new_resource = 0 THEN
+    -- Since the resource did not previously exist, make sure o_current_parameter_hash is NULL
+    SET o_current_parameter_hash = NULL;
+
+  ELSE
     -- as this is an existing resource, we need to know the current resource id.
     -- This is only available at the resource-specific logical_resources level
     PREPARE stmt FROM
diff --git a/fhir-persistence-schema/src/main/resources/db2/delete_resource_parameters.sql b/fhir-persistence-schema/src/main/resources/db2/delete_resource_parameters.sql
index 2ae2c86f58e..472f78810ad 100644
--- a/fhir-persistence-schema/src/main/resources/db2/delete_resource_parameters.sql
+++ b/fhir-persistence-schema/src/main/resources/db2/delete_resource_parameters.sql
@@ -45,6 +45,9 @@ BEGIN
     PREPARE d_stmt FROM 'DELETE FROM {{SCHEMA_NAME}}.' || p_resource_type || '_security            WHERE logical_resource_id = ?';
     EXECUTE d_stmt USING p_logical_resource_id;
 
+    PREPARE d_stmt FROM 'DELETE FROM {{SCHEMA_NAME}}.' || p_resource_type || '_ref_values          WHERE logical_resource_id = ?';
+    EXECUTE d_stmt USING p_logical_resource_id;
+
     PREPARE d_stmt FROM 'DELETE FROM {{SCHEMA_NAME}}.' || 'str_values                WHERE logical_resource_id = ?';
     EXECUTE d_stmt USING p_logical_resource_id;
 
diff --git a/fhir-persistence-schema/src/main/resources/postgres/add_any_resource.sql b/fhir-persistence-schema/src/main/resources/postgres/add_any_resource.sql
index 0130d2054e0..5cc07feede1 100644
--- a/fhir-persistence-schema/src/main/resources/postgres/add_any_resource.sql
+++ b/fhir-persistence-schema/src/main/resources/postgres/add_any_resource.sql
@@ -8,6 +8,9 @@
 -- Procedure to add a resource version and its associated parameters. These
 -- parameters only ever point to the latest version of a resource, never to
 -- previous versions, which are kept to support history queries.
+-- From V0027, we now use a logical_resource_ident table for locking. Records
+-- can be created in this table either by this procedure, or as part of
+-- reference parameter processing.
 -- implNote - Conventions:
 --           p_... prefix used to represent input parameters
 --           v_... prefix used to represent declared variables
@@ -58,10 +61,11 @@
   v_new_resource            INT := 0;
   v_duplicate               INT := 0;
   v_current_version         INT := 0;
+  v_ghost_resource          INT := 0;
   v_change_type            CHAR(1) := NULL;
   
   -- Because we don't really update any existing key, so use NO KEY UPDATE to achieve better concurrence performance. 
-  lock_cur CURSOR (t_resource_type_id INT, t_logical_id VARCHAR(255)) FOR SELECT logical_resource_id, parameter_hash, is_deleted FROM {{SCHEMA_NAME}}.logical_resources WHERE resource_type_id = t_resource_type_id AND logical_id = t_logical_id FOR NO KEY UPDATE;
+  lock_cur CURSOR (t_resource_type_id INT, t_logical_id VARCHAR(255)) FOR SELECT logical_resource_id FROM {{SCHEMA_NAME}}.logical_resource_ident WHERE resource_type_id = t_resource_type_id AND logical_id = t_logical_id FOR NO KEY UPDATE;
 
 BEGIN
   -- default value unless we hit If-None-Match
@@ -75,44 +79,77 @@ BEGIN
   -- Grab the new resource_id so that we can use it right away (and skip an update to xx_logical_resources later)
   SELECT NEXTVAL('{{SCHEMA_NAME}}.fhir_sequence') INTO v_resource_id;
 
-  -- Get a lock at the system-wide logical resource level
+  -- Get a lock on the logical resource identity record
   OPEN lock_cur(t_resource_type_id := v_resource_type_id, t_logical_id := p_logical_id);
-  FETCH lock_cur INTO v_logical_resource_id, o_current_parameter_hash, v_currently_deleted;
+  FETCH lock_cur INTO v_logical_resource_id;
   CLOSE lock_cur;
   
-  -- Create the resource if we don't have it already
+  -- Create the resource ident record if we don't have it already
   IF v_logical_resource_id IS NULL
   THEN
     SELECT nextval('{{SCHEMA_NAME}}.fhir_sequence') INTO v_logical_resource_id;
     -- remember that we have a concurrent system...so there is a possibility
-    -- that another thread snuck in before us and created the logical resource. This
-    -- is easy to handle, just turn around and read it
-    INSERT INTO {{SCHEMA_NAME}}.logical_resources (logical_resource_id, resource_type_id, logical_id, reindex_tstamp, is_deleted, last_updated, parameter_hash)
-         VALUES (v_logical_resource_id, v_resource_type_id, p_logical_id, '1970-01-01', p_is_deleted, p_last_updated, p_parameter_hash_b64) ON CONFLICT DO NOTHING;
-       
-      -- row exists, so we just need to obtain a lock on it. Because logical resource records are
-      -- never deleted, we don't need to worry about it disappearing again before we grab the row lock
-      OPEN lock_cur (t_resource_type_id := v_resource_type_id, t_logical_id := p_logical_id);
-      FETCH lock_cur INTO t_logical_resource_id, o_current_parameter_hash, v_currently_deleted;
-      CLOSE lock_cur;
-
-      -- Since the resource did not previously exist, set o_current_parameter_hash back to NULL
-      o_current_parameter_hash := NULL;
-      
+    -- that another thread snuck in before us and created the ident record. To
+    -- handle this in PostgreSQL, we INSERT...ON CONFLICT DO NOTHING, then turn
+    -- around and read again to check that the logical_resource_id in the table
+    -- matches the value we tried to insert.
+    INSERT INTO {{SCHEMA_NAME}}.logical_resource_ident (resource_type_id, logical_id, logical_resource_id)
+         VALUES (v_resource_type_id, p_logical_id, v_logical_resource_id) ON CONFLICT DO NOTHING;
+
+    -- Do a read so that we can verify that *we* did the insert
+    OPEN lock_cur(t_resource_type_id := v_resource_type_id, t_logical_id := p_logical_id);
+    FETCH lock_cur INTO t_logical_resource_id;
+    CLOSE lock_cur;
+
     IF v_logical_resource_id = t_logical_resource_id
     THEN
-      -- we created the logical resource and therefore we already own the lock. So now we can
-      -- safely create the corresponding record in the resource-type-specific logical_resources table
-      EXECUTE 'INSERT INTO ' || v_schema_name || '.' || p_resource_type || '_logical_resources (logical_resource_id, logical_id, is_deleted, last_updated, version_id, current_resource_id) '
-      || '     VALUES ($1, $2, $3, $4, $5, $6)' USING v_logical_resource_id, p_logical_id, p_is_deleted, p_last_updated, p_version, v_resource_id;
-      v_new_resource := 1;
+        -- we did the insert, so we know this is a new record
+        v_new_resource := 1;
     ELSE
-      v_logical_resource_id := t_logical_resource_id;
+      -- another thread created the resource.
+      -- New for V0027. Records in logical_resource_ident may be created because they
+      -- are the target of a reference. We therefore need to handle the case where
+      -- no logical_resources record exists.
+      SELECT logical_resource_id, parameter_hash, is_deleted 
+        INTO v_logical_resource_id, o_current_parameter_hash, v_currently_deleted
+        FROM {{SCHEMA_NAME}}.logical_resources 
+       WHERE logical_resource_id = t_logical_resource_id;
+       
+      IF (v_logical_resource_id IS NULL)
+      THEN
+        -- other thread only created the ident record, so we still need to treat
+        -- this as a new resource
+        v_logical_resource_id := t_logical_resource_id;
+        v_new_resource := 1;
+      END IF;
+    END IF;
+  ELSE
+    -- we have an ident record, but we still need to check if we have a logical_resources
+    -- record
+    SELECT logical_resource_id, parameter_hash, is_deleted 
+      INTO t_logical_resource_id, o_current_parameter_hash, v_currently_deleted
+      FROM {{SCHEMA_NAME}}.logical_resources 
+     WHERE logical_resource_id = v_logical_resource_id;
+    IF (t_logical_resource_id IS NULL)
+    THEN
+       v_new_resource := 1;
     END IF;
   END IF;
 
-  -- Remember everying is locked at the logical resource level, so we are thread-safe here
-  IF v_new_resource = 0 THEN
+  IF v_new_resource = 1
+  THEN
+    -- we already own the lock on the ident record, so we can safely create
+    -- the corresponding records in the logical_resources and resource-type-specific 
+    -- xx_logical_resources tables
+    INSERT INTO {{SCHEMA_NAME}}.logical_resources (logical_resource_id, resource_type_id, logical_id, reindex_tstamp, is_deleted, last_updated, parameter_hash)
+         VALUES (v_logical_resource_id, v_resource_type_id, p_logical_id, '1970-01-01', p_is_deleted, p_last_updated, p_parameter_hash_b64) ON CONFLICT DO NOTHING;
+
+    EXECUTE 'INSERT INTO ' || v_schema_name || '.' || p_resource_type || '_logical_resources (logical_resource_id, logical_id, is_deleted, last_updated, version_id, current_resource_id) '
+      || '     VALUES ($1, $2, $3, $4, $5, $6)' USING v_logical_resource_id, p_logical_id, p_is_deleted, p_last_updated, p_version, v_resource_id;
+
+    -- Since the resource did not previously exist, make sure o_current_parameter_hash is null
+    o_current_parameter_hash := NULL;
+  ELSE
     -- as this is an existing resource, we need to know the current resource id.
     -- This is only available at the resource-specific logical_resources level
     EXECUTE
@@ -153,19 +190,19 @@ BEGIN
 
     IF o_current_parameter_hash IS NULL OR p_parameter_hash_b64 != o_current_parameter_hash
     THEN
-	    -- existing resource, so need to delete all its parameters (select because it's a function, not a procedure)
-	    -- TODO patch parameter sets instead of all delete/all insert.
+        -- existing resource, so need to delete all its parameters (select because it's a function, not a procedure)
+        -- TODO patch parameter sets instead of all delete/all insert.
         EXECUTE 'SELECT {{SCHEMA_NAME}}.delete_resource_parameters($1, $2)'
         USING p_resource_type, v_logical_resource_id;
-	END IF; -- end if check parameter hash
+    END IF; -- end if check parameter hash
   END IF; -- end if existing resource
 
+  -- create the new resource version entry in xx_resources
   EXECUTE
          'INSERT INTO ' || v_schema_name || '.' || p_resource_type || '_resources (resource_id, logical_resource_id, version_id, data, last_updated, is_deleted, resource_payload_key) '
       || ' VALUES ($1, $2, $3, $4, $5, $6, $7)'
     USING v_resource_id, v_logical_resource_id, p_version, p_payload, p_last_updated, p_is_deleted, p_resource_payload_key;
 
-
   IF v_new_resource = 0 THEN
     -- As this is an existing logical resource, we need to update the xx_logical_resource values to match
     -- the values of the current resource. For new resources, these are added by the insert so we don't
@@ -199,4 +236,4 @@ BEGIN
   -- only the logical_resource_id is the target of any FK, so there's no need to return
   -- the resource_id (which is now private to the _resources tables).
   o_logical_resource_id := v_logical_resource_id;
-END $$;
\ No newline at end of file
+END $$;
diff --git a/fhir-persistence-schema/src/main/resources/postgres/delete_resource_parameters.sql b/fhir-persistence-schema/src/main/resources/postgres/delete_resource_parameters.sql
index 496ade054a1..292735c23ac 100644
--- a/fhir-persistence-schema/src/main/resources/postgres/delete_resource_parameters.sql
+++ b/fhir-persistence-schema/src/main/resources/postgres/delete_resource_parameters.sql
@@ -17,42 +17,44 @@
      AS $$
 
   DECLARE
-  v_schema_name         VARCHAR(128);
+    v_schema_name         VARCHAR(128);
 
 BEGIN
-  v_schema_name := '{{SCHEMA_NAME}}';
+    v_schema_name := '{{SCHEMA_NAME}}';
 
-	EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.' || p_resource_type || '_str_values          WHERE logical_resource_id = $1'
-	USING p_logical_resource_id;
-	EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.' || p_resource_type || '_number_values       WHERE logical_resource_id = $1'
-	USING p_logical_resource_id;
-	EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.' || p_resource_type || '_date_values         WHERE logical_resource_id = $1'
-	USING p_logical_resource_id;
-	EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.' || p_resource_type || '_latlng_values       WHERE logical_resource_id = $1'
-	USING p_logical_resource_id;
-	EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.' || p_resource_type || '_resource_token_refs WHERE logical_resource_id = $1'
-	USING p_logical_resource_id;
-	EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.' || p_resource_type || '_quantity_values     WHERE logical_resource_id = $1'
-	USING p_logical_resource_id;
-	EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.' || p_resource_type || '_profiles            WHERE logical_resource_id = $1'
-	USING p_logical_resource_id;
-	EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.' || p_resource_type || '_tags                WHERE logical_resource_id = $1'
-	USING p_logical_resource_id;
-	EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.' || p_resource_type || '_security            WHERE logical_resource_id = $1'
-	USING p_logical_resource_id;
-	EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.str_values                 WHERE logical_resource_id = $1'
-	USING p_logical_resource_id;
-	EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.date_values                WHERE logical_resource_id = $1'
-	USING p_logical_resource_id;
-	EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.resource_token_refs        WHERE logical_resource_id = $1'
-	USING p_logical_resource_id;
-	EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.logical_resource_profiles  WHERE logical_resource_id = $1'
-	USING p_logical_resource_id;
-	EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.logical_resource_tags      WHERE logical_resource_id = $1'
-	USING p_logical_resource_id;
-	EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.logical_resource_security  WHERE logical_resource_id = $1'
-	USING p_logical_resource_id;
-	
-	-- because we're a function, pass back a result
-	o_logical_resource_id := p_logical_resource_id;
+    EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.' || p_resource_type || '_str_values          WHERE logical_resource_id = $1'
+    USING p_logical_resource_id;
+    EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.' || p_resource_type || '_number_values       WHERE logical_resource_id = $1'
+    USING p_logical_resource_id;
+    EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.' || p_resource_type || '_date_values         WHERE logical_resource_id = $1'
+    USING p_logical_resource_id;
+    EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.' || p_resource_type || '_latlng_values       WHERE logical_resource_id = $1'
+    USING p_logical_resource_id;
+    EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.' || p_resource_type || '_resource_token_refs WHERE logical_resource_id = $1'
+    USING p_logical_resource_id;
+    EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.' || p_resource_type || '_quantity_values     WHERE logical_resource_id = $1'
+    USING p_logical_resource_id;
+    EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.' || p_resource_type || '_profiles            WHERE logical_resource_id = $1'
+    USING p_logical_resource_id;
+    EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.' || p_resource_type || '_tags                WHERE logical_resource_id = $1'
+    USING p_logical_resource_id;
+    EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.' || p_resource_type || '_security            WHERE logical_resource_id = $1'
+    USING p_logical_resource_id;
+    EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.' || p_resource_type || '_ref_values          WHERE logical_resource_id = $1'
+    USING p_logical_resource_id;
+    EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.str_values                 WHERE logical_resource_id = $1'
+    USING p_logical_resource_id;
+    EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.date_values                WHERE logical_resource_id = $1'
+    USING p_logical_resource_id;
+    EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.resource_token_refs        WHERE logical_resource_id = $1'
+    USING p_logical_resource_id;
+    EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.logical_resource_profiles  WHERE logical_resource_id = $1'
+    USING p_logical_resource_id;
+    EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.logical_resource_tags      WHERE logical_resource_id = $1'
+    USING p_logical_resource_id;
+    EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.logical_resource_security  WHERE logical_resource_id = $1'
+    USING p_logical_resource_id;
+
+    -- because we're a function, pass back a result
+    o_logical_resource_id := p_logical_resource_id;
 END $$;
\ No newline at end of file
diff --git a/fhir-persistence-schema/src/main/resources/postgres/delete_resource_parameters_sharded.sql b/fhir-persistence-schema/src/main/resources/postgres/delete_resource_parameters_sharded.sql
new file mode 100644
index 00000000000..e2278a70c0f
--- /dev/null
+++ b/fhir-persistence-schema/src/main/resources/postgres/delete_resource_parameters_sharded.sql
@@ -0,0 +1,65 @@
+-------------------------------------------------------------------------------
+-- (C) Copyright IBM Corp. 2021
+--
+-- SPDX-License-Identifier: Apache-2.0
+-------------------------------------------------------------------------------
+
+-- ----------------------------------------------------------------------------
+-- Procedure to delete all search parameters values for a given resource.
+-- This variant is for use with the distributed schema variant typically
+-- deployed to a distributed database service like Citus.
+-- 
+-- p_shard_key: the key used for distribution (sharding)
+-- p_resource_type: the resource type name
+-- p_logical_resource_id: the database id of the resource for which the parameters are to be deleted
+-- ----------------------------------------------------------------------------
+    (  IN p_shard_key          SMALLINT,
+       IN p_resource_type       VARCHAR( 36),
+       IN p_logical_resource_id  BIGINT,
+      OUT o_logical_resource_id  BIGINT)
+       RETURNS BIGINT
+    LANGUAGE plpgsql
+     AS $$
+
+  DECLARE
+  v_schema_name         VARCHAR(128);
+
+BEGIN
+  v_schema_name := '{{SCHEMA_NAME}}';
+
+    EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.' || p_resource_type || '_str_values          WHERE shard_key = $1 AND logical_resource_id = $2'
+    USING p_shard_key, p_logical_resource_id;
+    EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.' || p_resource_type || '_number_values       WHERE shard_key = $1 AND logical_resource_id = $2'
+    USING p_shard_key, p_logical_resource_id;
+    EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.' || p_resource_type || '_date_values         WHERE shard_key = $1 AND logical_resource_id = $2'
+    USING p_shard_key, p_logical_resource_id;
+    EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.' || p_resource_type || '_latlng_values       WHERE shard_key = $1 AND logical_resource_id = $2'
+    USING p_shard_key, p_logical_resource_id;
+    EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.' || p_resource_type || '_resource_token_refs WHERE shard_key = $1 AND logical_resource_id = $2'
+    USING p_shard_key, p_logical_resource_id;
+    EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.' || p_resource_type || '_quantity_values     WHERE shard_key = $1 AND logical_resource_id = $2'
+    USING p_shard_key, p_logical_resource_id;
+    EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.' || p_resource_type || '_profiles            WHERE shard_key = $1 AND logical_resource_id = $2'
+    USING p_shard_key, p_logical_resource_id;
+    EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.' || p_resource_type || '_tags                WHERE shard_key = $1 AND logical_resource_id = $2'
+    USING p_shard_key, p_logical_resource_id;
+    EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.' || p_resource_type || '_security            WHERE shard_key = $1 AND logical_resource_id = $2'
+    USING p_shard_key, p_logical_resource_id;
+    EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.' || p_resource_type || '_ref_values          WHERE shard_key = $1 AND logical_resource_id = $2'
+    USING p_logical_resource_id;
+    EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.str_values                 WHERE shard_key = $1 AND logical_resource_id = $2'
+    USING p_shard_key, p_logical_resource_id;
+    EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.date_values                WHERE shard_key = $1 AND logical_resource_id = $2'
+    USING p_shard_key, p_logical_resource_id;
+    EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.resource_token_refs        WHERE shard_key = $1 AND logical_resource_id = $2'
+    USING p_shard_key, p_logical_resource_id;
+    EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.logical_resource_profiles  WHERE shard_key = $1 AND logical_resource_id = $2'
+    USING p_shard_key, p_logical_resource_id;
+    EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.logical_resource_tags      WHERE shard_key = $1 AND logical_resource_id = $2'
+    USING p_shard_key, p_logical_resource_id;
+    EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.logical_resource_security  WHERE shard_key = $1 AND logical_resource_id = $2'
+	USING p_shard_key, p_logical_resource_id;
+	
+	-- because we're a function, pass back a result
+	o_logical_resource_id := p_logical_resource_id;
+END $$;
diff --git a/fhir-persistence-schema/src/main/resources/postgres/erase_resource_sharded.sql b/fhir-persistence-schema/src/main/resources/postgres/erase_resource_sharded.sql
new file mode 100644
index 00000000000..03967ad27fd
--- /dev/null
+++ b/fhir-persistence-schema/src/main/resources/postgres/erase_resource_sharded.sql
@@ -0,0 +1,89 @@
+-------------------------------------------------------------------------------
+-- (C) Copyright IBM Corp. 2021
+--
+-- SPDX-License-Identifier: Apache-2.0
+-------------------------------------------------------------------------------
+
+-- ----------------------------------------------------------------------------
+-- Procedure to remove a resource, history and parameters values
+-- 
+-- p_shard_key: the sharding key used for distribution
+-- p_resource_type: the resource type
+-- p_logical_id: the resource logical id
+-- o_deleted: the total number of resource versions that are deleted
+-- ----------------------------------------------------------------------------
+    ( IN p_shard_key                   SMALLINT,
+      IN p_resource_type                VARCHAR(  36),
+      IN p_logical_id                   VARCHAR( 255),
+      IN p_erased_resource_group_id      BIGINT,
+      OUT o_deleted                      BIGINT)
+    RETURNS BIGINT
+    LANGUAGE plpgsql
+     AS $$
+
+  DECLARE
+  v_schema_name         VARCHAR(128);
+  v_logical_resource_id BIGINT := NULL;
+  v_resource_type_id    BIGINT := -1;
+  v_total               BIGINT := 0;
+
+BEGIN
+  v_schema_name := '{{SCHEMA_NAME}}';
+
+  -- Prep 1: Get the v_resource_type_id
+  SELECT resource_type_id INTO v_resource_type_id 
+  FROM {{SCHEMA_NAME}}.resource_types
+  WHERE resource_type = p_resource_type;
+
+  -- Prep 2: Get the logical from the system-wide logical resource level
+  SELECT logical_resource_id INTO v_logical_resource_id 
+  FROM {{SCHEMA_NAME}}.logical_resources
+  WHERE shard_key = p_shard_key 
+    AND resource_type_id = v_resource_type_id 
+    AND logical_id = p_logical_id
+  FOR UPDATE;
+  
+  IF NOT FOUND
+  THEN
+    v_total := -1;
+  ELSE
+    -- Step 1: Delete from resource_change_log
+    -- Delete is done before the RESOURCES table entries disappear
+    -- This uses the primary_keys of each table to conditional-delete
+    EXECUTE 
+    'DELETE FROM {{SCHEMA_NAME}}.RESOURCE_CHANGE_LOG '
+    || '   WHERE SHARD_KEY = $1 AND RESOURCE_ID IN ( '
+    || '        SELECT RESOURCE_ID '
+    || '          FROM {{SCHEMA_NAME}}.' || p_resource_type || '_RESOURCES '
+    || '         WHERE SHARD_KEY = $2 '
+    || '           AND LOGICAL_RESOURCE_ID = $3) '
+    USING p_shard_key, p_shard_key, v_logical_resource_id;
+    
+    -- Step 1.1: Record the versions we need to delete if we are doing payload offload
+    EXECUTE 'INSERT INTO {{SCHEMA_NAME}}.erased_resources(shard_key, erased_resource_group_id, resource_type_id, logical_id, version_id) ' 
+        || '      SELECT SHARD_KEY, $1, $2, $3, version_id '
+        || '        FROM {{SCHEMA_NAME}}.' || p_resource_type || '_RESOURCES '
+        || '       WHERE SHARD_KEY = $4, LOGICAL_RESOURCE_ID = $5 '
+    USING p_erased_resource_group_id, v_resource_type_id, p_logical_id, p_shard_key, v_logical_resource_id;
+
+    -- Step 2: Delete All Versions from Resources Table 
+    EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.' || p_resource_type || '_RESOURCES WHERE SHARD_KEY = $1 AND LOGICAL_RESOURCE_ID = $2'
+    USING p_shard_key, v_logical_resource_id;
+    GET DIAGNOSTICS v_total = ROW_COUNT;
+
+    -- The delete_resource_parameters call is a function, so we have to use a select here, not call 
+    EXECUTE 'SELECT {{SCHEMA_NAME}}.delete_resource_parameters($1, $2, $3)'
+    USING p_shard_key, p_resource_type, v_logical_resource_id;
+    
+    -- Step 4: Delete from Logical Resources table 
+    EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.' || p_resource_type || '_LOGICAL_RESOURCES WHERE SHARD_KEY = $1 AND LOGICAL_RESOURCE_ID = $2'
+    USING p_shard_key, v_logical_resource_id;
+
+    -- Step 5: Delete from Global Logical Resources
+    EXECUTE 'DELETE FROM {{SCHEMA_NAME}}.LOGICAL_RESOURCES WHERE SHARD_KEY = $1 AND LOGICAL_RESOURCE_ID = $2 AND RESOURCE_TYPE_ID = $3'
+    USING p_shard_key, v_logical_resource_id, v_resource_type_id;
+  END IF;
+
+  -- Return the total number of deleted versions
+  o_deleted := v_total;
+END $$;
diff --git a/fhir-persistence-schema/src/test/java/com/ibm/fhir/schema/app/DataSchemaGeneratorTest.java b/fhir-persistence-schema/src/test/java/com/ibm/fhir/schema/app/DataSchemaGeneratorTest.java
index ffb936608f6..4ba936270e3 100644
--- a/fhir-persistence-schema/src/test/java/com/ibm/fhir/schema/app/DataSchemaGeneratorTest.java
+++ b/fhir-persistence-schema/src/test/java/com/ibm/fhir/schema/app/DataSchemaGeneratorTest.java
@@ -1,5 +1,5 @@
 /*
- * (C) Copyright IBM Corp. 2021
+ * (C) Copyright IBM Corp. 2021, 2022
  *
  * SPDX-License-Identifier: Apache-2.0
  */
@@ -8,7 +8,11 @@
 
 import org.testng.annotations.Test;
 
+import com.ibm.fhir.database.utils.api.ISchemaAdapter;
+import com.ibm.fhir.database.utils.api.SchemaApplyContext;
+import com.ibm.fhir.database.utils.api.SchemaType;
 import com.ibm.fhir.database.utils.common.JdbcTarget;
+import com.ibm.fhir.database.utils.common.PlainSchemaAdapter;
 import com.ibm.fhir.database.utils.db2.Db2Adapter;
 import com.ibm.fhir.database.utils.model.PhysicalDataModel;
 import com.ibm.fhir.database.utils.version.CreateVersionHistory;
@@ -24,9 +28,10 @@ public void testFHIRSchemaGeneratorCheckTags() {
         PrintConnection connection = test.new PrintConnection();
         JdbcTarget target = new JdbcTarget(connection);
         Db2Adapter adapter = new Db2Adapter(target);
+        ISchemaAdapter schemaAdapter = new PlainSchemaAdapter(adapter);
 
         // Set up the version history service first if it doesn't yet exist
-        CreateVersionHistory.createTableIfNeeded(Main.ADMIN_SCHEMANAME, adapter);
+        CreateVersionHistory.createTableIfNeeded(Main.ADMIN_SCHEMANAME, schemaAdapter);
 
         // Current version history for the database. This is used by applyWithHistory
         // to determine which updates to apply and to record the new changes as they
@@ -35,11 +40,12 @@ public void testFHIRSchemaGeneratorCheckTags() {
         vhs.setTarget(adapter);
 
         PhysicalDataModel pdm = new PhysicalDataModel();
-        FhirSchemaGenerator generator = new FhirSchemaGenerator(Main.ADMIN_SCHEMANAME, Main.DATA_SCHEMANAME, true);
+        FhirSchemaGenerator generator = new FhirSchemaGenerator(Main.ADMIN_SCHEMANAME, Main.DATA_SCHEMANAME, SchemaType.PLAIN);
         generator.buildSchema(pdm);
-        pdm.apply(adapter);
-        pdm.applyFunctions(adapter);
-        pdm.applyProcedures(adapter);
+        SchemaApplyContext context = SchemaApplyContext.getDefault();
+        pdm.apply(schemaAdapter, context);
+        pdm.applyFunctions(schemaAdapter, context);
+        pdm.applyProcedures(schemaAdapter, context);
 
         pdm.visit(new ConfirmTagsVisitor());
     }
diff --git a/fhir-persistence-schema/src/test/java/com/ibm/fhir/schema/app/JavaBatchSchemaGeneratorTest.java b/fhir-persistence-schema/src/test/java/com/ibm/fhir/schema/app/JavaBatchSchemaGeneratorTest.java
index 062ab9ba72e..c711f273c90 100644
--- a/fhir-persistence-schema/src/test/java/com/ibm/fhir/schema/app/JavaBatchSchemaGeneratorTest.java
+++ b/fhir-persistence-schema/src/test/java/com/ibm/fhir/schema/app/JavaBatchSchemaGeneratorTest.java
@@ -1,5 +1,5 @@
 /*
- * (C) Copyright IBM Corp. 2020,2021
+ * (C) Copyright IBM Corp. 2020, 2022
  *
  * SPDX-License-Identifier: Apache-2.0
  */
@@ -45,7 +45,10 @@
 
 import org.testng.annotations.Test;
 
+import com.ibm.fhir.database.utils.api.ISchemaAdapter;
+import com.ibm.fhir.database.utils.api.SchemaApplyContext;
 import com.ibm.fhir.database.utils.common.JdbcTarget;
+import com.ibm.fhir.database.utils.common.PlainSchemaAdapter;
 import com.ibm.fhir.database.utils.db2.Db2Adapter;
 import com.ibm.fhir.database.utils.model.AlterSequenceStartWith;
 import com.ibm.fhir.database.utils.model.AlterTableIdentityCache;
@@ -76,9 +79,10 @@ public void testJavaBatchSchemaGeneratorDb2() {
         PrintConnection connection = new PrintConnection();
         JdbcTarget target = new JdbcTarget(connection);
         Db2Adapter adapter = new Db2Adapter(target);
+        ISchemaAdapter schemaAdapter = new PlainSchemaAdapter(adapter);
 
         // Set up the version history service first if it doesn't yet exist
-        CreateVersionHistory.createTableIfNeeded(Main.ADMIN_SCHEMANAME, adapter);
+        CreateVersionHistory.createTableIfNeeded(Main.ADMIN_SCHEMANAME, schemaAdapter);
 
         // Current version history for the database. This is used by applyWithHistory
         // to determine which updates to apply and to record the new changes as they
@@ -89,9 +93,10 @@ public void testJavaBatchSchemaGeneratorDb2() {
         PhysicalDataModel pdm = new PhysicalDataModel();
         JavaBatchSchemaGenerator generator = new JavaBatchSchemaGenerator(Main.BATCH_SCHEMANAME);
         generator.buildJavaBatchSchema(pdm);
-        pdm.apply(adapter);
-        pdm.applyFunctions(adapter);
-        pdm.applyProcedures(adapter);
+        SchemaApplyContext context = SchemaApplyContext.getDefault();
+        pdm.apply(schemaAdapter, context);
+        pdm.applyFunctions(schemaAdapter, context);
+        pdm.applyProcedures(schemaAdapter, context);
 
         if (DEBUG) {
             for (Entry command : commands.entrySet()) {
@@ -107,9 +112,10 @@ public void testJavaBatchSchemaGeneratorPostgres() {
         PrintConnection connection = new PrintConnection();
         JdbcTarget target = new JdbcTarget(connection);
         PostgresAdapter adapter = new PostgresAdapter(target);
+        ISchemaAdapter schemaAdapter = new PlainSchemaAdapter(adapter);
 
         // Set up the version history service first if it doesn't yet exist
-        CreateVersionHistory.createTableIfNeeded(Main.ADMIN_SCHEMANAME, adapter);
+        CreateVersionHistory.createTableIfNeeded(Main.ADMIN_SCHEMANAME, schemaAdapter);
 
         // Current version history for the database. This is used by applyWithHistory
         // to determine which updates to apply and to record the new changes as they
@@ -120,8 +126,9 @@ public void testJavaBatchSchemaGeneratorPostgres() {
         PhysicalDataModel pdm = new PhysicalDataModel();
         JavaBatchSchemaGenerator generator = new JavaBatchSchemaGenerator(Main.BATCH_SCHEMANAME);
         generator.buildJavaBatchSchema(pdm);
-        pdm.apply(adapter);
-        pdm.applyFunctions(adapter);
+        SchemaApplyContext context = SchemaApplyContext.getDefault();
+        pdm.apply(schemaAdapter, context);
+        pdm.applyFunctions(schemaAdapter, context);
 
         if (DEBUG) {
             for (Entry command : commands.entrySet()) {
@@ -138,9 +145,10 @@ public void testJavaBatchSchemaGeneratorCheckTags() {
         PrintConnection connection = new PrintConnection();
         JdbcTarget target = new JdbcTarget(connection);
         Db2Adapter adapter = new Db2Adapter(target);
+        ISchemaAdapter schemaAdapter = new PlainSchemaAdapter(adapter);
 
         // Set up the version history service first if it doesn't yet exist
-        CreateVersionHistory.createTableIfNeeded(Main.ADMIN_SCHEMANAME, adapter);
+        CreateVersionHistory.createTableIfNeeded(Main.ADMIN_SCHEMANAME, schemaAdapter);
 
         // Current version history for the database. This is used by applyWithHistory
         // to determine which updates to apply and to record the new changes as they
@@ -151,9 +159,10 @@ public void testJavaBatchSchemaGeneratorCheckTags() {
         PhysicalDataModel pdm = new PhysicalDataModel();
         JavaBatchSchemaGenerator generator = new JavaBatchSchemaGenerator(Main.BATCH_SCHEMANAME);
         generator.buildJavaBatchSchema(pdm);
-        pdm.apply(adapter);
-        pdm.applyFunctions(adapter);
-        pdm.applyProcedures(adapter);
+        SchemaApplyContext context = SchemaApplyContext.getDefault();
+        pdm.apply(schemaAdapter, context);
+        pdm.applyFunctions(schemaAdapter, context);
+        pdm.applyProcedures(schemaAdapter, context);
 
         pdm.visit(new ConfirmTagsVisitor());
 
@@ -302,8 +311,15 @@ public Statement createStatement() throws SQLException {
 
         @Override
         public PreparedStatement prepareStatement(String sql) throws SQLException {
+
+            boolean hasRow = true;
+            if (sql.toUpperCase().startsWith("SELECT 1 FROM")) {
+                // this is one of our checks for the existing of a FK...which we want to
+                // say doesn't exist
+                hasRow = false;
+            }
             addCommand(sql);
-            return new PrintPreparedStatement();
+            return new PrintPreparedStatement(hasRow);
         }
 
         @Override
@@ -401,7 +417,7 @@ public Statement createStatement(int resultSetType, int resultSetConcurrency) th
 
         @Override
         public PreparedStatement prepareStatement(String sql, int resultSetType, int resultSetConcurrency) throws SQLException {
-            return new PrintPreparedStatement();
+            return new PrintPreparedStatement(true);
         }
 
         @Override
@@ -584,11 +600,15 @@ public int getNetworkTimeout() throws SQLException {
     }
 
     class PrintPreparedStatement implements java.sql.PreparedStatement {
+        private final boolean hasRow;
+        public PrintPreparedStatement(boolean hasRow) {
+            this.hasRow = hasRow;
+        }
 
         @Override
         public ResultSet executeQuery(String sql) throws SQLException {
             addCommand(sql);
-            return new PrintResultSet();
+            return new PrintResultSet(true);
         }
 
         @Override
@@ -838,7 +858,7 @@ public boolean isWrapperFor(Class iface) throws SQLException {
         @Override
         public ResultSet executeQuery() throws SQLException {
 
-            return new PrintResultSet();
+            return new PrintResultSet(this.hasRow);
         }
 
         @Override
@@ -1134,7 +1154,7 @@ public boolean isWrapperFor(Class iface) throws SQLException {
         @Override
         public ResultSet executeQuery(String sql) throws SQLException {
             addCommand(sql);
-            return new PrintResultSet();
+            return new PrintResultSet(true);
         }
 
         @Override
@@ -1372,7 +1392,11 @@ public boolean isCloseOnCompletion() throws SQLException {
     }
 
     class PrintResultSet implements java.sql.ResultSet {
+        private boolean hasRow;
 
+        public PrintResultSet(boolean hasRow) {
+            this.hasRow = hasRow;
+        }
         @Override
         public  T unwrap(Class iface) throws SQLException {
             return null;
@@ -1385,7 +1409,8 @@ public boolean isWrapperFor(Class iface) throws SQLException {
 
         @Override
         public boolean next() throws SQLException {
-            return true;
+            // pretend to have a row
+            return this.hasRow;
         }
 
         @Override
@@ -2425,7 +2450,7 @@ class PrintCallableStatement implements java.sql.CallableStatement {
 
         @Override
         public ResultSet executeQuery() throws SQLException {
-            return new PrintResultSet();
+            return new PrintResultSet(true);
         }
 
         @Override
@@ -2704,13 +2729,11 @@ public void setNClob(int parameterIndex, Reader reader) throws SQLException {
 
         @Override
         public ResultSet executeQuery(String sql) throws SQLException {
-
-            return new PrintResultSet();
+            return new PrintResultSet(true);
         }
 
         @Override
         public int executeUpdate(String sql) throws SQLException {
-
             return 0;
         }
 
diff --git a/fhir-persistence-schema/src/test/java/com/ibm/fhir/schema/app/OAuthSchemaGeneratorTest.java b/fhir-persistence-schema/src/test/java/com/ibm/fhir/schema/app/OAuthSchemaGeneratorTest.java
index 32e7cd491b9..8c21ec516ab 100644
--- a/fhir-persistence-schema/src/test/java/com/ibm/fhir/schema/app/OAuthSchemaGeneratorTest.java
+++ b/fhir-persistence-schema/src/test/java/com/ibm/fhir/schema/app/OAuthSchemaGeneratorTest.java
@@ -1,5 +1,5 @@
 /*
- * (C) Copyright IBM Corp. 2021
+ * (C) Copyright IBM Corp. 2021, 2022
  *
  * SPDX-License-Identifier: Apache-2.0
  */
@@ -8,7 +8,10 @@
 
 import org.testng.annotations.Test;
 
+import com.ibm.fhir.database.utils.api.ISchemaAdapter;
+import com.ibm.fhir.database.utils.api.SchemaApplyContext;
 import com.ibm.fhir.database.utils.common.JdbcTarget;
+import com.ibm.fhir.database.utils.common.PlainSchemaAdapter;
 import com.ibm.fhir.database.utils.db2.Db2Adapter;
 import com.ibm.fhir.database.utils.model.PhysicalDataModel;
 import com.ibm.fhir.database.utils.version.CreateVersionHistory;
@@ -24,9 +27,10 @@ public void testOAuthSchemaGeneratorCheckTags() {
         PrintConnection connection = test.new PrintConnection();
         JdbcTarget target = new JdbcTarget(connection);
         Db2Adapter adapter = new Db2Adapter(target);
+        ISchemaAdapter schemaAdapter = new PlainSchemaAdapter(adapter);
 
         // Set up the version history service first if it doesn't yet exist
-        CreateVersionHistory.createTableIfNeeded(Main.ADMIN_SCHEMANAME, adapter);
+        CreateVersionHistory.createTableIfNeeded(Main.ADMIN_SCHEMANAME, schemaAdapter);
 
         // Current version history for the database. This is used by applyWithHistory
         // to determine which updates to apply and to record the new changes as they
@@ -37,9 +41,10 @@ public void testOAuthSchemaGeneratorCheckTags() {
         PhysicalDataModel pdm = new PhysicalDataModel();
         OAuthSchemaGenerator generator = new OAuthSchemaGenerator(Main.OAUTH_SCHEMANAME);
         generator.buildOAuthSchema(pdm);
-        pdm.apply(adapter);
-        pdm.applyFunctions(adapter);
-        pdm.applyProcedures(adapter);
+        SchemaApplyContext context = SchemaApplyContext.getDefault();
+        pdm.apply(schemaAdapter, context);
+        pdm.applyFunctions(schemaAdapter, context);
+        pdm.applyProcedures(schemaAdapter, context);
 
         pdm.visit(new ConfirmTagsVisitor());
     }
diff --git a/fhir-persistence-schema/src/test/java/com/ibm/fhir/schema/control/FhirSchemaServiceTest.java b/fhir-persistence-schema/src/test/java/com/ibm/fhir/schema/control/FhirSchemaServiceTest.java
index 301652a01a7..fe041e08e12 100644
--- a/fhir-persistence-schema/src/test/java/com/ibm/fhir/schema/control/FhirSchemaServiceTest.java
+++ b/fhir-persistence-schema/src/test/java/com/ibm/fhir/schema/control/FhirSchemaServiceTest.java
@@ -1,5 +1,5 @@
 /*
- * (C) Copyright IBM Corp. 2019, 2020
+ * (C) Copyright IBM Corp. 2019, 2022
  *
  * SPDX-License-Identifier: Apache-2.0
  */
@@ -15,6 +15,10 @@
 
 import org.testng.annotations.Test;
 
+import com.ibm.fhir.database.utils.api.ISchemaAdapter;
+import com.ibm.fhir.database.utils.api.SchemaApplyContext;
+import com.ibm.fhir.database.utils.api.SchemaType;
+import com.ibm.fhir.database.utils.common.PlainSchemaAdapter;
 import com.ibm.fhir.database.utils.common.PrintTarget;
 import com.ibm.fhir.database.utils.db2.Db2Adapter;
 import com.ibm.fhir.database.utils.db2.Db2Translator;
@@ -22,8 +26,6 @@
 import com.ibm.fhir.database.utils.model.PhysicalDataModel;
 import com.ibm.fhir.database.utils.model.Table;
 import com.ibm.fhir.database.utils.version.CreateVersionHistory;
-import com.ibm.fhir.schema.control.FhirSchemaConstants;
-import com.ibm.fhir.schema.control.FhirSchemaGenerator;
 import com.ibm.fhir.task.api.ITaskCollector;
 import com.ibm.fhir.task.core.service.TaskService;
 
@@ -42,7 +44,7 @@ public void testDb2TableCreation() {
 
         // Create an instance of the service and use it to test creation
         // of the FHIR schema
-        FhirSchemaGenerator gen = new FhirSchemaGenerator(ADMIN_SCHEMA_NAME, SCHEMA_NAME, false);
+        FhirSchemaGenerator gen = new FhirSchemaGenerator(ADMIN_SCHEMA_NAME, SCHEMA_NAME, SchemaType.PLAIN);
         PhysicalDataModel model = new PhysicalDataModel();
         gen.buildSchema(model);
 
@@ -51,7 +53,9 @@ public void testDb2TableCreation() {
 
         // Pretend that our target is a DB2 database
         Db2Adapter adapter = new Db2Adapter(tgt);
-        model.apply(adapter);
+        ISchemaAdapter schemaAdapter = new PlainSchemaAdapter(adapter);
+        SchemaApplyContext context = SchemaApplyContext.getDefault();
+        model.apply(schemaAdapter, context);
     }
 
     @Test
@@ -60,7 +64,7 @@ public void testParallelTableCreation() {
 
         // Create an instance of the service and use it to test creation
         // of the FHIR schema
-        FhirSchemaGenerator gen = new FhirSchemaGenerator(ADMIN_SCHEMA_NAME, SCHEMA_NAME, false);
+        FhirSchemaGenerator gen = new FhirSchemaGenerator(ADMIN_SCHEMA_NAME, SCHEMA_NAME, SchemaType.PLAIN);
         PhysicalDataModel model = new PhysicalDataModel();
         gen.buildSchema(model);
 
@@ -71,7 +75,9 @@ public void testParallelTableCreation() {
         ITaskCollector collector = taskService.makeTaskCollector(pool);
         PrintTarget tgt = new PrintTarget(null, logger.isLoggable(Level.FINE));
         Db2Adapter adapter = new Db2Adapter(tgt);
-        model.collect(collector, adapter, new TransactionProviderTest(), vhs);
+        ISchemaAdapter schemaAdapter = new PlainSchemaAdapter(adapter);
+        SchemaApplyContext context = SchemaApplyContext.getDefault();
+        model.collect(collector, schemaAdapter, context, new TransactionProviderTest(), vhs);
 
         // FHIR in the hole!
         collector.startAndWait();
@@ -85,7 +91,7 @@ public void testDerbyTableCreation() {
 
         // Create an instance of the service and use it to test creation
         // of the FHIR schema
-        FhirSchemaGenerator gen = new FhirSchemaGenerator(ADMIN_SCHEMA_NAME, SCHEMA_NAME, false);
+        FhirSchemaGenerator gen = new FhirSchemaGenerator(ADMIN_SCHEMA_NAME, SCHEMA_NAME, SchemaType.PLAIN);
         PhysicalDataModel model = new PhysicalDataModel();
         gen.buildSchema(model);
 
@@ -94,7 +100,9 @@ public void testDerbyTableCreation() {
 
         // Pretend that our target is a Derby database
         DerbyAdapter adapter = new DerbyAdapter(tgt);
-        model.apply(adapter);
+        ISchemaAdapter schemaAdapter = new PlainSchemaAdapter(adapter);
+        SchemaApplyContext context = SchemaApplyContext.getDefault();
+        model.apply(schemaAdapter, context);
     }
 
     @Test
@@ -102,7 +110,7 @@ public void testTenantPartitioning() {
 
         // Create an instance of the service and use it to test creation
         // of the FHIR schema
-        FhirSchemaGenerator gen = new FhirSchemaGenerator(ADMIN_SCHEMA_NAME, SCHEMA_NAME, false);
+        FhirSchemaGenerator gen = new FhirSchemaGenerator(ADMIN_SCHEMA_NAME, SCHEMA_NAME, SchemaType.PLAIN);
         PhysicalDataModel model = new PhysicalDataModel();
         gen.buildSchema(model);
 
@@ -125,7 +133,7 @@ public void testDrop() {
 
         // Create an instance of the service and use it to test creation
         // of the FHIR schema
-        FhirSchemaGenerator gen = new FhirSchemaGenerator(ADMIN_SCHEMA_NAME, SCHEMA_NAME, false);
+        FhirSchemaGenerator gen = new FhirSchemaGenerator(ADMIN_SCHEMA_NAME, SCHEMA_NAME, SchemaType.PLAIN);
         PhysicalDataModel model = new PhysicalDataModel();
         gen.buildSchema(model);
 
@@ -134,7 +142,8 @@ public void testDrop() {
 
         // Pretend that our target is a DB2 database
         Db2Adapter adapter = new Db2Adapter(tgt);
-        model.drop(adapter);
+        ISchemaAdapter schemaAdapter = new PlainSchemaAdapter(adapter);
+        model.drop(schemaAdapter);
     }
 
     @Test
@@ -144,7 +153,8 @@ public void testVersionHistorySchema() {
 
         // Pretend that our target is a DB2 database
         Db2Adapter adapter = new Db2Adapter(tgt);
-        CreateVersionHistory.createTableIfNeeded(ADMIN_SCHEMA_NAME, adapter);
+        ISchemaAdapter schemaAdapter = new PlainSchemaAdapter(adapter);
+        CreateVersionHistory.createTableIfNeeded(ADMIN_SCHEMA_NAME, schemaAdapter);
 
     }
 }
diff --git a/fhir-persistence-schema/src/test/java/com/ibm/fhir/schema/control/ParallelBuildTest.java b/fhir-persistence-schema/src/test/java/com/ibm/fhir/schema/control/ParallelBuildTest.java
index 0fa95bb0c29..3427ace4e28 100644
--- a/fhir-persistence-schema/src/test/java/com/ibm/fhir/schema/control/ParallelBuildTest.java
+++ b/fhir-persistence-schema/src/test/java/com/ibm/fhir/schema/control/ParallelBuildTest.java
@@ -1,5 +1,5 @@
 /*
- * (C) Copyright IBM Corp. 2019, 2020
+ * (C) Copyright IBM Corp. 2019, 2022
  *
  * SPDX-License-Identifier: Apache-2.0
  */
@@ -13,10 +13,13 @@
 
 import org.testng.annotations.Test;
 
+import com.ibm.fhir.database.utils.api.ISchemaAdapter;
+import com.ibm.fhir.database.utils.api.SchemaApplyContext;
+import com.ibm.fhir.database.utils.api.SchemaType;
+import com.ibm.fhir.database.utils.common.PlainSchemaAdapter;
 import com.ibm.fhir.database.utils.common.PrintTarget;
 import com.ibm.fhir.database.utils.db2.Db2Adapter;
 import com.ibm.fhir.database.utils.model.PhysicalDataModel;
-import com.ibm.fhir.schema.control.FhirSchemaGenerator;
 import com.ibm.fhir.task.api.ITaskCollector;
 import com.ibm.fhir.task.core.service.TaskService;
 
@@ -34,7 +37,7 @@ public void testParallelTableCreation() {
 
         // Create an instance of the service and use it to test creation
         // of the FHIR schema
-        FhirSchemaGenerator gen = new FhirSchemaGenerator(ADMIN_SCHEMA_NAME, SCHEMA_NAME, false);
+        FhirSchemaGenerator gen = new FhirSchemaGenerator(ADMIN_SCHEMA_NAME, SCHEMA_NAME, SchemaType.PLAIN);
         PhysicalDataModel model = new PhysicalDataModel();
         gen.buildSchema(model);
 
@@ -45,7 +48,9 @@ public void testParallelTableCreation() {
         ITaskCollector collector = taskService.makeTaskCollector(pool);
         PrintTarget tgt = new PrintTarget(null, logger.isLoggable(Level.FINE));
         Db2Adapter adapter = new Db2Adapter(tgt);
-        model.collect(collector, adapter, new TransactionProviderTest(), vhs);
+        ISchemaAdapter schemaAdapter = new PlainSchemaAdapter(adapter);
+        SchemaApplyContext context = SchemaApplyContext.getDefault();
+        model.collect(collector, schemaAdapter, context, new TransactionProviderTest(), vhs);
 
         // FHIR in the hole!
         collector.startAndWait();
diff --git a/fhir-persistence-schema/src/test/java/com/ibm/fhir/schema/derby/DerbyFhirDatabaseTest.java b/fhir-persistence-schema/src/test/java/com/ibm/fhir/schema/derby/DerbyFhirDatabaseTest.java
index 2d4e75fc93d..ed732636169 100644
--- a/fhir-persistence-schema/src/test/java/com/ibm/fhir/schema/derby/DerbyFhirDatabaseTest.java
+++ b/fhir-persistence-schema/src/test/java/com/ibm/fhir/schema/derby/DerbyFhirDatabaseTest.java
@@ -1,5 +1,5 @@
 /*
- * (C) Copyright IBM Corp. 2019, 2021
+ * (C) Copyright IBM Corp. 2019, 2022
  *
  * SPDX-License-Identifier: Apache-2.0
  */
@@ -19,10 +19,13 @@
 import org.testng.annotations.Test;
 
 import com.ibm.fhir.database.utils.api.IConnectionProvider;
+import com.ibm.fhir.database.utils.api.ISchemaAdapter;
 import com.ibm.fhir.database.utils.api.ITransaction;
 import com.ibm.fhir.database.utils.api.ITransactionProvider;
+import com.ibm.fhir.database.utils.api.SchemaType;
 import com.ibm.fhir.database.utils.common.GetSequenceNextValueDAO;
 import com.ibm.fhir.database.utils.common.JdbcTarget;
+import com.ibm.fhir.database.utils.common.PlainSchemaAdapter;
 import com.ibm.fhir.database.utils.common.SchemaInfoObject;
 import com.ibm.fhir.database.utils.derby.DerbyAdapter;
 import com.ibm.fhir.database.utils.derby.DerbyMaster;
@@ -76,17 +79,18 @@ protected void testDrop(IConnectionProvider cp, String schemaName) throws SQLExc
         PoolConnectionProvider connectionPool = new PoolConnectionProvider(cp, 10);
         ITransactionProvider transactionProvider = new SimpleTransactionProvider(cp);
         DerbyAdapter adapter = new DerbyAdapter(connectionPool);
+        ISchemaAdapter schemaAdapter = new PlainSchemaAdapter(adapter);
         VersionHistoryService vhs = new VersionHistoryService(ADMIN_SCHEMA_NAME, schemaName);
         vhs.setTransactionProvider(transactionProvider);
         vhs.setTarget(adapter);
 
         try (ITransaction tx = transactionProvider.getTransaction()) {
             PhysicalDataModel pdm = new PhysicalDataModel();
-            FhirSchemaGenerator gen = new FhirSchemaGenerator(ADMIN_SCHEMA_NAME, schemaName, false);
+            FhirSchemaGenerator gen = new FhirSchemaGenerator(ADMIN_SCHEMA_NAME, schemaName, SchemaType.PLAIN);
             gen.buildSchema(pdm);
-            pdm.drop(adapter, FhirSchemaGenerator.SCHEMA_GROUP_TAG, FhirSchemaGenerator.FHIRDATA_GROUP);
+            pdm.drop(schemaAdapter, FhirSchemaGenerator.SCHEMA_GROUP_TAG, FhirSchemaGenerator.FHIRDATA_GROUP);
 
-            CreateWholeSchemaVersion.dropTable(schemaName, adapter);
+            CreateWholeSchemaVersion.dropTable(schemaName, schemaAdapter);
 
             // Check that the schema is empty
             List schemaObjects = adapter.listSchemaObjects(schemaName);
@@ -138,7 +142,7 @@ protected void checkDatabase(IConnectionProvider cp, String schemaName) throws S
 
                 // Check that we have the correct number of tables. This will need to be updated
                 // whenever tables, views or sequences are added or removed
-                assertEquals(adapter.listSchemaObjects(schemaName).size(), 2087);
+                assertEquals(adapter.listSchemaObjects(schemaName).size(), 2564);
                 c.commit();
             } catch (Throwable t) {
                 c.rollback();
diff --git a/fhir-persistence-schema/src/test/java/com/ibm/fhir/schema/derby/DerbyMigrationTest.java b/fhir-persistence-schema/src/test/java/com/ibm/fhir/schema/derby/DerbyMigrationTest.java
index 37c18593649..1aefbcd5075 100644
--- a/fhir-persistence-schema/src/test/java/com/ibm/fhir/schema/derby/DerbyMigrationTest.java
+++ b/fhir-persistence-schema/src/test/java/com/ibm/fhir/schema/derby/DerbyMigrationTest.java
@@ -30,6 +30,7 @@
 import com.ibm.fhir.database.utils.api.IDatabaseAdapter;
 import com.ibm.fhir.database.utils.api.ITransaction;
 import com.ibm.fhir.database.utils.api.ITransactionProvider;
+import com.ibm.fhir.database.utils.api.SchemaType;
 import com.ibm.fhir.database.utils.derby.DerbyAdapter;
 import com.ibm.fhir.database.utils.derby.DerbyConnectionProvider;
 import com.ibm.fhir.database.utils.derby.DerbyMaster;
@@ -174,7 +175,7 @@ public void testMigrateFhirSchema() throws Exception {
     private void createOrUpgradeSchema(DerbyMaster db, IConnectionProvider pool, VersionHistoryService vhs, Set resourceTypes) throws SQLException {
 
 
-        FhirSchemaGenerator gen = new FhirSchemaGenerator(ADMIN_SCHEMA_NAME, SCHEMA_NAME, false, resourceTypes);
+        FhirSchemaGenerator gen = new FhirSchemaGenerator(ADMIN_SCHEMA_NAME, SCHEMA_NAME, SchemaType.PLAIN, resourceTypes);
         PhysicalDataModel pdm = new PhysicalDataModel();
         gen.buildSchema(pdm);
 
diff --git a/fhir-persistence-schema/src/test/java/com/ibm/fhir/schema/derby/DerbySchemaVersionsTest.java b/fhir-persistence-schema/src/test/java/com/ibm/fhir/schema/derby/DerbySchemaVersionsTest.java
index ecc13bb7a6c..614d3bf60dc 100644
--- a/fhir-persistence-schema/src/test/java/com/ibm/fhir/schema/derby/DerbySchemaVersionsTest.java
+++ b/fhir-persistence-schema/src/test/java/com/ibm/fhir/schema/derby/DerbySchemaVersionsTest.java
@@ -52,7 +52,7 @@ public void test() throws Exception {
 
             // Make sure we can correctly determine the latest schema version value
             svm.updateSchemaVersion();
-            assertEquals(svm.getVersionForSchema(), FhirSchemaVersion.V0026.vid());
+            assertEquals(svm.getVersionForSchema(), FhirSchemaVersion.V0028.vid());
 
             assertFalse(svm.isSchemaOld());
        }
diff --git a/fhir-persistence/pom.xml b/fhir-persistence/pom.xml
index 6d22adfc846..0ba50f2d2cf 100644
--- a/fhir-persistence/pom.xml
+++ b/fhir-persistence/pom.xml
@@ -36,6 +36,10 @@
             fhir-search
             ${project.version}
         
+        
+            com.google.code.gson
+            gson
+        
         
             ${project.groupId}
             fhir-model
diff --git a/fhir-persistence/src/main/java/com/ibm/fhir/persistence/FHIRPersistence.java b/fhir-persistence/src/main/java/com/ibm/fhir/persistence/FHIRPersistence.java
index d69a3a3ba37..d7f32e8b807 100644
--- a/fhir-persistence/src/main/java/com/ibm/fhir/persistence/FHIRPersistence.java
+++ b/fhir-persistence/src/main/java/com/ibm/fhir/persistence/FHIRPersistence.java
@@ -211,11 +211,12 @@ default boolean isOffloadingSupported() {
      * @param indexIds list of index IDs of resources to reindex, or null
      * @param resourceLogicalId resourceType/logicalId value of a specific resource to reindex, or null;
      * this parameter is ignored if the indexIds parameter value is non-null
+     * @param force if true, always replace the stored parameters
      * @return count of the number of resources reindexed by this call
      * @throws FHIRPersistenceException
      */
     int reindex(FHIRPersistenceContext context, OperationOutcome.Builder operationOutcomeResult, java.time.Instant tstamp, List indexIds,
-        String resourceLogicalId) throws FHIRPersistenceException;
+        String resourceLogicalId, boolean force) throws FHIRPersistenceException;
 
     /**
      * Special function for high speed export of resource payloads. The process
@@ -248,6 +249,7 @@ default boolean isChangesSupported() {
     /**
      * Fetch up to resourceCount records from the RESOURCE_CHANGE_LOG table.
      *
+     * @param context the FHIRPersistenceContext associated with the current request
      * @param resourceCount the max number of resource change records to fetch
      * @param sinceLastModified filter records with record.lastUpdate >= sinceLastModified. Optional.
      * @param beforeLastModified filter records with record.lastUpdate <= beforeLastModified. Optional.
@@ -258,7 +260,7 @@ default boolean isChangesSupported() {
      * @return a list containing up to resourceCount elements describing resources which have changed
      * @throws FHIRPersistenceException
      */
-    List changes(int resourceCount, java.time.Instant sinceLastModified,
+    List changes(FHIRPersistenceContext context, int resourceCount, java.time.Instant sinceLastModified,
         java.time.Instant beforeLastModified, Long changeIdMarker, List resourceTypeNames,
         boolean excludeTransactionTimeoutWindow, HistorySortOrder historySortOrder)
         throws FHIRPersistenceException;
@@ -266,17 +268,19 @@ List changes(int resourceCount, java.time.Instant since
     /**
      * Erases part or a whole of a resource in the data layer.
      *
+     * @param context the FHIRPersistenceContext associated with this request
      * @param eraseDto the details of the user input
      * @return a record indicating the success or partial success of the erase
      * @throws FHIRPersistenceException
      */
-    default ResourceEraseRecord erase(EraseDTO eraseDto) throws FHIRPersistenceException {
+    default ResourceEraseRecord erase(FHIRPersistenceContext context, EraseDTO eraseDto) throws FHIRPersistenceException {
         throw new FHIRPersistenceException("Erase is not supported");
     }
 
     /**
      * Retrieves a list of index IDs available for reindexing.
      *
+     * @param context the FHIRPersistenceContext associated with this request
      * @param count the maximum nuber of index IDs to retrieve
      * @param notModifiedAfter only retrieve index IDs for resources not last updated after the specified timestamp
      * @param afterIndexId retrieve index IDs starting after this specified index ID, or null to start with first index ID
@@ -284,7 +288,7 @@ default ResourceEraseRecord erase(EraseDTO eraseDto) throws FHIRPersistenceExcep
      * @return list of index IDs available for reindexing
      * @throws FHIRPersistenceException
      */
-    List retrieveIndex(int count, java.time.Instant notModifiedAfter, Long afterIndexId, String resourceTypeName)
+    List retrieveIndex(FHIRPersistenceContext context, int count, java.time.Instant notModifiedAfter, Long afterIndexId, String resourceTypeName)
         throws FHIRPersistenceException;
 
     /**
diff --git a/fhir-persistence/src/main/java/com/ibm/fhir/persistence/context/FHIRPersistenceContext.java b/fhir-persistence/src/main/java/com/ibm/fhir/persistence/context/FHIRPersistenceContext.java
index 4f191c3944e..e9d823c06c2 100644
--- a/fhir-persistence/src/main/java/com/ibm/fhir/persistence/context/FHIRPersistenceContext.java
+++ b/fhir-persistence/src/main/java/com/ibm/fhir/persistence/context/FHIRPersistenceContext.java
@@ -49,4 +49,11 @@ public interface FHIRPersistenceContext {
      * @return
      */
     PayloadPersistenceResponse getOffloadResponse();
+
+    /**
+     * Get the key used for sharding used by the distributed schema variant. If
+     * the tenant is not configured for distribution, the value will be null
+     * @return the shard key value specified in the request
+     */
+    String getRequestShard();
 }
\ No newline at end of file
diff --git a/fhir-persistence/src/main/java/com/ibm/fhir/persistence/context/FHIRPersistenceContextFactory.java b/fhir-persistence/src/main/java/com/ibm/fhir/persistence/context/FHIRPersistenceContextFactory.java
index ff9e85f13c1..273703b04cc 100644
--- a/fhir-persistence/src/main/java/com/ibm/fhir/persistence/context/FHIRPersistenceContextFactory.java
+++ b/fhir-persistence/src/main/java/com/ibm/fhir/persistence/context/FHIRPersistenceContextFactory.java
@@ -60,10 +60,12 @@ public static FHIRPersistenceContext createPersistenceContext(FHIRPersistenceEve
      * Returns a FHIRPersistenceContext that contains a FHIRPersistenceEvent and a FHIRSearchContext.
      * @param event the FHIRPersistenceEvent instance to be contained in the FHIRPersistenceContext instance
      * @param searchContext the FHIRSearchContext instance to be contained in the FHIRPersistenceContext instance
+     * @param requestShard the requested shard; can be null
      */
-    public static FHIRPersistenceContext createPersistenceContext(FHIRPersistenceEvent event, FHIRSearchContext searchContext) {
+    public static FHIRPersistenceContext createPersistenceContext(FHIRPersistenceEvent event, FHIRSearchContext searchContext, String requestShard) {
         return FHIRPersistenceContextImpl.builder(event)
                 .withSearchContext(searchContext)
+                .withRequestShard(requestShard)
                 .build();
     }
 
diff --git a/fhir-persistence/src/main/java/com/ibm/fhir/persistence/context/impl/FHIRPersistenceContextImpl.java b/fhir-persistence/src/main/java/com/ibm/fhir/persistence/context/impl/FHIRPersistenceContextImpl.java
index 9458f171dcd..b43d4241b01 100644
--- a/fhir-persistence/src/main/java/com/ibm/fhir/persistence/context/impl/FHIRPersistenceContextImpl.java
+++ b/fhir-persistence/src/main/java/com/ibm/fhir/persistence/context/impl/FHIRPersistenceContextImpl.java
@@ -22,7 +22,8 @@ public class FHIRPersistenceContextImpl implements FHIRPersistenceContext {
     private FHIRHistoryContext historyContext;
     private FHIRSearchContext searchContext;
     private Integer ifNoneMatch;
-
+    private String requestShard;
+    
     // The response from the payload persistence (offloading) call, if any
     private PayloadPersistenceResponse offloadResponse;
 
@@ -44,7 +45,8 @@ public static class Builder {
         private FHIRSearchContext searchContext;
         private Integer ifNoneMatch;
         private PayloadPersistenceResponse offloadResponse;
-
+        private String requestShard;
+        
         /**
          * Protected constructor
          * @param event
@@ -69,7 +71,8 @@ public FHIRPersistenceContext build() {
             }
             impl.setIfNoneMatch(ifNoneMatch);
             impl.setOffloadResponse(offloadResponse);
-
+            impl.setRequestShard(requestShard);
+            
             return impl;
         }
 
@@ -103,6 +106,16 @@ public Builder withIfNoneMatch(Integer ifNoneMatch) {
             return this;
         }
 
+        /**
+         * Build with the requestShard value
+         * @param requestShard
+         * @return
+         */
+        public Builder withRequestShard(String requestShard) {
+            this.requestShard = requestShard;
+            return this;
+        }
+
         /**
          * Build with the given offloadResponse
          * @param offloadResponse
@@ -156,6 +169,19 @@ public FHIRSearchContext getSearchContext() {
         return this.searchContext;
     }
 
+    @Override
+    public String getRequestShard() {
+        return this.requestShard;
+    }
+
+    /**
+     * Set the shardKey value
+     * @param value
+     */
+    public void setRequestShard(String value) {
+        this.requestShard = value;
+    }
+
     /**
      * Setter for the If-None-Match header value
      * @param ifNoneMatch
diff --git a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/exception/FHIRPersistenceDataAccessException.java b/fhir-persistence/src/main/java/com/ibm/fhir/persistence/exception/FHIRPersistenceDataAccessException.java
similarity index 91%
rename from fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/exception/FHIRPersistenceDataAccessException.java
rename to fhir-persistence/src/main/java/com/ibm/fhir/persistence/exception/FHIRPersistenceDataAccessException.java
index c97e5fa5fcc..c59849c0cb7 100644
--- a/fhir-persistence-jdbc/src/main/java/com/ibm/fhir/persistence/jdbc/exception/FHIRPersistenceDataAccessException.java
+++ b/fhir-persistence/src/main/java/com/ibm/fhir/persistence/exception/FHIRPersistenceDataAccessException.java
@@ -1,15 +1,14 @@
 /*
- * (C) Copyright IBM Corp. 2017,2019
+ * (C) Copyright IBM Corp. 2017, 2022
  *
  * SPDX-License-Identifier: Apache-2.0
  */
 
-package com.ibm.fhir.persistence.jdbc.exception;
+package com.ibm.fhir.persistence.exception;
 
 import java.util.Collection;
 
 import com.ibm.fhir.model.resource.OperationOutcome;
-import com.ibm.fhir.persistence.exception.FHIRPersistenceException;
 
 /**
  * This exception class represents failures encountered while attempting to access (read, write) data in the FHIR DB. 
diff --git a/fhir-persistence/src/main/java/com/ibm/fhir/persistence/helper/RemoteIndexSupport.java b/fhir-persistence/src/main/java/com/ibm/fhir/persistence/helper/RemoteIndexSupport.java
new file mode 100644
index 00000000000..caa35512fca
--- /dev/null
+++ b/fhir-persistence/src/main/java/com/ibm/fhir/persistence/helper/RemoteIndexSupport.java
@@ -0,0 +1,73 @@
+/*
+ * (C) Copyright IBM Corp. 2022
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+ 
+package com.ibm.fhir.persistence.helper;
+
+import java.time.Instant;
+import java.time.format.DateTimeFormatter;
+import java.util.logging.Logger;
+
+import com.google.gson.Gson;
+import com.google.gson.GsonBuilder;
+import com.google.gson.JsonDeserializer;
+import com.google.gson.JsonPrimitive;
+import com.google.gson.JsonSerializer;
+import com.ibm.fhir.persistence.index.RemoteIndexMessage;
+
+/**
+ * Utility methods supporting the fhir-remote-index consumer
+ */
+public class RemoteIndexSupport {
+    private static final Logger logger = Logger.getLogger(RemoteIndexSupport.class.getName());
+    private static final DateTimeFormatter formatter = DateTimeFormatter.ISO_INSTANT;
+
+    /**
+     * Get an instance of Gson configured to support serialization/deserialization of
+     * remote index messages (sent through Kafka as strings)
+     * @return
+     */
+    public static Gson getGson() {
+        Gson gson = new GsonBuilder()
+                .registerTypeAdapter(Instant.class, (JsonSerializer) (value, type, context) ->
+                    new JsonPrimitive(formatter.format(value))
+                )
+                .registerTypeAdapter(Instant.class, (JsonDeserializer) (jsonElement, type, context) ->
+                        formatter.parse(jsonElement.getAsString(), Instant::from)
+                )
+                .create();
+        
+        return gson;
+    }
+
+    /**
+     * Unmarshall the JSON payload parameter as a RemoteIndexMessage 
+     * @param jsonPayload
+     * @return
+     */
+    public static RemoteIndexMessage unmarshall(String jsonPayload) {
+        try {
+            Gson gson = getGson();
+            return gson.fromJson(jsonPayload, RemoteIndexMessage.class);
+        } catch (Throwable t) {
+            // We need to sink this error to avoid poison messages from 
+            // blocking the queues.
+            // TODO. Perhaps push this to a dedicated error topic
+            logger.severe("Not a RemoteIndexMessage. Ignoring: '" + jsonPayload + "'");
+        }
+        return null;
+
+    }
+
+    /**
+     * Marshall the RemoteIndexMessage to a JSON string
+     * @param message
+     * @return
+     */
+    public static String marshallToString(RemoteIndexMessage message) {
+        Gson gson = getGson();
+        return gson.toJson(message);
+    }
+}
diff --git a/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/CanonicalSupport.java b/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/CanonicalSupport.java
new file mode 100644
index 00000000000..c8f9848b489
--- /dev/null
+++ b/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/CanonicalSupport.java
@@ -0,0 +1,87 @@
+/*
+ * (C) Copyright IBM Corp. 2022
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+ 
+package com.ibm.fhir.persistence.index;
+
+import com.ibm.fhir.persistence.exception.FHIRPersistenceException;
+
+/**
+ * Utility methods supporting the processing of profile search
+ * parameters which are stored using common_canonical_values
+ */
+public class CanonicalSupport {
+
+    /**
+     * Split the given string value to extract the profile url, version
+     * and fragment parts if they exist
+     * @param stringValue
+     * @return
+     */
+    public static ProfileParameter createProfileParameter(String name, String stringValue) {
+        ProfileParameter result;
+        try {
+            result = parseCanonicalValue(stringValue);
+        } catch (FHIRPersistenceException e) {
+            // Not a valid version/fragment format - just use the input string as the whole uri
+            result = new ProfileParameter();
+            result.setUrl(stringValue);
+        }
+        result.setName(name);
+        return result;
+    }
+
+    /**
+     * Parse the canonical value.
+     * @param canonicalValue
+     * @return
+     */
+    private static ProfileParameter parseCanonicalValue(String canonicalValue) throws FHIRPersistenceException {
+        String uri = canonicalValue;
+        String version = null;
+        String fragment = null;
+        
+        // Parse the canonical value to extract the URI|VERSION#FRAGMENT pieces
+        if (canonicalValue != null) {
+            int vindex = canonicalValue.indexOf('|');
+            int findex = canonicalValue.indexOf('#');
+            if (vindex == 0 || findex == 0 || vindex > findex && findex > -1) {
+                throw new FHIRPersistenceException("Invalid canonical URI");
+            }
+            
+            // Extract version if given
+            if (vindex > 0) {
+                if (findex > -1) {
+                    version = canonicalValue.substring(vindex+1, findex); // everything after the | but before the #
+                } else {
+                    version = canonicalValue.substring(vindex+1); // everything after the |
+                }
+                if (version.isEmpty()) {
+                    version = null;
+                }
+                uri = canonicalValue.substring(0, vindex); // everything before the |
+            }
+
+            // Extract fragment if given
+            if (findex > 0) {
+                fragment = canonicalValue.substring(findex+1);
+                if (fragment.isEmpty()) {
+                    fragment = null;
+                }
+
+                if (vindex < 0) {
+                    // fragment but no version
+                    uri = canonicalValue.substring(0, findex); // everything before the #
+                }
+            }
+        }
+        
+        ProfileParameter result = new ProfileParameter();
+        result.setUrl(uri);
+        result.setVersion(version);
+        result.setFragment(fragment);
+        return result;
+    }
+}
diff --git a/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/DateParameter.java b/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/DateParameter.java
new file mode 100644
index 00000000000..ad5d5d73a3a
--- /dev/null
+++ b/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/DateParameter.java
@@ -0,0 +1,59 @@
+/*
+ * (C) Copyright IBM Corp. 2022
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+ 
+package com.ibm.fhir.persistence.index;
+
+import java.time.Instant;
+
+/**
+ * A date search parameter value
+ */
+public class DateParameter extends SearchParameterValue {
+    private Instant valueDateStart;
+    private Instant valueDateEnd;
+
+    @Override
+    public String toString() {
+        StringBuilder result = new StringBuilder();
+        result.append("Date[");
+        addDescription(result);
+        result.append(",");
+        result.append(valueDateStart);
+        result.append(",");
+        result.append(valueDateEnd);
+        result.append("]");
+        return result.toString();
+    }
+
+    /**
+     * @return the valueDateStart
+     */
+    public Instant getValueDateStart() {
+        return valueDateStart;
+    }
+    
+    /**
+     * @param valueDateStart the valueDateStart to set
+     */
+    public void setValueDateStart(Instant valueDateStart) {
+        this.valueDateStart = valueDateStart;
+    }
+    
+    /**
+     * @return the valueDateEnd
+     */
+    public Instant getValueDateEnd() {
+        return valueDateEnd;
+    }
+    
+    /**
+     * @param valueDateEnd the valueDateEnd to set
+     */
+    public void setValueDateEnd(Instant valueDateEnd) {
+        this.valueDateEnd = valueDateEnd;
+    }
+
+}
diff --git a/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/FHIRIndexProvider.java b/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/FHIRIndexProvider.java
new file mode 100644
index 00000000000..e4347a470e8
--- /dev/null
+++ b/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/FHIRIndexProvider.java
@@ -0,0 +1,23 @@
+/*
+ * (C) Copyright IBM Corp. 2022
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+ 
+package com.ibm.fhir.persistence.index;
+
+import java.util.concurrent.CompletableFuture;
+
+/**
+ * Interface to support dispatching of resource parameter blocks to another
+ * service for offline processing.
+ */
+public interface FHIRIndexProvider {
+
+    /**
+     * Submit the index data request to the async indexing service we represent
+     * @param data
+     * @return A CompletableFuture which completes when the request is acknowledged to have been received by the async service
+     */
+    CompletableFuture submit(RemoteIndexData data);
+}
diff --git a/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/FHIRRemoteIndexService.java b/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/FHIRRemoteIndexService.java
new file mode 100644
index 00000000000..35188481b27
--- /dev/null
+++ b/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/FHIRRemoteIndexService.java
@@ -0,0 +1,46 @@
+/*
+ * (C) Copyright IBM Corp. 2022
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+ 
+package com.ibm.fhir.persistence.index;
+
+import com.ibm.fhir.config.FHIRRequestContext;
+
+/**
+ * Service interface to support shipping resource search parameter values to an
+ * external service where they can be loaded into the database asynchronously.
+ * Implementations are expected to be tenant aware. They must use the tenantId
+ * value from the current {@link FHIRRequestContext}
+ */
+public abstract class FHIRRemoteIndexService {
+
+    // For now we just publish this as a static service
+    // TODO we should be injecting these services to something like the request context
+    private static FHIRRemoteIndexService serviceInstance;
+
+    /**
+     * Initialize the serviceInstance value
+     * @param instance
+     */
+    public static void setServiceInstance(FHIRRemoteIndexService instance) {
+        serviceInstance = instance;
+    }
+
+    /**
+     * Get the serviceInstance value
+     * @return
+     */
+    public static FHIRRemoteIndexService getServiceInstance() {
+        return serviceInstance;
+    }
+
+    /**
+     * Submit the index data request to the async indexing service we represent
+     * @implNote implementations must use tenantId from {@link FHIRRequestContext}
+     * @param data
+     * @return A wrapper for a CompletableFuture which completes when the request is acknowledged to have been received by the async service
+     */
+    public abstract IndexProviderResponse submit(RemoteIndexData data);
+}
diff --git a/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/IndexProviderResponse.java b/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/IndexProviderResponse.java
new file mode 100644
index 00000000000..5242c5fc47f
--- /dev/null
+++ b/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/IndexProviderResponse.java
@@ -0,0 +1,45 @@
+/*
+ * (C) Copyright IBM Corp. 2022
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+ 
+package com.ibm.fhir.persistence.index;
+
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ExecutionException;
+
+/**
+ * Response from submitting IndexData to a FHIRIndexProvider implementation
+ */
+public class IndexProviderResponse {
+    private final RemoteIndexData data;
+    private final CompletableFuture ack;
+    /**
+     * Public constructor for a successful response
+     * @param tenantId
+     * @param data
+     */
+    public IndexProviderResponse(RemoteIndexData data, CompletableFuture ack) {
+        this.data = data;
+        this.ack = ack;
+    }
+
+    /**
+     * Get the request data for which this is the response
+     * @return
+     */
+    public RemoteIndexData getData() {
+        return this.data;
+    }
+
+    /**
+     * Get acknowledgement that the message was received by the service
+     * we sent it to
+     * @throws InterruptedException
+     * @throws ExecutionException
+     */
+    public void getAck() throws InterruptedException, ExecutionException {
+        ack.get();
+    }
+}
diff --git a/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/LocationParameter.java b/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/LocationParameter.java
new file mode 100644
index 00000000000..7e54303d046
--- /dev/null
+++ b/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/LocationParameter.java
@@ -0,0 +1,58 @@
+/*
+ * (C) Copyright IBM Corp. 2022
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+ 
+package com.ibm.fhir.persistence.index;
+
+
+/**
+ * A LatLng location search parameter
+ */
+public class LocationParameter extends SearchParameterValue {
+    private Double valueLatitude;
+    private Double valueLongitude;
+    
+    @Override
+    public String toString() {
+        StringBuilder result = new StringBuilder();
+        result.append("Location[");
+        addDescription(result);
+        result.append(",");
+        result.append(valueLatitude);
+        result.append(",");
+        result.append(valueLongitude);
+        result.append("]");
+        return result.toString();
+    }
+
+    /**
+     * @return the valueLatitude
+     */
+    public Double getValueLatitude() {
+        return valueLatitude;
+    }
+    
+    /**
+     * @param valueLatitude the valueLatitude to set
+     */
+    public void setValueLatitude(Double valueLatitude) {
+        this.valueLatitude = valueLatitude;
+    }
+    
+    /**
+     * @return the valueLongitude
+     */
+    public Double getValueLongitude() {
+        return valueLongitude;
+    }
+    
+    /**
+     * @param valueLongitude the valueLongitude to set
+     */
+    public void setValueLongitude(Double valueLongitude) {
+        this.valueLongitude = valueLongitude;
+    }
+
+}
diff --git a/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/NumberParameter.java b/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/NumberParameter.java
new file mode 100644
index 00000000000..e5fa2bdd317
--- /dev/null
+++ b/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/NumberParameter.java
@@ -0,0 +1,75 @@
+/*
+ * (C) Copyright IBM Corp. 2022
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+ 
+package com.ibm.fhir.persistence.index;
+
+import java.math.BigDecimal;
+
+/**
+ * A number search parameter
+ */
+public class NumberParameter extends SearchParameterValue {
+    private BigDecimal value;
+    private BigDecimal lowValue;
+    private BigDecimal highValue;
+    
+    @Override
+    public String toString() {
+        StringBuilder result = new StringBuilder();
+        result.append("Number[");
+        addDescription(result);
+        result.append(",");
+        result.append(value);
+        result.append(",");
+        result.append(lowValue);
+        result.append(",");
+        result.append(highValue);
+        result.append("]");
+        return result.toString();
+    }
+
+    /**
+     * @return the value
+     */
+    public BigDecimal getValue() {
+        return value;
+    }
+    
+    /**
+     * @param value the value to set
+     */
+    public void setValue(BigDecimal value) {
+        this.value = value;
+    }
+    
+    /**
+     * @return the lowValue
+     */
+    public BigDecimal getLowValue() {
+        return lowValue;
+    }
+    
+    /**
+     * @param lowValue the lowValue to set
+     */
+    public void setLowValue(BigDecimal lowValue) {
+        this.lowValue = lowValue;
+    }
+    
+    /**
+     * @return the highValue
+     */
+    public BigDecimal getHighValue() {
+        return highValue;
+    }
+    
+    /**
+     * @param highValue the highValue to set
+     */
+    public void setHighValue(BigDecimal highValue) {
+        this.highValue = highValue;
+    }
+}
diff --git a/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/ParameterValueVisitorAdapter.java b/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/ParameterValueVisitorAdapter.java
new file mode 100644
index 00000000000..984d03e8d49
--- /dev/null
+++ b/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/ParameterValueVisitorAdapter.java
@@ -0,0 +1,126 @@
+/*
+ * (C) Copyright IBM Corp. 2022
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+ 
+package com.ibm.fhir.persistence.index;
+
+import java.math.BigDecimal;
+import java.time.Instant;
+
+/**
+ * Used by a parameter value visitor to translate the parameter values
+ * to a new form
+ */
+public interface ParameterValueVisitorAdapter {
+
+    /**
+     * Process a string parameter
+     * 
+     * @param name
+     * @param valueString
+     * @param compositeId
+     * @param wholeSystem
+     */
+    void stringValue(String name, String valueString, Integer compositeId, boolean wholeSystem);
+
+    /**
+     * Process a number parameter
+     * 
+     * @param name
+     * @param valueNumber
+     * @param valueNumberLow
+     * @param valueNumberHigh
+     * @param compositeId
+     */
+    void numberValue(String name, BigDecimal valueNumber, BigDecimal valueNumberLow, BigDecimal valueNumberHigh, Integer compositeId);
+
+    /**
+     * Process a date parameter
+     * 
+     * @param name
+     * @param valueDateStart
+     * @param valueDateEnd
+     * @param compositeId
+     * @param wholeSystem
+     */
+    void dateValue(String name, Instant valueDateStart, Instant valueDateEnd, Integer compositeId, boolean wholeSystem);
+
+    /**
+     * Process a token parameter
+     * 
+     * @param name
+     * @param valueSystem
+     * @param valueCode
+     * @param compositeId
+     */
+    void tokenValue(String name, String valueSystem, String valueCode, Integer compositeId);
+
+    /**
+     * Process a tag parameter
+     * 
+     * @param name
+     * @param valueSystem
+     * @param valueCode
+     * @param compositeId
+     * @param wholeSystem
+     */
+    void tagValue(String name, String valueSystem, String valueCode, boolean wholeSystem);
+
+    /**
+     * Process a profile parameter
+     * 
+     * @param name
+     * @param url
+     * @param version
+     * @param fragment
+     * @param wholeSystem
+     */
+    void profileValue(String name, String url, String version, String fragment, boolean wholeSystem);
+
+    /**
+     * Process a security parameter
+     * 
+     * @param name
+     * @param valueSystem
+     * @param valueCode
+     * @param wholeSystem
+     */
+    void securityValue(String name, String valueSystem, String valueCode, boolean wholeSystem);
+    
+    /**
+     * Process a quantity parameter
+     * 
+     * @param name
+     * @param valueSystem
+     * @param valueCode
+     * @param valueNumber
+     * @param valueNumberLow
+     * @param valueNumberHigh
+     * @param compositeId
+     */
+    void quantityValue(String name, String valueSystem, String valueCode, BigDecimal valueNumber, BigDecimal valueNumberLow, BigDecimal valueNumberHigh,
+        Integer compositeId);
+
+    /**
+     * Process a location parameter
+     * 
+     * @param name
+     * @param valueLatitude
+     * @param valueLongitude
+     * @param compositeId
+     */
+    void locationValue(String name, Double valueLatitude, Double valueLongitude, Integer compositeId);
+
+    /**
+     * Process a reference parameter
+     * 
+     * @param name
+     * @param refResourceType
+     * @param refLogicalId
+     * @param refVersion
+     * @param compositeId
+     */
+    void referenceValue(String name, String refResourceType, String refLogicalId, Integer refVersion, Integer compositeId);
+}
diff --git a/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/ProfileParameter.java b/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/ProfileParameter.java
new file mode 100644
index 00000000000..5e19afb6516
--- /dev/null
+++ b/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/ProfileParameter.java
@@ -0,0 +1,79 @@
+/*
+ * (C) Copyright IBM Corp. 2022
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+ 
+package com.ibm.fhir.persistence.index;
+
+
+/**
+ * A profile search parameter value
+ */
+public class ProfileParameter extends SearchParameterValue {
+    
+    private String url;
+
+    // profile version value
+    private String version;
+
+    // profile fragment value
+    private String fragment;
+
+    @Override
+    public String toString() {
+        StringBuilder result = new StringBuilder();
+        result.append("Profile[");
+        addDescription(result);
+        result.append(",");
+        result.append(url);
+        result.append(",");
+        result.append(version);
+        result.append(",");
+        result.append(fragment);
+        result.append("]");
+        return result.toString();
+    }
+
+    /**
+     * @return the url
+     */
+    public String getUrl() {
+        return url;
+    }
+
+    /**
+     * @param url the url to set
+     */
+    public void setUrl(String url) {
+        this.url = url;
+    }
+
+    /**
+     * @return the version
+     */
+    public String getVersion() {
+        return version;
+    }
+
+    /**
+     * @param version the version to set
+     */
+    public void setVersion(String version) {
+        this.version = version;
+    }
+
+    /**
+     * @return the fragment
+     */
+    public String getFragment() {
+        return fragment;
+    }
+
+    /**
+     * @param fragment the fragment to set
+     */
+    public void setFragment(String fragment) {
+        this.fragment = fragment;
+    }
+}
diff --git a/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/QuantityParameter.java b/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/QuantityParameter.java
new file mode 100644
index 00000000000..b2386f0ed0e
--- /dev/null
+++ b/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/QuantityParameter.java
@@ -0,0 +1,109 @@
+/*
+ * (C) Copyright IBM Corp. 2022
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+ 
+package com.ibm.fhir.persistence.index;
+
+import java.math.BigDecimal;
+
+/**
+ * A quantity search parameter value
+ */
+public class QuantityParameter extends SearchParameterValue {
+    private BigDecimal valueNumber;
+    private BigDecimal valueNumberLow;
+    private BigDecimal valueNumberHigh;
+    private String valueSystem;
+    private String valueCode;
+
+    @Override
+    public String toString() {
+        StringBuilder result = new StringBuilder();
+        result.append("Quantity[");
+        addDescription(result);
+        result.append(",");
+        result.append(valueNumber);
+        result.append(",");
+        result.append(valueNumberLow);
+        result.append(",");
+        result.append(valueNumberHigh);
+        result.append(",");
+        result.append(valueSystem);
+        result.append(",");
+        result.append(valueCode);
+        result.append("]");
+        return result.toString();
+    }
+
+    /**
+     * @return the valueNumber
+     */
+    public BigDecimal getValueNumber() {
+        return valueNumber;
+    }
+    
+    /**
+     * @param valueNumber the valueNumber to set
+     */
+    public void setValueNumber(BigDecimal valueNumber) {
+        this.valueNumber = valueNumber;
+    }
+    
+    /**
+     * @return the valueNumberLow
+     */
+    public BigDecimal getValueNumberLow() {
+        return valueNumberLow;
+    }
+    
+    /**
+     * @param valueNumberLow the valueNumberLow to set
+     */
+    public void setValueNumberLow(BigDecimal valueNumberLow) {
+        this.valueNumberLow = valueNumberLow;
+    }
+    
+    /**
+     * @return the valueNumberHigh
+     */
+    public BigDecimal getValueNumberHigh() {
+        return valueNumberHigh;
+    }
+    
+    /**
+     * @param valueNumberHigh the valueNumberHigh to set
+     */
+    public void setValueNumberHigh(BigDecimal valueNumberHigh) {
+        this.valueNumberHigh = valueNumberHigh;
+    }
+    
+    /**
+     * @return the valueSystem
+     */
+    public String getValueSystem() {
+        return valueSystem;
+    }
+    
+    /**
+     * @param valueSystem the valueSystem to set
+     */
+    public void setValueSystem(String valueSystem) {
+        this.valueSystem = valueSystem;
+    }
+    
+    /**
+     * @return the valueCode
+     */
+    public String getValueCode() {
+        return valueCode;
+    }
+    
+    /**
+     * @param valueCode the valueCode to set
+     */
+    public void setValueCode(String valueCode) {
+        this.valueCode = valueCode;
+    }
+}
diff --git a/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/ReferenceParameter.java b/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/ReferenceParameter.java
new file mode 100644
index 00000000000..dbc2e45e202
--- /dev/null
+++ b/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/ReferenceParameter.java
@@ -0,0 +1,76 @@
+/*
+ * (C) Copyright IBM Corp. 2022
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+ 
+package com.ibm.fhir.persistence.index;
+
+
+/**
+ * A local reference search parameter value
+ */
+public class ReferenceParameter extends SearchParameterValue {
+    private String resourceType;
+    private String logicalId;
+
+    // for storing versioned references
+    private Integer refVersionId;
+
+    @Override
+    public String toString() {
+        StringBuilder result = new StringBuilder();
+        result.append("Reference[");
+        addDescription(result);
+        result.append(",");
+        result.append(resourceType);
+        result.append(",");
+        result.append(logicalId);
+        result.append(",");
+        result.append(refVersionId);
+        result.append("]");
+        return result.toString();
+    }
+    
+    /**
+     * @return the refVersionId
+     */
+    public Integer getRefVersionId() {
+        return refVersionId;
+    }
+
+    /**
+     * @param refVersionId the refVersionId to set
+     */
+    public void setRefVersionId(Integer refVersionId) {
+        this.refVersionId = refVersionId;
+    }
+
+    /**
+     * @return the resourceType
+     */
+    public String getResourceType() {
+        return resourceType;
+    }
+
+    /**
+     * @param resourceType the resourceType to set
+     */
+    public void setResourceType(String resourceType) {
+        this.resourceType = resourceType;
+    }
+
+    /**
+     * @return the logicalId
+     */
+    public String getLogicalId() {
+        return logicalId;
+    }
+
+    /**
+     * @param logicalId the logicalId to set
+     */
+    public void setLogicalId(String logicalId) {
+        this.logicalId = logicalId;
+    }
+}
\ No newline at end of file
diff --git a/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/RemoteIndexConstants.java b/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/RemoteIndexConstants.java
new file mode 100644
index 00000000000..34fd200749c
--- /dev/null
+++ b/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/RemoteIndexConstants.java
@@ -0,0 +1,17 @@
+/*
+ * (C) Copyright IBM Corp. 2022
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+ 
+package com.ibm.fhir.persistence.index;
+
+
+/**
+ * Constants associated with the remote index service
+ */
+public class RemoteIndexConstants {
+
+    // the current version of remote index messages we push to Kafka
+    public static final int MESSAGE_VERSION = 1;
+}
diff --git a/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/RemoteIndexData.java b/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/RemoteIndexData.java
new file mode 100644
index 00000000000..f9e79c496df
--- /dev/null
+++ b/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/RemoteIndexData.java
@@ -0,0 +1,57 @@
+/*
+ * (C) Copyright IBM Corp. 2022
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+ 
+package com.ibm.fhir.persistence.index;
+
+/**
+ * Supplier of index data
+ */
+public class RemoteIndexData {
+    // The partition key used to route the data
+    private final String partitionKey;
+
+    // The data object holding all the extracted search parameters
+    private final SearchParametersTransport searchParameters;
+
+    @Override
+    public String toString() {
+        final StringBuilder result = new StringBuilder();
+        result.append("partitionKey:[").append(partitionKey).append("]");
+        result.append(" resource:[");
+        result.append(searchParameters.getResourceType()).append("/").append(searchParameters.getLogicalId());
+        result.append("]");
+        return result.toString();
+    }
+
+    /**
+     * Public constructor
+     * @param partitionKey
+     * @param searchParameters
+     */
+    public RemoteIndexData(String partitionKey, SearchParametersTransport searchParameters) {
+        this.searchParameters = searchParameters;
+        this.partitionKey = partitionKey;
+    }
+
+    /**
+     * Get the search parameter block representing the data we want to send
+     * to the remote indexing service
+     * @return
+     */
+    public SearchParametersTransport getSearchParameters() {
+        return this.searchParameters;
+    }
+
+    /**
+     * Get the key used to select which partition we want to send to. Partitions
+     * are important because we want to see the IndexData processed in order within
+     * a particular partition
+     * @return
+     */
+    public String getPartitionKey() {
+        return this.partitionKey;
+    }
+}
\ No newline at end of file
diff --git a/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/RemoteIndexMessage.java b/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/RemoteIndexMessage.java
new file mode 100644
index 00000000000..663cd5337b3
--- /dev/null
+++ b/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/RemoteIndexMessage.java
@@ -0,0 +1,84 @@
+/*
+ * (C) Copyright IBM Corp. 2022
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+ 
+package com.ibm.fhir.persistence.index;
+
+/**
+ * The Kafka message we send to the remote index service
+ */
+public class RemoteIndexMessage {
+    private String tenantId;
+    private int messageVersion;
+    private String instanceIdentifier;
+    private SearchParametersTransport data;
+
+    @Override
+    public String toString() {
+        StringBuilder result = new StringBuilder();
+        result.append("tenant[");
+        result.append(tenantId);
+        result.append("] ");
+        result.append(data.toString());
+        return result.toString();
+    }
+
+    /**
+     * @return the tenantId
+     */
+    public String getTenantId() {
+        return tenantId;
+    }
+    
+    /**
+     * @param tenantId the tenantId to set
+     */
+    public void setTenantId(String tenantId) {
+        this.tenantId = tenantId;
+    }
+    
+    /**
+     * @return the data
+     */
+    public SearchParametersTransport getData() {
+        return data;
+    }
+    
+    /**
+     * @param data the data to set
+     */
+    public void setData(SearchParametersTransport data) {
+        this.data = data;
+    }
+
+    /**
+     * @return the messageVersion
+     */
+    public int getMessageVersion() {
+        return messageVersion;
+    }
+
+    /**
+     * @param messageVersion the messageVersion to set
+     */
+    public void setMessageVersion(int messageVersion) {
+        this.messageVersion = messageVersion;
+    }
+
+    /**
+     * @return the instanceIdentifier
+     */
+    public String getInstanceIdentifier() {
+        return instanceIdentifier;
+    }
+
+    /**
+     * @param instanceIdentifier the instanceIdentifier to set
+     */
+    public void setInstanceIdentifier(String instanceIdentifier) {
+        this.instanceIdentifier = instanceIdentifier;
+    }
+    
+}
diff --git a/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/SearchParameterValue.java b/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/SearchParameterValue.java
new file mode 100644
index 00000000000..c13bd06e634
--- /dev/null
+++ b/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/SearchParameterValue.java
@@ -0,0 +1,86 @@
+/*
+ * (C) Copyright IBM Corp. 2022
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+ 
+package com.ibm.fhir.persistence.index;
+
+
+/**
+ * The base class for our search parameter values. These index model classes
+ * are designed to reflect the raw values we want the remote indexing
+ * service to store
+ */
+public class SearchParameterValue {
+    // The name of the parameter
+    private String name;
+
+    // The composite id used to tie together values belonging to the same composite parameter. Null for ordinary params.
+    private Integer compositeId;
+
+    // True if this parameter should also be stored at the whole-system level
+    private Boolean wholeSystem;
+
+    /**
+     * Add the base description of this parameter to the given {@link StringBuilder}
+     * @param sb
+     */
+    protected void addDescription(StringBuilder sb) {
+        sb.append(name);
+        sb.append(",");
+        sb.append(compositeId);
+        sb.append(",");
+        sb.append(wholeSystem);
+    }
+
+    /**
+     * @return the name
+     */
+    public String getName() {
+        return name;
+    }
+
+    /**
+     * @param name the name to set
+     */
+    public void setName(String name) {
+        this.name = name;
+    }
+
+    /**
+     * @return the compositeId
+     */
+    public Integer getCompositeId() {
+        return compositeId;
+    }
+
+    /**
+     * @param compositeId the compositeId to set
+     */
+    public void setCompositeId(Integer compositeId) {
+        this.compositeId = compositeId;
+    }
+
+    /**
+     * @return the wholeSystem
+     */
+    public Boolean getWholeSystem() {
+        return wholeSystem;
+    }
+
+    /**
+     * Returns true iff the wholeSystem property is not null and true
+     * @return
+     */
+    public boolean isSystemParam() {
+        return this.wholeSystem != null && this.wholeSystem.booleanValue();
+    }
+
+    /**
+     * @param wholeSystem the wholeSystem to set
+     */
+    public void setWholeSystem(Boolean wholeSystem) {
+        this.wholeSystem = wholeSystem;
+    }
+}
diff --git a/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/SearchParametersTransport.java b/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/SearchParametersTransport.java
new file mode 100644
index 00000000000..dcd762c7d8a
--- /dev/null
+++ b/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/SearchParametersTransport.java
@@ -0,0 +1,602 @@
+/*
+ * (C) Copyright IBM Corp. 2022
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+ 
+package com.ibm.fhir.persistence.index;
+
+import java.time.Instant;
+import java.time.ZoneOffset;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Represents a collection of search parameters extracted from a FHIR resource
+ * held in a form that is easy to serialize/deserialize to a wire format
+ * (e.g. JSON) for sending to a remote/async indexing service.
+ * @implNote because we want to serialize/deserialize this object
+ * as JSON, we need to keep it simple
+ */
+public class SearchParametersTransport {
+
+    // The FHIR resource type name
+    private String resourceType;
+
+    // The logical id of the resource
+    private String logicalId;
+
+    // The database identifier assigned to this resource
+    private long logicalResourceId;
+    
+    // The current version of the resource
+    private int versionId;
+
+    // The parameter hash computed for this set of parameters
+    private String parameterHash;
+
+    // The last_updated time
+    private Instant lastUpdated;
+
+    // The key value used for sharding the data when using a distributed database
+    private String requestShard;
+
+    private List stringValues;
+    private List numberValues;
+    private List quantityValues;
+    private List tokenValues;
+    private List dateValues;
+    private List locationValues;
+    private List tagValues;
+    private List profileValues;
+    private List securityValues;
+    private List refValues;
+
+    /**
+     * Factory method to create a {@link Builder} instance
+     * @return
+     */
+    public static Builder builder() {
+        return new Builder();
+    }
+
+    @Override
+    public String toString() {
+        StringBuilder result = new StringBuilder();
+        result.append("resourceType[");
+        result.append(resourceType);
+        result.append("] ");
+        result.append("logicalId[");
+        result.append(logicalId);
+        result.append("] ");
+        result.append("versionId[");
+        result.append(versionId);
+        result.append("] ");
+        result.append("parameterHash[");
+        result.append(parameterHash);
+        result.append("] ");
+        result.append("lastUpdated[");
+        result.append(lastUpdated);
+        result.append("] ");
+        return result.toString();
+    }
+
+    /**
+     * A builder to make it easier to construct a {@link SearchParametersTransport}
+     */
+    public static class Builder {
+        private List stringValues = new ArrayList<>();
+        private List numberValues = new ArrayList<>();
+        private List quantityValues = new ArrayList<>();
+        private List tokenValues = new ArrayList<>();
+        private List dateValues = new ArrayList<>();
+        private List locationValues = new ArrayList<>();
+        private List tagValues = new ArrayList<>();
+        private List profileValues = new ArrayList<>();
+        private List securityValues = new ArrayList<>();
+        private List refValues = new ArrayList<>();
+    
+        private String resourceType;
+        private String logicalId;
+        private long logicalResourceId = -1;
+        private String requestShard;
+        private int versionId;
+        private String parameterHash;
+        private Instant lastUpdated;
+
+        /**
+         * Set the resourceType
+         * @param resourceType
+         * @return
+         */
+        public Builder withResourceType(String resourceType) {
+            this.resourceType = resourceType;
+            return this;
+        }
+
+        /**
+         * Set the parameterHash
+         * @param hash
+         * @return
+         */
+        public Builder withParameterHash(String hash) {
+            this.parameterHash = hash;
+            return this;
+        }
+
+        public Builder withLastUpdated(Instant lastUpdated) {
+            this.lastUpdated = lastUpdated;
+            return this;
+        }
+
+        /**
+         * Set the logicalId
+         * @param logicalId
+         * @return
+         */
+        public Builder withLogicalId(String logicalId) {
+            this.logicalId = logicalId;
+            return this;
+        }
+
+        /**
+         * Set the versionId
+         * @param versionId
+         * @return
+         */
+        public Builder withVersionId(int versionId) {
+            this.versionId = versionId;
+            return this;
+        }
+
+        /**
+         * Set the logicalResourceId
+         * @param logicalResourceId
+         * @return
+         */
+        public Builder withLogicalResourceId(long logicalResourceId) {
+            this.logicalResourceId = logicalResourceId;
+            return this;
+        }
+
+        /**
+         * Set the shardKey
+         * @param shardKey
+         * @return
+         */
+        public Builder withRequestShard(String shardValue) {
+            this.requestShard = shardValue;
+            return this;
+        }
+
+        /**
+         * Add a string parameter value
+         * @param value
+         * @return
+         */
+        public Builder addStringValue(StringParameter value) {
+            stringValues.add(value);
+            return this;
+        }
+
+        /**
+         * Add a number parameter value
+         * @param value
+         * @return
+         */
+        public Builder addNumberValue(NumberParameter value) {
+            numberValues.add(value);
+            return this;
+        }
+
+        /**
+         * Add a quantity parameter value
+         * @param value
+         * @return
+         */
+        public Builder addQuantityValue(QuantityParameter value) {
+            quantityValues.add(value);
+            return this;
+        }
+
+        /**
+         * Add a token parameter value
+         * @param value
+         * @return
+         */
+        public Builder addTokenValue(TokenParameter value) {
+            tokenValues.add(value);
+            return this;
+        }
+
+        /**
+         * Add a reference parameter value
+         * @param value
+         * @return
+         */
+        public Builder addReferenceValue(ReferenceParameter value) {
+            refValues.add(value);
+            return this;
+        }
+
+        /**
+         * Add a tag parameter value
+         * @param value
+         * @return
+         */
+        public Builder addTagValue(TagParameter value) {
+            tagValues.add(value);
+            return this;
+        }
+
+        /**
+         * Add a profile parameter value
+         * @param value
+         * @return
+         */
+        public Builder addProfileValue(ProfileParameter value) {
+            profileValues.add(value);
+            return this;
+        }
+
+        /**
+         * Add a security parameter value
+         * @param value
+         * @return
+         */
+        public Builder addSecurityValue(SecurityParameter value) {
+            securityValues.add(value);
+            return this;
+        }
+
+        /**
+         * Add a date parameter value
+         * @param value
+         * @return
+         */
+        public Builder addDateValue(DateParameter value) {
+            dateValues.add(value);
+            return this;
+        }
+
+        /**
+         * Add a location parameter value
+         * @param value
+         * @return
+         */
+        public Builder addLocationValue(LocationParameter value) {
+            locationValues.add(value);
+            return this;
+        }
+
+        /**
+         * Builder a new {@link SearchParametersTransport} instance based on the current state
+         * of this {@link Builder}.
+         * @return
+         */
+        public SearchParametersTransport build() {
+            if (this.logicalResourceId < 0) {
+                throw new IllegalStateException("Must set logicalResourceId");
+            }
+            if (this.resourceType == null) {
+                throw new IllegalStateException("Must set resourceType");
+            }
+            if (this.logicalId == null) {
+                throw new IllegalStateException("Must set logicalId");
+            }
+
+            SearchParametersTransport result = new SearchParametersTransport();
+            result.resourceType = this.resourceType;
+            result.logicalId = this.logicalId;
+            result.logicalResourceId = this.logicalResourceId;
+            result.setVersionId(this.versionId);
+            result.setRequestShard(this.requestShard);
+            result.setParameterHash(this.parameterHash);
+            result.setLastUpdated(this.lastUpdated);
+
+            if (this.stringValues.size() > 0) {
+                result.stringValues = new ArrayList<>(this.stringValues);
+            }
+            if (this.numberValues.size() > 0) {
+                result.numberValues = new ArrayList<>(this.numberValues);
+            }
+            if (this.quantityValues.size() > 0) {
+                result.quantityValues = new ArrayList<>(this.quantityValues);
+            }
+            if (this.tokenValues.size() > 0) {
+                result.tokenValues = new ArrayList<>(this.tokenValues);
+            }
+            if (this.dateValues.size() > 0) {
+                result.dateValues = new ArrayList<>(this.dateValues);
+            }
+            if (this.locationValues.size() > 0) {
+                result.locationValues = new ArrayList<>(this.locationValues);
+            }
+            if (this.tagValues.size() > 0) {
+                result.setTagValues(new ArrayList<>(this.tagValues));
+            }
+            if (this.profileValues.size() > 0) {
+                result.setProfileValues(new ArrayList<>(this.profileValues));
+            }
+            if (this.securityValues.size() > 0) {
+                result.setSecurityValues(new ArrayList<>(this.securityValues));
+            }
+            if (this.refValues.size() > 0) {
+                result.setRefValues(new ArrayList<>(this.refValues));
+            }
+            return result;
+        }
+    }
+    
+    /**
+     * @return the resourceType
+     */
+    public String getResourceType() {
+        return resourceType;
+    }
+
+    
+    /**
+     * @param resourceType the resourceType to set
+     */
+    public void setResourceType(String resourceType) {
+        this.resourceType = resourceType;
+    }
+
+    
+    /**
+     * @return the logicalId
+     */
+    public String getLogicalId() {
+        return logicalId;
+    }
+
+    
+    /**
+     * @param logicalId the logicalId to set
+     */
+    public void setLogicalId(String logicalId) {
+        this.logicalId = logicalId;
+    }
+
+    
+    /**
+     * @return the logicalResourceId
+     */
+    public long getLogicalResourceId() {
+        return logicalResourceId;
+    }
+
+    
+    /**
+     * @param logicalResourceId the logicalResourceId to set
+     */
+    public void setLogicalResourceId(long logicalResourceId) {
+        this.logicalResourceId = logicalResourceId;
+    }
+
+    
+    /**
+     * @return the stringValues
+     */
+    public List getStringValues() {
+        return stringValues;
+    }
+
+    
+    /**
+     * @param stringValues the stringValues to set
+     */
+    public void setStringValues(List stringValues) {
+        this.stringValues = stringValues;
+    }
+
+    
+    /**
+     * @return the numberValues
+     */
+    public List getNumberValues() {
+        return numberValues;
+    }
+
+    
+    /**
+     * @param numberValues the numberValues to set
+     */
+    public void setNumberValues(List numberValues) {
+        this.numberValues = numberValues;
+    }
+
+    
+    /**
+     * @return the quantityValues
+     */
+    public List getQuantityValues() {
+        return quantityValues;
+    }
+
+    
+    /**
+     * @param quantityValues the quantityValues to set
+     */
+    public void setQuantityValues(List quantityValues) {
+        this.quantityValues = quantityValues;
+    }
+
+    
+    /**
+     * @return the tokenValues
+     */
+    public List getTokenValues() {
+        return tokenValues;
+    }
+
+    
+    /**
+     * @param tokenValues the tokenValues to set
+     */
+    public void setTokenValues(List tokenValues) {
+        this.tokenValues = tokenValues;
+    }
+
+    
+    /**
+     * @return the dateValues
+     */
+    public List getDateValues() {
+        return dateValues;
+    }
+
+    
+    /**
+     * @param dateValues the dateValues to set
+     */
+    public void setDateValues(List dateValues) {
+        this.dateValues = dateValues;
+    }
+
+    
+    /**
+     * @return the locationValues
+     */
+    public List getLocationValues() {
+        return locationValues;
+    }
+
+    
+    /**
+     * @param locationValues the locationValues to set
+     */
+    public void setLocationValues(List locationValues) {
+        this.locationValues = locationValues;
+    }
+
+
+    /**
+     * @return the requestShard
+     */
+    public String getRequestShard() {
+        return requestShard;
+    }
+
+
+    /**
+     * @param shardValue the request shard value to set
+     */
+    public void setRequestShard(String shardValue) {
+        this.requestShard = shardValue;
+    }
+
+
+    /**
+     * @return the tagValues
+     */
+    public List getTagValues() {
+        return tagValues;
+    }
+
+
+    /**
+     * @param tagValues the tagValues to set
+     */
+    public void setTagValues(List tagValues) {
+        this.tagValues = tagValues;
+    }
+
+
+    /**
+     * @return the profileValues
+     */
+    public List getProfileValues() {
+        return profileValues;
+    }
+
+
+    /**
+     * @param profileValues the profileValues to set
+     */
+    public void setProfileValues(List profileValues) {
+        this.profileValues = profileValues;
+    }
+
+    /**
+     * @return the securityValues
+     */
+    public List getSecurityValues() {
+        return securityValues;
+    }
+
+    /**
+     * @param profileValues the profileValues to set
+     */
+    public void setSecurityValues(List securityValues) {
+        this.securityValues = securityValues;
+    }
+
+
+    /**
+     * @return the versionId
+     */
+    public int getVersionId() {
+        return versionId;
+    }
+
+
+    /**
+     * @param versionId the versionId to set
+     */
+    public void setVersionId(int versionId) {
+        this.versionId = versionId;
+    }
+
+    /**
+     * @return the parameterHash
+     */
+    public String getParameterHash() {
+        return parameterHash;
+    }
+
+    /**
+     * @param parameterHash the parameterHash to set
+     */
+    public void setParameterHash(String parameterHash) {
+        this.parameterHash = parameterHash;
+    }
+
+    /**
+     * @return the lastUpdated (UTC)
+     */
+    public Instant getLastUpdated() {
+        return lastUpdated;
+    }
+
+    /**
+     * @param lastUpdated the lastUpdated to set.
+     */
+    public void setLastUpdated(Instant lastUpdated) {
+        this.lastUpdated = lastUpdated;
+    }
+
+    /**
+     * Convenience function to get the lastUpdated time as an Instant. All our times are
+     * always UTC.
+     * @return
+     */
+    public Instant getLastUpdatedInstant() {
+        return Instant.from(lastUpdated.atOffset(ZoneOffset.UTC));
+    }
+
+    /**
+     * @return the refValues
+     */
+    public List getRefValues() {
+        return refValues;
+    }
+
+    /**
+     * @param refValues the refValues to set
+     */
+    public void setRefValues(List refValues) {
+        this.refValues = refValues;
+    }
+}
diff --git a/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/SearchParametersTransportAdapter.java b/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/SearchParametersTransportAdapter.java
new file mode 100644
index 00000000000..c156a994df2
--- /dev/null
+++ b/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/SearchParametersTransportAdapter.java
@@ -0,0 +1,162 @@
+/*
+ * (C) Copyright IBM Corp. 2022
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+ 
+package com.ibm.fhir.persistence.index;
+
+import java.math.BigDecimal;
+import java.time.Instant;
+
+
+/**
+ * Visitor adapter implementation to build an instance of {@link SearchParametersTransport} to
+ * provide support for shipping a set of search parameter values off to a remote
+ * index service. This allows the parameters to be stored in the database in a
+ * separate transaction, and allows the inserts to be batched together, providing
+ * improved throughput.
+ */
+public class SearchParametersTransportAdapter implements ParameterValueVisitorAdapter {
+
+    // The builder we use to collect all the visited parameter values
+    private final SearchParametersTransport.Builder builder;
+
+    /**
+     * Public constructor
+     * @param resourceType
+     * @param logicalId
+     * @param logicalResourceId
+     * @param versionId
+     * @param lastUpdated
+     * @param requestShard
+     * @param parameterHash
+     */
+    public SearchParametersTransportAdapter(String resourceType, String logicalId, long logicalResourceId, 
+            int versionId, Instant lastUpdated, String requestShard, String parameterHash) {
+        builder = SearchParametersTransport.builder()
+            .withResourceType(resourceType)
+            .withLogicalId(logicalId)
+            .withLogicalResourceId(logicalResourceId)
+            .withVersionId(versionId)
+            .withLastUpdated(lastUpdated)
+            .withRequestShard(requestShard)
+            .withParameterHash(parameterHash);
+    }
+
+    /**
+     * Build the SearchParametersTransport instance from the current state of builder
+     * @return
+     */
+    public SearchParametersTransport build() {
+        return builder.build();
+    }
+
+    @Override
+    public void stringValue(String name, String valueString, Integer compositeId, boolean wholeSystem) {
+        StringParameter value = new StringParameter();
+        value.setName(name);
+        value.setValue(valueString);
+        value.setCompositeId(compositeId);
+        value.setWholeSystem(wholeSystem);
+        builder.addStringValue(value);
+    }
+
+    @Override
+    public void numberValue(String name, BigDecimal valueNumber, BigDecimal valueNumberLow, BigDecimal valueNumberHigh, Integer compositeId) {
+        NumberParameter value = new NumberParameter();
+        value.setName(name);
+        value.setValue(valueNumber);
+        value.setLowValue(valueNumberLow);
+        value.setHighValue(valueNumberHigh);
+        value.setCompositeId(compositeId);
+        builder.addNumberValue(value);
+    }
+
+    @Override
+    public void dateValue(String name, Instant valueDateStart, Instant valueDateEnd, Integer compositeId, boolean wholeSystem) {
+        DateParameter value = new DateParameter();
+        value.setName(name);
+        value.setValueDateStart(valueDateStart);
+        value.setValueDateEnd(valueDateEnd);
+        value.setCompositeId(compositeId);
+        value.setWholeSystem(wholeSystem);
+        builder.addDateValue(value);
+    }
+
+    @Override
+    public void tokenValue(String name, String valueSystem, String valueCode, Integer compositeId) {
+        TokenParameter value = new TokenParameter();
+        value.setName(name);
+        value.setValueSystem(valueSystem);
+        value.setValueCode(valueCode);
+        value.setCompositeId(compositeId);
+        builder.addTokenValue(value);
+    }
+
+    @Override
+    public void tagValue(String name, String valueSystem, String valueCode, boolean wholeSystem) {
+        TagParameter value = new TagParameter();
+        value.setName(name);
+        value.setValueSystem(valueSystem);
+        value.setValueCode(valueCode);
+        value.setWholeSystem(wholeSystem);
+        builder.addTagValue(value);
+    }
+
+    @Override
+    public void profileValue(String name, String url, String version, String fragment, boolean wholeSystem) {
+        ProfileParameter value = new ProfileParameter();
+        value.setName(name);
+        value.setUrl(url);
+        value.setVersion(version);
+        value.setFragment(fragment);
+        value.setWholeSystem(wholeSystem);
+        builder.addProfileValue(value);
+    }
+
+    @Override
+    public void securityValue(String name, String valueSystem, String valueCode, boolean wholeSystem) {
+        SecurityParameter value = new SecurityParameter();
+        value.setName(name);
+        value.setValueSystem(valueSystem);
+        value.setValueCode(valueCode);
+        value.setWholeSystem(wholeSystem);
+        builder.addSecurityValue(value);
+    }
+
+    @Override
+    public void quantityValue(String name, String valueSystem, String valueCode, BigDecimal valueNumber, BigDecimal valueNumberLow, BigDecimal valueNumberHigh,
+        Integer compositeId) {
+        QuantityParameter value = new QuantityParameter();
+        value.setName(name);
+        value.setValueSystem(valueSystem);
+        value.setValueCode(valueCode);
+        value.setValueNumber(valueNumber);
+        value.setValueNumberLow(valueNumberLow);
+        value.setValueNumberHigh(valueNumberHigh);
+        value.setCompositeId(compositeId);
+        builder.addQuantityValue(value);
+    }
+
+    @Override
+    public void locationValue(String name, Double valueLatitude, Double valueLongitude, Integer compositeId) {
+        LocationParameter value = new LocationParameter();
+        value.setName(name);
+        value.setValueLatitude(valueLatitude);
+        value.setValueLongitude(valueLongitude);
+        value.setCompositeId(compositeId);
+        builder.addLocationValue(value);
+    }
+
+    @Override
+    public void referenceValue(String name, String refResourceType, String refLogicalId, Integer refVersion, Integer compositeId) {
+        ReferenceParameter value = new ReferenceParameter();
+        value.setName(name);
+        value.setResourceType(refResourceType);
+        value.setLogicalId(refLogicalId);
+        value.setRefVersionId(refVersion);
+        value.setCompositeId(compositeId);
+        builder.addReferenceValue(value);
+    }
+}
diff --git a/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/SecurityParameter.java b/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/SecurityParameter.java
new file mode 100644
index 00000000000..1d024762750
--- /dev/null
+++ b/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/SecurityParameter.java
@@ -0,0 +1,57 @@
+/*
+ * (C) Copyright IBM Corp. 2022
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+ 
+package com.ibm.fhir.persistence.index;
+
+
+/**
+ * A security search parameter value
+ */
+public class SecurityParameter extends SearchParameterValue {
+    private String valueSystem;
+    private String valueCode;
+
+    @Override
+    public String toString() {
+        StringBuilder result = new StringBuilder();
+        result.append("Security[");
+        addDescription(result);
+        result.append(",");
+        result.append(valueSystem);
+        result.append(",");
+        result.append(valueCode);
+        result.append("]");
+        return result.toString();
+    }
+    
+    /**
+     * @return the valueSystem
+     */
+    public String getValueSystem() {
+        return valueSystem;
+    }
+    
+    /**
+     * @param valueSystem the valueSystem to set
+     */
+    public void setValueSystem(String valueSystem) {
+        this.valueSystem = valueSystem;
+    }
+    
+    /**
+     * @return the valueCode
+     */
+    public String getValueCode() {
+        return valueCode;
+    }
+    
+    /**
+     * @param valueCode the valueCode to set
+     */
+    public void setValueCode(String valueCode) {
+        this.valueCode = valueCode;
+    }
+}
diff --git a/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/StringParameter.java b/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/StringParameter.java
new file mode 100644
index 00000000000..bb8077c041c
--- /dev/null
+++ b/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/StringParameter.java
@@ -0,0 +1,40 @@
+/*
+ * (C) Copyright IBM Corp. 2022
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+ 
+package com.ibm.fhir.persistence.index;
+
+
+/**
+ * A string search parameter used for transporting values for remote indexing
+ */
+public class StringParameter extends SearchParameterValue {
+    private String value;
+    
+    @Override
+    public String toString() {
+        StringBuilder result = new StringBuilder();
+        result.append("String[");
+        addDescription(result);
+        result.append(",");
+        result.append(value);
+        result.append("]");
+        return result.toString();
+    }
+
+    /**
+     * @return the value
+     */
+    public String getValue() {
+        return value;
+    }
+    
+    /**
+     * @param value the value to set
+     */
+    public void setValue(String value) {
+        this.value = value;
+    }
+}
diff --git a/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/TagParameter.java b/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/TagParameter.java
new file mode 100644
index 00000000000..12ba808966a
--- /dev/null
+++ b/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/TagParameter.java
@@ -0,0 +1,57 @@
+/*
+ * (C) Copyright IBM Corp. 2022
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+ 
+package com.ibm.fhir.persistence.index;
+
+
+/**
+ * A tag search parameter value
+ */
+public class TagParameter extends SearchParameterValue {
+    private String valueSystem;
+    private String valueCode;
+
+    @Override
+    public String toString() {
+        StringBuilder result = new StringBuilder();
+        result.append("Tag[");
+        addDescription(result);
+        result.append(",");
+        result.append(valueSystem);
+        result.append(",");
+        result.append(valueCode);
+        result.append("]");
+        return result.toString();
+    }
+    
+    /**
+     * @return the valueSystem
+     */
+    public String getValueSystem() {
+        return valueSystem;
+    }
+    
+    /**
+     * @param valueSystem the valueSystem to set
+     */
+    public void setValueSystem(String valueSystem) {
+        this.valueSystem = valueSystem;
+    }
+    
+    /**
+     * @return the valueCode
+     */
+    public String getValueCode() {
+        return valueCode;
+    }
+    
+    /**
+     * @param valueCode the valueCode to set
+     */
+    public void setValueCode(String valueCode) {
+        this.valueCode = valueCode;
+    }
+}
diff --git a/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/TokenParameter.java b/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/TokenParameter.java
new file mode 100644
index 00000000000..5e575784b71
--- /dev/null
+++ b/fhir-persistence/src/main/java/com/ibm/fhir/persistence/index/TokenParameter.java
@@ -0,0 +1,76 @@
+/*
+ * (C) Copyright IBM Corp. 2022
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+ 
+package com.ibm.fhir.persistence.index;
+
+
+/**
+ * A token search parameter value
+ */
+public class TokenParameter extends SearchParameterValue {
+    private String valueSystem;
+    private String valueCode;
+
+    // for storing versioned references
+    private Integer refVersionId;
+
+    @Override
+    public String toString() {
+        StringBuilder result = new StringBuilder();
+        result.append("Token[");
+        addDescription(result);
+        result.append(",");
+        result.append(valueSystem);
+        result.append(",");
+        result.append(valueCode);
+        result.append(",");
+        result.append(refVersionId);
+        result.append("]");
+        return result.toString();
+    }
+    
+    /**
+     * @return the valueSystem
+     */
+    public String getValueSystem() {
+        return valueSystem;
+    }
+    
+    /**
+     * @param valueSystem the valueSystem to set
+     */
+    public void setValueSystem(String valueSystem) {
+        this.valueSystem = valueSystem;
+    }
+    
+    /**
+     * @return the valueCode
+     */
+    public String getValueCode() {
+        return valueCode;
+    }
+    
+    /**
+     * @param valueCode the valueCode to set
+     */
+    public void setValueCode(String valueCode) {
+        this.valueCode = valueCode;
+    }
+
+    /**
+     * @return the refVersionId
+     */
+    public Integer getRefVersionId() {
+        return refVersionId;
+    }
+
+    /**
+     * @param refVersionId the refVersionId to set
+     */
+    public void setRefVersionId(Integer refVersionId) {
+        this.refVersionId = refVersionId;
+    }
+}
\ No newline at end of file
diff --git a/fhir-persistence/src/test/java/com/ibm/fhir/persistence/helper/MessageSerializationTest.java b/fhir-persistence/src/test/java/com/ibm/fhir/persistence/helper/MessageSerializationTest.java
new file mode 100644
index 00000000000..01fbb5e39bf
--- /dev/null
+++ b/fhir-persistence/src/test/java/com/ibm/fhir/persistence/helper/MessageSerializationTest.java
@@ -0,0 +1,128 @@
+/*
+ * (C) Copyright IBM Corp. 2022
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+ 
+package com.ibm.fhir.persistence.helper;
+
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertNotNull;
+
+import java.math.BigDecimal;
+import java.time.Instant;
+
+import org.testng.annotations.Test;
+
+import com.google.gson.Gson;
+import com.ibm.fhir.persistence.index.RemoteIndexConstants;
+import com.ibm.fhir.persistence.index.RemoteIndexMessage;
+import com.ibm.fhir.persistence.index.SearchParametersTransport;
+import com.ibm.fhir.persistence.index.SearchParametersTransportAdapter;
+
+/**
+ * Unit test for message serialization (for the payload sent over Kafka as a string)
+ */
+public class MessageSerializationTest {
+
+    @Test
+    public void testRoundtrip() throws Exception {
+        RemoteIndexMessage sent = new RemoteIndexMessage();
+        sent.setMessageVersion(RemoteIndexConstants.MESSAGE_VERSION);
+
+        final String resourceType = "Observation";
+        final String logicalId = "patientOne";
+        final long logicalResourceId = 1;
+        final int versionId = 1;
+        final Instant lastUpdated = Instant.now();
+        final String requestShard = null;
+        final String parameterHash = "1Z+NWYZb739Ava9Pd/d7wt2xecKmC2FkfLlCCml0I5M=";
+        final Instant ts1 = lastUpdated.plusMillis(1000);
+        final Instant ts2 = lastUpdated.plusMillis(2000);
+        final BigDecimal valueNumber = BigDecimal.valueOf(1.0);
+        final BigDecimal valueNumberLow = BigDecimal.valueOf(0.5);
+        final BigDecimal valueNumberHigh = BigDecimal.valueOf(1.5);
+        final String valueSystem = "system1";
+        final String valueCode = "code1";
+        final String refResourceType = "Patient";
+        final String refLogicalId = "pat1";
+        final Integer refVersion = 2;
+        final boolean wholeSystem = false;
+        final Integer compositeId = null;
+        final String valueString = "str1";
+        final String url = "http://some.profile/location";
+        final String profileVersion = "1.0";
+        
+        SearchParametersTransportAdapter adapter = new SearchParametersTransportAdapter(resourceType, logicalId, logicalResourceId, 
+            versionId, lastUpdated, requestShard, parameterHash);
+        adapter.dateValue("date-param", ts1, ts2, null, true);
+        adapter.locationValue("location-param", 0.1, 0.2, null);
+        adapter.numberValue("number-param", valueNumber, valueNumberLow, valueNumberHigh, null);
+        adapter.profileValue("profile-param", url, profileVersion, null, true);
+        adapter.quantityValue("quantity-param", valueSystem, valueCode, valueNumber, valueNumberLow, valueNumberHigh, compositeId);
+        adapter.referenceValue("reference-param", refResourceType, refLogicalId, refVersion, compositeId);
+        adapter.securityValue("security-param", valueSystem, valueCode, wholeSystem);
+        adapter.stringValue("string-param", valueString, compositeId, wholeSystem);
+        adapter.tagValue("tag-param", valueSystem, valueCode, wholeSystem);
+        adapter.tokenValue("token-param", valueSystem, valueCode, compositeId);
+
+        sent.setData(adapter.build());
+        final String payload = RemoteIndexSupport.marshallToString(sent);
+        // Now unmarshall the payload and check everything matches
+        RemoteIndexMessage rcvd = RemoteIndexSupport.unmarshall(payload);
+        assertNotNull(rcvd);
+        assertEquals(rcvd.getMessageVersion(), RemoteIndexConstants.MESSAGE_VERSION);
+
+        SearchParametersTransport data = rcvd.getData();
+        assertNotNull(data);
+        assertEquals(data.getParameterHash(), parameterHash);
+        assertEquals(data.getLastUpdatedInstant(), lastUpdated);
+        assertEquals(data.getLogicalResourceId(), logicalResourceId);
+        assertEquals(data.getResourceType(), resourceType);
+        assertEquals(data.getLogicalId(), logicalId);
+
+        assertEquals(data.getDateValues().size(), 1);
+        assertEquals(data.getDateValues().get(0).getName(), "date-param");
+        assertEquals(data.getDateValues().get(0).getValueDateStart(), ts1);
+        assertEquals(data.getDateValues().get(0).getValueDateEnd(), ts2);
+        assertEquals(data.getLocationValues().size(), 1);
+        assertEquals(data.getLocationValues().get(0).getValueLatitude(), 0.1);
+        assertEquals(data.getLocationValues().get(0).getValueLongitude(), 0.2);
+        assertEquals(data.getNumberValues().size(), 1);
+        assertEquals(data.getNumberValues().get(0).getValue(), valueNumber);
+        assertEquals(data.getNumberValues().get(0).getLowValue(), valueNumberLow);
+        assertEquals(data.getNumberValues().get(0).getHighValue(), valueNumberHigh);
+        assertEquals(data.getProfileValues().size(), 1);
+        assertEquals(data.getProfileValues().get(0).getUrl(), url);
+        assertEquals(data.getProfileValues().get(0).getVersion(), profileVersion);
+        assertEquals(data.getQuantityValues().size(), 1);
+        assertEquals(data.getQuantityValues().get(0).getValueNumber(), valueNumber);
+        assertEquals(data.getQuantityValues().get(0).getValueNumberLow(), valueNumberLow);
+        assertEquals(data.getQuantityValues().get(0).getValueNumberHigh(), valueNumberHigh);
+        assertEquals(data.getRefValues().size(), 1);
+        assertEquals(data.getRefValues().get(0).getResourceType(), refResourceType);
+        assertEquals(data.getRefValues().get(0).getLogicalId(), refLogicalId);
+        assertEquals(data.getSecurityValues().size(), 1);
+        assertEquals(data.getSecurityValues().get(0).getValueSystem(), valueSystem);
+        assertEquals(data.getSecurityValues().get(0).getValueCode(), valueCode);
+        assertEquals(data.getStringValues().size(), 1);
+        assertEquals(data.getStringValues().get(0).getValue(), valueString);
+        assertEquals(data.getTagValues().size(), 1);
+        assertEquals(data.getTagValues().get(0).getValueSystem(), valueSystem);
+        assertEquals(data.getTagValues().get(0).getValueCode(), valueCode);
+        assertEquals(data.getTokenValues().size(), 1);
+        assertEquals(data.getTokenValues().get(0).getValueSystem(), valueSystem);
+        assertEquals(data.getTokenValues().get(0).getValueCode(), valueCode);
+    }
+
+    @Test
+    public void testInstant() {
+        Gson gson = RemoteIndexSupport.getGson();
+        Instant x = Instant.now();
+        String value = gson.toJson(x);
+
+        // now try and convert the other way
+        Instant x2 = gson.fromJson(value, Instant.class);
+        assertEquals(x, x2);
+    }
+}
diff --git a/fhir-persistence/src/test/java/com/ibm/fhir/persistence/test/FHIRPersistenceContextTest.java b/fhir-persistence/src/test/java/com/ibm/fhir/persistence/test/FHIRPersistenceContextTest.java
index 8007fe958e8..1b44d5f1d6a 100644
--- a/fhir-persistence/src/test/java/com/ibm/fhir/persistence/test/FHIRPersistenceContextTest.java
+++ b/fhir-persistence/src/test/java/com/ibm/fhir/persistence/test/FHIRPersistenceContextTest.java
@@ -57,12 +57,13 @@ public void test3() {
         FHIRSearchContext sc = FHIRSearchContextFactory.createSearchContext();
         assertNotNull(sc);
 
-        FHIRPersistenceContext ctxt = FHIRPersistenceContextFactory.createPersistenceContext(pe, sc);
+        FHIRPersistenceContext ctxt = FHIRPersistenceContextFactory.createPersistenceContext(pe, sc, "pat42");
         assertNotNull(ctxt);
         assertNotNull(ctxt.getPersistenceEvent());
         assertEquals(pe, ctxt.getPersistenceEvent());
         assertNotNull(ctxt.getSearchContext());
         assertEquals(sc, ctxt.getSearchContext());
         assertNull(ctxt.getHistoryContext());
+        assertEquals("pat42", ctxt.getRequestShard());
     }
 }
diff --git a/fhir-persistence/src/test/java/com/ibm/fhir/persistence/test/MockPersistenceImpl.java b/fhir-persistence/src/test/java/com/ibm/fhir/persistence/test/MockPersistenceImpl.java
index 1211824fd45..d0e8f59f05d 100644
--- a/fhir-persistence/src/test/java/com/ibm/fhir/persistence/test/MockPersistenceImpl.java
+++ b/fhir-persistence/src/test/java/com/ibm/fhir/persistence/test/MockPersistenceImpl.java
@@ -75,7 +75,7 @@ public OperationOutcome getHealth() throws FHIRPersistenceException {
 
     @Override
     public int reindex(FHIRPersistenceContext context, OperationOutcome.Builder oob, Instant tstamp, List indexIds,
-            String resourceLogicalId) throws FHIRPersistenceException {
+            String resourceLogicalId, boolean force) throws FHIRPersistenceException {
         return 0;
     }
 
@@ -92,13 +92,13 @@ public ResourcePayload fetchResourcePayloads(Class resourceT
     }
 
     @Override
-    public List changes(int resourceCount, Instant fromLastModified, Instant beforeLastModified, Long afterResourceId, List resourceTypeNames,
+    public List changes(FHIRPersistenceContext context, int resourceCount, Instant fromLastModified, Instant beforeLastModified, Long afterResourceId, List resourceTypeNames,
             boolean excludeTransactionTimeoutWindow, HistorySortOrder historySortOrder) throws FHIRPersistenceException {
         return Collections.emptyList();
     }
 
     @Override
-    public List retrieveIndex(int count, java.time.Instant notModifiedAfter, Long afterIndexId, String resourceTypeName) throws FHIRPersistenceException {
+    public List retrieveIndex(FHIRPersistenceContext context, int count, java.time.Instant notModifiedAfter, Long afterIndexId, String resourceTypeName) throws FHIRPersistenceException {
         return Collections.emptyList();
     }
 
diff --git a/fhir-persistence/src/test/java/com/ibm/fhir/persistence/test/common/AbstractChangesTest.java b/fhir-persistence/src/test/java/com/ibm/fhir/persistence/test/common/AbstractChangesTest.java
index 6a6b13156f8..a26f1491579 100644
--- a/fhir-persistence/src/test/java/com/ibm/fhir/persistence/test/common/AbstractChangesTest.java
+++ b/fhir-persistence/src/test/java/com/ibm/fhir/persistence/test/common/AbstractChangesTest.java
@@ -104,7 +104,7 @@ public void testSomeData() throws Exception {
         final List resourceTypeNames = Collections.emptyList();
         final boolean excludeTransactionTimeoutWindow = false;
         final HistorySortOrder historySortOrder = HistorySortOrder.NONE;
-        List result = persistence.changes(100, sinceLastModified, beforeLastModified, changeIdMarker, resourceTypeNames, excludeTransactionTimeoutWindow, historySortOrder);
+        List result = persistence.changes(null, 100, sinceLastModified, beforeLastModified, changeIdMarker, resourceTypeNames, excludeTransactionTimeoutWindow, historySortOrder);
         assertNotNull(result);
         assertTrue(result.size() >= 7);
         assertTrue(result.size() <= 100);
@@ -119,7 +119,7 @@ public void testChanges() throws Exception {
         final Long afterResourceId = null;
         final String resourceTypeName = null;
 
-        List result = persistence.changes(7, sinceLastModified, null, null, null, false, HistorySortOrder.NONE);
+        List result = persistence.changes(null, 7, sinceLastModified, null, null, null, false, HistorySortOrder.NONE);
         assertNotNull(result);
 
         // 4 CREATE
@@ -160,7 +160,7 @@ public void testLimit() throws Exception {
         Long afterResourceId = null;
         final String resourceTypeName = null;
 
-        List result = persistence.changes(4, sinceLastModified, null, null, null, false, HistorySortOrder.NONE);
+        List result = persistence.changes(null, 4, sinceLastModified, null, null, null, false, HistorySortOrder.NONE);
         assertNotNull(result);
 
         // Limit was set to 4, so we should only get partial data
@@ -169,14 +169,14 @@ public void testLimit() throws Exception {
         // Make another call now to get the remaining 3 changes
         sinceLastModified = result.get(3).getChangeTstamp();
         afterResourceId = result.get(3).getChangeId();
-        result = persistence.changes(3, sinceLastModified, null, afterResourceId, null, false, HistorySortOrder.NONE);
+        result = persistence.changes(null, 3, sinceLastModified, null, afterResourceId, null, false, HistorySortOrder.NONE);
         assertNotNull(result);
         assertEquals(result.size(), 3);
 
         // And a final call to make sure we get nothing
         sinceLastModified = result.get(2).getChangeTstamp();
         afterResourceId = result.get(2).getChangeId();
-        result = persistence.changes(100, sinceLastModified, null, afterResourceId, null, false, HistorySortOrder.NONE);
+        result = persistence.changes(null, 100, sinceLastModified, null, afterResourceId, null, false, HistorySortOrder.NONE);
         assertNotNull(result);
         assertEquals(result.size(), 0);
     }
@@ -189,7 +189,7 @@ public void testResourceTypeFilter() throws Exception {
         final String resourceTypeName = resource1.getClass().getSimpleName();
 
         List resourceTypeNames = Arrays.asList(resourceTypeName);
-        List result = persistence.changes(10, sinceLastModified, null, afterResourceId, resourceTypeNames, false, HistorySortOrder.NONE);
+        List result = persistence.changes(null, 10, sinceLastModified, null, afterResourceId, resourceTypeNames, false, HistorySortOrder.NONE);
         assertNotNull(result);
         assertEquals(result.size(), 7);
     }
diff --git a/fhir-persistence/src/test/java/com/ibm/fhir/persistence/test/common/AbstractEraseTest.java b/fhir-persistence/src/test/java/com/ibm/fhir/persistence/test/common/AbstractEraseTest.java
index 448a97419cd..877df87e687 100644
--- a/fhir-persistence/src/test/java/com/ibm/fhir/persistence/test/common/AbstractEraseTest.java
+++ b/fhir-persistence/src/test/java/com/ibm/fhir/persistence/test/common/AbstractEraseTest.java
@@ -126,7 +126,7 @@ public void testEraseResourceWithHistory() throws Exception {
         EraseDTO dto = new EraseDTO();
         dto.setLogicalId(resource1.getId());
         dto.setResourceType("Basic");
-        ResourceEraseRecord eraseRecord = persistence.erase(dto);
+        ResourceEraseRecord eraseRecord = persistence.erase(null, dto);
         assertNotNull(eraseRecord);
         assertEquals((int) eraseRecord.getTotal(), 3);
         assertEquals(eraseRecord.getStatus(), ResourceEraseRecord.Status.DONE);
@@ -141,7 +141,7 @@ public void testEraseSingleResource() throws Exception {
         EraseDTO dto = new EraseDTO();
         dto.setLogicalId(resource1.getId());
         dto.setResourceType("Basic");
-        ResourceEraseRecord eraseRecord = persistence.erase(dto);
+        ResourceEraseRecord eraseRecord = persistence.erase(null, dto);
         assertNotNull(eraseRecord);
         assertEquals((int) eraseRecord.getTotal(), 1);
         assertEquals(eraseRecord.getStatus(), ResourceEraseRecord.Status.DONE);
@@ -158,7 +158,7 @@ public void testEraseLastIsDeleted() throws Exception {
         EraseDTO dto = new EraseDTO();
         dto.setLogicalId(resource1.getId());
         dto.setResourceType("Basic");
-        ResourceEraseRecord eraseRecord = persistence.erase(dto);
+        ResourceEraseRecord eraseRecord = persistence.erase(null, dto);
         assertNotNull(eraseRecord);
         assertEquals((int) eraseRecord.getTotal(), 2);
         assertEquals(eraseRecord.getStatus(), ResourceEraseRecord.Status.DONE);
@@ -171,7 +171,7 @@ public void testEraseNotExists() throws Exception {
         EraseDTO dto = new EraseDTO();
         dto.setLogicalId("---NOTEXISTS");
         dto.setResourceType("Basic");
-        ResourceEraseRecord eraseRecord = persistence.erase(dto);
+        ResourceEraseRecord eraseRecord = persistence.erase(null, dto);
         assertNotNull(eraseRecord);
         assertEquals(eraseRecord.getStatus(), ResourceEraseRecord.Status.NOT_FOUND);
     }
@@ -189,7 +189,7 @@ public void testEraseSpecificVersionGreater() throws Exception {
         dto.setLogicalId(resource1.getId());
         dto.setResourceType("Basic");
         dto.setVersion(4);
-        ResourceEraseRecord eraseRecord = persistence.erase(dto);
+        ResourceEraseRecord eraseRecord = persistence.erase(null, dto);
         assertNotNull(eraseRecord);
         assertEquals((int) eraseRecord.getTotal(), -1);
         assertEquals(eraseRecord.getStatus(), ResourceEraseRecord.Status.NOT_SUPPORTED_GREATER);
@@ -209,7 +209,7 @@ public void testEraseSpecificVersion() throws Exception {
         dto.setLogicalId(resource1.getId());
         dto.setResourceType("Basic");
         dto.setVersion(2);
-        ResourceEraseRecord eraseRecord = persistence.erase(dto);
+        ResourceEraseRecord eraseRecord = persistence.erase(null, dto);
         assertNotNull(eraseRecord);
         assertEquals((int) eraseRecord.getTotal(), 1);
         assertEquals(eraseRecord.getStatus(), ResourceEraseRecord.Status.VERSION);
@@ -232,7 +232,7 @@ public void testEraseLatestSpecificVersion() throws Exception {
         dto.setLogicalId(resource1.getId());
         dto.setResourceType("Basic");
         dto.setVersion(3);
-        ResourceEraseRecord eraseRecord = persistence.erase(dto);
+        ResourceEraseRecord eraseRecord = persistence.erase(null, dto);
         assertNotNull(eraseRecord);
         assertEquals((int) eraseRecord.getTotal(), -1);
         assertEquals(eraseRecord.getStatus(), ResourceEraseRecord.Status.NOT_SUPPORTED_LATEST);
@@ -248,7 +248,7 @@ public void testEraseSingleResourceWithVersion1() throws Exception {
         dto.setLogicalId(resource1.getId());
         dto.setResourceType("Basic");
         dto.setVersion(1);
-        ResourceEraseRecord eraseRecord = persistence.erase(dto);
+        ResourceEraseRecord eraseRecord = persistence.erase(null, dto);
         assertNotNull(eraseRecord);
         assertEquals((int) eraseRecord.getTotal(), 1);
         assertEquals(eraseRecord.getStatus(), ResourceEraseRecord.Status.DONE);
diff --git a/fhir-persistence/src/test/java/com/ibm/fhir/persistence/test/common/AbstractPersistenceTest.java b/fhir-persistence/src/test/java/com/ibm/fhir/persistence/test/common/AbstractPersistenceTest.java
index 68d34e1037c..8d3b73c9c64 100644
--- a/fhir-persistence/src/test/java/com/ibm/fhir/persistence/test/common/AbstractPersistenceTest.java
+++ b/fhir-persistence/src/test/java/com/ibm/fhir/persistence/test/common/AbstractPersistenceTest.java
@@ -98,7 +98,7 @@ protected FHIRPersistenceContext getPersistenceContextIfNoneMatch() throws Excep
         return FHIRPersistenceContextFactory.createPersistenceContext(null, ifNoneMatch);
     }
     protected FHIRPersistenceContext getPersistenceContextForSearch(FHIRSearchContext ctxt) {
-        return FHIRPersistenceContextFactory.createPersistenceContext(null, ctxt);
+        return FHIRPersistenceContextFactory.createPersistenceContext(null, ctxt, null);
     }
     protected FHIRPersistenceContext getPersistenceContextForHistory(FHIRHistoryContext ctxt) {
         return FHIRPersistenceContextFactory.createPersistenceContext(null, ctxt);
diff --git a/fhir-remote-index/README.md b/fhir-remote-index/README.md
new file mode 100644
index 00000000000..0214b77a358
--- /dev/null
+++ b/fhir-remote-index/README.md
@@ -0,0 +1,175 @@
+# Project: fhir-remote-index
+
+Project fhir-remote-index is a stand-alone application to support asynchronous storage of FHIR search parameters published by the IBM FHIR Server. When configured to do so, the IBM FHIR Server extracts search parameters from the incoming resource and instead of storing the parameters as part of the create/update transaction, it packages the parameters into a message which is then published to Kafka. The fhir-remote-index application consumes the messages from Kafka and stores the value in the resource parameter tables using efficient batch-based inserts.
+
+This pattern supports higher ingestion rates because:
+
+1. Any locking related to inserting normalized common parameter values is decoupled from the locking on logical resources (`logical_resource_ident` table) used to ensure correct versioning of the resource. Create and Update interactions should never see any contention unless the same logical resource is updated in parallel requests;
+2. If the same logical resource is updated in parallel, the contention will be reduced compared to synchronous parameter value storage because less work is performed while the lock is held, allowing the transaction to be completed sooner;
+3. The remote index consumer can build larger batches, with transactions not tied to the semantics of the original FHIR interaction;
+4. The remote index consumer uses batches for all the search parameter value inserts, making it more efficient than the current JDBC persistence layer implementation;
+5. The remote index consumer handles normalized value cache-miss lookups using bulk queries. This reduces the total number of database round-trips required to find the required foreign key values.
+
+In addition, this implementation eliminates any possibility of deadlocks occurring during the Insert/Update interaction. Deadlocks may still occur when processing the asynchronous remote index messages. As these will only occur in a backend process they will not be visible to IBM FHIR Server clients. Deadlocks are handle automatically using a rollback and retry mechanism. Care is taken to reduce the likelihood of deadlocks from occurring in the first place by sorting all record lists before they are processed.
+
+It is worth noting that using multiple Kafka topic partitions can increase throughput by allowing more resource parameter messages to be processed in parallel. If sufficient threads are allocated across all the fhir-remote-index consumer instances, each thread will read data from a single partition. There is no point allocating more total threads than the number of configured partitions. The partition key is a function of the {resource-type, logical-id} tuple which guarantees that changes related to a particular logical resource will be pushed to the same partition. This guarantees that these changes will be processed in order.
+
+## Processing Is Asynchronous
+
+Old search parameters are deleted whenever a resource is updated. When remote indexing is enabled, this means that a resource will not be searchable until the remote index service has received and processed the message. Carefully examine your interaction scenarios with the IBM FHIR Server to determine if this behavior is suitable. In particular, conditional updates are unlikely to work as expected because the search may not return the expected value, depending on timing.
+
+## Status
+
+At this time, fhir-remote-index should be considered experimental.
+
+## Build
+
+To build fhir-remote-index, clone the git repository and build as follows:
+
+```
+git clone git@github.com:IBM/FHIR.git
+cd FHIR
+mvn clean install -f fhir-examples
+mvn clean install -f fhir-parent
+mvn clean install -f fhir-remote-index
+```
+
+Note that at this time, fhir-remote-index is not built by default so its project must be built explicitly as shown above.
+
+## IBM FHIR Server Configuration
+
+To enable remote indexing of search parameters, add the following `remoteIndexService` entry to the default FHIR server configuration file `config/default/fhir-server-config.json`. This entry identifies the Kafka service and topic to use for sending remote index messages. When this entry is present, the JDBC persistence layer will skip storing the parameters and instead will package the parameters into a message and publish it to Kafka.
+
+```
+{
+    "__comment": "FHIR Server configuration",
+    "fhirServer": {
+       ...
+       "remoteIndexService": {
+            "type": "kafka",
+            "instanceIdenfier": "a-random-uuid-value",
+            "kafka": {
+                "mode": "ACTIVE",
+                "topicName": "FHIR_REMOTE_INDEX",
+                "connectionProperties": {
+                    "bootstrap.servers": "broker-0:9093, broker-1:9093, broker-2:9093, broker-3:9093, broker-4:9093, broker-5:9093",
+                    "sasl.jaas.config": "org.apache.kafka.common.security.plain.PlainLoginModule required username=\"token\" password=\"change-password\";",
+                    "sasl.mechanism": "PLAIN",
+                    "security.protocol": "SASL_SSL",
+                    "ssl.protocol": "TLSv1.2",
+                    "ssl.enabled.protocols": "TLSv1.2",
+                    "ssl.endpoint.identification.algorithm": "HTTPS"
+                }
+            }
+        },
+        ...
+```
+
+## Running the fhir-remote index consumer
+
+The fhir-remote-index application accepts two properties files defining access to Kafka and the target database:
+
+1. `--kafka-properties` the properties describing connection information for the upstream Kafka topic containing the resource search parameter messages sent by the upstream IBM FHIR Server;
+2. `--database-properties` the properties describing the location of the target database to which we will insert the search parameter records generated by the upstream IBM FHIR Server.
+
+```
+java -Djava.util.logging.config.file=logging.properties \
+  -jar /path/to/git/FHIR/fhir-remote-index/target/fhir-remote-index-*-cli.jar \
+  --db-type postgresql \
+  --database-properties database.properties \
+  --kafka-properties kafka.properties \
+  --topic-name FHIR_REMOTE_INDEX \
+  --consumer-count 3 \
+  --instance-identifier "a-random-uuid-value"
+```
+
+Logging uses standard `java.util.logging` (JUL) and can be configured as follows:
+
+```
+handlers=java.util.logging.ConsoleHandler,java.util.logging.FileHandler
+.level=INFO
+
+# Console output
+java.util.logging.ConsoleHandler.level = INFO
+java.util.logging.ConsoleHandler.formatter=com.ibm.fhir.database.utils.common.LogFormatter
+
+# What level do we want to see in the log file
+java.util.logging.FileHandler.level=INFO
+
+# Log retention: 50MB * 20 files ~= 1GB
+java.util.logging.FileHandler.formatter=com.ibm.fhir.database.utils.common.LogFormatter
+java.util.logging.FileHandler.limit=50000000
+java.util.logging.FileHandler.count=20
+java.util.logging.FileHandler.pattern=remoteindexservice-%u-%g.log
+```
+
+To configure the Kafka service, use a properties file as described by the [Kafka documentation](https://kafka.apache.org/documentation/#configuration):
+
+```
+bootstrap.servers=broker-0:9093, broker-1:9093, broker-2:9093, broker-3:9093, broker-4:9093, broker-5:9093
+sasl.jaas.config=org.apache.kafka.common.security.plain.PlainLoginModule required username="token" password="change-password";
+sasl.mechanism=PLAIN
+security.protocol=SASL_SSL
+ssl.protocol=TLSv1.2
+ssl.enabled.protocols=TLSv1.2
+ssl.endpoint.identification.algorithm=HTTPS
+```
+
+The database properties file:
+
+```
+db.host=your-db-host
+db.port=5432
+db.database=your-db-name
+user=fhirserver
+password=change-password
+currentSchema=fhirdata
+ssl=true
+sslmode=require
+sslcert=
+sslrootcert=postgres.crt
+sslkey=
+```
+
+| Property | DB Type | Description |
+| ------ | --- | ----------- |
+| db.host | All | The host name of the database containing the IBM FHIR Server schema | 
+| db.port | All | The database port number |
+| db.database | All | The database name |
+| user | All | The database credential user |
+| password | All | The database credential password |
+| currentSchema | All | The database schema name |
+| ssl | PostgreSQL | For PostgreSQL, use SSL (TLS) for the database connection |
+| sslmode | PostgreSQL | The PostgreSQL SSL connection mode |
+| sslcert | PostgreSQL | The PostgreSQL SSL certificate |
+| sslrootcert | PostgreSQL | The PostgreSQL SSL root certificate |
+| sslkey | PostgreSQL | The PostgreSQL SSL key |
+| sslConnection | Db2 | true or false |
+| sslTrustStoreLocation | Db2 | Location of the p12 trust store file containing the database server certificate |
+| sslTrustStorePassword | Db2 | Password for the p12 trust store |
+
+Note: Citus configuration is the same as PostgreSQL.
+
+## Command Line Options
+
+| Option | Description |
+| ------ | ----------- |
+| --consumer-count {n} | The number of Kafka consumer threads to start in this instance. Multiple instances of this service can be started. The total number of consumer threads across all instances should equal the number to the number of Kafka partitions on the topic. This should maximize throughput. |
+| --kafka-properties {properties-file} | A Java properties file containing connection details for the upstream Kafka service. |
+| --db-type {type} | The type of database. One of `postgresql`, `derby`, `db2` or `citus`. |
+| --database-properties {properties-file} | A Java properties file containing connection details for the downstream IBM FHIR Server database. |
+| --topic-name {topic} | The name of the Kafka topic to consume. Default `FHIR_REMOTE_INDEX`. |
+| --instance-identifier {uuid} | Each IBM FHIR Server cluster should be allocated a unique instance identifier. This identifier is added to each message sent over Kafka. The consumer will ignore messages unless they include the same instance identifier value. This helps to ensure that messages are processed from only intended sources. |
+| --consumer-group {grp} | Override the default Kafka consumer group (`group.id` value) for this application. Default `remote-index-service-cg`. |
+| --schema-type {type} | Set the schema type. One of `PLAIN` or `DISTRIBUTED`. Default is `PLAIN`. The schema type `DISTRIBUTED` is for use with Citus databases. |
+| --max-ready-time-ms {milliseconds} | The maximum number of milliseconds to wait for the database to contain the correct data for a particular set of consumed messages. Should be slightly longer than the configured Liberty transaction timeout value. |
+
+# Asynchronous Message Handling and Transaction Boundaries
+
+To guarantee delivery, the search parameter messages are posted to Kafka by the IBM FHIR Server before the transaction commits. The transaction will only be committed once all messages sent to Kafka have been acknowledged. This is important, because if the message were to be sent after the transaction, we could lose messages if a failure occured immediately after the transaction but before they were received by Kafka.
+
+Because messages are sent to Kafka before the transaction is committed, it is possible that a fhir-remote-index consumer may receive a search parameter message before the corresponding resource version record is visible in the database. The consumer therefore runs a query at the start of a batch to determine if the current resource version record matches the message content. The following logic is then applied:
+
+1. If the resource version doesn't yet exist in the database, the consumer will pause and wait for the transaction to be committed. The consumer will only wait up to the maximum transaction timeout window, at which point it will assume the transaction has failed and the message will be discarded.
+2. If the resource version matches, but the lastUpdated time does not match, it assumes the message came from an IBM FHIR Server which failed before the transaction was committed, but the request was processed successfully by another server. The message will be discarded because there will be another message waiting in the queue from the second attempt.
+3. If the resource version in the database already exceeds the version in the message, the message will be discarded because the information is already out of date. There will be another message waiting in the queue containing the search parameter values from the most recent resource.
diff --git a/fhir-remote-index/pom.xml b/fhir-remote-index/pom.xml
new file mode 100644
index 00000000000..aa9e079b7a6
--- /dev/null
+++ b/fhir-remote-index/pom.xml
@@ -0,0 +1,150 @@
+
+    4.0.0
+
+    fhir-remote-index
+    
+        com.ibm.fhir
+        fhir-parent
+        5.0.0-SNAPSHOT
+        ../fhir-parent
+    
+
+    
+        UTF-8
+    
+
+    
+        
+            ${project.groupId}
+            fhir-persistence
+            ${project.version}
+        
+        
+            ${project.groupId}
+            fhir-model
+            ${project.version}
+        
+        
+            ${project.groupId}
+            fhir-validation
+            ${project.version}
+        
+        
+            ${project.groupId}
+            fhir-config
+            ${project.version}
+            provided
+        
+        
+            ${project.groupId}
+            fhir-database-utils
+            ${project.version}
+        
+        
+            ${project.groupId}
+            fhir-search
+            ${project.version}
+        
+        
+            com.google.code.gson
+            gson
+        
+        
+        
+            org.apache.derby
+            derby
+            true
+        
+        
+            org.apache.derby
+            derbytools
+            true
+        
+        
+            com.ibm.db2
+            jcc
+            true
+        
+        
+            org.postgresql
+            postgresql
+            true
+        
+        
+        
+            org.apache.kafka
+            kafka-clients
+        
+        
+            com.github.ben-manes.caffeine
+            caffeine
+        
+        
+            org.testng
+            testng
+            test
+        
+        
+            ${project.groupId}
+            fhir-persistence-schema
+            ${project.version}
+            test
+        
+        
+            ${project.groupId}
+            fhir-model
+            ${project.version}
+            test-jar
+            test
+        
+        
+            org.eclipse.parsson
+            jakarta.json
+            test
+        
+        
+            org.skyscreamer
+            jsonassert
+            test
+        
+    
+
+    
+        
+            
+                org.apache.maven.plugins
+                maven-shade-plugin
+                
+                    
+                        package
+                        
+                            shade
+                        
+                        
+                            true
+                            cli
+                            
+                                
+                                    com.ibm.fhir.remote.index.app.Main
+                                
+                            
+                            
+                                
+                                    *:*
+                                    
+                                        META-INF/*.SF
+                                        META-INF/*.DSA
+                                        META-INF/*.RSA
+                                    
+                                
+                            
+                        
+                    
+                
+            
+        
+    
+
diff --git a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/api/BatchParameterProcessor.java b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/api/BatchParameterProcessor.java
new file mode 100644
index 00000000000..b93ec926e28
--- /dev/null
+++ b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/api/BatchParameterProcessor.java
@@ -0,0 +1,149 @@
+/*
+ * (C) Copyright IBM Corp. 2022
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+ 
+package com.ibm.fhir.remote.index.api;
+
+import com.ibm.fhir.persistence.exception.FHIRPersistenceException;
+import com.ibm.fhir.persistence.index.DateParameter;
+import com.ibm.fhir.persistence.index.LocationParameter;
+import com.ibm.fhir.persistence.index.NumberParameter;
+import com.ibm.fhir.persistence.index.ProfileParameter;
+import com.ibm.fhir.persistence.index.QuantityParameter;
+import com.ibm.fhir.persistence.index.ReferenceParameter;
+import com.ibm.fhir.persistence.index.SecurityParameter;
+import com.ibm.fhir.persistence.index.StringParameter;
+import com.ibm.fhir.persistence.index.TagParameter;
+import com.ibm.fhir.persistence.index.TokenParameter;
+import com.ibm.fhir.remote.index.database.CodeSystemValue;
+import com.ibm.fhir.remote.index.database.CommonCanonicalValue;
+import com.ibm.fhir.remote.index.database.CommonTokenValue;
+import com.ibm.fhir.remote.index.database.LogicalResourceIdentValue;
+import com.ibm.fhir.remote.index.database.ParameterNameValue;
+
+/**
+ * Processes batched parameters
+ */
+public interface BatchParameterProcessor {
+
+    /**
+     * Compute the shard key value use to distribute resources among nodes
+     * of the database
+     * @param requestShard
+     * @return
+     */
+    Short encodeShardKey(String requestShard);
+
+    /**
+     * @param requestShard
+     * @param resourceType
+     * @param logicalId
+     * @param logicalResourceId
+     * @param parameterNameValue
+     * @param parameter
+     */
+    void process(String requestShard, String resourceType, String logicalId, long logicalResourceId, ParameterNameValue parameterNameValue, StringParameter parameter) throws FHIRPersistenceException;
+
+    /**
+     * @param requestShard
+     * @param resourceType
+     * @param logicalId
+     * @param logicalResourceId
+     * @param parameterNameValue
+     * @param parameter
+     */
+    void process(String requestShard, String resourceType, String logicalId, long logicalResourceId, ParameterNameValue parameterNameValue, NumberParameter parameter) throws FHIRPersistenceException;
+
+    /**
+     * @param requestShard
+     * @param resourceType
+     * @param logicalId
+     * @param logicalResourceId
+     * @param parameterNameValue
+     * @param parameter
+     */
+    void process(String requestShard, String resourceType, String logicalId, long logicalResourceId, ParameterNameValue parameterNameValue, QuantityParameter parameter, CodeSystemValue codeSystemValue) throws FHIRPersistenceException;
+
+    /**
+     * @param requestShard
+     * @param resourceType
+     * @param logicalId
+     * @param logicalResourceId
+     * @param parameterNameValue
+     * @param parameter
+     */
+    void process(String requestShard, String resourceType, String logicalId, long logicalResourceId, ParameterNameValue parameterNameValue, LocationParameter parameter) throws FHIRPersistenceException;
+
+    /**
+     * @param requestShard
+     * @param resourceType
+     * @param logicalId
+     * @param logicalResourceId
+     * @param parameterNameValue
+     * @param parameter
+     */
+    void process(String requestShard, String resourceType, String logicalId, long logicalResourceId, ParameterNameValue parameterNameValue, DateParameter parameter) throws FHIRPersistenceException;
+
+    /**
+     * @param requestShard
+     * @param resourceType
+     * @param logicalId
+     * @param logicalResourceId
+     * @param parameterNameValue
+     * @param parameter
+     */
+    void process(String requestShard, String resourceType, String logicalId, long logicalResourceId, ParameterNameValue parameterNameValue, TokenParameter parameter, CommonTokenValue commonTokenValue) throws FHIRPersistenceException;
+
+    /**
+     * 
+     * @param requestShard
+     * @param resourceType
+     * @param logicalId
+     * @param logicalResourceId
+     * @param parameterNameValue
+     * @param parameter
+     * @param commonTokenValue
+     * @throws FHIRPersistenceException
+     */
+    void process(String requestShard, String resourceType, String logicalId, long logicalResourceId, ParameterNameValue parameterNameValue, TagParameter parameter, CommonTokenValue commonTokenValue) throws FHIRPersistenceException;
+
+    /**
+     * 
+     * @param requestShard
+     * @param resourceType
+     * @param logicalId
+     * @param logicalResourceId
+     * @param parameterNameValue
+     * @param parameter
+     * @param commonCanonicalValue
+     * @throws FHIRPersistenceException
+     */
+    void process(String requestShard, String resourceType, String logicalId, long logicalResourceId, ParameterNameValue parameterNameValue, ProfileParameter parameter, CommonCanonicalValue commonCanonicalValue) throws FHIRPersistenceException;
+
+    /**
+     * 
+     * @param requestShard
+     * @param resourceType
+     * @param logicalId
+     * @param logicalResourceId
+     * @param parameterNameValue
+     * @param parameter
+     * @param commonCanonicalValue
+     * @throws FHIRPersistenceException
+     */
+    void process(String requestShard, String resourceType, String logicalId, long logicalResourceId, ParameterNameValue parameterNameValue, SecurityParameter parameter, CommonTokenValue commonTokenValue) throws FHIRPersistenceException;
+
+    /**
+     * @param requestShard
+     * @param resourceType
+     * @param logicalId
+     * @param logicalResourceId
+     * @param parameterNameValue
+     * @param parameter
+     * @param refLogicalResourceId
+     */
+    void process(String requestShard, String resourceType, String logicalId, long logicalResourceId, ParameterNameValue parameterNameValue,
+        ReferenceParameter parameter, LogicalResourceIdentValue refLogicalResourceId) throws FHIRPersistenceException;
+}
diff --git a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/api/BatchParameterValue.java b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/api/BatchParameterValue.java
new file mode 100644
index 00000000000..5b8af6f9dcb
--- /dev/null
+++ b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/api/BatchParameterValue.java
@@ -0,0 +1,43 @@
+/*
+ * (C) Copyright IBM Corp. 2022
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+ 
+package com.ibm.fhir.remote.index.api;
+
+import com.ibm.fhir.persistence.exception.FHIRPersistenceException;
+import com.ibm.fhir.remote.index.database.ParameterNameValue;
+
+/**
+ * A parameter value batched for later processing
+ */
+public abstract class BatchParameterValue {
+    protected final String requestShard;
+    protected final ParameterNameValue parameterNameValue;
+    protected final String resourceType;
+    protected final String logicalId;
+    protected final long logicalResourceId;
+
+    /**
+     * Protected constructor
+     * @param requestShard
+     * @param resourceType
+     * @param logicalId
+     * @param logicalResourceId
+     * @param parameterNameValue
+     */
+    protected BatchParameterValue(String requestShard, String resourceType, String logicalId, long logicalResourceId, ParameterNameValue parameterNameValue) {
+        this.requestShard = requestShard;
+        this.resourceType = resourceType;
+        this.logicalId = logicalId;
+        this.logicalResourceId = logicalResourceId;
+        this.parameterNameValue = parameterNameValue;
+    }
+
+    /**
+     * Apply this parameter value to the target processor
+     * @param processor
+     */
+    public abstract void apply(BatchParameterProcessor processor) throws FHIRPersistenceException;
+}
diff --git a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/api/IMessageHandler.java b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/api/IMessageHandler.java
new file mode 100644
index 00000000000..4136b1355ee
--- /dev/null
+++ b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/api/IMessageHandler.java
@@ -0,0 +1,30 @@
+/*
+ * (C) Copyright IBM Corp. 2022
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+ 
+package com.ibm.fhir.remote.index.api;
+
+import java.util.List;
+
+import com.ibm.fhir.persistence.exception.FHIRPersistenceException;
+
+/**
+ * Our interface for handling messages received by the consumer. Used
+ * to decouple the Kafka consumer from the database persistence logic
+ */
+public interface IMessageHandler {
+
+    /**
+     * Ask the handler to process the list of messages.
+     * @param messages
+     * @throws FHIRPersistenceException
+     */
+    void process(List messages) throws FHIRPersistenceException;
+
+    /**
+     * Close any resources held by the handler
+     */
+    void close();
+}
diff --git a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/api/IdentityCache.java b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/api/IdentityCache.java
new file mode 100644
index 00000000000..786a3e952d5
--- /dev/null
+++ b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/api/IdentityCache.java
@@ -0,0 +1,99 @@
+/*
+ * (C) Copyright IBM Corp. 2022
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+ 
+package com.ibm.fhir.remote.index.api;
+
+
+/**
+ * Interface to hides the implementation of various caches we use during
+ * ingestion persistence
+ */
+public interface IdentityCache {
+    /**
+     * Get the parameter_name_id value for the given parameterName
+     * @param parameterName
+     * @return the parameter_name_id or null if the value is not found in the cache
+     */
+    Integer getParameterNameId(String parameterName);
+
+    /**
+     * Get the code_system_id value for the given codeSystem value
+     * @param codeSystem
+     * @return the code_system_id or null if the value is not found in the cache
+     */
+    Integer getCodeSystemId(String codeSystem);
+
+    /**
+     * Get the common_token_value_id for the given codeSystem and tokenValue
+     * @param shardKey
+     * @param codeSystem
+     * @param tokenValue
+     * @return the common_token_value_id or null if the value is not found in the cache
+     */
+    Long getCommonTokenValueId(short shardKey, String codeSystem, String tokenValue);
+
+    /**
+     * Add the given parameterName to parameterNameId mapping to the cache
+     * @param parameterName
+     * @param parameterNameId
+     */
+    void addParameterName(String parameterName, int parameterNameId);
+
+    /**
+     * @param shardKey
+     * @param url
+     * @return
+     */
+    Long getCommonCanonicalValueId(short shardKey, String url);
+
+    /**
+     * Add the common canonical value to the cache
+     * @param shardKey
+     * @param url
+     * @param commonCanonicalValueId
+     */
+    void addCommonCanonicalValue(short shardKey, String url, long commonCanonicalValueId);
+
+    /**
+     * Add the common token value to the cache
+     * @param shardKey
+     * @param codeSystem
+     * @param tokenValue
+     * @param commonTokenValueId
+     */
+    void addCommonTokenValue(short shardKey, String codeSystem, String tokenValue, long commonTokenValueId);
+
+    /**
+     * Add the code system value to the cache
+     * @param codeSystem
+     * @param codeSystemId
+     */
+    void addCodeSystem(String codeSystem, int codeSystemId);
+
+    /**
+     * Get the database resource_type_id value for the given resourceType value
+     * @param resourceType
+     * @return
+     * @throws IllegalArgumentException if resourceType is not a valid resource type name
+     */
+    int getResourceTypeId(String resourceType);
+
+    /**
+     * Get the database logical_resource_id for the given resourceType/logicalId tuple.
+     * @param resourceType
+     * @param logicalId
+     * @return
+     */
+    Long getLogicalResourceIdentId(String resourceType, String logicalId);
+
+    /**
+     * Add the logical_resource_ident mapping to the cache
+     * @param resourceType
+     * @param logicalId
+     * @param logicalResourceId
+     */
+    void addLogicalResourceIdent(String resourceType, String logicalId, long logicalResourceId);
+}
diff --git a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/app/Main.java b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/app/Main.java
new file mode 100644
index 00000000000..03e82148cae
--- /dev/null
+++ b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/app/Main.java
@@ -0,0 +1,493 @@
+/*
+ * (C) Copyright IBM Corp. 2022
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+ 
+package com.ibm.fhir.remote.index.app;
+
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.sql.Connection;
+import java.sql.SQLException;
+import java.time.Duration;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Objects;
+import java.util.Properties;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+import org.apache.kafka.clients.consumer.KafkaConsumer;
+import org.apache.kafka.common.PartitionInfo;
+
+import com.ibm.fhir.core.util.LogSupport;
+import com.ibm.fhir.database.utils.api.IConnectionProvider;
+import com.ibm.fhir.database.utils.api.IDatabaseTranslator;
+import com.ibm.fhir.database.utils.api.SchemaType;
+import com.ibm.fhir.database.utils.citus.CitusTranslator;
+import com.ibm.fhir.database.utils.citus.ConfigureConnectionDAO;
+import com.ibm.fhir.database.utils.common.JdbcConnectionProvider;
+import com.ibm.fhir.database.utils.derby.DerbyPropertyAdapter;
+import com.ibm.fhir.database.utils.derby.DerbyTranslator;
+import com.ibm.fhir.database.utils.model.DbType;
+import com.ibm.fhir.database.utils.postgres.PostgresPropertyAdapter;
+import com.ibm.fhir.database.utils.postgres.PostgresTranslator;
+import com.ibm.fhir.database.utils.thread.ThreadHandler;
+import com.ibm.fhir.persistence.exception.FHIRPersistenceException;
+import com.ibm.fhir.remote.index.api.IMessageHandler;
+import com.ibm.fhir.remote.index.cache.IdentityCacheImpl;
+import com.ibm.fhir.remote.index.database.CacheLoader;
+import com.ibm.fhir.remote.index.database.DistributedPostgresMessageHandler;
+import com.ibm.fhir.remote.index.database.PlainDerbyMessageHandler;
+import com.ibm.fhir.remote.index.database.PlainPostgresMessageHandler;
+import com.ibm.fhir.remote.index.kafka.RemoteIndexConsumer;
+import com.ibm.fhir.remote.index.sharded.ShardedPostgresMessageHandler;
+
+/**
+ * Main class for the FHIR remote index service Kafka consumer
+ */
+public class Main {
+    private static final Logger logger = Logger.getLogger(Main.class.getName());
+    // Properties holding the Kafka connection information
+    private final Properties kafkaProperties = new Properties();
+    // Properties holding the JDBC connection information
+    private final Properties databaseProperties = new Properties();
+
+    private String topicName;
+
+    // The standard consumer group which all the remote index consumers should use
+    private String consumerGroup = "remote-index-service-cg";
+
+    private int consumerCount = 1;
+
+    private Duration pollDuration = Duration.ofSeconds(10);
+    private long maxBatchCollectTimeMs = 5000;
+
+    // The max time we wait for the database to catch up with what was sent to Kafka
+    // Must be a little longer than the the Liberty transaction timeout
+    private long maxReadyTimeMs = 180000;
+
+    // the list of consumers
+    private final List consumers = new ArrayList<>();
+
+    // track the number of consumers that are still running
+    private AtomicInteger stillRunningCounter;
+
+    private volatile boolean running = true;
+
+    // Exit if we drop below this number of running consumers
+    private float minRunningConsumerRatio = 0.5f;
+    private int minRunningConsumerThreshold;
+    private IdentityCacheImpl identityCache;
+
+    // Database Configuration
+    private SchemaType schemaType = SchemaType.PLAIN;
+    private IDatabaseTranslator translator;
+    private IConnectionProvider connectionProvider;
+    private DbType dbType = DbType.POSTGRESQL;
+
+    // Make sure we process messages sent from only the FHIR servers we are configured for
+    private String instanceIdentifier;
+    
+    /**
+     * Parse the given command line arguments
+     * @param args
+     */
+    public void parseArgs(String[] args) {
+        int a = 0;
+        while (a < args.length) {
+            final String arg = args[a++];
+            switch (arg) {
+            case "--kafka-properties":
+                if (a < args.length && !args[a].startsWith("--")) {
+                    loadKafkaProperties(args[a++]);
+                } else {
+                    throw new IllegalArgumentException("Missing value for --kafka-properties");
+                }
+                break;
+            case "--database-properties":
+                if (a < args.length && !args[a].startsWith("--")) {
+                    loadDatabaseProperties(args[a++]);
+                } else {
+                    throw new IllegalArgumentException("Missing value for --database-properties");
+                }
+                break;
+            case "--db-type":
+                if (a < args.length && !args[a].startsWith("--")) {
+                    this.dbType = DbType.from(args[a++]);
+                } else {
+                    throw new IllegalArgumentException("Missing value for --db-type");
+                }
+                break;
+            case "--topic-name":
+                if (a < args.length && !args[a].startsWith("--")) {
+                    topicName = args[a++];
+                } else {
+                    throw new IllegalArgumentException("Missing value for --topic-name");
+                }
+                break;
+            case "--instance-identifier":
+                if (a < args.length && !args[a].startsWith("--")) {
+                    instanceIdentifier = args[a++];
+                } else {
+                    throw new IllegalArgumentException("Missing value for --instance-identifier");
+                }
+                break;
+            case "--consumer-group":
+                if (a < args.length && !args[a].startsWith("--")) {
+                    consumerGroup = args[a++];
+                } else {
+                    throw new IllegalArgumentException("Missing value for --consumer-group");
+                }
+                break;
+            case "--consumer-count":
+                if (a < args.length && !args[a].startsWith("--")) {
+                    consumerCount = Integer.parseInt(args[a++]);
+                } else {
+                    throw new IllegalArgumentException("Missing value for --consumer-count");
+                }
+                break;
+            case "--max-ready-time-ms":
+                if (a < args.length && !args[a].startsWith("--")) {
+                    maxReadyTimeMs = Long.parseLong(args[a++]);
+                } else {
+                    throw new IllegalArgumentException("Missing value for --max-ready-time-ms");
+                }
+                break;
+            case "--schema-type":
+                if (a < args.length && !args[a].startsWith("--")) {
+                    schemaType = SchemaType.valueOf(args[a++]);
+                } else {
+                    throw new IllegalArgumentException("Missing value for --schema-type");
+                }
+                break;
+            default:
+                throw new IllegalArgumentException("Bad arg: '" + arg + "'");
+            }
+        }
+    }
+
+    /**
+     * Read kafka properties from the given file
+     * @param filename
+     */
+    protected void loadKafkaProperties(String filename) {
+        try (InputStream is = new FileInputStream(filename)) {
+            kafkaProperties.load(is);
+        } catch (IOException x) {
+            throw new IllegalArgumentException(x);
+        }
+    }
+
+    /**
+     * Read database properties from the given file
+     * @param filename
+     */
+    protected void loadDatabaseProperties(String filename) {
+        try (InputStream is = new FileInputStream(filename)) {
+            databaseProperties.load(is);
+        } catch (IOException x) {
+            throw new IllegalArgumentException(x);
+        }
+    }
+
+    /**
+     * Get the configured schema name for where we need to use it explicitly
+     * @return
+     * @throws FHIRPersistenceException
+     */
+    private String getSchemaName() throws FHIRPersistenceException {
+        String result = databaseProperties.getProperty("currentSchema");
+        if (result == null) {
+            throw new FHIRPersistenceException("currentSchema value missing in database properties");
+        }
+        return result;
+    }
+
+    /**
+     * Keep consuming from Kafka forever...or until we see too many
+     * consumers fail
+     */
+    public void run() throws FHIRPersistenceException {
+        dumpProperties("kafka", kafkaProperties);
+        dumpProperties("database", databaseProperties);
+        configureDatabaseAccess();
+        initIdentityCache();
+
+        // Keep track of how many consumers are still running. If too many fail,
+        // we stop everything and exit which allows our operating environment
+        // to handle things perhaps by restarting us somewhere else
+        stillRunningCounter = new AtomicInteger(this.consumerCount);
+        this.minRunningConsumerThreshold = Math.max(1, Math.round(this.consumerCount * minRunningConsumerRatio));
+
+        // One thread per consumer
+        ExecutorService pool = Executors.newCachedThreadPool();
+        for (int i=0; i kc = buildConsumer();
+            if (i == 0) {
+                // use the first consumer to check we have partitions for the configured topic
+                doPartitionCheck(kc);
+            }
+            IMessageHandler handler = buildHandler();
+            RemoteIndexConsumer consumer = new RemoteIndexConsumer(kc, handler, () -> failedConsumerCallback(), topicName, maxBatchCollectTimeMs, pollDuration);
+            pool.submit(consumer);
+            // Keep track of the consumer, so that we can signal a shutdown if we need to
+            consumers.add(consumer);
+        }
+
+        // Hold in a slow poll loop, ideally forever. We only exit if too
+        // many consumers fail
+        while (running) {
+            ThreadHandler.safeSleep(1000);
+        }
+
+        // Make sure anything still running is stopped
+        logger.warning("Too many consumers have failed, so stopping everything");
+        for (RemoteIndexConsumer consumer: consumers) {
+            consumer.shutdown();
+        }
+
+        // Try to make the exit as clean as possible, although this may not happen
+        // because we're likely in some sort of failure scenario here (e.g. network
+        // partition, brokers failed, database down etc).
+        int waitForTerminationSeconds = 30;
+        logger.info("Waiting " + waitForTerminationSeconds + " seconds for consumers to stop");
+        pool.shutdown();
+        try {
+            pool.awaitTermination(waitForTerminationSeconds, TimeUnit.SECONDS);
+        } catch (InterruptedException x) {
+            logger.warning("Interrupted waiting for consumer pool to terminate");
+        }
+        logger.info("All consumers stopped");
+    }
+
+    /**
+     * Set up the identity cache and preload it with all the parameter_names
+     * currently in the database
+     * @throws FHIRPersistenceException
+     */
+    private void initIdentityCache() throws FHIRPersistenceException {
+        logger.info("Initializing identity cache");
+        identityCache = new IdentityCacheImpl(
+            1000, Duration.ofSeconds(86400),     // code systems
+            10000, Duration.ofSeconds(86400),    // common token values
+            1000, Duration.ofSeconds(86400),     // common canonical values
+            100000, Duration.ofSeconds(86400));  // logical resource idents
+        CacheLoader loader = new CacheLoader(identityCache);
+
+        // prefill the cache
+        try (Connection connection = connectionProvider.getConnection()) {
+            loader.apply(connection);
+            connection.commit();
+        } catch (SQLException x) {
+            throw new FHIRPersistenceException("cache init failed", x);
+        }
+    }
+
+    /**
+     * Create a new consumer
+     * @return
+     */
+    private KafkaConsumer buildConsumer() {
+            
+        Properties kp = new Properties();
+        kp.putAll(kafkaProperties);
+
+        // Inject the properties we want to force here
+        kp.put("enable.auto.commit", "false");
+        kp.put("key.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
+        kp.put("value.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
+        kp.put("group.id", this.consumerGroup);
+        kp.put("auto.offset.reset", "earliest");
+
+        KafkaConsumer consumer = new KafkaConsumer<>(kp);
+        return consumer;
+    }
+
+    /**
+     * Set up the database connection
+     */
+    private void configureDatabaseAccess() {
+        switch (this.dbType) {
+        case POSTGRESQL:
+        case CITUS:
+            configureForPostgres();
+            break;
+        case DERBY:
+            configureForDerby();
+            break;
+        default:
+            throw new IllegalArgumentException("Database type not supported: " + this.dbType);
+        }
+    }
+
+    /**
+     * Set things up to talk to a PostgreSQL database
+     */
+    private void configureForPostgres() {
+        this.translator = new PostgresTranslator();
+        try {
+            Class.forName(translator.getDriverClassName());
+        } catch (ClassNotFoundException e) {
+            throw new IllegalStateException(e);
+        }
+
+        PostgresPropertyAdapter propertyAdapter = new PostgresPropertyAdapter(databaseProperties);
+        connectionProvider = new JdbcConnectionProvider(translator, propertyAdapter);
+    }
+
+    /**
+     * Set things up to talk to a Derby database. Note that the in-memory
+     * instance of Derby supports only a single JVM and so the FHIR server
+     * instance would need to be stopped before running this fhir-remote-index
+     * application. Therefore, this is useful only for development work.
+     */
+    private void configureForDerby() {
+        this.translator = new DerbyTranslator();
+        try {
+            Class.forName(translator.getDriverClassName());
+        } catch (ClassNotFoundException e) {
+            throw new IllegalStateException(e);
+        }
+
+        DerbyPropertyAdapter propertyAdapter = new DerbyPropertyAdapter(databaseProperties);
+        connectionProvider = new JdbcConnectionProvider(translator, propertyAdapter);
+    }
+
+    /**
+     * Instantiate a new message handler for use by a consumer thread. Each handler gets
+     * its own database connection.
+     * @return
+     * @throws FHIRPersistenceException
+     */
+    private IMessageHandler buildHandler() throws FHIRPersistenceException {
+        Objects.requireNonNull(identityCache, "must set up identityCache first");
+        try {
+            // Each handler gets a dedicated database connection so we don't have
+            // to deal with contention when grabbing connections from a pool
+            Connection c = connectionProvider.getConnection();
+            if (this.dbType == DbType.CITUS) {
+                configureCitusConnection(c);
+            }
+            
+            switch (schemaType) {
+            case SHARDED:
+                return new ShardedPostgresMessageHandler(instanceIdentifier, c, getSchemaName(), identityCache, maxReadyTimeMs);
+            case PLAIN:
+                if (dbType == DbType.DERBY) {
+                    return new PlainDerbyMessageHandler(instanceIdentifier, c, getSchemaName(), identityCache, maxReadyTimeMs);                
+                } else {
+                    return new PlainPostgresMessageHandler(instanceIdentifier, c, getSchemaName(), identityCache, maxReadyTimeMs);                
+                }
+            case DISTRIBUTED:
+                return new DistributedPostgresMessageHandler(instanceIdentifier, c, getSchemaName(), identityCache, maxReadyTimeMs);                
+            default:
+                throw new FHIRPersistenceException("Schema type not supported: " + schemaType.name());
+            }
+        } catch (SQLException x) {
+            throw new FHIRPersistenceException("get connection failed", x);
+        }
+    }
+
+    /**
+     * Configure the connection by setting local properties required for Citus
+     * @param c
+     */
+    private static void configureCitusConnection(Connection c) {
+        logger.info("Citus: Configuring new database connection");
+        ConfigureConnectionDAO dao = new ConfigureConnectionDAO();
+        dao.run(new CitusTranslator(), c);
+     }
+
+    /**
+     * Get the partitions for the named topic to check if the topic actually exists
+     */
+    private void doPartitionCheck(KafkaConsumer consumer) {
+        // Checking for topic existence before subscribing
+        List partitions = consumer.partitionsFor(topicName);
+        if (partitions == null || partitions.isEmpty()) {
+            logger.severe("Topic not found: '" + topicName + "'");
+            throw new IllegalStateException("topic not found");
+        } else {
+            // dump the list of partitions configured for this topic
+            for (PartitionInfo pi: partitions) {
+                logger.info("Topic '" + topicName + "' has partition " + pi.toString());
+            }
+        }
+    }
+
+    
+    /**
+     * Log the properties which can help with debugging deployment issues.
+     * Hides secrets
+     * @param which the type of properties
+     * @param p the properties to dump
+     */
+    protected void dumpProperties(String which, Properties p) {
+
+        if (which != null && p != null) {
+            StringBuilder buffer = new StringBuilder();
+            buffer.append("{");
+            Iterator keys = p.keySet().iterator();
+            boolean first = true;
+            while (keys.hasNext()) {
+                String key = (String) keys.next();
+                String value = p.getProperty(key);
+                if (key.toLowerCase().contains("password")) {
+                    value = "[*******]";
+                }
+                // kill any passwords embedded within a more complex value string
+                value = LogSupport.hidePassword(value);
+                
+                if (first) {
+                    first = false;
+                } else {
+                    buffer.append(", ");
+                }
+                buffer.append("\"").append(key).append("\"");
+                buffer.append(": ");
+                buffer.append("\"").append(value).append("\"");
+            }
+            buffer.append("}");
+            logger.fine(which + ": " + buffer.toString());
+        }
+    }
+
+    /**
+     * Called from a consumer thread when it is about to exit
+     */
+    private void failedConsumerCallback() {
+        final int remainingConsumersStillRunning = stillRunningCounter.decrementAndGet();
+        if (remainingConsumersStillRunning < minRunningConsumerThreshold) {
+            // Signal termination of the entire program
+            logger.severe("Too many consumers have failed. Terminating");
+            this.running = false;
+        } else {
+            logger.info("Remaining consumer count: " + remainingConsumersStillRunning);
+        }
+    }
+
+    /**
+     * @param args
+     */
+    public static void main(String[] args) {
+        Main m = new Main();
+        try {
+            m.parseArgs(args);
+            m.run();
+        } catch (Throwable t) {
+            logger.log(Level.SEVERE, "terminating", t);
+        } finally {
+            // Any exit means something failed, so we call this an error so a container
+            // environment can react accordingly
+            System.exit(1);
+        }
+    }
+}
diff --git a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/batch/BatchDateParameter.java b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/batch/BatchDateParameter.java
new file mode 100644
index 00000000000..ed65a2f0a8b
--- /dev/null
+++ b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/batch/BatchDateParameter.java
@@ -0,0 +1,40 @@
+/*
+ * (C) Copyright IBM Corp. 2022
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+ 
+package com.ibm.fhir.remote.index.batch;
+
+import com.ibm.fhir.persistence.exception.FHIRPersistenceException;
+import com.ibm.fhir.persistence.index.DateParameter;
+import com.ibm.fhir.remote.index.api.BatchParameterProcessor;
+import com.ibm.fhir.remote.index.api.BatchParameterValue;
+import com.ibm.fhir.remote.index.database.ParameterNameValue;
+
+/**
+ * A date parameter we are collecting to batch
+ */
+public class BatchDateParameter extends BatchParameterValue {
+    private final DateParameter parameter;
+    
+    /**
+     * Canonical constructor
+     * 
+     * @param requestShard
+     * @param resourceType
+     * @param logicalId
+     * @param logicalResourceId
+     * @param parameterNameValue
+     * @param parameter
+     */
+    public BatchDateParameter(String requestShard, String resourceType, String logicalId, long logicalResourceId, ParameterNameValue parameterNameValue, DateParameter parameter) {
+        super(requestShard, resourceType, logicalId, logicalResourceId, parameterNameValue);
+        this.parameter = parameter;
+    }
+
+    @Override
+    public void apply(BatchParameterProcessor processor) throws FHIRPersistenceException {
+        processor.process(requestShard, resourceType, logicalId, logicalResourceId, parameterNameValue, parameter);
+    }
+}
diff --git a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/batch/BatchLocationParameter.java b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/batch/BatchLocationParameter.java
new file mode 100644
index 00000000000..3afcac8848c
--- /dev/null
+++ b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/batch/BatchLocationParameter.java
@@ -0,0 +1,40 @@
+/*
+ * (C) Copyright IBM Corp. 2022
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+ 
+package com.ibm.fhir.remote.index.batch;
+
+import com.ibm.fhir.persistence.exception.FHIRPersistenceException;
+import com.ibm.fhir.persistence.index.LocationParameter;
+import com.ibm.fhir.remote.index.api.BatchParameterProcessor;
+import com.ibm.fhir.remote.index.api.BatchParameterValue;
+import com.ibm.fhir.remote.index.database.ParameterNameValue;
+
+/**
+ * A location parameter we are collecting to batch
+ */
+public class BatchLocationParameter extends BatchParameterValue {
+    private final LocationParameter parameter;
+    
+    /**
+     * Canonical constructor
+     * 
+     * @param requestShard
+     * @param resourceType
+     * @param logicalId
+     * @param logicalResourceId
+     * @param parameterNameValue
+     * @param parameter
+     */
+    public BatchLocationParameter(String requestShard, String resourceType, String logicalId, long logicalResourceId, ParameterNameValue parameterNameValue, LocationParameter parameter) {
+        super(requestShard, resourceType, logicalId, logicalResourceId, parameterNameValue);
+        this.parameter = parameter;
+    }
+
+    @Override
+    public void apply(BatchParameterProcessor processor) throws FHIRPersistenceException {
+        processor.process(requestShard, resourceType, logicalId, logicalResourceId, parameterNameValue, parameter);
+    }
+}
diff --git a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/batch/BatchNumberParameter.java b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/batch/BatchNumberParameter.java
new file mode 100644
index 00000000000..3235dd08c8b
--- /dev/null
+++ b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/batch/BatchNumberParameter.java
@@ -0,0 +1,40 @@
+/*
+ * (C) Copyright IBM Corp. 2022
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+ 
+package com.ibm.fhir.remote.index.batch;
+
+import com.ibm.fhir.persistence.exception.FHIRPersistenceException;
+import com.ibm.fhir.persistence.index.NumberParameter;
+import com.ibm.fhir.remote.index.api.BatchParameterProcessor;
+import com.ibm.fhir.remote.index.api.BatchParameterValue;
+import com.ibm.fhir.remote.index.database.ParameterNameValue;
+
+/**
+ * A number parameter we are collecting to batch
+ */
+public class BatchNumberParameter extends BatchParameterValue {
+    private final NumberParameter parameter;
+    
+    /**
+     * Canonical constructor
+     * 
+     * @param requestShard
+     * @param resourceType
+     * @param logicalId
+     * @param logicalResourceId
+     * @param parameterNameValue
+     * @param parameter
+     */
+    public BatchNumberParameter(String requestShard, String resourceType, String logicalId, long logicalResourceId, ParameterNameValue parameterNameValue, NumberParameter parameter) {
+        super(requestShard, resourceType, logicalId, logicalResourceId, parameterNameValue);
+        this.parameter = parameter;
+    }
+
+    @Override
+    public void apply(BatchParameterProcessor processor) throws FHIRPersistenceException {
+        processor.process(requestShard, resourceType, logicalId, logicalResourceId, parameterNameValue, parameter);
+    }
+}
diff --git a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/batch/BatchProfileParameter.java b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/batch/BatchProfileParameter.java
new file mode 100644
index 00000000000..14c0e508693
--- /dev/null
+++ b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/batch/BatchProfileParameter.java
@@ -0,0 +1,45 @@
+/*
+ * (C) Copyright IBM Corp. 2022
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+ 
+package com.ibm.fhir.remote.index.batch;
+
+import com.ibm.fhir.persistence.exception.FHIRPersistenceException;
+import com.ibm.fhir.persistence.index.ProfileParameter;
+import com.ibm.fhir.remote.index.api.BatchParameterProcessor;
+import com.ibm.fhir.remote.index.api.BatchParameterValue;
+import com.ibm.fhir.remote.index.database.CommonCanonicalValue;
+import com.ibm.fhir.remote.index.database.ParameterNameValue;
+
+/**
+ * A profile parameter we are collecting to batch
+ */
+public class BatchProfileParameter extends BatchParameterValue {
+    private final ProfileParameter parameter;
+    private final CommonCanonicalValue commonCanonicalValue;
+    
+    /**
+     * Canonical constructor
+     * 
+     * @param requestShard
+     * @param resourceType
+     * @param logicalId
+     * @param logicalResourceId
+     * @param parameterNameValue
+     * @param parameter
+     * @param commonCanonicalValue
+     */
+    public BatchProfileParameter(String requestShard, String resourceType, String logicalId, long logicalResourceId, 
+            ParameterNameValue parameterNameValue, ProfileParameter parameter, CommonCanonicalValue commonCanonicalValue) {
+        super(requestShard, resourceType, logicalId, logicalResourceId, parameterNameValue);
+        this.parameter = parameter;
+        this.commonCanonicalValue = commonCanonicalValue;
+    }
+
+    @Override
+    public void apply(BatchParameterProcessor processor) throws FHIRPersistenceException {
+        processor.process(requestShard, resourceType, logicalId, logicalResourceId, parameterNameValue, parameter, commonCanonicalValue);
+    }
+}
diff --git a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/batch/BatchQuantityParameter.java b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/batch/BatchQuantityParameter.java
new file mode 100644
index 00000000000..80b568f1b95
--- /dev/null
+++ b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/batch/BatchQuantityParameter.java
@@ -0,0 +1,44 @@
+/*
+ * (C) Copyright IBM Corp. 2022
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+ 
+package com.ibm.fhir.remote.index.batch;
+
+import com.ibm.fhir.persistence.exception.FHIRPersistenceException;
+import com.ibm.fhir.persistence.index.QuantityParameter;
+import com.ibm.fhir.remote.index.api.BatchParameterProcessor;
+import com.ibm.fhir.remote.index.api.BatchParameterValue;
+import com.ibm.fhir.remote.index.database.CodeSystemValue;
+import com.ibm.fhir.remote.index.database.ParameterNameValue;
+
+/**
+ * A quantity parameter we are collecting to batch
+ */
+public class BatchQuantityParameter extends BatchParameterValue {
+    private final QuantityParameter parameter;
+    private final CodeSystemValue codeSystemValue;
+    
+    /**
+     * Canonical constructor
+     * 
+     * @param requestShard
+     * @param resourceType
+     * @param logicalId
+     * @param logicalResourceId
+     * @param parameterNameValue
+     * @param parameter
+     * @param csv
+     */
+    public BatchQuantityParameter(String requestShard, String resourceType, String logicalId, long logicalResourceId, ParameterNameValue parameterNameValue, QuantityParameter parameter, CodeSystemValue csv) {
+        super(requestShard, resourceType, logicalId, logicalResourceId, parameterNameValue);
+        this.parameter = parameter;
+        this.codeSystemValue = csv;
+    }
+
+    @Override
+    public void apply(BatchParameterProcessor processor) throws FHIRPersistenceException {
+        processor.process(requestShard, resourceType, logicalId, logicalResourceId, parameterNameValue, parameter, codeSystemValue);
+    }
+}
diff --git a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/batch/BatchReferenceParameter.java b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/batch/BatchReferenceParameter.java
new file mode 100644
index 00000000000..2aef5d8158c
--- /dev/null
+++ b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/batch/BatchReferenceParameter.java
@@ -0,0 +1,44 @@
+/*
+ * (C) Copyright IBM Corp. 2022
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+ 
+package com.ibm.fhir.remote.index.batch;
+
+import com.ibm.fhir.persistence.exception.FHIRPersistenceException;
+import com.ibm.fhir.persistence.index.ReferenceParameter;
+import com.ibm.fhir.remote.index.api.BatchParameterProcessor;
+import com.ibm.fhir.remote.index.api.BatchParameterValue;
+import com.ibm.fhir.remote.index.database.LogicalResourceIdentValue;
+import com.ibm.fhir.remote.index.database.ParameterNameValue;
+
+/**
+ * A reference parameter we are collecting to batch
+ */
+public class BatchReferenceParameter extends BatchParameterValue {
+    private final ReferenceParameter parameter;
+    private final LogicalResourceIdentValue refLogicalResourceId;
+    
+    /**
+     * Canonical constructor
+     * 
+     * @param requestShard
+     * @param resourceType
+     * @param logicalId
+     * @param logicalResourceId
+     * @param parameterNameValue
+     * @param parameter
+     * @param refLogicalResourceId
+     */
+    public BatchReferenceParameter(String requestShard, String resourceType, String logicalId, long logicalResourceId, ParameterNameValue parameterNameValue, ReferenceParameter parameter, LogicalResourceIdentValue refLogicalResourceId) {
+        super(requestShard, resourceType, logicalId, logicalResourceId, parameterNameValue);
+        this.parameter = parameter;
+        this.refLogicalResourceId = refLogicalResourceId;
+    }
+
+    @Override
+    public void apply(BatchParameterProcessor processor) throws FHIRPersistenceException {
+        processor.process(requestShard, resourceType, logicalId, logicalResourceId, parameterNameValue, parameter, refLogicalResourceId);
+    }
+}
diff --git a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/batch/BatchSecurityParameter.java b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/batch/BatchSecurityParameter.java
new file mode 100644
index 00000000000..eb5d86dfec3
--- /dev/null
+++ b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/batch/BatchSecurityParameter.java
@@ -0,0 +1,44 @@
+/*
+ * (C) Copyright IBM Corp. 2022
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+ 
+package com.ibm.fhir.remote.index.batch;
+
+import com.ibm.fhir.persistence.exception.FHIRPersistenceException;
+import com.ibm.fhir.persistence.index.SecurityParameter;
+import com.ibm.fhir.remote.index.api.BatchParameterProcessor;
+import com.ibm.fhir.remote.index.api.BatchParameterValue;
+import com.ibm.fhir.remote.index.database.CommonTokenValue;
+import com.ibm.fhir.remote.index.database.ParameterNameValue;
+
+/**
+ * A security parameter we are collecting to batch
+ */
+public class BatchSecurityParameter extends BatchParameterValue {
+    private final SecurityParameter parameter;
+    private final CommonTokenValue commonTokenValue;
+    
+    /**
+     * Canonical constructor
+     * 
+     * @param requestShard
+     * @param resourceType
+     * @param logicalId
+     * @param logicalResourceId
+     * @param parameterNameValue
+     * @param parameter
+     * @param commonTokenValue
+     */
+    public BatchSecurityParameter(String requestShard, String resourceType, String logicalId, long logicalResourceId, ParameterNameValue parameterNameValue, SecurityParameter parameter, CommonTokenValue commonTokenValue) {
+        super(requestShard, resourceType, logicalId, logicalResourceId, parameterNameValue);
+        this.parameter = parameter;
+        this.commonTokenValue = commonTokenValue;
+    }
+
+    @Override
+    public void apply(BatchParameterProcessor processor) throws FHIRPersistenceException {
+        processor.process(requestShard, resourceType, logicalId, logicalResourceId, parameterNameValue, parameter, commonTokenValue);
+    }
+}
diff --git a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/batch/BatchStringParameter.java b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/batch/BatchStringParameter.java
new file mode 100644
index 00000000000..bfd4a15feb0
--- /dev/null
+++ b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/batch/BatchStringParameter.java
@@ -0,0 +1,40 @@
+/*
+ * (C) Copyright IBM Corp. 2022
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+ 
+package com.ibm.fhir.remote.index.batch;
+
+import com.ibm.fhir.persistence.exception.FHIRPersistenceException;
+import com.ibm.fhir.persistence.index.StringParameter;
+import com.ibm.fhir.remote.index.api.BatchParameterProcessor;
+import com.ibm.fhir.remote.index.api.BatchParameterValue;
+import com.ibm.fhir.remote.index.database.ParameterNameValue;
+
+/**
+ * A string parameter we are collecting to batch
+ */
+public class BatchStringParameter extends BatchParameterValue {
+    private final StringParameter parameter;
+    
+    /**
+     * Canonical constructor
+     * 
+     * @param requestShard
+     * @param resourceType
+     * @param logicalId
+     * @param logicalResourceId
+     * @param parameterNameValue
+     * @param parameter
+     */
+    public BatchStringParameter(String requestShard, String resourceType, String logicalId, long logicalResourceId, ParameterNameValue parameterNameValue, StringParameter parameter) {
+        super(requestShard, resourceType, logicalId, logicalResourceId, parameterNameValue);
+        this.parameter = parameter;
+    }
+
+    @Override
+    public void apply(BatchParameterProcessor processor) throws FHIRPersistenceException {
+        processor.process(requestShard, resourceType, logicalId, logicalResourceId, parameterNameValue, parameter);
+    }
+}
diff --git a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/batch/BatchTagParameter.java b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/batch/BatchTagParameter.java
new file mode 100644
index 00000000000..d2a1aa1d982
--- /dev/null
+++ b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/batch/BatchTagParameter.java
@@ -0,0 +1,44 @@
+/*
+ * (C) Copyright IBM Corp. 2022
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+ 
+package com.ibm.fhir.remote.index.batch;
+
+import com.ibm.fhir.persistence.exception.FHIRPersistenceException;
+import com.ibm.fhir.persistence.index.TagParameter;
+import com.ibm.fhir.remote.index.api.BatchParameterProcessor;
+import com.ibm.fhir.remote.index.api.BatchParameterValue;
+import com.ibm.fhir.remote.index.database.CommonTokenValue;
+import com.ibm.fhir.remote.index.database.ParameterNameValue;
+
+/**
+ * A tag parameter we are collecting to batch
+ */
+public class BatchTagParameter extends BatchParameterValue {
+    private final TagParameter parameter;
+    private final CommonTokenValue commonTokenValue;
+    
+    /**
+     * Canonical constructor
+     * 
+     * @param requestShard
+     * @param resourceType
+     * @param logicalId
+     * @param logicalResourceId
+     * @param parameterNameValue
+     * @param parameter
+     * @param commonTokenValue
+     */
+    public BatchTagParameter(String requestShard, String resourceType, String logicalId, long logicalResourceId, ParameterNameValue parameterNameValue, TagParameter parameter, CommonTokenValue commonTokenValue) {
+        super(requestShard, resourceType, logicalId, logicalResourceId, parameterNameValue);
+        this.parameter = parameter;
+        this.commonTokenValue = commonTokenValue;
+    }
+
+    @Override
+    public void apply(BatchParameterProcessor processor) throws FHIRPersistenceException {
+        processor.process(requestShard, resourceType, logicalId, logicalResourceId, parameterNameValue, parameter, commonTokenValue);
+    }
+}
diff --git a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/batch/BatchTokenParameter.java b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/batch/BatchTokenParameter.java
new file mode 100644
index 00000000000..7dc6bfaa27a
--- /dev/null
+++ b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/batch/BatchTokenParameter.java
@@ -0,0 +1,44 @@
+/*
+ * (C) Copyright IBM Corp. 2022
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+ 
+package com.ibm.fhir.remote.index.batch;
+
+import com.ibm.fhir.persistence.exception.FHIRPersistenceException;
+import com.ibm.fhir.persistence.index.TokenParameter;
+import com.ibm.fhir.remote.index.api.BatchParameterProcessor;
+import com.ibm.fhir.remote.index.api.BatchParameterValue;
+import com.ibm.fhir.remote.index.database.CommonTokenValue;
+import com.ibm.fhir.remote.index.database.ParameterNameValue;
+
+/**
+ * A token parameter we are collecting to batch
+ */
+public class BatchTokenParameter extends BatchParameterValue {
+    private final TokenParameter parameter;
+    private final CommonTokenValue commonTokenValue;
+    
+    /**
+     * Canonical constructor
+     * 
+     * @param requestShard
+     * @param resourceType
+     * @param logicalId
+     * @param logicalResourceId
+     * @param parameterNameValue
+     * @param parameter
+     * @param commonTokenValue
+     */
+    public BatchTokenParameter(String requestShard, String resourceType, String logicalId, long logicalResourceId, ParameterNameValue parameterNameValue, TokenParameter parameter, CommonTokenValue commonTokenValue) {
+        super(requestShard, resourceType, logicalId, logicalResourceId, parameterNameValue);
+        this.parameter = parameter;
+        this.commonTokenValue = commonTokenValue;
+    }
+
+    @Override
+    public void apply(BatchParameterProcessor processor) throws FHIRPersistenceException {
+        processor.process(requestShard, resourceType, logicalId, logicalResourceId, parameterNameValue, parameter, commonTokenValue);
+    }
+}
diff --git a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/cache/IdentityCacheImpl.java b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/cache/IdentityCacheImpl.java
new file mode 100644
index 00000000000..9b73330e0f8
--- /dev/null
+++ b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/cache/IdentityCacheImpl.java
@@ -0,0 +1,130 @@
+/*
+ * (C) Copyright IBM Corp. 2022
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+ 
+package com.ibm.fhir.remote.index.cache;
+
+import java.time.Duration;
+import java.util.Collection;
+import java.util.concurrent.ConcurrentHashMap;
+
+import com.github.benmanes.caffeine.cache.Cache;
+import com.github.benmanes.caffeine.cache.Caffeine;
+import com.ibm.fhir.remote.index.api.IdentityCache;
+import com.ibm.fhir.remote.index.database.CommonCanonicalValueKey;
+import com.ibm.fhir.remote.index.database.CommonTokenValueKey;
+import com.ibm.fhir.remote.index.database.LogicalResourceIdentKey;
+import com.ibm.fhir.remote.index.database.ResourceTypeValue;
+
+/**
+ * Implementation of a cache we use to reduce the number of databases accesses
+ * required to find the id for a given object key
+ */
+public class IdentityCacheImpl implements IdentityCache {
+    private final ConcurrentHashMap parameterNames = new ConcurrentHashMap<>();
+    private final ConcurrentHashMap resourceTypes = new ConcurrentHashMap<>();
+    private final Cache codeSystemCache;
+    private final Cache commonTokenValueCache;
+    private final Cache commonCanonicalValueCache;
+    private final Cache logicalResourceIdentCache;
+    private static final Integer NULL_INT = null;
+    private static final Long NULL_LONG = null;
+
+    /**
+     * Public constructor
+     */
+    public IdentityCacheImpl(int maxCodeSystemCacheSize, Duration codeSystemCacheDuration,
+        long maxCommonTokenCacheSize, Duration commonTokenCacheDuration,
+        long maxCommonCanonicalCacheSize, Duration commonCanonicalCacheDuration,
+        long maxLogicalResourceIdentCacheSize, Duration logicalResourceIdentCacheDuration) {
+        codeSystemCache = Caffeine.newBuilder()
+                .maximumSize(maxCodeSystemCacheSize)
+                .expireAfterWrite(codeSystemCacheDuration)
+                .build();
+        commonTokenValueCache = Caffeine.newBuilder()
+                .maximumSize(maxCommonTokenCacheSize)
+                .expireAfterWrite(commonTokenCacheDuration)
+                .build();
+        commonCanonicalValueCache = Caffeine.newBuilder()
+                .maximumSize(maxCommonCanonicalCacheSize)
+                .expireAfterWrite(commonCanonicalCacheDuration)
+                .build();
+        logicalResourceIdentCache = Caffeine.newBuilder()
+                .maximumSize(maxLogicalResourceIdentCacheSize)
+                .expireAfterWrite(logicalResourceIdentCacheDuration)
+                .build();
+    }
+
+    /**
+     * Initialize the cache
+     * @param resourceTypeValues the complete list of resource types
+     */
+    public void init(Collection resourceTypeValues) {
+        for (ResourceTypeValue rtv: resourceTypeValues) {
+            resourceTypes.put(rtv.getResourceType(), rtv.getResourceTypeId());
+        }
+    }
+
+    @Override
+    public Integer getParameterNameId(String parameterName) {
+        // This should only miss if the parameter name value doesn't actually
+        // exist. Because the set is relatively small, we store everything.
+        return parameterNames.get(parameterName);
+    }
+
+    @Override
+    public Integer getCodeSystemId(String codeSystem) {
+        return codeSystemCache.get(codeSystem, k -> NULL_INT);
+    }
+
+    @Override
+    public Long getCommonTokenValueId(short shardKey, String codeSystem, String tokenValue) {
+        return commonTokenValueCache.get(new CommonTokenValueKey(shardKey, codeSystem, tokenValue), k -> NULL_LONG);
+    }
+
+    @Override
+    public void addParameterName(String parameterName, int parameterNameId) {
+        parameterNames.put(parameterName, parameterNameId);
+    }
+
+    @Override
+    public Long getCommonCanonicalValueId(short shardKey, String url) {
+        return commonCanonicalValueCache.get(new CommonCanonicalValueKey(shardKey, url), k -> NULL_LONG);
+    }
+
+    @Override
+    public int getResourceTypeId(String resourceType) {
+        Integer resourceTypeId = resourceTypes.get(resourceType);
+        if (resourceTypeId == null) {
+            throw new IllegalArgumentException("Not a valid resource type: " + resourceType);
+        }
+        return resourceTypeId;
+    }
+
+    @Override
+    public void addCommonCanonicalValue(short shardKey, String url, long commonCanonicalValueId) {
+        this.commonCanonicalValueCache.put(new CommonCanonicalValueKey(shardKey, url), commonCanonicalValueId);
+    }
+
+    @Override
+    public void addCommonTokenValue(short shardKey, String codeSystem, String tokenValue, long commonTokenValueId) {
+        this.commonTokenValueCache.put(new CommonTokenValueKey(shardKey, codeSystem, tokenValue), commonTokenValueId);
+    }
+
+    @Override
+    public void addCodeSystem(String codeSystem, int codeSystemId) {
+        this.codeSystemCache.put(codeSystem, codeSystemId);
+    }
+
+    @Override
+    public Long getLogicalResourceIdentId(String resourceType, String logicalId) {
+        return logicalResourceIdentCache.get(new LogicalResourceIdentKey(resourceType, logicalId), k -> NULL_LONG);
+    }
+
+    @Override
+    public void addLogicalResourceIdent(String resourceType, String logicalId, long logicalResourceId) {
+        logicalResourceIdentCache.put(new LogicalResourceIdentKey(resourceType, logicalId), logicalResourceId);
+    }
+}
\ No newline at end of file
diff --git a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/BaseMessageHandler.java b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/BaseMessageHandler.java
new file mode 100644
index 00000000000..059153ccfdf
--- /dev/null
+++ b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/BaseMessageHandler.java
@@ -0,0 +1,419 @@
+/*
+ * (C) Copyright IBM Corp. 2022
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+ 
+package com.ibm.fhir.remote.index.database;
+
+import java.security.SecureRandom;
+import java.time.format.DateTimeFormatter;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+import com.ibm.fhir.database.utils.thread.ThreadHandler;
+import com.ibm.fhir.persistence.exception.FHIRPersistenceDataAccessException;
+import com.ibm.fhir.persistence.exception.FHIRPersistenceException;
+import com.ibm.fhir.persistence.helper.RemoteIndexSupport;
+import com.ibm.fhir.persistence.index.DateParameter;
+import com.ibm.fhir.persistence.index.LocationParameter;
+import com.ibm.fhir.persistence.index.NumberParameter;
+import com.ibm.fhir.persistence.index.ProfileParameter;
+import com.ibm.fhir.persistence.index.QuantityParameter;
+import com.ibm.fhir.persistence.index.ReferenceParameter;
+import com.ibm.fhir.persistence.index.RemoteIndexMessage;
+import com.ibm.fhir.persistence.index.SearchParametersTransport;
+import com.ibm.fhir.persistence.index.SecurityParameter;
+import com.ibm.fhir.persistence.index.StringParameter;
+import com.ibm.fhir.persistence.index.TagParameter;
+import com.ibm.fhir.persistence.index.TokenParameter;
+import com.ibm.fhir.remote.index.api.IMessageHandler;
+
+
+/**
+ * Base for the Kafka message handler to load message data into
+ * a database via JDBC.
+ */
+public abstract class BaseMessageHandler implements IMessageHandler {
+    private final Logger logger = Logger.getLogger(BaseMessageHandler.class.getName());
+    private static final DateTimeFormatter formatter = DateTimeFormatter.ISO_INSTANT;
+    private static final int MIN_SUPPORTED_MESSAGE_VERSION = 1;
+
+    // If we fail 10 times due to deadlocks, then something is seriously wrong
+    private static final int MAX_TX_ATTEMPTS = 10;
+    private SecureRandom random = new SecureRandom();
+
+    private final long maxReadyWaitMs;
+
+    // Process messages only from a known origin
+    private final String instanceIdentifier;
+
+    /**
+     * Protected constructor
+     * @param maxReadyWaitMs the max time in ms to wait for the upstream transaction to make the data ready
+     */
+    protected BaseMessageHandler(String instanceIdentifier, long maxReadyWaitMs) {
+        if (instanceIdentifier == null || instanceIdentifier.isEmpty()) {
+            throw new IllegalArgumentException("Must specify an instanceIdentifier value");
+        }
+        this.instanceIdentifier = instanceIdentifier;
+        this.maxReadyWaitMs = maxReadyWaitMs;
+    }
+
+    @Override
+    public void process(List messages) throws FHIRPersistenceException {
+        List unmarshalled = new ArrayList<>(messages.size());
+        for (String payload: messages) {
+            if (logger.isLoggable(Level.FINEST)) {
+                logger.finest("Processing message payload: " + payload);
+            }
+            RemoteIndexMessage message = RemoteIndexSupport.unmarshall(payload);
+            if (message != null) {
+                if (message.getMessageVersion() >= MIN_SUPPORTED_MESSAGE_VERSION) {
+                    // check to make sure that the instanceIdentifier matches our configuration. This protects us
+                    // from messages accidentally sent over the same topic from another instance
+                    if (this.instanceIdentifier.equals(message.getInstanceIdentifier())) {
+                        unmarshalled.add(message);
+                    } else {
+                        logger.warning("Message from unknown origin, ignoring payload=[" + payload + "]");
+                    }
+                } else {
+                    logger.warning("Message version [" + message.getMessageVersion() + "] not supported, ignoring payload=[" + payload + "]");
+                }
+            }
+        }
+
+        if (unmarshalled.size() > 0) {
+            processWithRetry(unmarshalled);
+        }
+    }
+
+    /**
+     * Process the batch of messages with support for retries in the case
+     * of a retryable error such as a database deadlock
+     * 
+     * @param messages
+     * @throws FHIRPersistenceException
+     */
+    private void processWithRetry(List messages) throws FHIRPersistenceException {
+        int attempt = 1;
+        do {
+            try {
+                if (attempt > 1) {
+                    // introduce a random delay before we re-attempt to process the batch. This
+                    // may help to avoid subsequent deadlocks if there are multiple transactions
+                    // involved
+                    final long delay = random.nextInt(10) * 1000l;
+                    logger.fine(() -> "Deadlock retry backoff ms: " + delay);
+                    ThreadHandler.safeSleep(delay);
+                }
+                startBatch();
+                processMessages(messages);
+                pushBatch();
+
+                attempt = MAX_TX_ATTEMPTS; // exit our do...while
+            } catch (FHIRPersistenceDataAccessException x) {
+                setRollbackOnly();
+                // see if this is a retryable error
+                if (x.isTransactionRetryable() && attempt++ < MAX_TX_ATTEMPTS) {
+                    logger.warning("tx failed, but retry permitted: " + x.getMessage());
+                    resetBatch(); // clear up any cruft from the previous attempt
+                } else {
+                    throw x;
+                }
+            } catch (Throwable t) {
+                setRollbackOnly();
+                logger.log(Level.SEVERE, "batch failed", t);
+                throw t;
+            } finally {
+                endTransaction();
+            }
+        } while (attempt < MAX_TX_ATTEMPTS);
+    }
+
+    /**
+     * Called before we start processing a new batch of messages
+     */
+    protected abstract void startBatch();
+
+    /**
+     * Mark the transaction for rollback
+     */
+    protected abstract void setRollbackOnly();
+
+    /**
+     * Reset the state of the handler following a failure so that the batch can
+     * be retried
+     */
+    protected abstract void resetBatch();
+
+    /**
+     * Push any data we've accumulated from processing messages.
+     */
+    protected abstract void pushBatch() throws FHIRPersistenceException;
+
+    /**
+     * Process the list of messages
+     * @param messages
+     * @throws FHIRPersistenceException
+     */
+    private void processMessages(List messages) throws FHIRPersistenceException {
+        // We need to do a quick scan of all the messages to make sure that
+        // the logical resource records for each already exist. If the check
+        // returns anything in the notReady list, it means one of two things:
+        // 1. we received the message before the server transaction committed
+        // 2. the server transaction failed/rolled back, so we'll never be ready
+        long timeoutTime = System.nanoTime() + this.maxReadyWaitMs * 1000000;
+
+        // Messages which match the current version info in the database
+        List okToProcess = new ArrayList<>();
+
+        // resources which don't yet exist of their version is older than the message
+        List notReady = new ArrayList<>();
+
+        // make at least one attempt
+        do {
+            if (okToProcess.size() > 0) {
+                okToProcess.clear(); // reset ready for next prepare call
+            }
+            if (notReady.size() > 0) {
+                notReady.clear(); // reset ready for next prepare call
+            }
+
+            // Ask the handle to check which messages match the database
+            // and are therefore ready to be processed
+            checkReady(messages, okToProcess, notReady);
+
+            // If the ready check fails just sleep for a bit because we need
+            // to wait until the upstream transaction commits. This means we
+            // may need to keep waiting for a long time which unfortunately
+            // stalls processing this partition
+            if (notReady.size() > 0) {
+                long snoozeMs = Math.min(1000l, (timeoutTime - System.nanoTime()) / 1000000);
+                // short sleep to wait for the upstream transaction to complete
+                if (snoozeMs > 0) {
+                    ThreadHandler.safeSleep(snoozeMs);
+                }
+            }
+        } while (notReady.size() > 0 && System.nanoTime() < timeoutTime);
+
+        // okToProcess contains those messages for which we see the upstream transaction
+        // has committed.
+        for (RemoteIndexMessage message: okToProcess) {
+            process(message);
+        }
+
+        // Make a note of which messages we were unable to process because the upstream
+        // transaction did not commit before our maxReadyWaitMs timeout
+        for (RemoteIndexMessage message: notReady) {
+            logger.warning("Timed out waiting for upstream transaction to commit data for: " + message.toString());
+        }
+    }
+
+    /**
+     * Check to see if the database is ready to process the messages
+     * @param  IN: messages to check
+     * @param OUT: okToMessages the messages matching the current database
+     * @param OUT: notReady the messages for which the upstream transaction has yet to commit
+     */
+    protected abstract void checkReady(List messages, List okToProcess, List notReady) throws FHIRPersistenceException;
+
+    /**
+     * Process the data 
+     * @param message
+     */
+    private void process(RemoteIndexMessage message) throws FHIRPersistenceException {
+        SearchParametersTransport params = message.getData();
+        if (params.getStringValues() != null) {
+            for (StringParameter p: params.getStringValues()) {
+                process(message.getTenantId(), params.getRequestShard(), params.getResourceType(), params.getLogicalId(), params.getLogicalResourceId(), p);
+            }
+        }
+
+        if (params.getDateValues() != null) {
+            for (DateParameter p: params.getDateValues()) {
+                process(message.getTenantId(), params.getRequestShard(), params.getResourceType(), params.getLogicalId(), params.getLogicalResourceId(), p);
+            }
+        }
+
+        if (params.getNumberValues() != null) {
+            for (NumberParameter p: params.getNumberValues()) {
+                process(message.getTenantId(), params.getRequestShard(), params.getResourceType(), params.getLogicalId(), params.getLogicalResourceId(), p);
+            }
+        }
+
+        if (params.getQuantityValues() != null) {
+            for (QuantityParameter p: params.getQuantityValues()) {
+                process(message.getTenantId(), params.getRequestShard(), params.getResourceType(), params.getLogicalId(), params.getLogicalResourceId(), p);                
+            }
+        }
+
+        if (params.getTokenValues() != null) {
+            for (TokenParameter p: params.getTokenValues()) {
+                process(message.getTenantId(), params.getRequestShard(), params.getResourceType(), params.getLogicalId(), params.getLogicalResourceId(), p);
+            }
+        }
+
+        if (params.getLocationValues() != null) {
+            for (LocationParameter p: params.getLocationValues()) {
+                process(message.getTenantId(), params.getRequestShard(), params.getResourceType(), params.getLogicalId(), params.getLogicalResourceId(), p);
+            }
+        }
+
+        if (params.getTagValues() != null) {
+            for (TagParameter p: params.getTagValues()) {
+                process(message.getTenantId(), params.getRequestShard(), params.getResourceType(), params.getLogicalId(), params.getLogicalResourceId(), p);
+            }
+        }
+
+        if (params.getProfileValues() != null) {
+            for (ProfileParameter p: params.getProfileValues()) {
+                process(message.getTenantId(), params.getRequestShard(), params.getResourceType(), params.getLogicalId(), params.getLogicalResourceId(), p);
+            }
+        }
+
+        if (params.getSecurityValues() != null) {
+            for (SecurityParameter p: params.getSecurityValues()) {
+                process(message.getTenantId(), params.getRequestShard(), params.getResourceType(), params.getLogicalId(), params.getLogicalResourceId(), p);
+            }
+        }
+
+        if (params.getRefValues() != null) {
+            for (ReferenceParameter p: params.getRefValues()) {
+                process(message.getTenantId(), params.getRequestShard(), params.getResourceType(), params.getLogicalId(), params.getLogicalResourceId(), p);
+            }
+        }
+    }
+
+    /**
+     * Process the given LocationParameter p
+     * 
+     * @param tenantId
+     * @param requestShard
+     * @param resourceType
+     * @param logicalId
+     * @param logicalResourceId
+     * @param p
+     */
+    protected abstract void process(String tenantId, String requestShard, String resourceType, String logicalId, long logicalResourceId, LocationParameter p) throws FHIRPersistenceException;
+
+    /**
+     * Process the given TokenParameter p
+     * 
+     * @param tenantId
+     * @param requestShard
+     * @param resourceType
+     * @param logicalId
+     * @param logicalResourceId
+     * @param p
+     */
+    protected abstract void process(String tenantId, String requestShard, String resourceType, String logicalId, long logicalResourceId, TokenParameter p) throws FHIRPersistenceException;
+
+    /**
+     * Process the given TagParameter p
+     * 
+     * @param tenantId
+     * @param requestShard
+     * @param resourceType
+     * @param logicalId
+     * @param logicalResourceId
+     * @param p
+     * @throws FHIRPersistenceException
+     */
+    protected abstract void process(String tenantId, String requestShard, String resourceType, String logicalId, long logicalResourceId, TagParameter p) throws FHIRPersistenceException;
+
+    /**
+     * Process the given ProfileParameter p
+     * 
+     * @param tenantId
+     * @param requestShard
+     * @param resourceType
+     * @param logicalId
+     * @param logicalResourceId
+     * @param p
+     * @throws FHIRPersistenceException
+     */
+    protected abstract void process(String tenantId, String requestShard, String resourceType, String logicalId, long logicalResourceId, ProfileParameter p) throws FHIRPersistenceException;
+
+    /**
+     * Proces the given SecurityParameter p
+     * 
+     * @param tenantId
+     * @param requestShard
+     * @param resourceType
+     * @param logicalId
+     * @param logicalResourceId
+     * @param p
+     * @throws FHIRPersistenceException
+     */
+    protected abstract void process(String tenantId, String requestShard, String resourceType, String logicalId, long logicalResourceId, SecurityParameter p) throws FHIRPersistenceException;
+
+    /**
+     * Process the given QuantityParameter p
+     * 
+     * @param tenantId
+     * @param requestShard
+     * @param resourceType
+     * @param logicalId
+     * @param logicalResourceId
+     * @param p
+     */
+    protected abstract void process(String tenantId, String requestShard, String resourceType, String logicalId, long logicalResourceId, QuantityParameter p) throws FHIRPersistenceException;
+
+    /**
+     * Process the given NumberParameter p
+     * 
+     * @param tenantId
+     * @param requestShard
+     * @param resourceType
+     * @param logicalId
+     * @param logicalResourceId
+     * @param p
+     */
+    protected abstract void process(String tenantId, String requestShard, String resourceType, String logicalId, long logicalResourceId, NumberParameter p) throws FHIRPersistenceException;
+
+    /**
+     * Process the given DateParameter p
+     * 
+     * @param tenantId
+     * @param requestShard
+     * @param resourceType
+     * @param logicalId
+     * @param logicalResourceId
+     * @param p
+     */
+    protected abstract void process(String tenantId, String requestShard, String resourceType, String logicalId, long logicalResourceId, DateParameter p) throws FHIRPersistenceException;
+
+    /**
+     * Process the given ReferenceParameter p
+     * 
+     * @param tenantId
+     * @param requestShard
+     * @param resourceType
+     * @param logicalId
+     * @param logicalResourceId
+     * @param p
+     * @throws FHIRPersistenceException
+     */
+    protected abstract void process(String tenantId, String requestShard, String resourceType, String logicalId, long logicalResourceId, ReferenceParameter p) throws FHIRPersistenceException;
+
+    /**
+     * @param tenantId
+     * @param requestShard
+     * @param resourceType
+     * @param logicalId
+     * @param logicalResourceId
+     * @param p
+     */
+    protected abstract void process(String tenantId, String requestShard, String resourceType, String logicalId, long logicalResourceId, StringParameter p) throws FHIRPersistenceException;
+
+    /**
+     * Tell the persistence layer to commit the current transaction, or perform a rollback
+     * if setRollbackOnly() has been called.
+     * 
+     * @throws FHIRPersistenceException
+     */
+    protected abstract void endTransaction() throws FHIRPersistenceException;
+}
diff --git a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/CacheLoader.java b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/CacheLoader.java
new file mode 100644
index 00000000000..7d66a6bcae8
--- /dev/null
+++ b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/CacheLoader.java
@@ -0,0 +1,64 @@
+/*
+ * (C) Copyright IBM Corp. 2022
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+ 
+package com.ibm.fhir.remote.index.database;
+
+import java.sql.Connection;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.util.ArrayList;
+import java.util.List;
+
+import com.ibm.fhir.persistence.exception.FHIRPersistenceException;
+import com.ibm.fhir.remote.index.cache.IdentityCacheImpl;
+
+/**
+ * Preload the cache
+ */
+public class CacheLoader {
+    private final IdentityCacheImpl cache;
+
+    /**
+     * Public constructor
+     * @param cache
+     */
+    public CacheLoader(IdentityCacheImpl cache) {
+        this.cache = cache;
+    }
+
+    /**
+     * Read records from the database using the given connection and apply the
+     * values to the configured cache object.
+     * @param connection
+     * @throws FHIRPersistenceException
+     */
+    public void apply(Connection connection) throws FHIRPersistenceException {
+        // load the static list of resource types
+        List resourceTypes = new ArrayList<>();
+        final String SELECT_RESOURCE_TYPES = "SELECT resource_type, resource_type_id FROM resource_types";
+        try (PreparedStatement ps = connection.prepareStatement(SELECT_RESOURCE_TYPES)) {
+            ResultSet rs = ps.executeQuery();
+            while (rs.next()) {
+                resourceTypes.add(new ResourceTypeValue(rs.getString(1), rs.getInt(2)));
+            }
+        } catch (SQLException x) {
+            throw new FHIRPersistenceException("fetch parameter names failed", x);
+        }
+        cache.init(resourceTypes);
+
+        // also seed the cache with all the parameter_names we know so far
+        final String SQL = "SELECT parameter_name, parameter_name_id FROM parameter_names";
+        try (PreparedStatement ps = connection.prepareStatement(SQL)) {
+            ResultSet rs = ps.executeQuery();
+            while (rs.next()) {
+                cache.addParameterName(rs.getString(1), rs.getInt(2));
+            }
+        } catch (SQLException x) {
+            throw new FHIRPersistenceException("fetch parameter names failed", x);
+        }
+    }
+}
\ No newline at end of file
diff --git a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/CodeSystemValue.java b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/CodeSystemValue.java
new file mode 100644
index 00000000000..472fa977d10
--- /dev/null
+++ b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/CodeSystemValue.java
@@ -0,0 +1,48 @@
+/*
+ * (C) Copyright IBM Corp. 2022
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+ 
+package com.ibm.fhir.remote.index.database;
+
+
+/**
+ * Holder for the code_system_id obtained from the database. This is
+ * modelled as a mutable record so that we can reference this record
+ * many times, and resolve it once (either from cache lookup, database
+ * lookup, or a database create).
+ */
+public class CodeSystemValue {
+    private final String codeSystem;
+    private Integer codeSystemId;
+
+    /**
+     * Public constructor
+     * @param codeSystem
+     */
+    public CodeSystemValue(String codeSystem) {
+        this.codeSystem = codeSystem;
+    }
+
+    /**
+     * @return the codeSystemId or null if it is current unknown
+     */
+    public Integer getCodeSystemId() {
+        return codeSystemId;
+    }
+
+    /**
+     * @param codeSystemId the codeSystemId to set
+     */
+    public void setCodeSystemId(int codeSystemId) {
+        this.codeSystemId = codeSystemId;
+    }
+
+    /**
+     * @return the codeSystem
+     */
+    public String getCodeSystem() {
+        return codeSystem;
+    }
+}
\ No newline at end of file
diff --git a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/CommonCanonicalValue.java b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/CommonCanonicalValue.java
new file mode 100644
index 00000000000..449d43d36ed
--- /dev/null
+++ b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/CommonCanonicalValue.java
@@ -0,0 +1,67 @@
+/*
+ * (C) Copyright IBM Corp. 2022
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+ 
+package com.ibm.fhir.remote.index.database;
+
+/**
+ * Represents a common_canonical_value record which may or may not yet exist
+ * in the database. If it exists in the database, we may not yet have
+ * retrieved its canonical_id.
+ */
+public class CommonCanonicalValue {
+    private final short shardKey;
+    private final String url;
+    private Long canonicalId;
+
+    @Override
+    public String toString() {
+        StringBuilder result = new StringBuilder();
+        result.append(shardKey);
+        result.append(",");
+        result.append(url);
+        result.append(",");
+        result.append(canonicalId);
+        return result.toString();
+    }
+    /**
+     * Public constructor
+     * @param shardKey
+     * @param codeSystemValue
+     * @param tokenValue
+     */
+    public CommonCanonicalValue(short shardKey, String url) {
+        this.shardKey = shardKey;
+        this.url = url;
+    }
+
+    /**
+     * @return the canonicalId
+     */
+    public Long getCanonicalId() {
+        return canonicalId;
+    }
+
+    /**
+     * @param canonicalId the canonicalId to set
+     */
+    public void setCanonicalId(Long canonicalId) {
+        this.canonicalId = canonicalId;
+    }
+
+    /**
+     * @return the shardKey
+     */
+    public short getShardKey() {
+        return shardKey;
+    }
+
+    /**
+     * @return the url
+     */
+    public String getUrl() {
+        return url;
+    }
+}
\ No newline at end of file
diff --git a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/CommonCanonicalValueKey.java b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/CommonCanonicalValueKey.java
new file mode 100644
index 00000000000..a6d3b8d19e6
--- /dev/null
+++ b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/CommonCanonicalValueKey.java
@@ -0,0 +1,44 @@
+/*
+ * (C) Copyright IBM Corp. 2022
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+ 
+package com.ibm.fhir.remote.index.database;
+
+import java.util.Objects;
+
+/**
+ * A key used to identify a common_canonical_value record in our distributed schema
+ * variant
+ */
+public class CommonCanonicalValueKey {
+    private final short shardKey;
+    private final String url;
+
+    /**
+     * Public constructor
+     * @param shardKey
+     * @param codeSystem
+     * @param tokenValue
+     */
+    public CommonCanonicalValueKey(short shardKey, String url) {
+        this.shardKey = shardKey;
+        this.url = Objects.requireNonNull(url);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(url, shardKey);
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (obj instanceof CommonCanonicalValueKey) {
+            CommonCanonicalValueKey that = (CommonCanonicalValueKey)obj;
+            return this.shardKey == that.shardKey
+                    && this.url.equals(that.url);
+        }
+        return false;
+    }
+}
\ No newline at end of file
diff --git a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/CommonTokenValue.java b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/CommonTokenValue.java
new file mode 100644
index 00000000000..4a271217aae
--- /dev/null
+++ b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/CommonTokenValue.java
@@ -0,0 +1,66 @@
+/*
+ * (C) Copyright IBM Corp. 2022
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+ 
+package com.ibm.fhir.remote.index.database;
+
+/**
+ * Represents a common_token_value record which may or may not yet exist
+ * in the database. If it exists in the database, we may not yet have
+ * retrieved its common_token_value_id.
+ */
+public class CommonTokenValue {
+    private final short shardKey;
+    private final CodeSystemValue codeSystemValue;
+    private final String tokenValue;
+    private Long commonTokenValueId;
+
+    /**
+     * Public constructor
+     * @param shardKey
+     * @param codeSystemValue
+     * @param tokenValue
+     */
+    public CommonTokenValue(short shardKey, CodeSystemValue codeSystemValue, String tokenValue) {
+        this.shardKey = shardKey;
+        this.codeSystemValue = codeSystemValue;
+        this.tokenValue = tokenValue;
+    }
+
+    /**
+     * @return the commonTokenValueId
+     */
+    public Long getCommonTokenValueId() {
+        return commonTokenValueId;
+    }
+
+    /**
+     * @param commonTokenValueId the commonTokenValueId to set
+     */
+    public void setCommonTokenValueId(Long commonTokenValueId) {
+        this.commonTokenValueId = commonTokenValueId;
+    }
+
+    /**
+     * @return the codeSystemValue
+     */
+    public CodeSystemValue getCodeSystemValue() {
+        return codeSystemValue;
+    }
+
+    /**
+     * @return the tokenValue
+     */
+    public String getTokenValue() {
+        return tokenValue;
+    }
+
+    /**
+     * @return the shardKey
+     */
+    public short getShardKey() {
+        return shardKey;
+    }
+}
\ No newline at end of file
diff --git a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/CommonTokenValueKey.java b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/CommonTokenValueKey.java
new file mode 100644
index 00000000000..de3b6fa6907
--- /dev/null
+++ b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/CommonTokenValueKey.java
@@ -0,0 +1,47 @@
+/*
+ * (C) Copyright IBM Corp. 2022
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+ 
+package com.ibm.fhir.remote.index.database;
+
+import java.util.Objects;
+
+/**
+ * A key used to identify a common_token_value record in our distributed schema
+ * variant
+ */
+public class CommonTokenValueKey {
+    private final short shardKey;
+    private final String codeSystem;
+    private final String tokenValue;
+
+    /**
+     * Public constructor
+     * @param shardKey
+     * @param codeSystem
+     * @param tokenValue
+     */
+    public CommonTokenValueKey(short shardKey, String codeSystem, String tokenValue) {
+        this.shardKey = shardKey;
+        this.codeSystem = Objects.requireNonNull(codeSystem);
+        this.tokenValue = Objects.requireNonNull(tokenValue);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(codeSystem, tokenValue, shardKey);
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (obj instanceof CommonTokenValueKey) {
+            CommonTokenValueKey that = (CommonTokenValueKey)obj;
+            return this.shardKey == that.shardKey
+                    && this.codeSystem.equals(that.codeSystem)
+                    && this.tokenValue.equals(that.tokenValue);
+        }
+        return false;
+    }
+}
\ No newline at end of file
diff --git a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/DistributedPostgresMessageHandler.java b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/DistributedPostgresMessageHandler.java
new file mode 100644
index 00000000000..dd679de3fc2
--- /dev/null
+++ b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/DistributedPostgresMessageHandler.java
@@ -0,0 +1,122 @@
+/*
+ * (C) Copyright IBM Corp. 2022
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+ 
+package com.ibm.fhir.remote.index.database;
+
+import java.sql.Connection;
+import java.sql.PreparedStatement;
+import java.sql.SQLException;
+import java.util.List;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+import com.ibm.fhir.database.utils.postgres.PostgresTranslator;
+import com.ibm.fhir.persistence.exception.FHIRPersistenceException;
+import com.ibm.fhir.remote.index.api.IdentityCache;
+
+/**
+ * For the DISTRIBUTED schema variant used on databases such as Citus, we
+ * can't use IDENTITY columns. Instead we have to use values generated
+ * by a sequence, which means a slightly different INSERT statement
+ * in certain cases
+ */
+public class DistributedPostgresMessageHandler extends PlainMessageHandler {
+    private static final Logger logger = Logger.getLogger(DistributedPostgresMessageHandler.class.getName());
+
+    /**
+     * Public constructor
+     * 
+     * @param instanceIdentifier
+     * @param connection
+     * @param schemaName
+     * @param cache
+     * @param maxReadyTimeMs
+     */
+    public DistributedPostgresMessageHandler(String instanceIdentifier, Connection connection, String schemaName, IdentityCache cache, long maxReadyTimeMs) {
+        super(instanceIdentifier, new PostgresTranslator(), connection, schemaName, cache, maxReadyTimeMs);
+    }
+
+    @Override
+    protected String onConflict() {
+        return "ON CONFLICT DO NOTHING";
+    }
+
+    @Override
+    protected void addMissingCommonTokenValues(List missing) throws FHIRPersistenceException {
+
+        // Need to use our own sequence number because distributed databases don't
+        // like generated identity columns
+        final String nextVal = translator.nextValue(schemaName, "fhir_sequence");
+        StringBuilder insert = new StringBuilder();
+        insert.append("INSERT INTO common_token_values (code_system_id, token_value, common_token_value_id) ");
+        insert.append(" OVERRIDING SYSTEM VALUE "); // we want to use our sequence number
+        insert.append("     VALUES (?,?,");
+        insert.append(nextVal); // next sequence value
+        insert.append(") ");
+        insert.append(onConflict());
+
+        try (PreparedStatement ps = connection.prepareStatement(insert.toString())) {
+            int count = 0;
+            for (CommonTokenValue ctv: missing) {
+                ps.setInt(1, ctv.getCodeSystemValue().getCodeSystemId());
+                ps.setString(2, ctv.getTokenValue());
+                ps.addBatch();
+                if (++count == this.maxCommonTokenValuesPerStatement) {
+                    // not too many statements in a single batch
+                    ps.executeBatch();
+                    count = 0;
+                }
+            }
+            if (count > 0) {
+                // final batch
+                ps.executeBatch();
+            }
+        } catch (SQLException x) {
+            logger.log(Level.SEVERE, "failed: " + insert.toString(), x);
+            throw new FHIRPersistenceException("failed inserting new common token values");
+        }
+    }
+
+    @Override
+    protected void addMissingCommonCanonicalValues(List missing) throws FHIRPersistenceException {
+
+        // Need to use our own sequence number because distributed databases don't
+        // like generated identity columns
+        final String nextVal = translator.nextValue(schemaName, "fhir_sequence");
+        StringBuilder insert = new StringBuilder();
+        insert.append("INSERT INTO common_canonical_values (url, canonical_id) ");
+        insert.append(" OVERRIDING SYSTEM VALUE "); // we want to use our sequence number
+        insert.append("     VALUES (?,");
+        insert.append(nextVal); // next sequence value
+        insert.append(") ");
+        insert.append(onConflict());
+
+        final String DML = insert.toString();
+        if (logger.isLoggable(Level.FINE)) {
+            logger.fine("addMissingCanonicalIds: " + DML);
+        }
+        try (PreparedStatement ps = connection.prepareStatement(DML)) {
+            int count = 0;
+            for (CommonCanonicalValue ctv: missing) {
+                logger.finest(() -> "Adding canonical value [" + ctv.toString() + "]");
+                ps.setString(1, ctv.getUrl());
+                ps.addBatch();
+                if (++count == this.maxCommonCanonicalValuesPerStatement) {
+                    // not too many statements in a single batch
+                    ps.executeBatch();
+                    count = 0;
+                }
+            }
+            if (count > 0) {
+                // final batch
+                ps.executeBatch();
+            }
+        } catch (SQLException x) {
+            logger.log(Level.SEVERE, "failed: " + insert.toString(), x);
+            throw new FHIRPersistenceException("failed inserting new common canonical values");
+        }
+    }
+}
\ No newline at end of file
diff --git a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/LogicalResourceIdentKey.java b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/LogicalResourceIdentKey.java
new file mode 100644
index 00000000000..5c8e205f4fd
--- /dev/null
+++ b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/LogicalResourceIdentKey.java
@@ -0,0 +1,42 @@
+/*
+ * (C) Copyright IBM Corp. 2022
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+ 
+package com.ibm.fhir.remote.index.database;
+
+import java.util.Objects;
+
+/**
+ * Key used to uniquely identify a logical resource
+ */
+public class LogicalResourceIdentKey {
+    private final String resourceType;
+    private final String logicalId;
+
+    /**
+     * Canonical constructor
+     * @param resourceType
+     * @param logicalId
+     */
+    public LogicalResourceIdentKey(String resourceType, String logicalId) {
+        this.resourceType = resourceType;
+        this.logicalId = logicalId;
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(resourceType, logicalId);
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (obj instanceof LogicalResourceIdentKey) {
+            LogicalResourceIdentKey that = (LogicalResourceIdentKey)obj;
+            return this.resourceType.equals(that.resourceType)
+                    && this.logicalId.equals(that.logicalId);
+        }
+        return false;
+    }
+}
diff --git a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/LogicalResourceIdentValue.java b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/LogicalResourceIdentValue.java
new file mode 100644
index 00000000000..7e882366599
--- /dev/null
+++ b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/LogicalResourceIdentValue.java
@@ -0,0 +1,159 @@
+/*
+ * (C) Copyright IBM Corp. 2022
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+ 
+package com.ibm.fhir.remote.index.database;
+
+import java.util.Objects;
+
+/**
+ * A DTO representing a record from logical_resource_ident
+ */
+public class LogicalResourceIdentValue implements Comparable {
+    private final int resourceTypeId;
+    private final String resourceType;
+    private final String logicalId;
+    private Long logicalResourceId;
+
+    @Override
+    public String toString() {
+        StringBuilder result = new StringBuilder();
+        result.append("'");
+        result.append(resourceType);
+        result.append("/");
+        result.append(logicalId);
+        result.append("'");
+        result.append(" => ");
+        result.append(resourceTypeId);
+        result.append("/");
+        result.append(logicalResourceId);
+        return result.toString();
+    }
+
+    /**
+     * Builder for fluent creation of LogicalResourceIdentValue objects
+     */
+    public static class Builder {
+        private int resourceTypeId;
+        private String resourceType;
+        private String logicalId;
+        private Long logicalResourceId;
+
+        /**
+         * Set the resourceTypeId
+         * @param resourceTypeId
+         * @return
+         */
+        public Builder withResourceTypeId(int resourceTypeId) {
+            this.resourceTypeId = resourceTypeId;
+            return this;
+        }
+
+        /**
+         * Set the logicalResourceId
+         * @param logicalResourceId
+         * @return
+         */
+        public Builder withLogicalResourceId(long logicalResourceId) {
+            this.logicalResourceId = logicalResourceId;
+            return this;
+        }
+
+        /**
+         * Set the resourceType
+         * @param resourceType
+         * @return
+         */
+        public Builder withResourceType(String resourceType) {
+            this.resourceType = resourceType;
+            return this;
+        }
+
+        /**
+         * Set the logicalId
+         * @param logicalId
+         * @return
+         */
+        public Builder withLogicalId(String logicalId) {
+            this.logicalId = logicalId;
+            return this;
+        }
+
+        /**
+         * Create a new {@link LogicalResourceValue} using the current state of this {@link Builder}
+         * @return
+         */
+        public LogicalResourceIdentValue build() {
+            return new LogicalResourceIdentValue(resourceTypeId, resourceType, logicalId, logicalResourceId);
+        }
+    }
+
+    /**
+     * Factor function to create a fresh instance of a {@link Builder}
+     * @return
+     */
+    public static Builder builder() {
+        return new Builder();
+    }
+
+    /**
+     * Public constructor
+     * @param resourceTypeId
+     * @param resourceType
+     * @param logicalId
+     * @param logicalResourceId
+     */
+    public LogicalResourceIdentValue(int resourceTypeId, String resourceType, String logicalId, Long logicalResourceId) {
+        this.resourceTypeId = resourceTypeId;
+        this.resourceType = resourceType;
+        this.logicalId = Objects.requireNonNull(logicalId);
+        this.logicalResourceId = logicalResourceId;
+    }
+
+    /**
+     * @return the logicalResourceId
+     */
+    public Long getLogicalResourceId() {
+        return logicalResourceId;
+    }
+
+    /**
+     * @return the resourceType
+     */
+    public String getResourceType() {
+        return resourceType;
+    }
+
+    /**
+     * @return the logicalId
+     */
+    public String getLogicalId() {
+        return logicalId;
+    }
+
+    /**
+     * Set the logicalResourceId value
+     * @param logicalResourceId
+     */
+    public void setLogicalResourceId(Long logicalResourceId) {
+        this.logicalResourceId = logicalResourceId;
+    }
+
+    @Override
+    public int compareTo(LogicalResourceIdentValue that) {
+        int result = this.resourceType.compareTo(that.resourceType);
+        if (0 == result) {
+            result = this.logicalId.compareTo(that.logicalId);
+        }
+        return result;
+    }
+
+    /**
+     * @return the resourceTypeId
+     */
+    public Integer getResourceTypeId() {
+        return resourceTypeId;
+    }
+}
\ No newline at end of file
diff --git a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/LogicalResourceValue.java b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/LogicalResourceValue.java
new file mode 100644
index 00000000000..1c6dc14cc7d
--- /dev/null
+++ b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/LogicalResourceValue.java
@@ -0,0 +1,197 @@
+/*
+ * (C) Copyright IBM Corp. 2022
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+ 
+package com.ibm.fhir.remote.index.database;
+
+import java.sql.Timestamp;
+
+/**
+ * A DTO representing a record from logical_resources
+ */
+public class LogicalResourceValue {
+    private final short shardKey;
+    private final long logicalResourceId;
+    private final String resourceType;
+    private final String logicalId;
+    private final int versionId;
+    private final Timestamp lastUpdated;
+    private final String parameterHash;
+
+    /**
+     * Builder for fluent creation of LogicalResourceValue objects
+     */
+    public static class Builder {
+        private short shardKey;
+        private long logicalResourceId;
+        private String resourceType;
+        private String logicalId;
+        private int versionId;
+        private Timestamp lastUpdated;
+        private String parameterHash;
+
+        /**
+         * Set the shardKey
+         * @param shardKey
+         * @return
+         */
+        public Builder withShardKey(short shardKey) {
+            this.shardKey = shardKey;
+            return this;
+        }
+
+        /**
+         * Set the logicalResourceId value
+         * @param logicalResourceId
+         * @return
+         */
+        public Builder withLogicalResourceId(long logicalResourceId) {
+            this.logicalResourceId = logicalResourceId;
+            return this;
+        }
+
+        /**
+         * Set the resourceType value
+         * @param resourceType
+         * @return
+         */
+        public Builder withResourceType(String resourceType) {
+            this.resourceType = resourceType;
+            return this;
+        }
+
+        /**
+         * Set the logicalId value
+         * @param logicalId
+         * @return
+         */
+        public Builder withLogicalId(String logicalId) {
+            this.logicalId = logicalId;
+            return this;
+        }
+
+        /**
+         * Set the versionId value
+         * @param versionId
+         * @return
+         */
+        public Builder withVersionId(int versionId) {
+            this.versionId = versionId;
+            return this;
+        }
+
+        /**
+         * Set the lastUpdated value
+         * @param lastUpdated
+         * @return
+         */
+        public Builder withLastUpdated(Timestamp lastUpdated) {
+            this.lastUpdated = lastUpdated;
+            return this;
+        }
+
+        /**
+         * Set the parameterHash value
+         * @param parameterHash
+         * @return
+         */
+        public Builder withParameterHash(String parameterHash) {
+            this.parameterHash = parameterHash;
+            return this;
+        }
+
+        /**
+         * Create a new {@link LogicalResourceValue} using the current state of this {@link Builder}
+         * @return
+         */
+        public LogicalResourceValue build() {
+            return new LogicalResourceValue(shardKey, logicalResourceId, resourceType, logicalId, versionId, lastUpdated, parameterHash);
+        }
+    }
+
+    /**
+     * Factory function to create a fresh instance of a {@link Builder}
+     * @return
+     */
+    public static Builder builder() {
+        return new Builder();
+    }
+
+    /**
+     * Canonical constructor
+     * 
+     * @param shardKey
+     * @param logicalResourceId
+     * @param resourceType
+     * @param logicalId
+     * @param lastUpdated
+     * @param parameterHash
+     */
+    public LogicalResourceValue(short shardKey, long logicalResourceId, String resourceType, String logicalId, int versionId, Timestamp lastUpdated, String parameterHash) {
+        this.shardKey = shardKey;
+        this.logicalResourceId = logicalResourceId;
+        this.resourceType = resourceType;
+        this.logicalId = logicalId;
+        this.versionId = versionId;
+        this.lastUpdated = lastUpdated;
+        this.parameterHash = parameterHash;
+    }
+
+    
+    /**
+     * @return the shardKey
+     */
+    public short getShardKey() {
+        return shardKey;
+    }
+
+    
+    /**
+     * @return the logicalResourceId
+     */
+    public long getLogicalResourceId() {
+        return logicalResourceId;
+    }
+
+    
+    /**
+     * @return the resourceType
+     */
+    public String getResourceType() {
+        return resourceType;
+    }
+
+    
+    /**
+     * @return the logicalId
+     */
+    public String getLogicalId() {
+        return logicalId;
+    }
+
+    
+    /**
+     * @return the lastUpdated
+     */
+    public Timestamp getLastUpdated() {
+        return lastUpdated;
+    }
+
+    
+    /**
+     * @return the parameterHash
+     */
+    public String getParameterHash() {
+        return parameterHash;
+    }
+
+    /**
+     * @return the versionId
+     */
+    public int getVersionId() {
+        return versionId;
+    }
+
+}
diff --git a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/ParameterNameValue.java b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/ParameterNameValue.java
new file mode 100644
index 00000000000..8abe2d46457
--- /dev/null
+++ b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/ParameterNameValue.java
@@ -0,0 +1,47 @@
+/*
+ * (C) Copyright IBM Corp. 2022
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+ 
+package com.ibm.fhir.remote.index.database;
+
+
+/**
+ * Represents a record in parameter_names for which we don't know
+ * the parameter_name_id value, or which we need to create
+ */
+public class ParameterNameValue {
+    private final String parameterName;
+    private Integer parameterNameId;
+
+    /**
+     * Public constructor
+     * @param parameterName
+     */
+    public ParameterNameValue(String parameterName) {
+        this.parameterName = parameterName;
+    }
+
+    /**
+     * @return the parameterName
+     */
+    public String getParameterName() {
+        return parameterName;
+    }
+
+    /**
+     * @return the parameterNameId
+     */
+    public Integer getParameterNameId() {
+        return parameterNameId;
+    }
+
+    /**
+     * @param parameterNameId the parameterNameId to set
+     */
+    public void setParameterNameId(Integer parameterNameId) {
+        this.parameterNameId = parameterNameId;
+    }
+
+}
diff --git a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/PlainBatchParameterProcessor.java b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/PlainBatchParameterProcessor.java
new file mode 100644
index 00000000000..77763ad8df5
--- /dev/null
+++ b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/PlainBatchParameterProcessor.java
@@ -0,0 +1,328 @@
+/*
+ * (C) Copyright IBM Corp. 2022
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+ 
+package com.ibm.fhir.remote.index.database;
+
+import java.sql.Connection;
+import java.sql.SQLException;
+import java.sql.Timestamp;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+import com.ibm.fhir.persistence.exception.FHIRPersistenceException;
+import com.ibm.fhir.persistence.index.DateParameter;
+import com.ibm.fhir.persistence.index.LocationParameter;
+import com.ibm.fhir.persistence.index.NumberParameter;
+import com.ibm.fhir.persistence.index.ProfileParameter;
+import com.ibm.fhir.persistence.index.QuantityParameter;
+import com.ibm.fhir.persistence.index.ReferenceParameter;
+import com.ibm.fhir.persistence.index.SecurityParameter;
+import com.ibm.fhir.persistence.index.StringParameter;
+import com.ibm.fhir.persistence.index.TagParameter;
+import com.ibm.fhir.persistence.index.TokenParameter;
+import com.ibm.fhir.remote.index.api.BatchParameterProcessor;
+
+
+/**
+ * Processes batched parameters by pushing the values to various
+ * JDBC statements based on the plain variant of the schema
+ */
+public class PlainBatchParameterProcessor implements BatchParameterProcessor {
+    private static final Logger logger = Logger.getLogger(PlainBatchParameterProcessor.class.getName());
+
+    // A cache of the resource-type specific DAOs we've created
+    private final Map daoMap = new HashMap<>();
+
+    // Encapculates the statements for inserting whole-system level search params
+    private final PlainPostgresSystemParameterBatch systemDao;
+
+    // Resource types we've touched in the current batch
+    private final Set resourceTypesInBatch = new HashSet<>();
+
+    // The database connection this consumer thread is using
+    private final Connection connection;
+
+    /**
+     * Public constructor
+     * @param connection
+     */
+    public PlainBatchParameterProcessor(Connection connection) {
+        this.connection = connection;
+        this.systemDao = new PlainPostgresSystemParameterBatch(connection);        
+    }
+
+    /**
+     * Close any resources we're holding to support a cleaner exit
+     */
+    public void close() {
+        for (Map.Entry entry: daoMap.entrySet()) {
+            entry.getValue().close();
+        }
+        systemDao.close();
+    }
+
+    /**
+     * Start processing a new batch
+     */
+    public void startBatch() {
+        resourceTypesInBatch.clear();
+    }
+
+    /**
+     * Make sure that each statement that may contain data is cleared before we
+     * retry a batch
+     */
+    public void reset() {
+        for (String resourceType: resourceTypesInBatch) {
+            PlainPostgresParameterBatch dao = daoMap.get(resourceType);
+            dao.close();
+        }
+        systemDao.close();
+    }
+
+    @Override
+    public Short encodeShardKey(String requestShard) {
+        // This implementation doesn't get involved in application-based sharding
+        return null;
+    }
+
+    /**
+     * Push any statements that have been batched but not yet executed
+     * @throws FHIRPersistenceException
+     */
+    public void pushBatch() throws FHIRPersistenceException {
+        try {
+            for (String resourceType: resourceTypesInBatch) {
+                if (logger.isLoggable(Level.FINE)) {
+                    logger.fine("Pushing batch for [" + resourceType + "]");
+                }
+                PlainPostgresParameterBatch dao = daoMap.get(resourceType);
+                try {
+                    dao.pushBatch();
+                } catch (SQLException x) {
+                    throw new FHIRPersistenceException("pushBatch failed for '" + resourceType + "'");
+                }
+            }
+
+            try {
+                logger.fine("Pushing batch for whole-system parameters");
+                systemDao.pushBatch();
+            } catch (SQLException x) {
+                throw new FHIRPersistenceException("batch insert for whole-system parameters", x);
+            }
+        } finally {
+            // Reset the set of active resource-types ready for the next batch
+            resourceTypesInBatch.clear();
+        }
+    }
+
+    /**
+     * Get the DAO used to batch parameter inserts for the given resourceType.
+     * This method also tracks the unique set of resource types seen for a
+     * collection of messages.
+     * @param resourceType
+     * @return
+     */
+    private PlainPostgresParameterBatch getParameterBatchDao(String resourceType) {
+        resourceTypesInBatch.add(resourceType);
+        PlainPostgresParameterBatch dao = daoMap.get(resourceType);
+        if (dao == null) {
+            dao = new PlainPostgresParameterBatch(connection, resourceType);
+            daoMap.put(resourceType, dao);
+        }
+        return dao;
+    }
+
+    @Override
+    public void process(String requestShard, String resourceType, String logicalId, long logicalResourceId, ParameterNameValue parameterNameValue, StringParameter parameter) throws FHIRPersistenceException {
+        if (logger.isLoggable(Level.FINE)) {
+            logger.fine("process string parameter [" + resourceType + "] [" + logicalId + "] [" + logicalResourceId + "] [" + parameterNameValue.getParameterName() + "] ["
+                + parameter.toString() + "]");
+        }
+
+        try {
+            PlainPostgresParameterBatch dao = getParameterBatchDao(resourceType);
+            dao.addString(logicalResourceId, parameterNameValue.getParameterNameId(), parameter.getValue(), parameter.getValue().toLowerCase(), parameter.getCompositeId());
+
+            if (parameter.isSystemParam()) {
+                systemDao.addString(logicalResourceId, parameterNameValue.getParameterNameId(), parameter.getValue(), parameter.getValue().toLowerCase());
+            }
+        } catch (SQLException x) {
+            logger.log(Level.SEVERE, "StringParameter", x);
+            throw new FHIRPersistenceException("Failed inserting string params for '" + resourceType + "'");
+        }
+    }
+
+    @Override
+    public void process(String requestShard, String resourceType, String logicalId, long logicalResourceId, ParameterNameValue parameterNameValue, NumberParameter p) throws FHIRPersistenceException {
+        if (logger.isLoggable(Level.FINE)) {
+            logger.fine("process number parameter [" + resourceType + "] [" + logicalId + "] [" + logicalResourceId + "] [" + parameterNameValue.getParameterName() + "] ["
+                    + p.toString() + "]");
+        }
+
+        try {
+            PlainPostgresParameterBatch dao = getParameterBatchDao(resourceType);
+            dao.addNumber(logicalResourceId, parameterNameValue.getParameterNameId(), p.getValue(), p.getLowValue(), p.getHighValue(), p.getCompositeId());
+        } catch (SQLException x) {
+            throw new FHIRPersistenceException("Failed inserting string params for '" + resourceType + "'");
+        }
+    }
+
+    @Override
+    public void process(String requestShard, String resourceType, String logicalId, long logicalResourceId, ParameterNameValue parameterNameValue, QuantityParameter p, CodeSystemValue codeSystemValue) throws FHIRPersistenceException {
+        if (logger.isLoggable(Level.FINE)) {
+            logger.fine("process quantity parameter [" + resourceType + "] [" + logicalId + "] [" + logicalResourceId + "] [" + parameterNameValue.getParameterName() + "] ["
+                    + p.toString() + "]");
+        }
+
+        try {
+            PlainPostgresParameterBatch dao = getParameterBatchDao(resourceType);
+            dao.addQuantity(logicalResourceId, parameterNameValue.getParameterNameId(), codeSystemValue.getCodeSystemId(), p.getValueCode(), p.getValueNumber(), p.getValueNumberLow(), p.getValueNumberHigh(), p.getCompositeId());
+        } catch (SQLException x) {
+            throw new FHIRPersistenceException("Failed inserting quantity params for '" + resourceType + "'");
+        }
+    }
+
+    @Override
+    public void process(String requestShard, String resourceType, String logicalId, long logicalResourceId, ParameterNameValue parameterNameValue, LocationParameter p) throws FHIRPersistenceException {
+        if (logger.isLoggable(Level.FINE)) {
+            logger.fine("process location parameter [" + resourceType + "] [" + logicalId + "] [" + logicalResourceId + "] [" + parameterNameValue.getParameterName() + "] ["
+                    + p.toString() + "]");
+        }
+
+        try {
+            PlainPostgresParameterBatch dao = getParameterBatchDao(resourceType);
+            dao.addLocation(logicalResourceId, parameterNameValue.getParameterNameId(), p.getValueLatitude(), p.getValueLongitude(), p.getCompositeId());
+        } catch (SQLException x) {
+            throw new FHIRPersistenceException("Failed inserting location params for '" + resourceType + "'");
+        }
+    }
+
+    @Override
+    public void process(String requestShard, String resourceType, String logicalId, long logicalResourceId, ParameterNameValue parameterNameValue, DateParameter p) throws FHIRPersistenceException {
+        if (logger.isLoggable(Level.FINE)) {
+            logger.fine("process date parameter [" + resourceType + "] [" + logicalId + "] [" + logicalResourceId + "] [" + parameterNameValue.getParameterName() + "] ["
+                    + p.toString() + "]");
+        }
+
+        try {
+            PlainPostgresParameterBatch dao = getParameterBatchDao(resourceType);
+            final Timestamp valueDateStart = Timestamp.from(p.getValueDateStart());
+            final Timestamp valueDateEnd = Timestamp.from(p.getValueDateEnd());
+            dao.addDate(logicalResourceId, parameterNameValue.getParameterNameId(), valueDateStart, valueDateEnd, p.getCompositeId());
+            if (p.isSystemParam()) {
+                systemDao.addDate(logicalResourceId, parameterNameValue.getParameterNameId(), valueDateStart, valueDateEnd, p.getCompositeId());
+            }
+        } catch (SQLException x) {
+            logger.log(Level.SEVERE, "DateParameter", x);
+            throw new FHIRPersistenceException("Failed inserting date params for '" + resourceType + "'");
+        }
+    }
+
+    @Override
+    public void process(String requestShard, String resourceType, String logicalId, long logicalResourceId, ParameterNameValue parameterNameValue, TokenParameter p,
+        CommonTokenValue commonTokenValue) throws FHIRPersistenceException {
+
+        if (logger.isLoggable(Level.FINE)) {
+            logger.fine("process token parameter [" + resourceType + "] [" + logicalId + "] [" + logicalResourceId + "] [" + parameterNameValue.getParameterName() + "] ["
+                    + p.toString() + "] [" + commonTokenValue.getCommonTokenValueId() + "]");
+        }
+
+        try {
+            PlainPostgresParameterBatch dao = getParameterBatchDao(resourceType);
+            dao.addResourceTokenRef(logicalResourceId, parameterNameValue.getParameterNameId(), commonTokenValue.getCommonTokenValueId(), p.getRefVersionId(), p.getCompositeId());
+            if (p.isSystemParam()) {
+                // Currently we store _tag:text as a token value, and because it's also whole-system, we need to add it here
+                systemDao.addResourceTokenRef(logicalResourceId, parameterNameValue.getParameterNameId(), commonTokenValue.getCommonTokenValueId());
+            }
+        } catch (SQLException x) {
+            throw new FHIRPersistenceException("Failed inserting token params for '" + resourceType + "'");
+        }
+    }
+
+    @Override
+    public void process(String requestShard, String resourceType, String logicalId, long logicalResourceId, ParameterNameValue parameterNameValue, TagParameter p,
+        CommonTokenValue commonTokenValue) throws FHIRPersistenceException {
+
+        if (logger.isLoggable(Level.FINE)) {
+            logger.fine("process tag parameter [" + resourceType + "] [" + logicalId + "] [" + logicalResourceId + "] [" + parameterNameValue.getParameterName() + "] ["
+                    + p.toString() + "] [" + commonTokenValue.getCommonTokenValueId() + "]");
+        }
+
+        try {
+            PlainPostgresParameterBatch dao = getParameterBatchDao(resourceType);
+            dao.addTag(logicalResourceId, commonTokenValue.getCommonTokenValueId());
+            
+            if (p.isSystemParam()) {
+                systemDao.addTag(logicalResourceId, commonTokenValue.getCommonTokenValueId());
+            }
+        } catch (SQLException x) {
+            throw new FHIRPersistenceException("Failed inserting tag params for '" + resourceType + "'");
+        }
+    }
+
+    @Override
+    public void process(String requestShard, String resourceType, String logicalId, long logicalResourceId, ParameterNameValue parameterNameValue, ProfileParameter p,
+        CommonCanonicalValue commonCanonicalValue) throws FHIRPersistenceException {
+
+        if (logger.isLoggable(Level.FINE)) {
+            logger.fine("process profile parameter [" + resourceType + "] [" + logicalId + "] [" + logicalResourceId + "] [" + parameterNameValue.getParameterName() + "] ["
+                    + p.toString() + "] [" + commonCanonicalValue.getCanonicalId() + "]");
+        }
+
+        try {
+            PlainPostgresParameterBatch dao = getParameterBatchDao(resourceType);
+            dao.addProfile(logicalResourceId, commonCanonicalValue.getCanonicalId(), p.getVersion(), p.getFragment());
+            if (p.isSystemParam()) {
+                systemDao.addProfile(logicalResourceId, commonCanonicalValue.getCanonicalId(), p.getVersion(), p.getFragment());
+            }
+        } catch (SQLException x) {
+            throw new FHIRPersistenceException("Failed inserting profile params for '" + resourceType + "'");
+        }
+    }
+
+    @Override
+    public void process(String requestShard, String resourceType, String logicalId, long logicalResourceId, ParameterNameValue parameterNameValue, SecurityParameter p,
+        CommonTokenValue commonTokenValue) throws FHIRPersistenceException {
+
+        if (logger.isLoggable(Level.FINE)) {
+            logger.fine("process security parameter [" + resourceType + "] [" + logicalId + "] [" + logicalResourceId + "] [" + parameterNameValue.getParameterName() + "] ["
+                    + p.toString() + "] [" + commonTokenValue.getCommonTokenValueId() + "]");
+        }
+
+        try {
+            PlainPostgresParameterBatch dao = getParameterBatchDao(resourceType);
+            dao.addSecurity(logicalResourceId, commonTokenValue.getCommonTokenValueId());
+            
+            if (p.isSystemParam()) {
+                systemDao.addSecurity(logicalResourceId, commonTokenValue.getCommonTokenValueId());
+            }
+        } catch (SQLException x) {
+            throw new FHIRPersistenceException("Failed inserting security params for '" + resourceType + "'");
+        }
+    }
+
+    @Override
+    public void process(String requestShard, String resourceType, String logicalId, long logicalResourceId, ParameterNameValue parameterNameValue,
+        ReferenceParameter parameter, LogicalResourceIdentValue refLogicalResourceId) throws FHIRPersistenceException {
+
+        if (logger.isLoggable(Level.FINE)) {
+            logger.fine("process reference parameter [" + resourceType + "] [" + logicalId + "] [" + logicalResourceId + "] [" + parameterNameValue.getParameterName() + "] ["
+                    + parameter.toString() + "] [" + refLogicalResourceId.getLogicalResourceId() + "]");
+        }
+
+        try {
+            PlainPostgresParameterBatch dao = getParameterBatchDao(resourceType);
+            dao.addReference(logicalResourceId, parameterNameValue.getParameterNameId(), refLogicalResourceId.getLogicalResourceId(), parameter.getRefVersionId());
+        } catch (SQLException x) {
+            throw new FHIRPersistenceException("Failed inserting security params for '" + resourceType + "'");
+        }
+    }
+}
diff --git a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/PlainDerbyMessageHandler.java b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/PlainDerbyMessageHandler.java
new file mode 100644
index 00000000000..ad5e36052b6
--- /dev/null
+++ b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/PlainDerbyMessageHandler.java
@@ -0,0 +1,152 @@
+/*
+ * (C) Copyright IBM Corp. 2022
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+ 
+package com.ibm.fhir.remote.index.database;
+
+import java.sql.Connection;
+import java.sql.PreparedStatement;
+import java.sql.SQLException;
+import java.util.List;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+import com.ibm.fhir.database.utils.derby.DerbyTranslator;
+import com.ibm.fhir.persistence.exception.FHIRPersistenceException;
+import com.ibm.fhir.remote.index.api.IdentityCache;
+
+/**
+ * Derby variant of the plain schema message handler which is needed because Derby
+ * needs slightly different syntax for some queries
+ */
+public class PlainDerbyMessageHandler extends PlainMessageHandler {
+    private static final Logger logger = Logger.getLogger(PlainDerbyMessageHandler.class.getName());
+
+    /**
+     * Public constructor
+     * @param instanceIdentifier
+     * @param connection
+     * @param schemaName
+     * @param cache
+     * @param maxReadyTimeMs
+     */
+    public PlainDerbyMessageHandler(String instanceIdentifier, Connection connection, String schemaName, IdentityCache cache, long maxReadyTimeMs) {
+        super(instanceIdentifier, new DerbyTranslator(), connection, schemaName, cache, maxReadyTimeMs);
+    }
+
+    @Override
+    protected String onConflict() {
+        return "";
+    }
+
+    @Override
+    protected PreparedStatement buildLogicalResourceIdentSelectStatement(List values) throws SQLException {
+        StringBuilder query = new StringBuilder();
+        query.append("SELECT rt.resource_type, lri.logical_id, lri.logical_resource_id ");
+        query.append("  FROM logical_resource_ident AS lri ");
+        query.append("  JOIN resource_types AS rt ON (rt.resource_type_id = lri.resource_type_id)");
+        query.append(" WHERE ");
+        for (int i=0; i 0) {
+                query.append(" OR ");
+            }
+            query.append("(lri.resource_type_id = ? AND lri.logical_id = ?)");
+        }
+        PreparedStatement ps = connection.prepareStatement(query.toString());
+        // bind the parameter values
+        int param = 1;
+        for (LogicalResourceIdentValue val: values) {
+            ps.setInt(param++, val.getResourceTypeId());
+            ps.setString(param++, val.getLogicalId());
+        }
+        logger.fine(() -> "logicalResourceIdents: " + query.toString());
+        return ps;
+    }
+
+    @Override
+    protected Integer createParameterName(String parameterName) throws SQLException {
+        Integer parameterNameId = getNextRefId();
+        final String insertParameterName = ""
+                + "INSERT INTO parameter_names (parameter_name_id, parameter_name) "
+                + "     VALUES (?, ?)";
+        try (PreparedStatement stmt = connection.prepareStatement(insertParameterName)) {
+            stmt.setInt(1, parameterNameId);
+            stmt.setString(2, parameterName);
+            stmt.execute();
+        }
+
+        return parameterNameId;
+    }
+
+    @Override
+    protected PreparedStatementWrapper buildCommonTokenValueSelectStatement(List values) throws SQLException {
+        StringBuilder query = new StringBuilder();
+        // need the code_system name - so we join back to the code_systems table as well
+        query.append("SELECT cs.code_system_name, c.token_value, c.common_token_value_id ");
+        query.append("  FROM common_token_values c");
+        query.append("  JOIN code_systems cs ON (cs.code_system_id = c.code_system_id)");
+        query.append(" WHERE ");
+
+        // Create a (codeSystem, tokenValue) tuple for each of the CommonTokenValue records
+        boolean first = true;
+        for (CommonTokenValue ctv: values) {
+            if (first) {
+                first = false;
+            } else {
+                query.append(" OR ");
+            }
+            query.append("(c.code_system_id = ");
+            query.append(ctv.getCodeSystemValue().getCodeSystemId()); // literal for code_system_id
+            query.append(" AND c.token_value = ?)");
+        }
+
+        // Create the prepared statement and bind the values
+        final String statementText = query.toString();
+        PreparedStatement ps = connection.prepareStatement(statementText);
+
+        // bind the parameter values
+        int param = 1;
+        for (CommonTokenValue ctv: values) {
+            ps.setString(param++, ctv.getTokenValue());
+        }
+        return new PreparedStatementWrapper(statementText, ps);
+    }
+    @Override
+    protected void addMissingCommonCanonicalValues(List missing) throws FHIRPersistenceException {
+
+        final String nextVal = translator.nextValue(schemaName, "fhir_sequence");
+        StringBuilder insert = new StringBuilder();
+        insert.append("INSERT INTO common_canonical_values (url, canonical_id) ");
+        insert.append("     VALUES (?,");
+        insert.append(nextVal); // next sequence value
+        insert.append(") ");
+
+        final String DML = insert.toString();
+        if (logger.isLoggable(Level.FINE)) {
+            logger.fine("addMissingCanonicalIds: " + DML);
+        }
+        try (PreparedStatement ps = connection.prepareStatement(DML)) {
+            int count = 0;
+            for (CommonCanonicalValue ctv: missing) {
+                logger.finest(() -> "Adding canonical value [" + ctv.toString() + "]");
+                ps.setString(1, ctv.getUrl());
+                ps.addBatch();
+                if (++count == this.maxCommonCanonicalValuesPerStatement) {
+                    // not too many statements in a single batch
+                    ps.executeBatch();
+                    count = 0;
+                }
+            }
+            if (count > 0) {
+                // final batch
+                ps.executeBatch();
+            }
+        } catch (SQLException x) {
+            logger.log(Level.SEVERE, "failed: " + insert.toString(), x);
+            throw new FHIRPersistenceException("failed inserting new common canonical values");
+        }
+    }
+
+}
\ No newline at end of file
diff --git a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/PlainMessageHandler.java b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/PlainMessageHandler.java
new file mode 100644
index 00000000000..5db820a8209
--- /dev/null
+++ b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/PlainMessageHandler.java
@@ -0,0 +1,1359 @@
+/*
+ * (C) Copyright IBM Corp. 2022
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+ 
+package com.ibm.fhir.remote.index.database;
+
+import java.sql.CallableStatement;
+import java.sql.Connection;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.sql.Types;
+import java.time.Instant;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+import java.util.stream.Collectors;
+
+import com.ibm.fhir.database.utils.api.IDatabaseTranslator;
+import com.ibm.fhir.database.utils.common.ResultSetReader;
+import com.ibm.fhir.persistence.exception.FHIRPersistenceException;
+import com.ibm.fhir.persistence.index.DateParameter;
+import com.ibm.fhir.persistence.index.LocationParameter;
+import com.ibm.fhir.persistence.index.NumberParameter;
+import com.ibm.fhir.persistence.index.ProfileParameter;
+import com.ibm.fhir.persistence.index.QuantityParameter;
+import com.ibm.fhir.persistence.index.ReferenceParameter;
+import com.ibm.fhir.persistence.index.RemoteIndexMessage;
+import com.ibm.fhir.persistence.index.SearchParameterValue;
+import com.ibm.fhir.persistence.index.SecurityParameter;
+import com.ibm.fhir.persistence.index.StringParameter;
+import com.ibm.fhir.persistence.index.TagParameter;
+import com.ibm.fhir.persistence.index.TokenParameter;
+import com.ibm.fhir.remote.index.api.BatchParameterValue;
+import com.ibm.fhir.remote.index.api.IdentityCache;
+import com.ibm.fhir.remote.index.batch.BatchDateParameter;
+import com.ibm.fhir.remote.index.batch.BatchLocationParameter;
+import com.ibm.fhir.remote.index.batch.BatchNumberParameter;
+import com.ibm.fhir.remote.index.batch.BatchProfileParameter;
+import com.ibm.fhir.remote.index.batch.BatchQuantityParameter;
+import com.ibm.fhir.remote.index.batch.BatchReferenceParameter;
+import com.ibm.fhir.remote.index.batch.BatchSecurityParameter;
+import com.ibm.fhir.remote.index.batch.BatchStringParameter;
+import com.ibm.fhir.remote.index.batch.BatchTagParameter;
+import com.ibm.fhir.remote.index.batch.BatchTokenParameter;
+
+/**
+ * Loads search parameter values into the target PostgreSQL database using
+ * the plain (non-sharded) schema variant.
+ */
+public abstract class PlainMessageHandler extends BaseMessageHandler {
+    private static final String CLASSNAME = PlainMessageHandler.class.getName();
+    private static final Logger logger = Logger.getLogger(PlainMessageHandler.class.getName());
+    private static final short FIXED_SHARD = 0;
+
+    // the connection to use for the inserts
+    protected final Connection connection;
+
+    // The translator to handle variations in SQL syntax
+    protected final IDatabaseTranslator translator;
+
+    // The FHIR data schema
+    protected final String schemaName;
+
+    // the cache we use for various lookups
+    protected final IdentityCache identityCache;
+
+    // All logical_resource_ident values we've seen
+    private final Map logicalResourceIdentMap = new HashMap<>();
+
+    // All parameter names we've seen (cleared if there's a rollback)
+    private final Map parameterNameMap = new HashMap<>();
+
+    // A map of code system name to the value holding its codeSystemId from the database
+    private final Map codeSystemValueMap = new HashMap<>();
+
+    // A map to support lookup of CommonTokenValue records by key
+    private final Map commonTokenValueMap = new HashMap<>();
+
+    // A map to support lookup of CommonCanonicalValue records by key
+    private final Map commonCanonicalValueMap = new HashMap<>();
+
+    // A list of all the logical_resource_ident values for which we don't yet know the logical_resource_id
+    private final List unresolvedLogicalResourceIdents = new ArrayList<>();
+
+    // All parameter names in the current transaction for which we don't yet know the parameter_name_id
+    private final List unresolvedParameterNames = new ArrayList<>();
+
+    // A list of all the CodeSystemValues for which we don't yet know the code_system_id
+    private final List unresolvedSystemValues = new ArrayList<>();
+
+    // A list of all the CommonTokenValues for which we don't yet know the common_token_value_id
+    private final List unresolvedTokenValues = new ArrayList<>();
+
+    // A list of all the CommonCanonicalValues for which we don't yet know the canonical_id
+    private final List unresolvedCanonicalValues = new ArrayList<>();
+    
+    // The processed values we've collected
+    private final List batchedParameterValues = new ArrayList<>();
+
+    // The processor used to process the batched parameter values after all the reference values are created
+    private final PlainBatchParameterProcessor batchProcessor;
+
+    protected final int maxLogicalResourcesPerStatement = 256;
+    protected final int maxCodeSystemsPerStatement = 512;
+    protected final int maxCommonTokenValuesPerStatement = 256;
+    protected final int maxCommonCanonicalValuesPerStatement = 256;
+    private boolean rollbackOnly;
+
+    /**
+     * Public constructor
+     * 
+     * @param instanceIdentifier
+     * @param translator
+     * @param connection
+     * @param schemaName
+     * @param cache
+     * @param maxReadyTimeMs
+     */
+    public PlainMessageHandler(String instanceIdentifier, IDatabaseTranslator translator, Connection connection, String schemaName, IdentityCache cache, long maxReadyTimeMs) {
+        super(instanceIdentifier, maxReadyTimeMs);
+        this.translator = translator;
+        this.connection = connection;
+        this.schemaName = schemaName;
+        this.identityCache = cache;
+        this.batchProcessor = new PlainBatchParameterProcessor(connection);
+    }
+
+    @Override
+    protected void startBatch() {
+        // always start with a clean slate
+        batchedParameterValues.clear();
+        unresolvedLogicalResourceIdents.clear();
+        unresolvedParameterNames.clear();
+        unresolvedSystemValues.clear();
+        unresolvedTokenValues.clear();
+        unresolvedCanonicalValues.clear();
+        batchProcessor.startBatch();
+    }
+
+    @Override
+    protected void setRollbackOnly() {
+        this.rollbackOnly = true;
+    }
+
+    @Override
+    public void close() {
+        try {
+            batchProcessor.close();
+        } catch (Throwable t) {
+            logger.log(Level.SEVERE, "close batchProcessor failed" , t);
+        }
+    }
+
+    @Override
+    protected void endTransaction() throws FHIRPersistenceException {
+        boolean committed = false;
+        try {
+            if (!this.rollbackOnly) {
+                logger.fine("Committing transaction");
+                connection.commit();
+                committed = true;
+
+                // any values from parameter_names, code_systems and common_token_values
+                // are now committed to the database, so we can publish their record ids
+                // to the shared cache which makes them accessible from other threads 
+                publishValuesToCache();
+            } else {
+                // something went wrong...try to roll back the transaction before we close
+                // everything
+                try {
+                    connection.rollback();
+                } catch (SQLException x) {
+                    // It could very well be that we've lost touch with the database in which case
+                    // the rollback will also fail. Not much we can do, although we don't bother
+                    // with a stack trace here because it's just more noise for the log file, and
+                    // the exception that triggered the rollback is already going to be propagated
+                    // and logged.
+                    logger.severe("Rollback failed; reason=[" + x.getMessage() + "]");
+                }
+            }
+        } catch (SQLException x) {
+            throw new FHIRPersistenceException("commit failed", x);
+        } finally {
+            // always clear these maps because otherwise they could grow unbounded. Values
+            // are cached by the identityCache
+            this.logicalResourceIdentMap.clear();
+            this.parameterNameMap.clear();
+            this.codeSystemValueMap.clear();
+            this.commonTokenValueMap.clear();
+            this.commonCanonicalValueMap.clear();
+        }
+    }
+
+    /**
+     * After the transaction has been committed, we can publish certain values to the
+     * shared identity caches allowing them to be used by other threads
+     */
+    private void publishValuesToCache() {
+        for (ParameterNameValue pnv: this.unresolvedParameterNames) {
+            logger.fine(() -> "Adding parameter-name to cache: '" + pnv.getParameterName() + "' -> " + pnv.getParameterNameId());
+            identityCache.addParameterName(pnv.getParameterName(), pnv.getParameterNameId());
+        }
+
+        for (CommonCanonicalValue value: this.unresolvedCanonicalValues) {
+            identityCache.addCommonCanonicalValue(FIXED_SHARD, value.getUrl(), value.getCanonicalId());
+        }
+
+        for (CodeSystemValue value: this.unresolvedSystemValues) {
+            identityCache.addCodeSystem(value.getCodeSystem(), value.getCodeSystemId());
+        }
+
+        for (CommonTokenValue value: this.unresolvedTokenValues) {
+            identityCache.addCommonTokenValue(FIXED_SHARD, value.getCodeSystemValue().getCodeSystem(), value.getTokenValue(), value.getCommonTokenValueId());
+        }
+
+        for (LogicalResourceIdentValue value: this.unresolvedLogicalResourceIdents) {
+            identityCache.addLogicalResourceIdent(value.getResourceType(), value.getLogicalId(), value.getLogicalResourceId());
+        }
+    }
+
+    @Override
+    protected void pushBatch() throws FHIRPersistenceException {
+        // Push any data we've accumulated so far. This may occur
+        // if we cross a volume threshold, and will always occur as
+        // the last step before the current transaction is committed,
+        // Process the token values so that we can establish
+        // any entries we need for common_token_values
+        logger.fine("pushBatch: resolving all ids");
+        resolveLogicalResourceIdents();
+        resolveParameterNames();
+        resolveCodeSystems();
+        resolveCommonTokenValues();
+        resolveCommonCanonicalValues();
+
+        // Now that all the lookup values should've been resolved, we can go ahead
+        // and push the parameters to the JDBC batch insert statements via the
+        // batchProcessor
+        logger.fine("pushBatch: processing collected parameters");
+        for (BatchParameterValue v: this.batchedParameterValues) {
+            v.apply(batchProcessor);
+        }
+
+        logger.fine("pushBatch: executing final batch statements");
+        batchProcessor.pushBatch();
+        logger.fine("pushBatch completed");
+    }
+
+    /**
+     * Get the parameter name value for the given parameter value
+     * @param p
+     * @return
+     */
+    private ParameterNameValue getParameterNameId(SearchParameterValue p) throws FHIRPersistenceException {
+        if (logger.isLoggable(Level.FINEST)) {
+            logger.finest("get ParameterNameValue for [" + p.toString() + "]");
+        }
+        ParameterNameValue result = parameterNameMap.get(p.getName());
+        if (result == null) {
+            result = new ParameterNameValue(p.getName());
+            parameterNameMap.put(p.getName(), result);
+
+            // let's see if the id is available in the shared identity cache
+            Integer parameterNameId = identityCache.getParameterNameId(p.getName());
+            if (parameterNameId != null) {
+                result.setParameterNameId(parameterNameId);
+            } else {
+                // ids will be created later (so that we can process them in order)
+                unresolvedParameterNames.add(result);
+            }
+        }
+        return result;
+    }
+    
+    @Override
+    protected void process(String tenantId, String requestShard, String resourceType, String logicalId, long logicalResourceId, StringParameter p) throws FHIRPersistenceException {
+        ParameterNameValue parameterNameValue = getParameterNameId(p);
+        this.batchedParameterValues.add(new BatchStringParameter(requestShard, resourceType, logicalId, logicalResourceId, parameterNameValue, p));
+    }
+
+    @Override
+    protected void process(String tenantId, String requestShard, String resourceType, String logicalId, long logicalResourceId, LocationParameter p) throws FHIRPersistenceException {
+        ParameterNameValue parameterNameValue = getParameterNameId(p);
+        this.batchedParameterValues.add(new BatchLocationParameter(requestShard, resourceType, logicalId, logicalResourceId, parameterNameValue, p));
+    }
+
+    @Override
+    protected void process(String tenantId, String requestShard, String resourceType, String logicalId, long logicalResourceId, TokenParameter p) throws FHIRPersistenceException {
+        CommonTokenValue ctv = lookupCommonTokenValue(p.getValueSystem(), p.getValueCode());
+        ParameterNameValue parameterNameValue = getParameterNameId(p);
+        this.batchedParameterValues.add(new BatchTokenParameter(requestShard, resourceType, logicalId, logicalResourceId, parameterNameValue, p, ctv));
+    }
+
+    @Override
+    protected void process(String tenantId, String requestShard, String resourceType, String logicalId, long logicalResourceId, TagParameter p) throws FHIRPersistenceException {
+        CommonTokenValue ctv = lookupCommonTokenValue(p.getValueSystem(), p.getValueCode());
+        ParameterNameValue parameterNameValue = getParameterNameId(p);
+        this.batchedParameterValues.add(new BatchTagParameter(requestShard, resourceType, logicalId, logicalResourceId, parameterNameValue, p, ctv));
+    }
+
+    @Override
+    protected void process(String tenantId, String requestShard, String resourceType, String logicalId, long logicalResourceId, SecurityParameter p) throws FHIRPersistenceException {
+        CommonTokenValue ctv = lookupCommonTokenValue(p.getValueSystem(), p.getValueCode());
+        ParameterNameValue parameterNameValue = getParameterNameId(p);
+        this.batchedParameterValues.add(new BatchSecurityParameter(requestShard, resourceType, logicalId, logicalResourceId, parameterNameValue, p, ctv));
+    }
+
+    @Override
+    protected void process(String tenantId, String requestShard, String resourceType, String logicalId, long logicalResourceId, ProfileParameter p) throws FHIRPersistenceException {
+        CommonCanonicalValue ctv = lookupCommonCanonicalValue(p.getUrl());
+        ParameterNameValue parameterNameValue = getParameterNameId(p);
+        this.batchedParameterValues.add(new BatchProfileParameter(requestShard, resourceType, logicalId, logicalResourceId, parameterNameValue, p, ctv));
+    }
+
+    @Override
+    protected void process(String tenantId, String requestShard, String resourceType, String logicalId, long logicalResourceId, QuantityParameter p) throws FHIRPersistenceException {
+        ParameterNameValue parameterNameValue = getParameterNameId(p);
+        CodeSystemValue csv = lookupCodeSystemValue(p.getValueSystem());
+        this.batchedParameterValues.add(new BatchQuantityParameter(requestShard, resourceType, logicalId, logicalResourceId, parameterNameValue, p, csv));
+    }
+
+    @Override
+    protected void process(String tenantId, String requestShard, String resourceType, String logicalId, long logicalResourceId, NumberParameter p) throws FHIRPersistenceException {
+        ParameterNameValue parameterNameValue = getParameterNameId(p);
+        this.batchedParameterValues.add(new BatchNumberParameter(requestShard, resourceType, logicalId, logicalResourceId, parameterNameValue, p));
+    }
+
+    @Override
+    protected void process(String tenantId, String requestShard, String resourceType, String logicalId, long logicalResourceId, DateParameter p) throws FHIRPersistenceException {
+        ParameterNameValue parameterNameValue = getParameterNameId(p);
+        this.batchedParameterValues.add(new BatchDateParameter(requestShard, resourceType, logicalId, logicalResourceId, parameterNameValue, p));
+    }
+
+    @Override
+    protected void process(String tenantId, String requestShard, String resourceType, String logicalId, long logicalResourceId, ReferenceParameter p) throws FHIRPersistenceException {
+        logger.fine(() -> "Processing reference parameter value:" + p.toString());
+        ParameterNameValue parameterNameValue = getParameterNameId(p);
+        LogicalResourceIdentValue lriv = lookupLogicalResourceIdentValue(p.getResourceType(), p.getLogicalId());
+        this.batchedParameterValues.add(new BatchReferenceParameter(requestShard, resourceType, logicalId, logicalResourceId, parameterNameValue, p, lriv));
+    }
+
+    /**
+     * Get the CodeSystemValue we've assigned for the given codeSystem value. This
+     * may not yet have the actual code_system_id from the database yet - any values
+     * we don't have will be assigned in a later phase (so we can do things neatly
+     * in bulk).
+     * @param codeSystem
+     * @return
+     */
+    private CodeSystemValue lookupCodeSystemValue(String codeSystem) {
+        CodeSystemValue result = this.codeSystemValueMap.get(codeSystem);
+        if (result == null) {
+            result = new CodeSystemValue(codeSystem);
+            this.codeSystemValueMap.put(codeSystem, result);
+
+            // Take this opportunity to see if we have a cached value for this codeSystem
+            Integer codeSystemId = identityCache.getCodeSystemId(codeSystem);
+            if (codeSystemId != null) {
+                result.setCodeSystemId(codeSystemId);
+            } else {
+                // Stash for later resolution
+                this.unresolvedSystemValues.add(result);
+            }
+        }
+        return result;
+    }
+
+    /**
+     * Get the CommonTokenValue we've assigned for the given (codeSystem, tokenValue) tuple.
+     * The returned value may not yet have the actual common_token_value_id yet - we fetch
+     * these values later and create new database records as necessary.
+     * @param codeSystem
+     * @param tokenValue
+     * @return
+     */
+    private CommonTokenValue lookupCommonTokenValue(String codeSystem, String tokenValue) {
+        CommonTokenValueKey key = new CommonTokenValueKey(FIXED_SHARD, codeSystem, tokenValue);
+        CommonTokenValue result = this.commonTokenValueMap.get(key);
+        if (result == null) {
+            CodeSystemValue csv = lookupCodeSystemValue(codeSystem);
+            result = new CommonTokenValue(FIXED_SHARD, csv, tokenValue);
+            this.commonTokenValueMap.put(key, result);
+
+            // Take this opportunity to see if we have a cached value for this common token value
+            Long commonTokenValueId = identityCache.getCommonTokenValueId(FIXED_SHARD, codeSystem, tokenValue);
+            if (commonTokenValueId != null) {
+                result.setCommonTokenValueId(commonTokenValueId);
+            } else {
+                this.unresolvedTokenValues.add(result);
+            }
+        }
+        return result;
+    }
+
+    /**
+     * Get the LogicalResourceIdentValue we've assigned for the given (resourceType, logicalId)
+     * tuple. The returned value may not yet have the actual logical_resource_id yet - we fetch
+     * these values later and create new database records as necessary
+     * @param resourceType
+     * @param logicalId
+     * @return
+     */
+    private LogicalResourceIdentValue lookupLogicalResourceIdentValue(String resourceType, String logicalId) {
+        LogicalResourceIdentKey key = new LogicalResourceIdentKey(resourceType, logicalId);
+        LogicalResourceIdentValue result = this.logicalResourceIdentMap.get(key);
+        if (result == null) {
+            result = LogicalResourceIdentValue.builder()
+                    .withResourceTypeId(identityCache.getResourceTypeId(resourceType))
+                    .withResourceType(resourceType)
+                    .withLogicalId(logicalId)
+                    .build();
+            this.logicalResourceIdentMap.put(key, result);
+
+            // see if we can find the logical_resource_id from the cache
+            Long logicalResourceId = identityCache.getLogicalResourceIdentId(resourceType, logicalId);
+            if (logicalResourceId != null) {
+                result.setLogicalResourceId(logicalResourceId);
+            } else {
+                // Add to the unresolved list to look up later
+                this.unresolvedLogicalResourceIdents.add(result);
+            }
+        }
+        return result;
+    }
+
+    /**
+     * Get the CommonCanonicalValue we've assigned for the given url value.
+     * The returned value may not yet have the actual canonical_id yet - we fetch
+     * these values later and create new database records as necessary.
+     * @param url
+     * @return
+     */
+    private CommonCanonicalValue lookupCommonCanonicalValue(String url) {
+        CommonCanonicalValueKey key = new CommonCanonicalValueKey(FIXED_SHARD, url);
+        CommonCanonicalValue result = this.commonCanonicalValueMap.get(key);
+        if (result == null) {
+            result = new CommonCanonicalValue(FIXED_SHARD, url);
+            this.commonCanonicalValueMap.put(key, result);
+
+            // Take this opportunity to see if we have a cached value for this common token value
+            Long canonicalId = identityCache.getCommonCanonicalValueId(FIXED_SHARD, url);
+            if (canonicalId != null) {
+                result.setCanonicalId(canonicalId);
+            } else {
+                this.unresolvedCanonicalValues.add(result);
+            }
+        }
+        return result;
+    }
+
+    /**
+     * Make sure we have values for all the code_systems we have collected
+     * in the current
+     * batch
+     * @throws FHIRPersistenceException
+     */
+    private void resolveCodeSystems() throws FHIRPersistenceException {
+        // identify which values aren't yet in the database
+        List missing = fetchCodeSystemIds(unresolvedSystemValues);
+
+        if (!missing.isEmpty()) {
+            addMissingCodeSystems(missing);
+        }
+
+        // All the previously missing values should now be in the database. We need to fetch them again,
+        // possibly having to use multiple queries
+        List bad = fetchCodeSystemIds(missing);
+
+        if (!bad.isEmpty()) {
+            // shouldn't happend, but let's protected against it anyway
+            throw new FHIRPersistenceException("Failed to create all code system values");
+        }
+    }
+
+    /**
+     * Build and prepare a statement to fetch the code_system_id and code_system_name
+     * from the code_systems table for all the given (unresolved) code system values
+     * @param values
+     * @return
+     * @throws SQLException
+     */
+    private PreparedStatement buildCodeSystemSelectStatement(List values) throws SQLException {
+        StringBuilder query = new StringBuilder();
+        query.append("SELECT code_system_id, code_system_name FROM code_systems WHERE code_system_name IN (");
+        for (int i=0; i 0) {
+                query.append(",");
+            }
+            query.append("?");
+        }
+        query.append(")");
+        PreparedStatement ps = connection.prepareStatement(query.toString());
+        // bind the parameter values
+        int param = 1;
+        for (CodeSystemValue csv: values) {
+            ps.setString(param++, csv.getCodeSystem());
+        }
+        return ps;
+    }
+
+    protected abstract String onConflict();
+
+    /**
+     * These code systems weren't found in the database, so we need to try and add them.
+     * We have to deal with concurrency here - there's a chance another thread could also
+     * be trying to add them. To avoid deadlocks, it's important to do any inserts in a
+     * consistent order. At the end, we should be able to read back values for each entry
+     * @param missing
+     */
+    protected void addMissingCodeSystems(List missing) throws FHIRPersistenceException {
+        List values = missing.stream().map(csv -> csv.getCodeSystem()).collect(Collectors.toList());
+        // Sort the code system values first to help avoid deadlocks
+        Collections.sort(values); // natural ordering for String is fine here
+
+        final String nextVal = translator.nextValue(schemaName, "fhir_ref_sequence");
+        StringBuilder insert = new StringBuilder();
+        insert.append("INSERT INTO code_systems (code_system_id, code_system_name) VALUES (");
+        insert.append(nextVal); // next sequence value
+        insert.append(",?) ");
+        insert.append(onConflict());
+
+        try (PreparedStatement ps = connection.prepareStatement(insert.toString())) {
+            int count = 0;
+            for (String codeSystem: values) {
+                ps.setString(1, codeSystem);
+                ps.addBatch();
+                if (++count == this.maxCodeSystemsPerStatement) {
+                    // not too many statements in a single batch
+                    ps.executeBatch();
+                    count = 0;
+                }
+            }
+            if (count > 0) {
+                // final batch
+                ps.executeBatch();
+            }
+        } catch (SQLException x) {
+            logger.log(Level.SEVERE, "code systems fetch failed: " + insert.toString(), x);
+            throw new FHIRPersistenceException("code systems fetch failed");
+        }
+    }
+
+    /**
+     * Fetch all the code_system_id values for the given list of CodeSystemValue objects.
+     * @param unresolved
+     * @return
+     * @throws FHIRPersistenceException
+     */
+    private List fetchCodeSystemIds(List unresolved) throws FHIRPersistenceException {
+        // track which values aren't yet in the database
+        List missing = new ArrayList<>();
+
+        int offset = 0;
+        while (offset < unresolved.size()) {
+            int remaining = unresolved.size() - offset;
+            int subSize = Math.min(remaining, this.maxCodeSystemsPerStatement);
+            List sub = unresolved.subList(offset, offset+subSize); // remember toIndex is exclusive
+            offset += subSize; // set up for the next iteration
+            try (PreparedStatement ps = buildCodeSystemSelectStatement(sub)) {
+                ResultSet rs = ps.executeQuery();
+                // We can't rely on the order of result rows matching the order of the in-list,
+                // so we have to go back to our map to look up each CodeSystemValue
+                int resultCount = 0;
+                while (rs.next()) {
+                    resultCount++;
+                    CodeSystemValue csv = this.codeSystemValueMap.get(rs.getString(2));
+                    if (csv != null) {
+                        csv.setCodeSystemId(rs.getInt(1));
+                    } else {
+                        // can't really happen, but be defensive
+                        throw new FHIRPersistenceException("code systems query returned an unexpected value");
+                    }
+                }
+
+                // Most of the time we'll get everything, so we can bypass the check for
+                // missing values
+                if (resultCount == 0) {
+                    // 100% miss
+                    missing.addAll(sub);
+                } else if (resultCount < subSize) {
+                    // need to scan the sub list and see which values we don't yet have ids for
+                    for (CodeSystemValue csv: sub) {
+                        if (csv.getCodeSystemId() == null) {
+                            missing.add(csv);
+                        }
+                    }
+                }
+            } catch (SQLException x) {
+                logger.log(Level.SEVERE, "code systems fetch failed", x);
+                throw new FHIRPersistenceException("code systems fetch failed");
+            }
+        }
+
+        // Return the list of CodeSystemValues which don't yet have a database entry
+        return missing;
+    }
+
+    /**
+     * Make sure we have values for all the common_token_value records we have collected
+     * in the current batch
+     * @throws FHIRPersistenceException
+     */
+    private void resolveCommonTokenValues() throws FHIRPersistenceException {
+        // identify which values aren't yet in the database
+        List missing = fetchCommonTokenValueIds(unresolvedTokenValues);
+
+        if (!missing.isEmpty()) {
+            // Sort first to minimize deadlocks
+            Collections.sort(missing, (a,b) -> {
+                int result = a.getTokenValue().compareTo(b.getTokenValue());
+                if (result == 0) {
+                    result = Integer.compare(a.getCodeSystemValue().getCodeSystemId(), b.getCodeSystemValue().getCodeSystemId());
+                    if (result == 0) {
+                        result = Short.compare(a.getShardKey(), b.getShardKey());
+                    }
+                }
+                return result;
+            });
+            addMissingCommonTokenValues(missing);
+        }
+
+        // All the previously missing values should now be in the database. We need to fetch them again,
+        // possibly having to use multiple queries
+        List bad = fetchCommonTokenValueIds(missing);
+
+        if (!bad.isEmpty()) {
+            // shouldn't happend, but let's protected against it anyway
+            throw new FHIRPersistenceException("Failed to create all common token values");
+        }
+    }
+
+    /**
+     * Build and prepare a statement to fetch the common_token_value records
+     * for all the given (unresolved) code system values
+     * @param values
+     * @return SELECT code_system, token_value, common_token_value_id
+     * @throws SQLException
+     */
+    protected PreparedStatementWrapper buildCommonTokenValueSelectStatement(List values) throws SQLException {
+        StringBuilder query = new StringBuilder();
+        // need the code_system name - so we join back to the code_systems table as well
+        query.append("SELECT cs.code_system_name, c.token_value, c.common_token_value_id ");
+        query.append("  FROM common_token_values c");
+        query.append("  JOIN code_systems cs ON (cs.code_system_id = c.code_system_id)");
+        query.append("  JOIN (VALUES ");
+
+        // Create a (codeSystem, tokenValue) tuple for each of the CommonTokenValue records
+        boolean first = true;
+        for (CommonTokenValue ctv: values) {
+            if (first) {
+                first = false;
+            } else {
+                query.append(",");
+            }
+            query.append("(");
+            query.append(ctv.getCodeSystemValue().getCodeSystemId()); // literal for code_system_id
+            query.append(",?)"); // bind variable for the token-value
+        }
+        query.append(") AS v(code_system_id, token_value) ");
+        query.append(" ON (c.code_system_id = v.code_system_id AND c.token_value = v.token_value)");
+
+        // Create the prepared statement and bind the values
+        final String statementText = query.toString();
+        PreparedStatement ps = connection.prepareStatement(statementText);
+
+        // bind the parameter values
+        int param = 1;
+        for (CommonTokenValue ctv: values) {
+            ps.setString(param++, ctv.getTokenValue());
+        }
+        return new PreparedStatementWrapper(statementText, ps);
+    }
+
+    /**
+     * Fetch the common_token_value_id values for the given list of CommonTokenValue objects.
+     * @param unresolved
+     * @return
+     * @throws FHIRPersistenceException
+     */
+    private List fetchCommonTokenValueIds(List unresolved) throws FHIRPersistenceException {
+        // track which values aren't yet in the database
+        List missing = new ArrayList<>();
+
+        int offset = 0;
+        while (offset < unresolved.size()) {
+            int remaining = unresolved.size() - offset;
+            int subSize = Math.min(remaining, this.maxCommonTokenValuesPerStatement);
+            List sub = unresolved.subList(offset, offset+subSize); // remember toIndex is exclusive
+            offset += subSize; // set up for the next iteration
+            String sql = null; // the SQL text for logging when there's an error
+            try (PreparedStatementWrapper ps = buildCommonTokenValueSelectStatement(sub)) {
+                sql = ps.getStatementText();
+                ResultSet rs = ps.executeQuery();
+                // We can't rely on the order of result rows matching the order of the in-list,
+                // so we have to go back to our map to look up each CodeSystemValue
+                int resultCount = 0;
+                while (rs.next()) {
+                    resultCount++;
+                    CommonTokenValueKey key = new CommonTokenValueKey(FIXED_SHARD, rs.getString(1), rs.getString(2));
+                    CommonTokenValue ctv = this.commonTokenValueMap.get(key);
+                    if (ctv != null) {
+                        ctv.setCommonTokenValueId(rs.getLong(3));
+                    } else {
+                        // can't really happen, but be defensive
+                        throw new FHIRPersistenceException("common token values query returned an unexpected value");
+                    }
+                }
+
+                // Optimize the check for missing values
+                if (resultCount == 0) {
+                    // 100% miss
+                    missing.addAll(sub);
+                } else if (resultCount < subSize) {
+                    // need to scan the sub list and see which values we don't yet have ids for
+                    for (CommonTokenValue ctv: sub) {
+                        if (ctv.getCommonTokenValueId() == null) {
+                            missing.add(ctv);
+                        }
+                    }
+                }
+            } catch (SQLException x) {
+                logger.log(Level.SEVERE, "common token values fetch failed. SQL=[" + sql + "]", x);
+                throw new FHIRPersistenceException("common token values fetch failed");
+            }
+        }
+
+        // Return the list of CodeSystemValues which don't yet have a database entry
+        return missing;
+    }
+
+    /**
+     * Add the values we think are missing from the database. The given list should be
+     * sorted to reduce deadlocks
+     * @param missing
+     * @throws FHIRPersistenceException
+     */
+    protected void addMissingCommonTokenValues(List missing) throws FHIRPersistenceException {
+
+        StringBuilder insert = new StringBuilder();
+        insert.append("INSERT INTO common_token_values (code_system_id, token_value) ");
+        insert.append("     VALUES (?,?) ");
+        insert.append(onConflict());
+
+        try (PreparedStatement ps = connection.prepareStatement(insert.toString())) {
+            int count = 0;
+            for (CommonTokenValue ctv: missing) {
+                ps.setInt(1, ctv.getCodeSystemValue().getCodeSystemId());
+                ps.setString(2, ctv.getTokenValue());
+                ps.addBatch();
+                if (++count == this.maxCommonTokenValuesPerStatement) {
+                    // not too many statements in a single batch
+                    ps.executeBatch();
+                    count = 0;
+                }
+            }
+            if (count > 0) {
+                // final batch
+                ps.executeBatch();
+            }
+        } catch (SQLException x) {
+            logger.log(Level.SEVERE, "failed: " + insert.toString(), x);
+            throw new FHIRPersistenceException("failed inserting new common token values");
+        }
+    }
+
+    /**
+     * Make sure we have values for all the common_canonical_value records we have collected
+     * in the current batch
+     * @throws FHIRPersistenceException
+     */
+    private void resolveCommonCanonicalValues() throws FHIRPersistenceException {
+        // identify which values aren't yet in the database
+        List missing = fetchCanonicalIds(unresolvedCanonicalValues);
+
+        if (!missing.isEmpty()) {
+            // Sort on url to minimize deadlocks
+            Collections.sort(missing, (a,b) -> {
+                return a.getUrl().compareTo(b.getUrl());
+            });
+            addMissingCommonCanonicalValues(missing);
+        }
+
+        // All the previously missing values should now be in the database. We need to fetch them again,
+        // possibly having to use multiple queries
+        List bad = fetchCanonicalIds(missing);
+
+        if (!bad.isEmpty()) {
+            // shouldn't happen, but let's protected against it anyway
+            throw new FHIRPersistenceException("Failed to create all canonical values");
+        }
+    }
+
+    /**
+     * Fetch the common_canonical_id values for the given list of CommonCanonicalValue objects.
+     * @param unresolved
+     * @return
+     * @throws FHIRPersistenceException
+     */
+    private List fetchCanonicalIds(List unresolved) throws FHIRPersistenceException {
+        // track which values aren't yet in the database
+        List missing = new ArrayList<>();
+
+        int offset = 0;
+        while (offset < unresolved.size()) {
+            int remaining = unresolved.size() - offset;
+            int subSize = Math.min(remaining, this.maxCommonCanonicalValuesPerStatement);
+            List sub = unresolved.subList(offset, offset+subSize); // remember toIndex is exclusive
+            offset += subSize; // set up for the next iteration
+            String sql = null; // the SQL text for logging when there's an error
+            try (PreparedStatementWrapper ps = buildCommonCanonicalValueSelectStatement(sub)) {
+                sql = ps.getStatementText();
+                ResultSet rs = ps.executeQuery();
+                // We can't rely on the order of result rows matching the order of the in-list,
+                // so we have to go back to our map to look up each CodeSystemValue
+                int resultCount = 0;
+                while (rs.next()) {
+                    resultCount++;
+                    CommonCanonicalValueKey key = new CommonCanonicalValueKey(FIXED_SHARD, rs.getString(1));
+                    CommonCanonicalValue ctv = this.commonCanonicalValueMap.get(key);
+                    if (ctv != null) {
+                        ctv.setCanonicalId(rs.getLong(2));
+                    } else {
+                        // can't really happen, but be defensive
+                        throw new FHIRPersistenceException("common canonical values query returned an unexpected value");
+                    }
+                }
+
+                // Optimize the check for missing values
+                if (resultCount == 0) {
+                    // 100% miss
+                    missing.addAll(sub);
+                } else if (resultCount < subSize) {
+                    // need to scan the sub list and see which values we don't yet have ids for
+                    for (CommonCanonicalValue ctv: sub) {
+                        if (ctv.getCanonicalId() == null) {
+                            missing.add(ctv);
+                        }
+                    }
+                }
+            } catch (SQLException x) {
+                logger.log(Level.SEVERE, "common canonical values fetch failed. SQL=[" + sql + "]", x);
+                throw new FHIRPersistenceException("common canonical values fetch failed");
+            }
+        }
+
+        // Return the list of CodeSystemValues which don't yet have a database entry
+        return missing;
+    }
+
+    /**
+     * Build and prepare a statement to fetch the common_token_value records
+     * for all the given (unresolved) code system values
+     * @param values
+     * @return SELECT code_system, token_value, common_token_value_id
+     * @throws SQLException
+     */
+    private PreparedStatementWrapper buildCommonCanonicalValueSelectStatement(List values) throws SQLException {
+        StringBuilder query = new StringBuilder();
+        query.append("SELECT c.url, c.canonical_id ");
+        query.append("  FROM common_canonical_values c ");
+        query.append("  WHERE c.url IN (");
+
+        // add bind variables for each url we need to fetch
+        boolean first = true;
+        for (CommonCanonicalValue ctv: values) {
+            if (first) {
+                first = false;
+            } else {
+                query.append(",");
+            }
+            query.append("?"); // bind variable for the url
+        }
+        query.append(")");
+
+        // Create the prepared statement and bind the values
+        final String statementText = query.toString();
+        logger.finer(() -> "fetch common canonical values [" + statementText + "]");
+        PreparedStatement ps = connection.prepareStatement(statementText);
+
+        // bind the parameter values
+        int param = 1;
+        for (CommonCanonicalValue ctv: values) {
+            ps.setString(param++, ctv.getUrl());
+        }
+        return new PreparedStatementWrapper(statementText, ps);
+    }
+
+    /**
+     * Add the values we think are missing from the database. The given list should be
+     * sorted to reduce deadlocks
+     * @param missing
+     * @throws FHIRPersistenceException
+     */
+    protected void addMissingCommonCanonicalValues(List missing) throws FHIRPersistenceException {
+
+        StringBuilder insert = new StringBuilder();
+        insert.append("INSERT INTO common_canonical_values (url) VALUES (?) ");
+        insert.append(onConflict());
+
+        final String DML = insert.toString();
+        if (logger.isLoggable(Level.FINE)) {
+            logger.fine("addMissingCanonicalIds: " + DML);
+        }
+        try (PreparedStatement ps = connection.prepareStatement(DML)) {
+            int count = 0;
+            for (CommonCanonicalValue ctv: missing) {
+                logger.finest(() -> "Adding canonical value [" + ctv.toString() + "]");
+                ps.setString(1, ctv.getUrl());
+                ps.addBatch();
+                if (++count == this.maxCommonCanonicalValuesPerStatement) {
+                    // not too many statements in a single batch
+                    ps.executeBatch();
+                    count = 0;
+                }
+            }
+            if (count > 0) {
+                // final batch
+                ps.executeBatch();
+            }
+        } catch (SQLException x) {
+            logger.log(Level.SEVERE, "failed: " + insert.toString(), x);
+            throw new FHIRPersistenceException("failed inserting new common canonical values");
+        }
+    }
+
+    /**
+     * Make sure all the parameter names we've seen in the batch exist
+     * in the database and have ids.
+     * @throws FHIRPersistenceException
+     */
+    private void resolveParameterNames() throws FHIRPersistenceException {
+        // We expect parameter names to have a very high cache hit rate and
+        // so we simplify processing by simply iterating one-by-one for the
+        // values we still need to resolve. The most important point here is
+        // to do this in a sorted order to avoid deadlock issues because this
+        // could be happening across multiple consumer threads at the same time.
+        logger.fine("resolveParameterNames: sorting unresolved names");
+        Collections.sort(this.unresolvedParameterNames, (a,b) -> {
+            return a.getParameterName().compareTo(b.getParameterName());
+        });
+
+        try {
+            for (ParameterNameValue pnv: this.unresolvedParameterNames) {
+                logger.finer(() -> "fetching parameter_name_id for '" + pnv.getParameterName() + "'");
+                Integer parameterNameId = getParameterNameIdFromDatabase(pnv.getParameterName());
+                if (parameterNameId == null) {
+                    parameterNameId = createParameterName(pnv.getParameterName());
+                    if (logger.isLoggable(Level.FINER)) {
+                        logger.finer("assigned parameter_name_id '" + pnv.getParameterName() + "' = " + parameterNameId);
+                    }
+
+                    if (parameterNameId == null) {
+                        // be defensive
+                        throw new FHIRPersistenceException("parameter_name_id not assigned for '" + pnv.getParameterName());
+                    }
+                } else if (logger.isLoggable(Level.FINER)) {
+                    logger.finer("read parameter_name_id '" + pnv.getParameterName() + "' = " + parameterNameId);
+                }
+                pnv.setParameterNameId(parameterNameId);
+            }
+        } catch (SQLException x) {
+            throw new FHIRPersistenceException("error resolving parameter names", x);
+        } finally {
+            logger.exiting(CLASSNAME, "resolveParameterNames");
+        }
+    }
+
+    /**
+     * Fetch the parameter_name_id for the given parameterName value
+     * @param parameterName
+     * @return
+     * @throws SQLException
+     */
+    private Integer getParameterNameIdFromDatabase(String parameterName) throws SQLException {
+        String SQL = "SELECT parameter_name_id FROM parameter_names WHERE parameter_name = ?";
+        try (PreparedStatement ps = connection.prepareStatement(SQL)) {
+            ps.setString(1, parameterName);
+            ResultSet rs = ps.executeQuery();
+            if (rs.next()) {
+                return rs.getInt(1);
+            }
+        }
+
+        // no entry in parameter_names
+        return null;
+    }
+
+    /**
+     * Create the parameter name using the stored procedure which handles any concurrency
+     * issue we may have
+     * @param parameterName
+     * @return
+     */
+    protected Integer createParameterName(String parameterName) throws SQLException {
+        final String CALL = "{CALL " + schemaName + ".add_parameter_name(?, ?)}";
+        Integer parameterNameId;
+        try (CallableStatement stmt = connection.prepareCall(CALL)) {
+            stmt.setString(1, parameterName);
+            stmt.registerOutParameter(2, Types.INTEGER);
+            stmt.execute();
+            parameterNameId = stmt.getInt(2);
+        }
+
+        return parameterNameId;
+    }
+
+    @Override
+    protected void resetBatch() {
+        // Called when a transaction has been rolled back because of a deadlock
+        // or other retryable error and we want to try and process the batch again
+        batchProcessor.reset();
+    }
+
+    /**
+     * Build the check ready query
+     * @param messagesByResourceType
+     * @return
+     */
+    private String buildCheckReadyQuery(Map> messagesByResourceType) {
+        // The trouble here is that we'll end up with a unique query for every single
+        // batch of messages we process (which the database then need to parse etc).
+        // This may introduce scaling issues, in which case we should consider 
+        // individual queries for each resource type using bind variables, perhaps
+        // going so far as using multiple statements with a power-of-2 number of bind
+        // variables. But JDBC doesn't support batching of select statements, so
+        // the alternative there would be to insert-as-select into a global temp table
+        // and then simply select from that. Fairly straightforward, but a lot more
+        // work so only worth doing if we identify contention here.
+
+        StringBuilder select = new StringBuilder();
+        // SELECT lr.logical_resource_id, lr.resource_type, lr.logical_id, xlr.version_id, lr.last_updated, lr.parameter_hash
+        //   FROM logical_resources AS lr,
+        //        patient_logical_resources AS xlr
+        //  WHERE lr.logical_resource_id = xlr.logical_resource_id
+        //    AND xlr.logical_resource_id IN (1,2,3,4)
+        //  UNION ALL
+        // SELECT lr.logical_resource_id, lr.resource_type, lr.logical_id, xlr.version_id, lr.last_updated, lr.parameter_hash
+        //   FROM logical_resources AS lr,
+        //        observation_logical_resources AS xlr
+        //  WHERE lr.logical_resource_id = xlr.logical_resource_id
+        //    AND xlr.logical_resource_id IN (5,6,7)
+        boolean first = true;
+        for (Map.Entry> entry: messagesByResourceType.entrySet()) {
+            final String resourceType = entry.getKey();
+            final List messages = entry.getValue();
+            final String inlist = messages.stream().map(m -> Long.toString(m.getData().getLogicalResourceId())).collect(Collectors.joining(","));
+            if (first) {
+                first = false;
+            } else {
+                select.append(" UNION ALL ");
+            }
+            select.append(" SELECT lr.logical_resource_id, '" + resourceType + "' AS resource_type, lr.logical_id, xlr.version_id, lr.last_updated, lr.parameter_hash ");
+            select.append("   FROM logical_resources AS lr, ");
+            select.append(resourceType).append("_logical_resources AS xlr ");
+            select.append("  WHERE lr.logical_resource_id = xlr.logical_resource_id ");
+            select.append("    AND xlr.logical_resource_id IN (").append(inlist).append(")");
+        }
+        
+        return select.toString();
+    }
+
+    @Override
+    protected void checkReady(List messages, List okToProcess, List notReady) throws FHIRPersistenceException {
+        // Get a list of all the resources for which we can see the current logical resource data.
+        // If the resource doesn't yet exist or its version meta doesn't the message
+        // then we add to the notReady list. If the resource version meta already
+        // exceeds the message, then we'll skip processing altogether because it
+        // means that there should be another message in the queue with more
+        // up-to-date parameters
+        Map messageMap = new HashMap<>();
+        Map> messagesByResourceType = new HashMap<>();
+        for (RemoteIndexMessage msg: messages) {
+            Long logicalResourceId = msg.getData().getLogicalResourceId();
+            messageMap.put(logicalResourceId, msg);
+
+            // split out the messages per resource type because we need to read from xx_logical_resources
+            List values = messagesByResourceType.computeIfAbsent(msg.getData().getResourceType(), k -> new ArrayList<>());
+            values.add(msg);
+        }
+
+        Set found = new HashSet<>();
+        final String checkReadyQuery = buildCheckReadyQuery(messagesByResourceType);
+        logger.fine(() -> "check ready query: " + checkReadyQuery);
+        try (PreparedStatement ps = connection.prepareStatement(checkReadyQuery)) {
+            ResultSet rs = ps.executeQuery();
+            // wrap the ResultSet in a reader for easier consumption
+            ResultSetReader rsReader = new ResultSetReader(rs);
+            while (rsReader.next()) {
+                LogicalResourceValue lrv = LogicalResourceValue.builder()
+                        .withLogicalResourceId(rsReader.getLong())
+                        .withResourceType(rsReader.getString())
+                        .withLogicalId(rsReader.getString())
+                        .withVersionId(rsReader.getInt())
+                        .withLastUpdated(rsReader.getTimestamp())
+                        .withParameterHash(rsReader.getString())
+                        .build();
+                RemoteIndexMessage m = messageMap.get(lrv.getLogicalResourceId());
+                if (m == null) {
+                    throw new IllegalStateException("query returned a logical resource which we didn't request");
+                }
+
+                // Check the values from the database to see if they match
+                // the information in the message.
+                if (m.getData().getVersionId() == lrv.getVersionId()) {
+                    // only process this message if the parameter hash and lastUpdated
+                    // times match - which is a good check that we're storing parameters
+                    // from the correct transaction. If these don't match, we can simply
+                    // say we found the data but don't need to process the message.
+                    final Instant dbLastUpdated = lrv.getLastUpdated().toInstant();
+                    final Instant msgLastUpdated = m.getData().getLastUpdatedInstant();
+                    if (lrv.getParameterHash().equals(m.getData().getParameterHash()) 
+                            && dbLastUpdated.equals(msgLastUpdated)) {
+                        okToProcess.add(m);
+                    } else {
+                        logger.warning("Parameter message must match both parameter_hash and last_updated. Must be from an uncommitted transaction so ignoring: " + m.toString());
+                    }
+                    found.add(lrv.getLogicalResourceId()); // won't be marked as missing
+                } else if (m.getData().getVersionId() > lrv.getVersionId()) {
+                    // we can skip processing this record because the database has already
+                    // been updated with a newer version. Identify the record as having been
+                    // found so we don't keep waiting for it
+                    found.add(lrv.getLogicalResourceId());
+                }
+                // if the version in the database is prior to version in the message we
+                // received it means that the server transaction hasn't been committed...
+                // so we have to wait just as though it were missing altogether
+            }
+        } catch (SQLException x) {
+            logger.log(Level.SEVERE, "prepare failed: " + checkReadyQuery, x);
+            throw new FHIRPersistenceException("prepare query failed");
+        }
+
+        if (found.size() < messages.size()) {
+            // identify the missing records and add to the notReady list
+            for (RemoteIndexMessage m: messages) {
+                if (!found.contains(m.getData().getLogicalResourceId())) {
+                    notReady.add(m);
+                }
+            }
+        }
+    }
+    
+    
+    
+    
+    /**
+     * Make sure we have values for all the logical_resource_ident values
+     * we have collected in the current batch. Need to make sure these are
+     * added in order to minimize deadlocks. Note that because we may create
+     * new logical_resource_ident records, we could be blocked by the main
+     * add_any_resource procedure run within the server CREATE/UPDATE
+     * transaction.
+     * @throws FHIRPersistenceException
+     */
+    private void resolveLogicalResourceIdents() throws FHIRPersistenceException {
+        logger.fine("resolveLogicalResourceIdents: fetching ids for unresolved LogicalResourceIdent records");
+        // identify which values aren't yet in the database
+        List missing = fetchLogicalResourceIdentIds(unresolvedLogicalResourceIdents);
+
+        if (!missing.isEmpty()) {
+            logger.fine("resolveLogicalResourceIdents: add missing LogicalResourceIdent records");
+            addMissingLogicalResourceIdents(missing);
+        }
+
+        // All the previously missing values should now be in the database. We need to fetch them again,
+        // possibly having to use multiple queries
+        logger.fine("resolveLogicalResourceIdents: fetch ids for missing LogicalResourceIdent records");
+        List bad = fetchLogicalResourceIdentIds(missing);
+
+        if (!bad.isEmpty()) {
+            // shouldn't happen, but let's protected against it anyway
+            throw new FHIRPersistenceException("Failed to create all logical_resource_ident values");
+        }
+        logger.fine("resolveLogicalResourceIdents: all resolved");
+    }
+
+    /**
+     * Build and prepare a statement to fetch the code_system_id and code_system_name
+     * from the code_systems table for all the given (unresolved) code system values
+     * @param values
+     * @return
+     * @throws SQLException
+     */
+    protected PreparedStatement buildLogicalResourceIdentSelectStatement(List values) throws SQLException {
+        StringBuilder query = new StringBuilder();
+        query.append("SELECT rt.resource_type, lri.logical_id, lri.logical_resource_id ");
+        query.append("  FROM logical_resource_ident AS lri ");
+        query.append("  JOIN (VALUES ");
+        for (int i=0; i 0) {
+                query.append(",");
+            }
+            query.append("(?,?)");
+        }
+        query.append(") AS v(resource_type_id, logical_id) ");
+        query.append("    ON (lri.resource_type_id = v.resource_type_id AND lri.logical_id = v.logical_id)");
+        query.append("  JOIN resource_types AS rt ON (rt.resource_type_id = v.resource_type_id)"); // convenient to get the resource type name here
+        PreparedStatement ps = connection.prepareStatement(query.toString());
+        // bind the parameter values
+        int param = 1;
+        for (LogicalResourceIdentValue val: values) {
+            ps.setInt(param++, val.getResourceTypeId());
+            ps.setString(param++, val.getLogicalId());
+        }
+        logger.fine(() -> "logicalResourceIdents: " + query.toString());
+        return ps;
+    }
+
+    /**
+     * These logical_resource_ident values weren't found in the database, so we need to try and add them.
+     * We have to deal with concurrency here - there's a chance another thread could also
+     * be trying to add them. To avoid deadlocks, it's important to do any inserts in a
+     * consistent order. At the end, we should be able to read back values for each entry
+     * @param missing
+     */
+    protected void addMissingLogicalResourceIdents(List missing) throws FHIRPersistenceException {
+        // Sort the values first to help avoid deadlocks
+        Collections.sort(missing, (a,b) -> {
+            return a.compareTo(b);
+        });
+
+        final String nextVal = translator.nextValue(schemaName, "fhir_sequence");
+        StringBuilder insert = new StringBuilder();
+        insert.append("INSERT INTO logical_resource_ident (resource_type_id, logical_id, logical_resource_id) VALUES (?,?,");
+        insert.append(nextVal); // next sequence value
+        insert.append(") ");
+        insert.append(onConflict());
+
+        try (PreparedStatement ps = connection.prepareStatement(insert.toString())) {
+            int count = 0;
+            for (LogicalResourceIdentValue value: missing) {
+                if (value.getResourceTypeId() == null) {
+                    logger.severe("bad value: " + value);
+                }
+                ps.setInt(1, value.getResourceTypeId());
+                ps.setString(2, value.getLogicalId());
+                ps.addBatch();
+                if (++count == this.maxLogicalResourcesPerStatement) {
+                    // not too many statements in a single batch
+                    ps.executeBatch();
+                    count = 0;
+                }
+            }
+            if (count > 0) {
+                // final batch
+                ps.executeBatch();
+            }
+        } catch (SQLException x) {
+            logger.log(Level.SEVERE, "logical_resource_ident insert failed: " + insert.toString(), x);
+            throw new FHIRPersistenceException("logical_resource_ident insert failed");
+        }
+    }
+
+    /**
+     * Fetch logical_resource_id values for the given list of LogicalResourceIdent objects.
+     * @param unresolved
+     * @return
+     * @throws FHIRPersistenceException
+     */
+    private List fetchLogicalResourceIdentIds(List unresolved) throws FHIRPersistenceException {
+        // track which values aren't yet in the database
+        List missing = new ArrayList<>();
+
+        int offset = 0;
+        while (offset < unresolved.size()) {
+            int remaining = unresolved.size() - offset;
+            int subSize = Math.min(remaining, this.maxCodeSystemsPerStatement);
+            List sub = unresolved.subList(offset, offset+subSize); // remember toIndex is exclusive
+            offset += subSize; // set up for the next iteration
+            try (PreparedStatement ps = buildLogicalResourceIdentSelectStatement(sub)) {
+                ResultSet rs = ps.executeQuery();
+                // We can't rely on the order of result rows matching the order of the in-list,
+                // so we have to go back to our map to look up each LogicalResourceIdentValue
+                int resultCount = 0;
+                while (rs.next()) {
+                    resultCount++;
+                    LogicalResourceIdentKey key = new LogicalResourceIdentKey(rs.getString(1), rs.getString(2));
+                    LogicalResourceIdentValue csv = this.logicalResourceIdentMap.get(key);
+                    if (csv != null) {
+                        csv.setLogicalResourceId(rs.getLong(3));
+                    } else {
+                        // can't really happen, but be defensive
+                        throw new FHIRPersistenceException("logical resource ident query returned an unexpected value");
+                    }
+                }
+
+                // Most of the time we'll get everything, so we can bypass the check for
+                // missing values
+                if (resultCount == 0) {
+                    // 100% miss
+                    missing.addAll(sub);
+                } else if (resultCount < subSize) {
+                    // need to scan the sub list and see which values we don't yet have ids for
+                    for (LogicalResourceIdentValue csv: sub) {
+                        if (csv.getLogicalResourceId() == null) {
+                            missing.add(csv);
+                        }
+                    }
+                }
+            } catch (SQLException x) {
+                logger.log(Level.SEVERE, "logical resource ident fetch failed", x);
+                throw new FHIRPersistenceException("logical resource ident fetch failed");
+            }
+        }
+
+        // Return the list of CodeSystemValues which don't yet have a database entry
+        return missing;
+    }
+
+    /**
+     * Get the next value from fhir_ref_sequence
+     * @param c
+     * @return
+     * @throws SQLException
+     */
+    protected Integer getNextRefId() throws SQLException {
+        final String select = translator.selectSequenceNextValue(schemaName, "fhir_ref_sequence");
+        Integer result;
+        try (Statement s = connection.createStatement()) {
+            ResultSet rs = s.executeQuery(select);
+            if (rs.next()) {
+                result = rs.getInt(1);
+            } else {
+                throw new IllegalStateException("no row from '" + select + "'");
+            }
+        }
+        return result;
+    }    
+
+    /**
+     * Get the next value from fhir_sequence
+     * @param c
+     * @return
+     * @throws SQLException
+     */
+    protected Long getNextId() throws SQLException {
+        final String select = translator.selectSequenceNextValue(schemaName, "fhir_sequence");
+        Long result;
+        try (Statement s = connection.createStatement()) {
+            ResultSet rs = s.executeQuery(select);
+            if (rs.next()) {
+                result = rs.getLong(1);
+            } else {
+                throw new IllegalStateException("no row from '" + select + "'");
+            }
+        }
+        return result;
+    }
+
+}
\ No newline at end of file
diff --git a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/PlainPostgresMessageHandler.java b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/PlainPostgresMessageHandler.java
new file mode 100644
index 00000000000..04e43094465
--- /dev/null
+++ b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/PlainPostgresMessageHandler.java
@@ -0,0 +1,37 @@
+/*
+ * (C) Copyright IBM Corp. 2022
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+ 
+package com.ibm.fhir.remote.index.database;
+
+import java.sql.Connection;
+
+import com.ibm.fhir.database.utils.postgres.PostgresTranslator;
+import com.ibm.fhir.remote.index.api.IdentityCache;
+
+/**
+ * PostgreSQL variant of the remote index message handler
+ */
+public class PlainPostgresMessageHandler extends PlainMessageHandler {
+
+    /**
+     * Public constructor
+     * 
+     * @param instanceIdentifier
+     * @param connection
+     * @param schemaName
+     * @param cache
+     * @param maxReadyTimeMs
+     */
+    public PlainPostgresMessageHandler(String instanceIdentifier, Connection connection, String schemaName, IdentityCache cache, long maxReadyTimeMs) {
+        super(instanceIdentifier, new PostgresTranslator(), connection, schemaName, cache, maxReadyTimeMs);
+    }
+
+    @Override
+    protected String onConflict() {
+        return "ON CONFLICT DO NOTHING";
+    }
+
+}
diff --git a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/PlainPostgresParameterBatch.java b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/PlainPostgresParameterBatch.java
new file mode 100644
index 00000000000..eb9a4a6209f
--- /dev/null
+++ b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/PlainPostgresParameterBatch.java
@@ -0,0 +1,499 @@
+/*
+ * (C) Copyright IBM Corp. 2022
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+ 
+package com.ibm.fhir.remote.index.database;
+
+import java.math.BigDecimal;
+import java.sql.Connection;
+import java.sql.PreparedStatement;
+import java.sql.SQLException;
+import java.sql.Timestamp;
+import java.sql.Types;
+import java.util.Calendar;
+import java.util.logging.Logger;
+
+import com.ibm.fhir.database.utils.common.CalendarHelper;
+import com.ibm.fhir.database.utils.common.PreparedStatementHelper;
+
+/**
+ * Parameter batch statements configured for a given resource type
+ * using the plain schema variant
+ */
+public class PlainPostgresParameterBatch {
+    private static final Logger logger = Logger.getLogger(PlainPostgresParameterBatch.class.getName());
+
+    private final Connection connection;
+    private final String resourceType;
+
+    private PreparedStatement strings;
+    private int stringCount;
+
+    private PreparedStatement numbers;
+    private int numberCount;
+
+    private PreparedStatement dates;
+    private int dateCount;
+
+    private PreparedStatement quantities;
+    private int quantityCount;
+
+    private PreparedStatement locations;
+    private int locationCount;
+
+    private PreparedStatement resourceTokenRefs;
+    private int resourceTokenRefCount;
+
+    private PreparedStatement tags;
+    private int tagCount;
+
+    private PreparedStatement profiles;
+    private int profileCount;
+
+    private PreparedStatement security;
+    private int securityCount;
+
+    private PreparedStatement refs;
+    private int refCount;
+
+    /**
+     * Public constructor
+     * @param c
+     * @param resourceType
+     */
+    public PlainPostgresParameterBatch(Connection c, String resourceType) {
+        this.connection = c;
+        this.resourceType = resourceType;
+    }
+
+    /**
+     * Push the current batch
+     */
+    public void pushBatch() throws SQLException {
+        if (stringCount > 0) {
+            strings.executeBatch();
+            stringCount = 0;
+        }
+        if (numberCount > 0) {
+            numbers.executeBatch();
+            numberCount = 0;
+        }
+        if (dateCount > 0) {
+            dates.executeBatch();
+            dateCount = 0;
+        }
+        if (quantityCount > 0) {
+            quantities.executeBatch();
+            quantityCount = 0;
+        }
+        if (locationCount > 0) {
+            locations.executeBatch();
+            locationCount = 0;
+        }
+        if (resourceTokenRefCount > 0) {
+            resourceTokenRefs.executeBatch();
+            resourceTokenRefCount = 0;
+        }
+        if (tagCount > 0) {
+            tags.executeBatch();
+            tagCount = 0;
+        }
+        if (profileCount > 0) {
+            profiles.executeBatch();
+            profileCount = 0;
+        }
+        if (securityCount > 0) {
+            security.executeBatch();
+            securityCount = 0;
+        }
+        if (refCount > 0) {
+            refs.executeBatch();
+            refCount = 0;
+        }
+    }
+
+    /**
+     * Resets the state of the DAO by closing all statements and
+     * setting any batch counts to 0
+     */
+    public void close() {
+        if (strings != null) {
+            try {
+                strings.close();
+            } catch (SQLException x) {
+                // NOP
+            } finally {
+                strings = null;
+                stringCount = 0;
+            }
+        }
+
+        if (numbers != null) {
+            try {
+                numbers.close();
+            } catch (SQLException x) {
+                // NOP
+            } finally {
+                numbers = null;
+                numberCount = 0;
+            }
+        }
+
+        if (dates != null) {
+            try {
+                dates.close();
+            } catch (SQLException x) {
+                // NOP
+            } finally {
+                dates = null;
+                dateCount = 0;
+            }
+        }
+
+        if (quantities != null) {
+            try {
+                quantities.close();
+            } catch (SQLException x) {
+                // NOP
+            } finally {
+                quantities = null;
+                quantityCount = 0;
+            }
+        }
+
+        if (locations != null) {
+            try {
+                locations.close();
+            } catch (SQLException x) {
+                // NOP
+            } finally {
+                locations = null;
+                locationCount = 0;
+            }
+        }
+
+        if (resourceTokenRefs != null) {
+            try {
+                resourceTokenRefs.close();
+            } catch (SQLException x) {
+                // NOP
+            } finally {
+                resourceTokenRefs = null;
+                resourceTokenRefCount = 0;
+            }
+        }
+        if (tags != null) {
+            try {
+                tags.close();
+            } catch (SQLException x) {
+                // NOP
+            } finally {
+                tags = null;
+                tagCount = 0;
+            }            
+        }
+        if (profiles != null) {
+            try {
+                profiles.close();
+            } catch (SQLException x) {
+                // NOP
+            } finally {
+                profiles = null;
+                profileCount = 0;
+            }            
+        }
+        if (security != null) {
+            try {
+                security.close();
+            } catch (SQLException x) {
+                // NOP
+            } finally {
+                security = null;
+                securityCount = 0;
+            }
+        }
+        if (refs != null) {
+            try {
+                refs.close();
+            } catch (SQLException x) {
+                // NOP
+            } finally {
+                refs = null;
+                refCount = 0;
+            }
+        }
+    }
+
+    /**
+     * Set the compositeId on the given PreparedStatement, handling a value if necessary
+     * @param ps
+     * @param index
+     * @param compositeId
+     * @throws SQLException
+     */
+    private void setComposite(PreparedStatement ps, int index, Integer compositeId) throws SQLException {
+        if (compositeId != null) {
+            ps.setInt(index, compositeId);
+        } else {
+            ps.setNull(index, Types.INTEGER);
+        }
+    }
+    /**
+     * Utility method to set a string value and handle null
+     * @param ps
+     * @param index
+     * @param value
+     * @throws SQLException
+     */
+    private void setString(PreparedStatement ps, int index, String value) throws SQLException {
+        if (value == null) {
+            ps.setNull(index, Types.VARCHAR);
+        } else {
+            ps.setString(index, value);
+        }
+    }
+
+    /**
+     * Add a string parameter value to the batch statement
+     * @param logicalResourceId
+     * @param parameterNameId
+     * @param strValue
+     * @param strValueLower
+     * @param compositeId
+     * @throws SQLException
+     */
+    public void addString(long logicalResourceId, int parameterNameId, String strValue, String strValueLower, Integer compositeId) throws SQLException {
+        if (strings == null) {
+            final String tablePrefix = resourceType.toLowerCase();
+            final String insertString = "INSERT INTO " + tablePrefix + "_str_values (parameter_name_id, str_value, str_value_lcase, logical_resource_id, composite_id) VALUES (?,?,?,?,?)";
+            strings = connection.prepareStatement(insertString);
+        }
+
+        strings.setInt(1, parameterNameId);
+        strings.setString(2, strValue);
+        strings.setString(3, strValueLower);
+        strings.setLong(4, logicalResourceId);
+        setComposite(strings, 5, compositeId);
+        strings.addBatch();
+        stringCount++;
+    }
+
+    /**
+     * Add a number parameter value to the batch statement
+     * @param logicalResourceId
+     * @param parameterNameId
+     * @param value
+     * @param valueLow
+     * @param valueHigh
+     * @param compositeId
+     * @throws SQLException
+     */
+    public void addNumber(long logicalResourceId, int parameterNameId, BigDecimal value, BigDecimal valueLow, BigDecimal valueHigh, Integer compositeId) throws SQLException {
+        if (numbers == null) {
+            final String tablePrefix = resourceType.toLowerCase();
+            final String insertNumber = "INSERT INTO " + tablePrefix + "_number_values (parameter_name_id, number_value, number_value_low, number_value_high, logical_resource_id, composite_id) VALUES (?,?,?,?,?,?)";
+            numbers = connection.prepareStatement(insertNumber);
+        }
+        numbers.setInt(1, parameterNameId);
+        numbers.setBigDecimal(2, value);
+        numbers.setBigDecimal(3, valueLow);
+        numbers.setBigDecimal(4, valueHigh);
+        numbers.setLong(5, logicalResourceId);
+        setComposite(numbers, 6, compositeId);
+        numbers.addBatch();
+        numberCount++;
+    }
+
+    /**
+     * Add a date parameter value to the batch statement
+     * @param logicalResourceId
+     * @param parameterNameId
+     * @param dateStart
+     * @param dateEnd
+     * @param compositeId
+     * @throws SQLException
+     */
+    public void addDate(long logicalResourceId, int parameterNameId, Timestamp dateStart, Timestamp dateEnd, Integer compositeId) throws SQLException {
+        if (dates == null) {
+            final String tablePrefix = resourceType.toLowerCase();
+            final String insertDate = "INSERT INTO " + tablePrefix + "_date_values (parameter_name_id, date_start, date_end, logical_resource_id, composite_id) VALUES (?,?,?,?,?)";
+            dates = connection.prepareStatement(insertDate);
+        }
+
+        final Calendar UTC = CalendarHelper.getCalendarForUTC();
+        dates.setInt(1, parameterNameId);
+        dates.setTimestamp(2, dateStart, UTC);
+        dates.setTimestamp(3, dateEnd, UTC);
+        dates.setLong(4, logicalResourceId);
+        setComposite(dates, 5, compositeId);
+        dates.addBatch();
+        dateCount++;
+    }
+
+    /**
+     * Add a quantity parameter value to the batch statement
+     * @param logicalResourceId
+     * @param parameterNameId
+     * @param codeSystemId
+     * @param valueCode
+     * @param valueNumber
+     * @param valueNumberLow
+     * @param valueNumberHigh
+     * @param compositeId
+     * @throws SQLException
+     */
+    public void addQuantity(long logicalResourceId, int parameterNameId, Integer codeSystemId, String valueCode, BigDecimal valueNumber, BigDecimal valueNumberLow, BigDecimal valueNumberHigh, Integer compositeId) throws SQLException {
+        if (quantities == null) {
+            final String tablePrefix = resourceType.toLowerCase();
+            final String insertQuantity = "INSERT INTO " + tablePrefix + "_quantity_values (parameter_name_id, code_system_id, code, quantity_value, quantity_value_low, quantity_value_high, logical_resource_id, composite_id) VALUES (?,?,?,?,?,?,?,?)";
+            quantities = connection.prepareStatement(insertQuantity);
+        }
+
+        quantities.setInt(1, parameterNameId);
+        quantities.setInt(2, codeSystemId);
+        quantities.setString(3, valueCode);
+        quantities.setBigDecimal(4, valueNumber);
+        quantities.setBigDecimal(5, valueNumberLow);
+        quantities.setBigDecimal(6, valueNumberHigh);
+        quantities.setLong(7, logicalResourceId);
+        setComposite(quantities, 8, compositeId);
+        quantities.addBatch();
+        quantityCount++;
+    }
+
+    /**
+     * Add a location parameter value to the batch statement
+     * 
+     * @param logicalResourceId
+     * @param parameterNameId
+     * @param lat
+     * @param lng
+     * @param compositeId
+     * @throws SQLException
+     */
+    public void addLocation(long logicalResourceId, int parameterNameId, Double lat, Double lng, Integer compositeId) throws SQLException {
+        if (locations == null) {
+            final String tablePrefix = resourceType.toLowerCase();
+            final String insertLocation = "INSERT INTO " + tablePrefix + "_latlng_values (parameter_name_id, latitude_value, longitude_value, logical_resource_id, composite_id) VALUES (?,?,?,?,?)";
+            locations = connection.prepareStatement(insertLocation);
+        }
+
+        locations.setInt(1, parameterNameId);
+        locations.setDouble(2, lat);
+        locations.setDouble(3, lng);
+        locations.setLong(4, logicalResourceId);
+        setComposite(locations, 5, compositeId);
+        locations.addBatch();
+        locationCount++;
+    }
+
+    /**
+     * Add a token parameter value to the batch statement
+     * 
+     * @param logicalResourceId
+     * @param parameterNameId
+     * @param commonTokenValueId
+     * @param refVersionId
+     * @param compositeId
+     * @throws SQLException
+     */
+    public void addResourceTokenRef(long logicalResourceId, int parameterNameId, long commonTokenValueId, Integer refVersionId, Integer compositeId) throws SQLException {
+        if (resourceTokenRefs == null) {
+            final String tablePrefix = resourceType.toLowerCase();
+            final String tokenString = "INSERT INTO " + tablePrefix + "_resource_token_refs (parameter_name_id, common_token_value_id, logical_resource_id, composite_id) VALUES (?,?,?,?)";
+            resourceTokenRefs = connection.prepareStatement(tokenString);
+        }
+        resourceTokenRefs.setInt(1, parameterNameId);
+        resourceTokenRefs.setLong(2, commonTokenValueId);
+        resourceTokenRefs.setLong(3, logicalResourceId);
+        setComposite(resourceTokenRefs, 4, compositeId);
+        resourceTokenRefs.addBatch();
+        resourceTokenRefCount++;
+    }
+
+    /**
+     * Add a tag parameter value to the batch statement
+     * 
+     * @param logicalResourceId
+     * @param commonTokenValueId
+     * @throws SQLException
+     */
+    public void addTag(long logicalResourceId, long commonTokenValueId) throws SQLException {
+        if (tags == null) {
+            final String tablePrefix = resourceType.toLowerCase();
+            final String tokenString = "INSERT INTO " + tablePrefix + "_tags (common_token_value_id, logical_resource_id) VALUES (?,?)";
+            tags = connection.prepareStatement(tokenString);
+        }
+        tags.setLong(1, commonTokenValueId);
+        tags.setLong(2, logicalResourceId);
+        tags.addBatch();
+        tagCount++;
+    }
+
+    /**
+     * Add a profile parameter value to the batch statement
+     * 
+     * @param logicalResourceId
+     * @param canonicalId
+     * @param version
+     * @param fragment
+     * @throws SQLException
+     */
+    public void addProfile(long logicalResourceId, long canonicalId, String version, String fragment) throws SQLException {
+        if (profiles == null) {
+            final String tablePrefix = resourceType.toLowerCase();
+            final String tokenString = "INSERT INTO " + tablePrefix + "_profiles (canonical_id, logical_resource_id, version, fragment) VALUES (?,?,?,?)";
+            profiles = connection.prepareStatement(tokenString);
+        }
+        profiles.setLong(1, canonicalId);
+        profiles.setLong(2, logicalResourceId);
+        setString(profiles, 3, version);
+        setString(profiles, 4, fragment);
+        profiles.addBatch();
+        profileCount++;
+    }
+
+    /**
+     * Add a security parameter value to the batch statement
+     * @param logicalResourceId
+     * @param commonTokenValueId
+     * @throws SQLException
+     */
+    public void addSecurity(long logicalResourceId, long commonTokenValueId) throws SQLException {
+        if (security == null) {
+            final String tablePrefix = resourceType.toLowerCase();
+            final String INS = "INSERT INTO " + tablePrefix + "_security (common_token_value_id, logical_resource_id) VALUES (?,?)";
+            security = connection.prepareStatement(INS);
+        }
+        PreparedStatementHelper psh = new PreparedStatementHelper(security);
+        psh.setLong(commonTokenValueId)
+            .setLong(logicalResourceId)
+            .addBatch();
+        securityCount++;
+    }
+
+    /**
+     * Add a reference parameter value to the batch statement
+     * 
+     * @param logicalResourceId
+     * @param refLogicalResourceId
+     * @param refVersionId
+     */
+    public void addReference(long logicalResourceId, int parameterNameId, long refLogicalResourceId, Integer refVersionId) throws SQLException {
+        logger.fine(() -> "Adding reference: parameterNameId:" + parameterNameId + " refLogicalResourceId:" + refLogicalResourceId + " refVersionId:" + refVersionId);
+        if (refs == null) {
+            final String tablePrefix = resourceType.toLowerCase();
+            final String insertString = "INSERT INTO " + tablePrefix + "_ref_values (parameter_name_id, logical_resource_id, ref_logical_resource_id, ref_version_id) VALUES (?,?,?,?)";
+            refs = connection.prepareStatement(insertString);
+        }
+        PreparedStatementHelper psh = new PreparedStatementHelper(refs);
+        psh.setInt(parameterNameId)
+            .setLong(logicalResourceId)
+            .setLong(refLogicalResourceId)
+            .setInt(refVersionId)
+            .addBatch();
+        refCount++;
+    }
+}
\ No newline at end of file
diff --git a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/PlainPostgresSystemParameterBatch.java b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/PlainPostgresSystemParameterBatch.java
new file mode 100644
index 00000000000..d350684f7a5
--- /dev/null
+++ b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/PlainPostgresSystemParameterBatch.java
@@ -0,0 +1,274 @@
+/*
+ * (C) Copyright IBM Corp. 2022
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+ 
+package com.ibm.fhir.remote.index.database;
+
+import java.sql.Connection;
+import java.sql.PreparedStatement;
+import java.sql.SQLException;
+import java.sql.Timestamp;
+import java.util.Calendar;
+
+import com.ibm.fhir.database.utils.common.CalendarHelper;
+import com.ibm.fhir.database.utils.common.PreparedStatementHelper;
+
+/**
+ * Batch insert statements for system-level parameters
+ * @implNote targets the plain variant of the schema
+ * without an explicit sharding column
+ */
+public class PlainPostgresSystemParameterBatch {
+    private final Connection connection;
+
+    private PreparedStatement systemStrings;
+    private int systemStringCount;
+
+    private PreparedStatement systemDates;
+    private int systemDateCount;
+
+    private PreparedStatement systemResourceTokenRefs;
+    private int systemResourceTokenRefCount;
+
+    private PreparedStatement systemProfiles;
+    private int systemProfileCount;
+
+    private PreparedStatement systemTags;
+    private int systemTagCount;
+
+    private PreparedStatement systemSecurity;
+    private int systemSecurityCount;
+
+    /**
+     * Public constructor
+     * @param c
+     */
+    public PlainPostgresSystemParameterBatch(Connection c) {
+        this.connection = c;
+    }
+
+    /**
+     * Push the current batch
+     */
+    public void pushBatch() throws SQLException {
+        if (systemStringCount > 0) {
+            systemStrings.executeBatch();
+            systemStringCount = 0;
+        }
+        if (systemDateCount > 0) {
+            systemDates.executeBatch();
+            systemDateCount = 0;
+        }
+        if (systemResourceTokenRefCount > 0) {
+            systemResourceTokenRefs.executeBatch();
+            systemResourceTokenRefCount = 0;
+        }
+        if (systemTagCount > 0) {
+            systemTags.executeBatch();
+            systemTagCount = 0;
+        }
+        if (systemProfileCount > 0) {
+            systemProfiles.executeBatch();
+            systemProfileCount = 0;
+        }
+        if (systemSecurityCount > 0) {
+            systemSecurity.executeBatch();
+            systemSecurityCount = 0;
+        }
+    }
+
+    /**
+     * Closes all the statements currently open
+     */
+    public void close() {
+
+        if (systemStrings != null) {
+            try {
+                systemStrings.close();
+            } catch (SQLException x) {
+                // NOP
+            } finally {
+                systemStrings = null;
+                systemStringCount = 0;
+            }
+        }
+
+        if (systemDates != null) {
+            try {
+                systemDates.close();
+            } catch (SQLException x) {
+                // NOP
+            } finally {
+                systemDates = null;
+                systemDateCount = 0;
+            }
+        }
+        if (systemResourceTokenRefs != null) {
+            try {
+                systemResourceTokenRefs.close();
+            } catch (SQLException x) {
+                // NOP
+            } finally {
+                systemResourceTokenRefs = null;
+                systemResourceTokenRefCount = 0;
+            }
+        }
+        if (systemTags != null) {
+            try {
+                systemTags.close();
+            } catch (SQLException x) {
+                // NOP
+            } finally {
+                systemTags = null;
+                systemTagCount = 0;
+            }
+        }
+        if (systemProfiles != null) {
+            try {
+                systemProfiles.close();
+            } catch (SQLException x) {
+                // NOP
+            } finally {
+                systemProfiles = null;
+                systemProfileCount = 0;
+            }
+        }
+        if (systemSecurity != null) {
+            try {
+                systemSecurity.close();
+            } catch (SQLException x) {
+                // NOP
+            } finally {
+                systemSecurity = null;
+                systemProfileCount = 0;
+            }
+        }
+    }
+
+    /**
+     * Add a string parameter value to the whole-system batch statement
+     * 
+     * @param logicalResourceId
+     * @param parameterNameId
+     * @param strValue
+     * @param strValueLower
+     * @throws SQLException
+     */
+    public void addString(long logicalResourceId, int parameterNameId, String strValue, String strValueLower) throws SQLException {
+            // System level string attributes
+            if (systemStrings == null) {
+                final String insertSystemString = "INSERT INTO str_values (parameter_name_id, str_value, str_value_lcase, logical_resource_id) VALUES (?,?,?,?)";
+                systemStrings = connection.prepareStatement(insertSystemString);
+            }
+            systemStrings.setInt(1, parameterNameId);
+            systemStrings.setString(2, strValue);
+            systemStrings.setString(3, strValueLower);
+            systemStrings.setLong(4, logicalResourceId);
+            systemStrings.addBatch();
+            systemStringCount++;
+    }
+
+    /**
+     * Add a date parameter value to the whole-system batch statement
+     * @param logicalResourceId
+     * @param parameterNameId
+     * @param dateStart
+     * @param dateEnd
+     * @param compositeId
+     * @throws SQLException
+     */
+    public void addDate(long logicalResourceId, int parameterNameId, Timestamp dateStart, Timestamp dateEnd, Integer compositeId) throws SQLException {
+        if (systemDates == null) {
+            final String insertSystemDate = "INSERT INTO date_values (parameter_name_id, date_start, date_end, logical_resource_id) VALUES (?,?,?,?)";
+            systemDates = connection.prepareStatement(insertSystemDate);
+        }
+        final Calendar UTC = CalendarHelper.getCalendarForUTC();
+        systemDates.setInt(1, parameterNameId);
+        systemDates.setTimestamp(2, dateStart, UTC);
+        systemDates.setTimestamp(3, dateEnd, UTC);
+        systemDates.setLong(4, logicalResourceId);
+        systemDates.addBatch();
+        systemDateCount++;
+    }
+
+    /**
+     * Add a token parameter value to the batch statement
+     * 
+     * @param logicalResourceId
+     * @param parameterNameId
+     * @param commonTokenValueId
+     * @throws SQLException
+     */
+    public void addResourceTokenRef(long logicalResourceId, int parameterNameId, long commonTokenValueId) throws SQLException {
+        if (systemResourceTokenRefs == null) {
+            final String tokenString = "INSERT INTO resource_token_refs (parameter_name_id, common_token_value_id, logical_resource_id) VALUES (?,?,?)";
+            systemResourceTokenRefs = connection.prepareStatement(tokenString);
+        }
+        systemResourceTokenRefs.setInt(1, parameterNameId);
+        systemResourceTokenRefs.setLong(2, commonTokenValueId);
+        systemResourceTokenRefs.setLong(3, logicalResourceId);
+        systemResourceTokenRefs.addBatch();
+        systemResourceTokenRefCount++;
+    }
+
+    /**
+     * Add a tag parameter value to the whole-system batch statement
+     * 
+     * @param logicalResourceId
+     * @param commonTokenValueId
+     * @throws SQLException
+     */
+    public void addTag(long logicalResourceId, long commonTokenValueId) throws SQLException {
+        if (systemTags == null) {
+            final String INS = "INSERT INTO logical_resource_tags(common_token_value_id, logical_resource_id) VALUES (?,?)";
+            systemTags = connection.prepareStatement(INS);
+        }
+        PreparedStatementHelper psh = new PreparedStatementHelper(systemTags);
+        psh.setLong(commonTokenValueId)
+            .setLong(logicalResourceId)
+            .addBatch();
+        systemTagCount++;
+    }
+
+    /**
+     * Add a profile parameter value to the whole-system batch statement
+     * @param logicalResourceId
+     * @param canonicalId
+     * @param version
+     * @param fragment
+     * @throws SQLException
+     */
+    public void addProfile(long logicalResourceId, long canonicalId, String version, String fragment) throws SQLException {
+        if (systemProfiles == null) {
+            final String INS = "INSERT INTO logical_resource_profiles(canonical_id, logical_resource_id, version, fragment) VALUES (?,?,?,?)";
+            systemProfiles = connection.prepareStatement(INS);
+        }
+        PreparedStatementHelper psh = new PreparedStatementHelper(systemProfiles);
+        psh.setLong(canonicalId)
+            .setLong(logicalResourceId)
+            .setString(version)
+            .setString(fragment)
+            .addBatch();
+        systemProfileCount++;
+    }
+
+    /**
+     * Add a security parameter value to the whole-system batch statement
+     * @param logicalResourceId
+     * @param commonTokenValueId
+     * @throws SQLException
+     */
+    public void addSecurity(long logicalResourceId, long commonTokenValueId) throws SQLException {
+        if (systemSecurity == null) {
+            final String INS = "INSERT INTO logical_resource_security(common_token_value_id, logical_resource_id) VALUES (?,?)";
+            systemSecurity = connection.prepareStatement(INS);
+        }
+        PreparedStatementHelper psh = new PreparedStatementHelper(systemSecurity);
+        psh.setLong(commonTokenValueId)
+            .setLong(logicalResourceId)
+            .addBatch();
+        systemSecurityCount++;
+    }
+}
\ No newline at end of file
diff --git a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/PreparedStatementWrapper.java b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/PreparedStatementWrapper.java
new file mode 100644
index 00000000000..4705a8f4a26
--- /dev/null
+++ b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/PreparedStatementWrapper.java
@@ -0,0 +1,58 @@
+/*
+ * (C) Copyright IBM Corp. 2022
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+ 
+package com.ibm.fhir.remote.index.database;
+
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+
+/**
+ * Wraps a {@link PreparedStatement} together with the statement text for easier
+ * logging when there are errors
+ */
+public class PreparedStatementWrapper implements AutoCloseable {
+    private final String statementText;
+    private final PreparedStatement preparedStatement;
+
+    /**
+     * Canonical constructor
+     * @param statementText
+     * @param ps
+     */
+    public PreparedStatementWrapper(String statementText, PreparedStatement ps) {
+        this.statementText = statementText;
+        this.preparedStatement = ps;
+    }
+
+    /**
+     * @return the statementText
+     */
+    public String getStatementText() {
+        return statementText;
+    }
+
+    /**
+     * @return the preparedStatement
+     */
+    public PreparedStatement getPreparedStatement() {
+        return preparedStatement;
+    }
+
+    @Override
+    public void close() throws SQLException {
+        this.preparedStatement.close();
+    }
+
+    /**
+     * Convenience method to delegate the call to the wrapped statement
+     * @return
+     * @throws SQLException
+     */
+    public ResultSet executeQuery() throws SQLException {
+        return preparedStatement.executeQuery();
+    }
+}
diff --git a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/ResourceTokenValue.java b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/ResourceTokenValue.java
new file mode 100644
index 00000000000..3e287e51d39
--- /dev/null
+++ b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/ResourceTokenValue.java
@@ -0,0 +1,76 @@
+/*
+ * (C) Copyright IBM Corp. 2022
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+ 
+package com.ibm.fhir.remote.index.database;
+
+import com.ibm.fhir.persistence.index.TokenParameter;
+
+/**
+ * Record representing a new row in _resource_token_refs
+ */
+public class ResourceTokenValue {
+    private final String resourceType;
+    private final String logicalId;
+    private final long logicalResourceId;
+    private final TokenParameter tokenParameter;
+    private final CommonTokenValue commonTokenValue;
+
+    /**
+     * Public constructor
+     * @param resourceType
+     * @param logicalId
+     * @param logicalResourceId
+     * @param tokenParameter
+     * @param commonTokenValue
+     */
+    public ResourceTokenValue(String resourceType, String logicalId, long logicalResourceId, TokenParameter tokenParameter, CommonTokenValue commonTokenValue) {
+        this.resourceType = resourceType;
+        this.logicalId = logicalId;
+        this.logicalResourceId = logicalResourceId;
+        this.tokenParameter = tokenParameter;
+        this.commonTokenValue = commonTokenValue;
+    }
+
+    
+    /**
+     * @return the resourceType
+     */
+    public String getResourceType() {
+        return resourceType;
+    }
+
+    
+    /**
+     * @return the logicalId
+     */
+    public String getLogicalId() {
+        return logicalId;
+    }
+
+    
+    /**
+     * @return the logicalResourceId
+     */
+    public long getLogicalResourceId() {
+        return logicalResourceId;
+    }
+
+    
+    /**
+     * @return the tokenParameter
+     */
+    public TokenParameter getTokenParameter() {
+        return tokenParameter;
+    }
+
+    
+    /**
+     * @return the commonTokenValue
+     */
+    public CommonTokenValue getCommonTokenValue() {
+        return commonTokenValue;
+    }
+}
diff --git a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/ResourceTypeValue.java b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/ResourceTypeValue.java
new file mode 100644
index 00000000000..ae640d61f87
--- /dev/null
+++ b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/database/ResourceTypeValue.java
@@ -0,0 +1,42 @@
+/*
+ * (C) Copyright IBM Corp. 2022
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+ 
+package com.ibm.fhir.remote.index.database;
+
+
+/**
+ * A DTO representing a record from the resource_types table
+ */
+public class ResourceTypeValue {
+    private final String resourceType;
+    private final int resourceTypeId;
+
+    /**
+     * Canonical constructor
+     * @param resourceType
+     * @param resourceTypeId
+     */
+    public ResourceTypeValue(String resourceType, int resourceTypeId) {
+        this.resourceType = resourceType;
+        this.resourceTypeId = resourceTypeId;
+    }
+
+    
+    /**
+     * @return the resourceType
+     */
+    public String getResourceType() {
+        return resourceType;
+    }
+
+    
+    /**
+     * @return the resourceTypeId
+     */
+    public int getResourceTypeId() {
+        return resourceTypeId;
+    }
+}
diff --git a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/kafka/RemoteIndexConsumer.java b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/kafka/RemoteIndexConsumer.java
new file mode 100644
index 00000000000..4884c5a54ed
--- /dev/null
+++ b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/kafka/RemoteIndexConsumer.java
@@ -0,0 +1,192 @@
+/*
+ * (C) Copyright IBM Corp. 2022
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+package com.ibm.fhir.remote.index.kafka;
+
+import java.time.Duration;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+import org.apache.kafka.clients.consumer.ConsumerRebalanceListener;
+import org.apache.kafka.clients.consumer.ConsumerRecord;
+import org.apache.kafka.clients.consumer.ConsumerRecords;
+import org.apache.kafka.clients.consumer.KafkaConsumer;
+import org.apache.kafka.clients.consumer.OffsetAndMetadata;
+import org.apache.kafka.clients.consumer.OffsetCommitCallback;
+import org.apache.kafka.common.TopicPartition;
+
+import com.ibm.fhir.persistence.exception.FHIRPersistenceException;
+import com.ibm.fhir.remote.index.api.IMessageHandler;
+
+/**
+ * Kafka consumer reading remote index messages, batches the data and
+ * loads it into the configured database.
+ */
+public class RemoteIndexConsumer implements Runnable, OffsetCommitCallback {
+
+    private static final Logger logger = Logger.getLogger(RemoteIndexConsumer.class.getName());
+
+    // Nanoseconds in a second
+    private static final long NANOS = 1000000000L;
+
+    // the remote index service Kafka topic name
+    private final String topicName;
+
+    // The max time to spend collecting a batch before submitting
+    private final long maxBatchCollectTimeMs;
+
+    // The consumer object representing the connection to Kafka
+    private final KafkaConsumer kafkaConsumer;
+
+    // The handler we use to process messages we receive from Kafka
+    private final IMessageHandler messageHandler;
+
+    private final Duration pollWaitTime;
+
+    // Flag used to exit kafka loop on termination
+    private volatile boolean running = true;
+
+    // The number of commit failures since the last successful one
+    private int sequentialCommitFailures = 0;
+
+    // If we get 5 failures in a row, we disconnect
+    private static final int COMMIT_FAILURE_DISCONNECT_THRESHOLD = 5;
+
+    // A callback used to signal that this consumer has failed
+    private final Runnable consumerFailedCallback;
+
+    /**
+     * A listener to track the partitions assigned to this cluster member
+     * Callbacks happen as part of the consumer#poll() call, so no concurrency concerns
+     */
+    private class PartitionChangeListener implements ConsumerRebalanceListener {
+
+        @Override
+        public void onPartitionsRevoked(Collection partitions) {
+            for (TopicPartition tp : partitions) {
+                logger.info("Revoking partition: " + tp.topic() + ":" + tp.partition());
+            }
+        }
+
+        @Override
+        public void onPartitionsAssigned(Collection partitions) {
+            for (TopicPartition tp : partitions) {
+                logger.info("Assigning partition: " + tp.topic() + ":" + tp.partition());
+            }
+        }
+    }
+
+    /**
+     * Public constructor
+     * 
+     * @param kafkaConsumer
+     * @param messageHandler
+     * @param consumerFailedCallback
+     * @param topicName
+     * @param maxBatchCollectTimeMs
+     * @param pollWaitTime
+     */
+    public RemoteIndexConsumer(KafkaConsumer kafkaConsumer, IMessageHandler messageHandler,
+            Runnable consumerFailedCallback, String topicName, long maxBatchCollectTimeMs, Duration pollWaitTime) {
+        this.kafkaConsumer = kafkaConsumer;
+        this.messageHandler = messageHandler;
+        this.consumerFailedCallback = consumerFailedCallback;
+        this.topicName = topicName;
+        this.maxBatchCollectTimeMs = maxBatchCollectTimeMs;
+        this.pollWaitTime = pollWaitTime;
+    }
+
+    @Override
+    public void run() {
+        logger.info("Subscribing consumer to topic '" + this.topicName + "'");
+        kafkaConsumer.subscribe(Collections.singletonList(topicName), new PartitionChangeListener());
+
+        logger.info("Starting consumer loop");
+        while (running) {
+            try {
+                if (this.sequentialCommitFailures > COMMIT_FAILURE_DISCONNECT_THRESHOLD) {
+                    logger.severe("Too many commit failures. Stopping consumer");
+                    this.running = false;
+                } else {
+                    consume();
+                }
+            } catch (Throwable t) {
+                // If we end up here, it means we think it's an unrecoverable error so
+                // clear the running flag allowing us to exit
+                logger.log(Level.SEVERE, "unexpected error in consumer loop", t);
+                this.running = false;
+            } finally {
+                if (!running) {
+                    logger.warning("Stopping consumer loop");
+
+                    // explicitly closing the consumer here should allow for faster error recovery
+                    // (assuming, of course, that the brokers are still reachable from this node)
+                    kafkaConsumer.close();
+
+                    // close the handler, cleaning up any resources it is holding onto
+                    messageHandler.close();
+
+                    // signal back to the main controller. If enough consumer
+                    // threads fail, this will lead to the whole program to terminate
+                    // which in typical deployments will mean that the container will
+                    // hopefully be restarted
+                    consumerFailedCallback.run();
+                }
+            }
+        }
+        logger.info("Consumer closed and thread terminated");
+    }
+
+    /**
+     * poll the consumer and forward any messages we receive to the message handler
+     */
+    private void consume() throws FHIRPersistenceException {
+        logger.finer("Polling Kafka");
+        ConsumerRecords records = kafkaConsumer.poll(pollWaitTime);
+        logger.finer(() -> "Kafka poll records count: " + records.count());
+
+        // Extract the message payloads from each ConsumerRecord
+        List messages = new ArrayList<>();
+        for (ConsumerRecord record : records) {
+            messages.add(record.value());
+        }
+        if (messages.size() > 0) {
+            messageHandler.process(messages);
+        }
+        kafkaConsumer.commitAsync(this);
+    }
+
+    /**
+     * Shut down this consumer, interrupting the Kafka poll wait
+     */
+    public void shutdown() {
+        this.running = false;
+        try {
+            kafkaConsumer.wakeup();
+        } catch (Throwable x) {
+            logger.warning("Error waking up kafka consumer: " + x.getMessage());
+        }
+    }
+
+    @Override
+    public void onComplete(Map offsets, Exception exception) {
+        // called on this consumer's thread, so no need for any synchronization
+        if (exception == null) {
+            // successful commit, so reset the failure counter
+            this.sequentialCommitFailures = 0;
+        } else {
+            this.sequentialCommitFailures++;
+            logger.warning("Commit failure sequential=[" + this.sequentialCommitFailures + "] reason=[" + exception.getMessage() + "]");
+            if (logger.isLoggable(Level.FINE)) {
+                logger.log(Level.FINE, "sequentialCommitFailures=" + sequentialCommitFailures, exception);
+            }
+        }
+    }
+}
\ No newline at end of file
diff --git a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/sharded/ShardedBatchParameterProcessor.java b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/sharded/ShardedBatchParameterProcessor.java
new file mode 100644
index 00000000000..bb4e0a3b6c1
--- /dev/null
+++ b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/sharded/ShardedBatchParameterProcessor.java
@@ -0,0 +1,332 @@
+/*
+ * (C) Copyright IBM Corp. 2022
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+ 
+package com.ibm.fhir.remote.index.sharded;
+
+import java.sql.Connection;
+import java.sql.SQLException;
+import java.sql.Timestamp;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+import com.ibm.fhir.persistence.exception.FHIRPersistenceException;
+import com.ibm.fhir.persistence.index.DateParameter;
+import com.ibm.fhir.persistence.index.LocationParameter;
+import com.ibm.fhir.persistence.index.NumberParameter;
+import com.ibm.fhir.persistence.index.ProfileParameter;
+import com.ibm.fhir.persistence.index.QuantityParameter;
+import com.ibm.fhir.persistence.index.ReferenceParameter;
+import com.ibm.fhir.persistence.index.SecurityParameter;
+import com.ibm.fhir.persistence.index.StringParameter;
+import com.ibm.fhir.persistence.index.TagParameter;
+import com.ibm.fhir.persistence.index.TokenParameter;
+import com.ibm.fhir.remote.index.api.BatchParameterProcessor;
+import com.ibm.fhir.remote.index.database.CodeSystemValue;
+import com.ibm.fhir.remote.index.database.CommonCanonicalValue;
+import com.ibm.fhir.remote.index.database.CommonTokenValue;
+import com.ibm.fhir.remote.index.database.LogicalResourceIdentValue;
+import com.ibm.fhir.remote.index.database.ParameterNameValue;
+
+
+/**
+ * Processes batched parameters by pushing the values to various
+ * JDBC statements based on the distributed (shard_key) variant
+ * of the schema
+ */
+public class ShardedBatchParameterProcessor implements BatchParameterProcessor {
+    private static final Logger logger = Logger.getLogger(ShardedBatchParameterProcessor.class.getName());
+
+    // A cache of the resource-type specific DAOs we've created
+    private final Map daoMap = new HashMap<>();
+
+    // Encapculates the statements for inserting whole-system level search params
+    private final ShardedPostgresSystemParameterBatch systemDao;
+
+    // Resource types we've touched in the current batch
+    private final Set resourceTypesInBatch = new HashSet<>();
+
+    // The database connection this consumer thread is using
+    private final Connection connection;
+
+    /**
+     * Public constructor
+     * @param connection
+     */
+    public ShardedBatchParameterProcessor(Connection connection) {
+        this.connection = connection;
+        this.systemDao = new ShardedPostgresSystemParameterBatch(connection);        
+    }
+
+    /**
+     * Close any resources we're holding to support a cleaner exit
+     */
+    public void close() {
+        for (Map.Entry entry: daoMap.entrySet()) {
+            entry.getValue().close();
+        }
+        systemDao.close();
+    }
+
+    /**
+     * Start processing a new batch
+     */
+    public void startBatch() {
+        resourceTypesInBatch.clear();
+    }
+
+    /**
+     * Make sure that each statement that may contain data is cleared before we
+     * retry a batch
+     */
+    public void reset() {
+        for (String resourceType: resourceTypesInBatch) {
+            ShardedPostgresParameterBatch dao = daoMap.get(resourceType);
+            dao.close();
+        }
+        systemDao.close();
+    }
+    /**
+     * Push any statements that have been batched but not yet executed
+     * @throws FHIRPersistenceException
+     */
+    public void pushBatch() throws FHIRPersistenceException {
+        try {
+            for (String resourceType: resourceTypesInBatch) {
+                if (logger.isLoggable(Level.FINE)) {
+                    logger.fine("Pushing batch for [" + resourceType + "]");
+                }
+                ShardedPostgresParameterBatch dao = daoMap.get(resourceType);
+                try {
+                    dao.pushBatch();
+                } catch (SQLException x) {
+                    throw new FHIRPersistenceException("pushBatch failed for '" + resourceType + "'");
+                }
+            }
+
+            try {
+                logger.fine("Pushing batch for whole-system parameters");
+                systemDao.pushBatch();
+            } catch (SQLException x) {
+                throw new FHIRPersistenceException("batch insert for whole-system parameters", x);
+            }
+        } finally {
+            // Reset the set of active resource-types ready for the next batch
+            resourceTypesInBatch.clear();
+        }
+    }
+
+    private ShardedPostgresParameterBatch getParameterBatchDao(String resourceType) {
+        resourceTypesInBatch.add(resourceType);
+        ShardedPostgresParameterBatch dao = daoMap.get(resourceType);
+        if (dao == null) {
+            dao = new ShardedPostgresParameterBatch(connection, resourceType);
+            daoMap.put(resourceType, dao);
+        }
+        return dao;
+    }
+
+    @Override
+    public Short encodeShardKey(String requestShard) {
+        if (requestShard != null) {
+            return Short.valueOf((short)requestShard.hashCode());
+        } else {
+            return null;
+        }
+    }
+
+    @Override
+    public void process(String requestShard, String resourceType, String logicalId, long logicalResourceId, ParameterNameValue parameterNameValue, StringParameter parameter) throws FHIRPersistenceException {
+        if (logger.isLoggable(Level.FINE)) {
+            logger.fine("process string parameter [" + requestShard + "] [" + resourceType + "] [" + logicalId + "] [" + logicalResourceId + "] [" + parameterNameValue.getParameterName() + "] ["
+                + parameter.toString() + "]");
+        }
+
+        try {
+            ShardedPostgresParameterBatch dao = getParameterBatchDao(resourceType);
+            final Short shardKey = encodeShardKey(requestShard);
+            dao.addString(logicalResourceId, parameterNameValue.getParameterNameId(), parameter.getValue(), parameter.getValue().toLowerCase(), parameter.getCompositeId(), shardKey);
+
+            if (parameter.isSystemParam()) {
+                systemDao.addString(logicalResourceId, parameterNameValue.getParameterNameId(), parameter.getValue(), parameter.getValue().toLowerCase(), parameter.getCompositeId(), shardKey);
+            }
+        } catch (SQLException x) {
+            throw new FHIRPersistenceException("Failed inserting string params for '" + resourceType + "'");
+        }
+    }
+
+    @Override
+    public void process(String requestShard, String resourceType, String logicalId, long logicalResourceId, ParameterNameValue parameterNameValue, NumberParameter p) throws FHIRPersistenceException {
+        if (logger.isLoggable(Level.FINE)) {
+            logger.fine("process number parameter [" + requestShard + "] [" + resourceType + "] [" + logicalId + "] [" + logicalResourceId + "] [" + parameterNameValue.getParameterName() + "] ["
+                    + p.toString() + "]");
+        }
+
+        try {
+            ShardedPostgresParameterBatch dao = getParameterBatchDao(resourceType);
+            final Short shardKey = encodeShardKey(requestShard);
+            dao.addNumber(logicalResourceId, parameterNameValue.getParameterNameId(), p.getValue(), p.getLowValue(), p.getHighValue(), p.getCompositeId(), shardKey);
+        } catch (SQLException x) {
+            throw new FHIRPersistenceException("Failed inserting string params for '" + resourceType + "'");
+        }
+    }
+
+    @Override
+    public void process(String requestShard, String resourceType, String logicalId, long logicalResourceId, ParameterNameValue parameterNameValue, QuantityParameter p, CodeSystemValue codeSystemValue) throws FHIRPersistenceException {
+        if (logger.isLoggable(Level.FINE)) {
+            logger.fine("process quantity parameter [" + requestShard + "] [" + resourceType + "] [" + logicalId + "] [" + logicalResourceId + "] [" + parameterNameValue.getParameterName() + "] ["
+                    + p.toString() + "]");
+        }
+
+        try {
+            ShardedPostgresParameterBatch dao = getParameterBatchDao(resourceType);
+            final Short shardKey = encodeShardKey(requestShard);
+            dao.addQuantity(logicalResourceId, parameterNameValue.getParameterNameId(), codeSystemValue.getCodeSystemId(), p.getValueCode(), p.getValueNumber(), p.getValueNumberLow(), p.getValueNumberHigh(), p.getCompositeId(), shardKey);
+        } catch (SQLException x) {
+            throw new FHIRPersistenceException("Failed inserting quantity params for '" + resourceType + "'");
+        }
+    }
+
+    @Override
+    public void process(String requestShard, String resourceType, String logicalId, long logicalResourceId, ParameterNameValue parameterNameValue, LocationParameter p) throws FHIRPersistenceException {
+        if (logger.isLoggable(Level.FINE)) {
+            logger.fine("process location parameter [" + requestShard + "] [" + resourceType + "] [" + logicalId + "] [" + logicalResourceId + "] [" + parameterNameValue.getParameterName() + "] ["
+                    + p.toString() + "]");
+        }
+
+        try {
+            ShardedPostgresParameterBatch dao = getParameterBatchDao(resourceType);
+            final Short shardKey = encodeShardKey(requestShard);
+            dao.addLocation(logicalResourceId, parameterNameValue.getParameterNameId(), p.getValueLatitude(), p.getValueLongitude(), p.getCompositeId(), shardKey);
+        } catch (SQLException x) {
+            throw new FHIRPersistenceException("Failed inserting location params for '" + resourceType + "'");
+        }
+    }
+
+    @Override
+    public void process(String requestShard, String resourceType, String logicalId, long logicalResourceId, ParameterNameValue parameterNameValue, DateParameter p) throws FHIRPersistenceException {
+        if (logger.isLoggable(Level.FINE)) {
+            logger.fine("process date parameter [" + requestShard + "] [" + resourceType + "] [" + logicalId + "] [" + logicalResourceId + "] [" + parameterNameValue.getParameterName() + "] ["
+                    + p.toString() + "]");
+        }
+
+        try {
+            ShardedPostgresParameterBatch dao = getParameterBatchDao(resourceType);
+            final Short shardKey = encodeShardKey(requestShard);
+            final Timestamp valueDateStart = Timestamp.from(p.getValueDateStart());
+            final Timestamp valueDateEnd = Timestamp.from(p.getValueDateEnd());
+            dao.addDate(logicalResourceId, parameterNameValue.getParameterNameId(), valueDateStart, valueDateEnd, p.getCompositeId(), shardKey);
+            if (p.isSystemParam()) {
+                systemDao.addDate(logicalResourceId, parameterNameValue.getParameterNameId(), valueDateStart, valueDateEnd, p.getCompositeId(), shardKey);
+            }
+        } catch (SQLException x) {
+            throw new FHIRPersistenceException("Failed inserting date params for '" + resourceType + "'");
+        }
+    }
+
+    @Override
+    public void process(String requestShard, String resourceType, String logicalId, long logicalResourceId, ParameterNameValue parameterNameValue, TokenParameter p,
+        CommonTokenValue commonTokenValue) throws FHIRPersistenceException {
+
+        if (logger.isLoggable(Level.FINE)) {
+            logger.fine("process token parameter [" + requestShard + "] [" + resourceType + "] [" + logicalId + "] [" + logicalResourceId + "] [" + parameterNameValue.getParameterName() + "] ["
+                    + p.toString() + "] [" + commonTokenValue.getCommonTokenValueId() + "]");
+        }
+
+        try {
+            ShardedPostgresParameterBatch dao = getParameterBatchDao(resourceType);
+            final Short shardKey = encodeShardKey(requestShard);
+            dao.addResourceTokenRef(logicalResourceId, parameterNameValue.getParameterNameId(), commonTokenValue.getCommonTokenValueId(), p.getRefVersionId(), p.getCompositeId(), shardKey);
+        } catch (SQLException x) {
+            throw new FHIRPersistenceException("Failed inserting token params for '" + resourceType + "'");
+        }
+    }
+
+    @Override
+    public void process(String requestShard, String resourceType, String logicalId, long logicalResourceId, ParameterNameValue parameterNameValue, TagParameter p,
+        CommonTokenValue commonTokenValue) throws FHIRPersistenceException {
+
+        if (logger.isLoggable(Level.FINE)) {
+            logger.fine("process tag parameter [" + requestShard + "] [" + resourceType + "] [" + logicalId + "] [" + logicalResourceId + "] [" + parameterNameValue.getParameterName() + "] ["
+                    + p.toString() + "] [" + commonTokenValue.getCommonTokenValueId() + "]");
+        }
+
+        try {
+            ShardedPostgresParameterBatch dao = getParameterBatchDao(resourceType);
+            final Short shardKey = encodeShardKey(requestShard);
+            dao.addTag(logicalResourceId, commonTokenValue.getCommonTokenValueId(), shardKey);
+            
+            if (p.isSystemParam()) {
+                systemDao.addTag(logicalResourceId, commonTokenValue.getCommonTokenValueId(), shardKey);
+            }
+        } catch (SQLException x) {
+            throw new FHIRPersistenceException("Failed inserting tag params for '" + resourceType + "'");
+        }
+    }
+
+    @Override
+    public void process(String requestShard, String resourceType, String logicalId, long logicalResourceId, ParameterNameValue parameterNameValue, ProfileParameter p,
+        CommonCanonicalValue commonCanonicalValue) throws FHIRPersistenceException {
+
+        if (logger.isLoggable(Level.FINE)) {
+            logger.fine("process profile parameter [" + requestShard + "] [" + resourceType + "] [" + logicalId + "] [" + logicalResourceId + "] [" + parameterNameValue.getParameterName() + "] ["
+                    + p.toString() + "] [" + commonCanonicalValue.getCanonicalId() + "]");
+        }
+
+        try {
+            ShardedPostgresParameterBatch dao = getParameterBatchDao(resourceType);
+            final Short shardKey = encodeShardKey(requestShard);
+            dao.addProfile(logicalResourceId, commonCanonicalValue.getCanonicalId(), p.getVersion(), p.getFragment(), shardKey);
+            if (p.isSystemParam()) {
+                systemDao.addProfile(logicalResourceId, commonCanonicalValue.getCanonicalId(), p.getVersion(), p.getFragment(), shardKey);
+            }
+        } catch (SQLException x) {
+            throw new FHIRPersistenceException("Failed inserting profile params for '" + resourceType + "'");
+        }
+    }
+
+    @Override
+    public void process(String requestShard, String resourceType, String logicalId, long logicalResourceId, ParameterNameValue parameterNameValue, SecurityParameter p,
+        CommonTokenValue commonTokenValue) throws FHIRPersistenceException {
+
+        if (logger.isLoggable(Level.FINE)) {
+            logger.fine("process security parameter [" + requestShard + "] [" + resourceType + "] [" + logicalId + "] [" + logicalResourceId + "] [" + parameterNameValue.getParameterName() + "] ["
+                    + p.toString() + "] [" + commonTokenValue.getCommonTokenValueId() + "]");
+        }
+
+        try {
+            ShardedPostgresParameterBatch dao = getParameterBatchDao(resourceType);
+            final Short shardKey = encodeShardKey(requestShard);
+            dao.addSecurity(logicalResourceId, commonTokenValue.getCommonTokenValueId(), shardKey);
+            
+            if (p.isSystemParam()) {
+                systemDao.addSecurity(logicalResourceId, commonTokenValue.getCommonTokenValueId(), shardKey);
+            }
+        } catch (SQLException x) {
+            throw new FHIRPersistenceException("Failed inserting security params for '" + resourceType + "'");
+        }
+    }
+
+    @Override
+    public void process(String requestShard, String resourceType, String logicalId, long logicalResourceId, ParameterNameValue parameterNameValue,
+        ReferenceParameter parameter, LogicalResourceIdentValue refLogicalResourceId) throws FHIRPersistenceException {
+        if (logger.isLoggable(Level.FINE)) {
+            logger.fine("process ref parameter [" + requestShard + "] [" + resourceType + "] [" + logicalId + "] [" + logicalResourceId + "] [" + parameterNameValue.getParameterName() + "] ["
+                    + parameter.toString() + "] [" + refLogicalResourceId.getLogicalResourceId() + "]");
+        }
+
+        try {
+            ShardedPostgresParameterBatch dao = getParameterBatchDao(resourceType);
+            final Short shardKey = encodeShardKey(requestShard);
+            dao.addReference(logicalResourceId, parameterNameValue.getParameterNameId(), refLogicalResourceId.getLogicalResourceId(), parameter.getRefVersionId(), shardKey);
+        } catch (SQLException x) {
+            throw new FHIRPersistenceException("Failed inserting ref param for '" + resourceType + "'");
+        }
+    }
+}
diff --git a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/sharded/ShardedPostgresMessageHandler.java b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/sharded/ShardedPostgresMessageHandler.java
new file mode 100644
index 00000000000..afc4755ab31
--- /dev/null
+++ b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/sharded/ShardedPostgresMessageHandler.java
@@ -0,0 +1,1089 @@
+/*
+ * (C) Copyright IBM Corp. 2022
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+ 
+package com.ibm.fhir.remote.index.sharded;
+
+import java.sql.CallableStatement;
+import java.sql.Connection;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Types;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+import java.util.stream.Collectors;
+
+import com.ibm.fhir.database.utils.api.IDatabaseTranslator;
+import com.ibm.fhir.database.utils.common.ResultSetReader;
+import com.ibm.fhir.database.utils.postgres.PostgresTranslator;
+import com.ibm.fhir.persistence.exception.FHIRPersistenceException;
+import com.ibm.fhir.persistence.index.DateParameter;
+import com.ibm.fhir.persistence.index.LocationParameter;
+import com.ibm.fhir.persistence.index.NumberParameter;
+import com.ibm.fhir.persistence.index.ProfileParameter;
+import com.ibm.fhir.persistence.index.QuantityParameter;
+import com.ibm.fhir.persistence.index.ReferenceParameter;
+import com.ibm.fhir.persistence.index.RemoteIndexMessage;
+import com.ibm.fhir.persistence.index.SearchParameterValue;
+import com.ibm.fhir.persistence.index.SecurityParameter;
+import com.ibm.fhir.persistence.index.StringParameter;
+import com.ibm.fhir.persistence.index.TagParameter;
+import com.ibm.fhir.persistence.index.TokenParameter;
+import com.ibm.fhir.remote.index.api.BatchParameterValue;
+import com.ibm.fhir.remote.index.api.IdentityCache;
+import com.ibm.fhir.remote.index.batch.BatchDateParameter;
+import com.ibm.fhir.remote.index.batch.BatchLocationParameter;
+import com.ibm.fhir.remote.index.batch.BatchNumberParameter;
+import com.ibm.fhir.remote.index.batch.BatchProfileParameter;
+import com.ibm.fhir.remote.index.batch.BatchQuantityParameter;
+import com.ibm.fhir.remote.index.batch.BatchReferenceParameter;
+import com.ibm.fhir.remote.index.batch.BatchSecurityParameter;
+import com.ibm.fhir.remote.index.batch.BatchStringParameter;
+import com.ibm.fhir.remote.index.batch.BatchTagParameter;
+import com.ibm.fhir.remote.index.batch.BatchTokenParameter;
+import com.ibm.fhir.remote.index.database.BaseMessageHandler;
+import com.ibm.fhir.remote.index.database.CodeSystemValue;
+import com.ibm.fhir.remote.index.database.CommonCanonicalValue;
+import com.ibm.fhir.remote.index.database.CommonCanonicalValueKey;
+import com.ibm.fhir.remote.index.database.CommonTokenValue;
+import com.ibm.fhir.remote.index.database.CommonTokenValueKey;
+import com.ibm.fhir.remote.index.database.LogicalResourceIdentKey;
+import com.ibm.fhir.remote.index.database.LogicalResourceIdentValue;
+import com.ibm.fhir.remote.index.database.LogicalResourceValue;
+import com.ibm.fhir.remote.index.database.ParameterNameValue;
+import com.ibm.fhir.remote.index.database.PlainMessageHandler;
+import com.ibm.fhir.remote.index.database.PreparedStatementWrapper;
+
+/**
+ * Loads search parameter values into the target FHIR schema on
+ * a PostgreSQL database.
+ * TODO refactor to try and share more processing with the {@link PlainMessageHandler}
+ */
+public class ShardedPostgresMessageHandler extends BaseMessageHandler {
+    private static final Logger logger = Logger.getLogger(ShardedPostgresMessageHandler.class.getName());
+
+    // the connection to use for the inserts
+    private final Connection connection;
+
+    // We're a PostgreSQL DAO, so we now which translator to use
+    private final IDatabaseTranslator translator = new PostgresTranslator();
+
+    // The FHIR data schema
+    private final String schemaName;
+
+    // the cache we use for various lookups
+    private final IdentityCache identityCache;
+
+    // All logical_resource_ident values we've seen
+    private final Map logicalResourceIdentMap = new HashMap<>();
+
+    // All parameter names we've seen (cleared if there's a rollback)
+    private final Map parameterNameMap = new HashMap<>();
+
+    // A map of code system name to the value holding its codeSystemId from the database
+    private final Map codeSystemValueMap = new HashMap<>();
+
+    // A map to support lookup of CommonTokenValue records by key
+    private final Map commonTokenValueMap = new HashMap<>();
+
+    // A map to support lookup of CommonCanonicalValue records by key
+    private final Map commonCanonicalValueMap = new HashMap<>();
+
+    // A list of all the logical_resource_ident values for which we don't yet know the logical_resource_id
+    private final List unresolvedLogicalResourceIdents = new ArrayList<>();
+
+    // All parameter names in the current transaction for which we don't yet know the parameter_name_id
+    private final List unresolvedParameterNames = new ArrayList<>();
+
+    // A list of all the CodeSystemValues for which we don't yet know the code_system_id
+    private final List unresolvedSystemValues = new ArrayList<>();
+
+    // A list of all the CommonTokenValues for which we don't yet know the common_token_value_id
+    private final List unresolvedTokenValues = new ArrayList<>();
+
+    // A list of all the CommonCanonicalValues for which we don't yet know the canonical_id
+    private final List unresolvedCanonicalValues = new ArrayList<>();
+    
+    // The processed values we've collected
+    private final List batchedParameterValues = new ArrayList<>();
+
+    // The processor used to process the batched parameter values after all the reference values are created
+    private final ShardedBatchParameterProcessor batchProcessor;
+
+    private final int maxCodeSystemsPerStatement = 512;
+    private final int maxCommonTokenValuesPerStatement = 256;
+    private final int maxCommonCanonicalValuesPerStatement = 256;
+    private boolean rollbackOnly;
+
+    /**
+     * Public constructor
+     * 
+     * @param instanceIdentifier
+     * @param connection
+     * @param schemaName
+     * @param cache
+     * @param maxReadyTimeMs
+     */
+    public ShardedPostgresMessageHandler(String instanceIdentifier, Connection connection, String schemaName, IdentityCache cache, long maxReadyTimeMs) {
+        super(instanceIdentifier, maxReadyTimeMs);
+        this.connection = connection;
+        this.schemaName = schemaName;
+        this.identityCache = cache;
+        this.batchProcessor = new ShardedBatchParameterProcessor(connection);
+    }
+
+    @Override
+    protected void startBatch() {
+        // always start with a clean slate
+        batchedParameterValues.clear();
+        unresolvedParameterNames.clear();
+        unresolvedSystemValues.clear();
+        unresolvedTokenValues.clear();
+        unresolvedCanonicalValues.clear();
+        batchProcessor.startBatch();
+    }
+
+    @Override
+    protected void setRollbackOnly() {
+        this.rollbackOnly = true;
+    }
+
+    @Override
+    public void close() {
+        try {
+            batchProcessor.close();
+        } catch (Throwable t) {
+            logger.log(Level.SEVERE, "close batchProcessor failed" , t);
+        }
+    }
+
+    @Override
+    protected void endTransaction() throws FHIRPersistenceException {
+        boolean committed = false;
+        try {
+            if (!this.rollbackOnly) {
+                logger.fine("Committing transaction");
+                connection.commit();
+                committed = true;
+
+                // any values from parameter_names, code_systems and common_token_values
+                // are now committed to the database, so we can publish their record ids
+                // to the shared cache which makes them accessible from other threads 
+                publishCachedValues();
+            } else {
+                // something went wrong...try to roll back the transaction before we close
+                // everything
+                try {
+                    connection.rollback();
+                } catch (SQLException x) {
+                    // It could very well be that we've lost touch with the database in which case
+                    // the rollback will also fail. Not much we can do, although we don't bother
+                    // with a stack trace here because it's just more noise for the log file, and
+                    // the exception that triggered the rollback is already going to be propagated
+                    // and logged.
+                    logger.severe("Rollback failed; reason=[" + x.getMessage() + "]");
+                }
+            }
+        } catch (SQLException x) {
+            throw new FHIRPersistenceException("commit failed", x);
+        } finally {
+            if (!committed) {
+                // The maps may contain ids that were not committed to the database so
+                // we should clean them out in case we decide to reuse this consumer
+                this.parameterNameMap.clear();
+                this.codeSystemValueMap.clear();
+                this.commonTokenValueMap.clear();
+                this.commonCanonicalValueMap.clear();
+            }
+        }
+    }
+
+    /**
+     * After the transaction has been committed, we can publish certain values to the
+     * shared identity caches
+     */
+    public void publishCachedValues() {
+        // all the unresolvedParameterNames should be resolved at this point
+        for (ParameterNameValue pnv: this.unresolvedParameterNames) {
+            identityCache.addParameterName(pnv.getParameterName(), pnv.getParameterNameId());
+        }
+    }
+
+    @Override
+    protected void pushBatch() throws FHIRPersistenceException {
+        // Push any data we've accumulated so far. This may occur
+        // if we cross a volume threshold, and will always occur as
+        // the last step before the current transaction is committed,
+        // Process the token values so that we can establish
+        // any entries we need for common_token_values
+        resolveParameterNames();
+        resolveCodeSystems();
+        resolveCommonTokenValues();
+        resolveCommonCanonicalValues();
+
+        // Now that all the lookup values should've been resolved, we can go ahead
+        // and push the parameters to the JDBC batch insert statements via the
+        // batchProcessor
+        for (BatchParameterValue v: this.batchedParameterValues) {
+            v.apply(batchProcessor);
+        }
+        batchProcessor.pushBatch();
+    }
+
+    /**
+     * Get the parameter name value for the given parameter value
+     * @param p
+     * @return
+     */
+    private ParameterNameValue getParameterNameId(SearchParameterValue p) throws FHIRPersistenceException {
+        if (logger.isLoggable(Level.FINEST)) {
+            logger.finest("get ParameterNameValue for [" + p.toString() + "]");
+        }
+        ParameterNameValue result = parameterNameMap.get(p.getName());
+        if (result == null) {
+            result = new ParameterNameValue(p.getName());
+            parameterNameMap.put(p.getName(), result);
+
+            // let's see if the id is available in the shared identity cache
+            Integer parameterNameId = identityCache.getParameterNameId(p.getName());
+            if (parameterNameId != null) {
+                result.setParameterNameId(parameterNameId);
+            } else {
+                // ids will be created later (so that we can process them in order)
+                unresolvedParameterNames.add(result);
+            }
+        }
+        return result;
+    }
+
+    private LogicalResourceIdentValue lookupLogicalResourceIdentValue(String resourceType, String logicalId) {
+        LogicalResourceIdentKey key = new LogicalResourceIdentKey(resourceType, logicalId);
+        LogicalResourceIdentValue result = this.logicalResourceIdentMap.get(key);
+        if (result == null) {
+            result = LogicalResourceIdentValue.builder()
+                    .withResourceType(resourceType)
+                    .withLogicalId(logicalId)
+                    .build();
+            this.logicalResourceIdentMap.put(key, result);
+            this.unresolvedLogicalResourceIdents.add(result);
+        }
+        return result;
+    }
+    
+    @Override
+    protected void process(String tenantId, String requestShard, String resourceType, String logicalId, long logicalResourceId, StringParameter p) throws FHIRPersistenceException {
+        ParameterNameValue parameterNameValue = getParameterNameId(p);
+        this.batchedParameterValues.add(new BatchStringParameter(requestShard, resourceType, logicalId, logicalResourceId, parameterNameValue, p));
+    }
+
+    @Override
+    protected void process(String tenantId, String requestShard, String resourceType, String logicalId, long logicalResourceId, LocationParameter p) throws FHIRPersistenceException {
+        ParameterNameValue parameterNameValue = getParameterNameId(p);
+        this.batchedParameterValues.add(new BatchLocationParameter(requestShard, resourceType, logicalId, logicalResourceId, parameterNameValue, p));
+    }
+
+    @Override
+    protected void process(String tenantId, String requestShard, String resourceType, String logicalId, long logicalResourceId, TokenParameter p) throws FHIRPersistenceException {
+        Short shardKey = batchProcessor.encodeShardKey(requestShard);
+        CommonTokenValue ctv = lookupCommonTokenValue(shardKey, p.getValueSystem(), p.getValueCode());
+        ParameterNameValue parameterNameValue = getParameterNameId(p);
+        this.batchedParameterValues.add(new BatchTokenParameter(requestShard, resourceType, logicalId, logicalResourceId, parameterNameValue, p, ctv));
+    }
+
+    @Override
+    protected void process(String tenantId, String requestShard, String resourceType, String logicalId, long logicalResourceId, TagParameter p) throws FHIRPersistenceException {
+        Short shardKey = batchProcessor.encodeShardKey(requestShard);
+        CommonTokenValue ctv = lookupCommonTokenValue(shardKey, p.getValueSystem(), p.getValueCode());
+        ParameterNameValue parameterNameValue = getParameterNameId(p);
+        this.batchedParameterValues.add(new BatchTagParameter(requestShard, resourceType, logicalId, logicalResourceId, parameterNameValue, p, ctv));
+    }
+
+    @Override
+    protected void process(String tenantId, String requestShard, String resourceType, String logicalId, long logicalResourceId, SecurityParameter p) throws FHIRPersistenceException {
+        Short shardKey = batchProcessor.encodeShardKey(requestShard);
+        CommonTokenValue ctv = lookupCommonTokenValue(shardKey, p.getValueSystem(), p.getValueCode());
+        ParameterNameValue parameterNameValue = getParameterNameId(p);
+        this.batchedParameterValues.add(new BatchSecurityParameter(requestShard, resourceType, logicalId, logicalResourceId, parameterNameValue, p, ctv));
+    }
+
+    @Override
+    protected void process(String tenantId, String requestShard, String resourceType, String logicalId, long logicalResourceId, ProfileParameter p) throws FHIRPersistenceException {
+        Short shardKey = batchProcessor.encodeShardKey(requestShard);
+        CommonCanonicalValue ctv = lookupCommonCanonicalValue(shardKey, p.getUrl());
+        ParameterNameValue parameterNameValue = getParameterNameId(p);
+        this.batchedParameterValues.add(new BatchProfileParameter(requestShard, resourceType, logicalId, logicalResourceId, parameterNameValue, p, ctv));
+    }
+
+    @Override
+    protected void process(String tenantId, String requestShard, String resourceType, String logicalId, long logicalResourceId, QuantityParameter p) throws FHIRPersistenceException {
+        ParameterNameValue parameterNameValue = getParameterNameId(p);
+        CodeSystemValue csv = lookupCodeSystemValue(p.getValueSystem());
+        this.batchedParameterValues.add(new BatchQuantityParameter(requestShard, resourceType, logicalId, logicalResourceId, parameterNameValue, p, csv));
+    }
+
+    @Override
+    protected void process(String tenantId, String requestShard, String resourceType, String logicalId, long logicalResourceId, NumberParameter p) throws FHIRPersistenceException {
+        ParameterNameValue parameterNameValue = getParameterNameId(p);
+        this.batchedParameterValues.add(new BatchNumberParameter(requestShard, resourceType, logicalId, logicalResourceId, parameterNameValue, p));
+    }
+
+    @Override
+    protected void process(String tenantId, String requestShard, String resourceType, String logicalId, long logicalResourceId, DateParameter p) throws FHIRPersistenceException {
+        ParameterNameValue parameterNameValue = getParameterNameId(p);
+        this.batchedParameterValues.add(new BatchDateParameter(requestShard, resourceType, logicalId, logicalResourceId, parameterNameValue, p));
+    }
+
+    @Override
+    protected void process(String tenantId, String requestShard, String resourceType, String logicalId, long logicalResourceId, ReferenceParameter p) throws FHIRPersistenceException {
+        ParameterNameValue parameterNameValue = getParameterNameId(p);
+        LogicalResourceIdentValue lriv = lookupLogicalResourceIdentValue(p.getResourceType(), p.getLogicalId());
+        this.batchedParameterValues.add(new BatchReferenceParameter(requestShard, resourceType, logicalId, logicalResourceId, parameterNameValue, p, lriv));
+    }
+
+    /**
+     * Get the CodeSystemValue we've assigned for the given codeSystem value. This
+     * may not yet have the actual code_system_id from the database yet - any values
+     * we don't have will be assigned in a later phase (so we can do things neatly
+     * in bulk).
+     * @param codeSystem
+     * @return
+     */
+    private CodeSystemValue lookupCodeSystemValue(String codeSystem) {
+        CodeSystemValue result = this.codeSystemValueMap.get(codeSystem);
+        if (result == null) {
+            result = new CodeSystemValue(codeSystem);
+            this.codeSystemValueMap.put(codeSystem, result);
+
+            // Take this opportunity to see if we have a cached value for this codeSystem
+            Integer codeSystemId = identityCache.getCodeSystemId(codeSystem);
+            if (codeSystemId != null) {
+                result.setCodeSystemId(codeSystemId);
+            } else {
+                // Stash for later resolution
+                this.unresolvedSystemValues.add(result);
+            }
+        }
+        return result;
+    }
+
+    /**
+     * Get the CommonTokenValue we've assigned for the given (codeSystem, tokenValue) tuple.
+     * The returned value may not yet have the actual common_token_value_id yet - we fetch
+     * these values later and create new database records as necessary.
+     * @param codeSystem
+     * @param tokenValue
+     * @return
+     */
+    private CommonTokenValue lookupCommonTokenValue(short shardKey, String codeSystem, String tokenValue) {
+        CommonTokenValueKey key = new CommonTokenValueKey(shardKey, codeSystem, tokenValue);
+        CommonTokenValue result = this.commonTokenValueMap.get(key);
+        if (result == null) {
+            CodeSystemValue csv = lookupCodeSystemValue(codeSystem);
+            result = new CommonTokenValue(shardKey, csv, tokenValue);
+            this.commonTokenValueMap.put(key, result);
+
+            // Take this opportunity to see if we have a cached value for this common token value
+            Long commonTokenValueId = identityCache.getCommonTokenValueId(shardKey, codeSystem, tokenValue);
+            if (commonTokenValueId != null) {
+                result.setCommonTokenValueId(commonTokenValueId);
+            } else {
+                this.unresolvedTokenValues.add(result);
+            }
+        }
+        return result;
+    }
+
+    private CommonCanonicalValue lookupCommonCanonicalValue(short shardKey, String url) {
+        CommonCanonicalValueKey key = new CommonCanonicalValueKey(shardKey, url);
+        CommonCanonicalValue result = this.commonCanonicalValueMap.get(key);
+        if (result == null) {
+            result = new CommonCanonicalValue(shardKey, url);
+            this.commonCanonicalValueMap.put(key, result);
+
+            // Take this opportunity to see if we have a cached value for this common token value
+            Long canonicalId = identityCache.getCommonCanonicalValueId(shardKey, url);
+            if (canonicalId != null) {
+                result.setCanonicalId(canonicalId);
+            } else {
+                this.unresolvedCanonicalValues.add(result);
+            }
+        }
+        return result;
+    }
+
+    /**
+     * Make sure we have values for all the code_systems we have collected
+     * in the current
+     * batch
+     * @throws FHIRPersistenceException
+     */
+    private void resolveCodeSystems() throws FHIRPersistenceException {
+        // identify which values aren't yet in the database
+        List missing = fetchCodeSystemIds(unresolvedSystemValues);
+
+        if (!missing.isEmpty()) {
+            addMissingCodeSystems(missing);
+        }
+
+        // All the previously missing values should now be in the database. We need to fetch them again,
+        // possibly having to use multiple queries
+        List bad = fetchCodeSystemIds(missing);
+
+        if (!bad.isEmpty()) {
+            // shouldn't happend, but let's protected against it anyway
+            throw new FHIRPersistenceException("Failed to create all code system values");
+        }
+    }
+
+    /**
+     * Build and prepare a statement to fetch the code_system_id and code_system_name
+     * from the code_systems table for all the given (unresolved) code system values
+     * @param values
+     * @return
+     * @throws SQLException
+     */
+    private PreparedStatement buildCodeSystemSelectStatement(List values) throws SQLException {
+        StringBuilder query = new StringBuilder();
+        query.append("SELECT code_system_id, code_system_name FROM code_systems WHERE code_system_name IN (");
+        for (int i=0; i 0) {
+                query.append(",");
+            }
+            query.append("?");
+        }
+        query.append(")");
+        PreparedStatement ps = connection.prepareStatement(query.toString());
+        // bind the parameter values
+        int param = 1;
+        for (CodeSystemValue csv: values) {
+            ps.setString(param++, csv.getCodeSystem());
+        }
+        return ps;
+    }
+
+    /**
+     * These code systems weren't found in the database, so we need to try and add them.
+     * We have to deal with concurrency here - there's a chance another thread could also
+     * be trying to add them. To avoid deadlocks, it's important to do any inserts in a
+     * consistent order. At the end, we should be able to read back values for each entry
+     * @param missing
+     */
+    private void addMissingCodeSystems(List missing) throws FHIRPersistenceException {
+        List values = missing.stream().map(csv -> csv.getCodeSystem()).collect(Collectors.toList());
+        // Sort the code system values first to help avoid deadlocks
+        Collections.sort(values); // natural ordering for String is fine here
+
+        final String nextVal = translator.nextValue(schemaName, "fhir_ref_sequence");
+        StringBuilder insert = new StringBuilder();
+        insert.append("INSERT INTO code_systems (code_system_id, code_system_name) VALUES (");
+        insert.append(nextVal); // next sequence value
+        insert.append(",?) ON CONFLICT DO NOTHING");
+
+        try (PreparedStatement ps = connection.prepareStatement(insert.toString())) {
+            int count = 0;
+            for (String codeSystem: values) {
+                ps.setString(1, codeSystem);
+                ps.addBatch();
+                if (++count == this.maxCodeSystemsPerStatement) {
+                    // not too many statements in a single batch
+                    ps.executeBatch();
+                    count = 0;
+                }
+            }
+            if (count > 0) {
+                // final batch
+                ps.executeBatch();
+            }
+        } catch (SQLException x) {
+            logger.log(Level.SEVERE, "code systems fetch failed: " + insert.toString(), x);
+            throw new FHIRPersistenceException("code systems fetch failed");
+        }
+    }
+
+    private List fetchCodeSystemIds(List unresolved) throws FHIRPersistenceException {
+        // track which values aren't yet in the database
+        List missing = new ArrayList<>();
+
+        int offset = 0;
+        while (offset < unresolved.size()) {
+            int remaining = unresolved.size() - offset;
+            int subSize = Math.min(remaining, this.maxCodeSystemsPerStatement);
+            List sub = unresolved.subList(offset, offset+subSize); // remember toIndex is exclusive
+            offset += subSize; // set up for the next iteration
+            try (PreparedStatement ps = buildCodeSystemSelectStatement(sub)) {
+                ResultSet rs = ps.executeQuery();
+                // We can't rely on the order of result rows matching the order of the in-list,
+                // so we have to go back to our map to look up each CodeSystemValue
+                int resultCount = 0;
+                while (rs.next()) {
+                    resultCount++;
+                    CodeSystemValue csv = this.codeSystemValueMap.get(rs.getString(2));
+                    if (csv != null) {
+                        csv.setCodeSystemId(rs.getInt(1));
+                    } else {
+                        // can't really happen, but be defensive
+                        throw new FHIRPersistenceException("code systems query returned an unexpected value");
+                    }
+                }
+
+                // Most of the time we'll get everything, so we can bypass the check for
+                // missing values
+                if (resultCount == 0) {
+                    // 100% miss
+                    missing.addAll(sub);
+                } else if (resultCount < subSize) {
+                    // need to scan the sub list and see which values we don't yet have ids for
+                    for (CodeSystemValue csv: sub) {
+                        if (csv.getCodeSystemId() == null) {
+                            missing.add(csv);
+                        }
+                    }
+                }
+            } catch (SQLException x) {
+                logger.log(Level.SEVERE, "code systems fetch failed", x);
+                throw new FHIRPersistenceException("code systems fetch failed");
+            }
+        }
+
+        // Return the list of CodeSystemValues which don't yet have a database entry
+        return missing;
+    }
+
+    /**
+     * Make sure we have values for all the common_token_value records we have collected
+     * in the current batch
+     * @throws FHIRPersistenceException
+     */
+    private void resolveCommonTokenValues() throws FHIRPersistenceException {
+        // identify which values aren't yet in the database
+        List missing = fetchCommonTokenValueIds(unresolvedTokenValues);
+
+        if (!missing.isEmpty()) {
+            // Sort first to minimize deadlocks
+            Collections.sort(missing, (a,b) -> {
+                int result = a.getTokenValue().compareTo(b.getTokenValue());
+                if (result == 0) {
+                    result = Integer.compare(a.getCodeSystemValue().getCodeSystemId(), b.getCodeSystemValue().getCodeSystemId());
+                    if (result == 0) {
+                        result = Short.compare(a.getShardKey(), b.getShardKey());
+                    }
+                }
+                return result;
+            });
+            addMissingCommonTokenValues(missing);
+        }
+
+        // All the previously missing values should now be in the database. We need to fetch them again,
+        // possibly having to use multiple queries
+        List bad = fetchCommonTokenValueIds(missing);
+
+        if (!bad.isEmpty()) {
+            // shouldn't happend, but let's protected against it anyway
+            throw new FHIRPersistenceException("Failed to create all common token values");
+        }
+    }
+
+    /**
+     * Build and prepare a statement to fetch the common_token_value records
+     * for all the given (unresolved) code system values
+     * @param values
+     * @return SELECT shard_key, code_system, token_value, common_token_value_id
+     * @throws SQLException
+     */
+    private PreparedStatementWrapper buildCommonTokenValueSelectStatement(List values) throws SQLException {
+        StringBuilder query = new StringBuilder();
+        // need the code_system name - so we join back to the code_systems table as well
+        query.append("SELECT c.shard_key, cs.code_system_name, c.token_value, c.common_token_value_id ");
+        query.append("  FROM common_token_values c");
+        query.append("  JOIN code_systems cs ON (cs.code_system_id = c.code_system_id)");
+        query.append("  JOIN (VALUES ");
+
+        // Create a (codeSystem, shardKey, tokenValue) tuple for each of the CommonTokenValue records
+        boolean first = true;
+        for (CommonTokenValue ctv: values) {
+            if (first) {
+                first = false;
+            } else {
+                query.append(",");
+            }
+            query.append("(");
+            query.append(ctv.getCodeSystemValue().getCodeSystemId()); // literal for code_system_id
+            query.append(",").append(ctv.getShardKey()); // literal for shard_key
+            query.append(",?)"); // bind variable for the token-value
+        }
+        query.append(") AS v(code_system_id, shard_key, token_value) ");
+        query.append(" ON (c.code_system_id = v.code_system_id AND c.token_value = v.token_value AND c.shard_key = v.shard_key)");
+
+        // Create the prepared statement and bind the values
+        final String statementText = query.toString();
+        PreparedStatement ps = connection.prepareStatement(statementText);
+
+        // bind the parameter values
+        int param = 1;
+        for (CommonTokenValue ctv: values) {
+            ps.setString(param++, ctv.getTokenValue());
+        }
+        return new PreparedStatementWrapper(statementText, ps);
+    }
+    
+    private List fetchCommonTokenValueIds(List unresolved) throws FHIRPersistenceException {
+        // track which values aren't yet in the database
+        List missing = new ArrayList<>();
+
+        int offset = 0;
+        while (offset < unresolved.size()) {
+            int remaining = unresolved.size() - offset;
+            int subSize = Math.min(remaining, this.maxCommonTokenValuesPerStatement);
+            List sub = unresolved.subList(offset, offset+subSize); // remember toIndex is exclusive
+            offset += subSize; // set up for the next iteration
+            String sql = null; // the SQL text for logging when there's an error
+            try (PreparedStatementWrapper ps = buildCommonTokenValueSelectStatement(sub)) {
+                sql = ps.getStatementText();
+                ResultSet rs = ps.executeQuery();
+                // We can't rely on the order of result rows matching the order of the in-list,
+                // so we have to go back to our map to look up each CodeSystemValue
+                int resultCount = 0;
+                while (rs.next()) {
+                    resultCount++;
+                    CommonTokenValueKey key = new CommonTokenValueKey(rs.getShort(1), rs.getString(2), rs.getString(3));
+                    CommonTokenValue ctv = this.commonTokenValueMap.get(key);
+                    if (ctv != null) {
+                        ctv.setCommonTokenValueId(rs.getLong(4));
+                    } else {
+                        // can't really happen, but be defensive
+                        throw new FHIRPersistenceException("common token values query returned an unexpected value");
+                    }
+                }
+
+                // Optimize the check for missing values
+                if (resultCount == 0) {
+                    // 100% miss
+                    missing.addAll(sub);
+                } else if (resultCount < subSize) {
+                    // need to scan the sub list and see which values we don't yet have ids for
+                    for (CommonTokenValue ctv: sub) {
+                        if (ctv.getCommonTokenValueId() == null) {
+                            missing.add(ctv);
+                        }
+                    }
+                }
+            } catch (SQLException x) {
+                logger.log(Level.SEVERE, "common token values fetch failed. SQL=[" + sql + "]", x);
+                throw new FHIRPersistenceException("common token values fetch failed");
+            }
+        }
+
+        // Return the list of CodeSystemValues which don't yet have a database entry
+        return missing;
+    }
+
+    /**
+     * Add the values we think are missing from the database. The given list should be
+     * sorted to reduce deadlocks
+     * @param missing
+     * @throws FHIRPersistenceException
+     */
+    private void addMissingCommonTokenValues(List missing) throws FHIRPersistenceException {
+
+        final String nextVal = translator.nextValue(schemaName, "fhir_sequence");
+        StringBuilder insert = new StringBuilder();
+        insert.append("INSERT INTO common_token_values (shard_key, code_system_id, token_value, common_token_value_id) ");
+        insert.append(" OVERRIDING SYSTEM VALUE "); // we want to use our sequence number
+        insert.append("     VALUES (?,?,?,");
+        insert.append(nextVal); // next sequence value
+        insert.append(") ON CONFLICT DO NOTHING");
+
+        try (PreparedStatement ps = connection.prepareStatement(insert.toString())) {
+            int count = 0;
+            for (CommonTokenValue ctv: missing) {
+                ps.setShort(1, ctv.getShardKey());
+                ps.setInt(2, ctv.getCodeSystemValue().getCodeSystemId());
+                ps.setString(3, ctv.getTokenValue());
+                ps.addBatch();
+                if (++count == this.maxCommonTokenValuesPerStatement) {
+                    // not too many statements in a single batch
+                    ps.executeBatch();
+                    count = 0;
+                }
+            }
+            if (count > 0) {
+                // final batch
+                ps.executeBatch();
+            }
+        } catch (SQLException x) {
+            logger.log(Level.SEVERE, "failed: " + insert.toString(), x);
+            throw new FHIRPersistenceException("failed inserting new common token values");
+        }
+    }
+
+    /**
+     * Make sure we have values for all the common_canonical_value records we have collected
+     * in the current batch
+     * @throws FHIRPersistenceException
+     */
+    private void resolveCommonCanonicalValues() throws FHIRPersistenceException {
+        // identify which values aren't yet in the database
+        List missing = fetchCanonicalIds(unresolvedCanonicalValues);
+
+        if (!missing.isEmpty()) {
+            // Sort on (url, shard_key) to minimize deadlocks
+            Collections.sort(missing, (a,b) -> {
+                int result = a.getUrl().compareTo(b.getUrl());
+                if (result == 0) {
+                    result = Short.compare(a.getShardKey(), b.getShardKey());
+                }
+                return result;
+            });
+            addMissingCommonCanonicalValues(missing);
+        }
+
+        // All the previously missing values should now be in the database. We need to fetch them again,
+        // possibly having to use multiple queries
+        List bad = fetchCanonicalIds(missing);
+
+        if (!bad.isEmpty()) {
+            // shouldn't happen, but let's protected against it anyway
+            throw new FHIRPersistenceException("Failed to create all canonical values");
+        }
+    }
+
+    private List fetchCanonicalIds(List unresolved) throws FHIRPersistenceException {
+        // track which values aren't yet in the database
+        List missing = new ArrayList<>();
+
+        int offset = 0;
+        while (offset < unresolved.size()) {
+            int remaining = unresolved.size() - offset;
+            int subSize = Math.min(remaining, this.maxCommonCanonicalValuesPerStatement);
+            List sub = unresolved.subList(offset, offset+subSize); // remember toIndex is exclusive
+            offset += subSize; // set up for the next iteration
+            String sql = null; // the SQL text for logging when there's an error
+            try (PreparedStatementWrapper ps = buildCommonCanonicalValueSelectStatement(sub)) {
+                sql = ps.getStatementText();
+                ResultSet rs = ps.executeQuery();
+                // We can't rely on the order of result rows matching the order of the in-list,
+                // so we have to go back to our map to look up each CodeSystemValue
+                int resultCount = 0;
+                while (rs.next()) {
+                    resultCount++;
+                    CommonCanonicalValueKey key = new CommonCanonicalValueKey(rs.getShort(1), rs.getString(2));
+                    CommonCanonicalValue ctv = this.commonCanonicalValueMap.get(key);
+                    if (ctv != null) {
+                        ctv.setCanonicalId(rs.getLong(3));
+                    } else {
+                        // can't really happen, but be defensive
+                        throw new FHIRPersistenceException("common canonical values query returned an unexpected value");
+                    }
+                }
+
+                // Optimize the check for missing values
+                if (resultCount == 0) {
+                    // 100% miss
+                    missing.addAll(sub);
+                } else if (resultCount < subSize) {
+                    // need to scan the sub list and see which values we don't yet have ids for
+                    for (CommonCanonicalValue ctv: sub) {
+                        if (ctv.getCanonicalId() == null) {
+                            missing.add(ctv);
+                        }
+                    }
+                }
+            } catch (SQLException x) {
+                logger.log(Level.SEVERE, "common canonical values fetch failed. SQL=[" + sql + "]", x);
+                throw new FHIRPersistenceException("common canonical values fetch failed");
+            }
+        }
+
+        // Return the list of CodeSystemValues which don't yet have a database entry
+        return missing;
+    }
+
+    /**
+     * Build and prepare a statement to fetch the common_token_value records
+     * for all the given (unresolved) code system values
+     * @param values
+     * @return SELECT shard_key, code_system, token_value, common_token_value_id
+     * @throws SQLException
+     */
+    private PreparedStatementWrapper buildCommonCanonicalValueSelectStatement(List values) throws SQLException {
+        StringBuilder query = new StringBuilder();
+        query.append("SELECT c.shard_key, c.url, c.canonical_id ");
+        query.append("  FROM common_canonical_values c ");
+        query.append("  JOIN (VALUES ");
+
+        // Create a (shardKey, url) tuple for each of the CommonCanonicalValue records
+        boolean first = true;
+        for (CommonCanonicalValue ctv: values) {
+            if (first) {
+                first = false;
+            } else {
+                query.append(",");
+            }
+            query.append("(");
+            query.append(ctv.getShardKey()); // literal for shard_key
+            query.append(",?)"); // bind variable for the uri
+        }
+        query.append(") AS v(shard_key, url) ");
+        query.append(" ON (c.url = v.url AND c.shard_key = v.shard_key)");
+
+        // Create the prepared statement and bind the values
+        final String statementText = query.toString();
+        logger.finer(() -> "fetch common canonical values [" + statementText + "]");
+        PreparedStatement ps = connection.prepareStatement(statementText);
+
+        // bind the parameter values
+        int param = 1;
+        for (CommonCanonicalValue ctv: values) {
+            ps.setString(param++, ctv.getUrl());
+        }
+        return new PreparedStatementWrapper(statementText, ps);
+    }
+
+    /**
+     * Add the values we think are missing from the database. The given list should be
+     * sorted to reduce deadlocks
+     * @param missing
+     * @throws FHIRPersistenceException
+     */
+    private void addMissingCommonCanonicalValues(List missing) throws FHIRPersistenceException {
+
+        final String nextVal = translator.nextValue(schemaName, "fhir_sequence");
+        StringBuilder insert = new StringBuilder();
+        insert.append("INSERT INTO common_canonical_values (shard_key, url, canonical_id) ");
+        insert.append(" OVERRIDING SYSTEM VALUE "); // we want to use our sequence number
+        insert.append("     VALUES (?,?,");
+        insert.append(nextVal); // next sequence value
+        insert.append(") ON CONFLICT DO NOTHING");
+
+        final String DML = insert.toString();
+        if (logger.isLoggable(Level.FINE)) {
+            logger.fine("addMissingCanonicalIds: " + DML);
+        }
+        try (PreparedStatement ps = connection.prepareStatement(DML)) {
+            int count = 0;
+            for (CommonCanonicalValue ctv: missing) {
+                logger.finest(() -> "Adding canonical value [" + ctv.toString() + "]");
+                ps.setShort(1, ctv.getShardKey());
+                ps.setString(2, ctv.getUrl());
+                ps.addBatch();
+                if (++count == this.maxCommonCanonicalValuesPerStatement) {
+                    // not too many statements in a single batch
+                    ps.executeBatch();
+                    count = 0;
+                }
+            }
+            if (count > 0) {
+                // final batch
+                ps.executeBatch();
+            }
+        } catch (SQLException x) {
+            logger.log(Level.SEVERE, "failed: " + insert.toString(), x);
+            throw new FHIRPersistenceException("failed inserting new common canonical values");
+        }
+    }
+
+    /**
+     * Make sure all the parameter names we've seen in the batch exist
+     * in the database and have ids.
+     * @throws FHIRPersistenceException
+     */
+    private void resolveParameterNames() throws FHIRPersistenceException {
+        // We expect parameter names to have a very high cache hit rate and
+        // so we simplify processing by simply iterating one-by-one for the
+        // values we still need to resolve. The most important point here is
+        // to do this in a sorted order to avoid deadlock issues because this
+        // could be happening across multiple consumer threads at the same time.
+        Collections.sort(this.unresolvedParameterNames, (a,b) -> {
+            return a.getParameterName().compareTo(b.getParameterName());
+        });
+
+        try {
+            for (ParameterNameValue pnv: this.unresolvedParameterNames) {
+                Integer parameterNameId = getParameterNameIdFromDatabase(pnv.getParameterName());
+                if (parameterNameId == null) {
+                    parameterNameId = createParameterName(pnv.getParameterName());
+                }
+                pnv.setParameterNameId(parameterNameId);
+            }
+        } catch (SQLException x) {
+            throw new FHIRPersistenceException("error resolving parameter names", x);
+        }
+    }
+
+    private Integer getParameterNameIdFromDatabase(String parameterName) throws SQLException {
+        String SQL = "SELECT parameter_name_id FROM parameter_names WHERE parameter_name = ?";
+        try (PreparedStatement ps = connection.prepareStatement(SQL)) {
+            ps.setString(1, parameterName);
+            ResultSet rs = ps.executeQuery();
+            if (rs.next()) {
+                return rs.getInt(1);
+            }
+        }
+
+        // no entry in parameter_names
+        return null;
+    }
+
+    /**
+     * Create the parameter name using the stored procedure which handles any concurrency
+     * issue we may have
+     * @param parameterName
+     * @return
+     */
+    private Integer createParameterName(String parameterName) throws SQLException {
+        final String CALL = "{CALL " + schemaName + ".add_parameter_name(?, ?)}";
+        Integer parameterNameId;
+        try (CallableStatement stmt = connection.prepareCall(CALL)) {
+            stmt.setString(1, parameterName);
+            stmt.registerOutParameter(2, Types.INTEGER);
+            stmt.execute();
+            parameterNameId = stmt.getInt(2);
+        }
+
+        return parameterNameId;
+    }
+
+    @Override
+    protected void resetBatch() {
+        // Called when a transaction has been rolled back because of a deadlock
+        // or other retryable error and we want to try and process the batch again
+        batchProcessor.reset();
+    }
+
+    /**
+     * Build the check ready query
+     * @param messagesByResourceType
+     * @return
+     */
+    private String buildCheckReadyQuery(Map> messagesByResourceType) {
+        // The trouble here is that we'll end up with a unique query for every single
+        // batch of messages we process (which the database then need to parse etc).
+        // This may introduce scaling issues, in which case we should consider 
+        // individual queries for each resource type using bind variables, perhaps
+        // going so far as using multiple statements with a power-of-2 number of bind
+        // variables. But JDBC doesn't support batching of select statements, so
+        // the alternative there would be to insert-as-select into a global temp table
+        // and then simply select from that. Fairly straightforward, but a lot more
+        // work so only worth doing if we identify contention here.
+
+        StringBuilder select = new StringBuilder();
+        // SELECT lr.shard_key, lr.logical_resource_id, lr.resource_type, lr.logical_id, xlr.version_id, lr.last_updated, lr.parameter_hash
+        //   FROM logical_resources AS lr,
+        //        patient_logical_resources AS xlr
+        //  WHERE lr.logical_resource_id = xlr.logical_resource_id
+        //    AND xlr.logical_resource_id IN (1,2,3,4)
+        //  UNION ALL
+        // SELECT lr.shard_key, lr.logical_resource_id, lr.resource_type, lr.logical_id, xlr.version_id, lr.last_updated, lr.parameter_hash
+        //   FROM logical_resources AS lr,
+        //        observation_logical_resources AS xlr
+        //  WHERE lr.logical_resource_id = xlr.logical_resource_id
+        //    AND xlr.logical_resource_id IN (5,6,7)
+        boolean first = true;
+        for (Map.Entry> entry: messagesByResourceType.entrySet()) {
+            final String resourceType = entry.getKey();
+            final List messages = entry.getValue();
+            final String inlist = messages.stream().map(m -> Long.toString(m.getData().getLogicalResourceId())).collect(Collectors.joining(","));
+            if (first) {
+                first = false;
+            } else {
+                select.append(" UNION ALL ");
+            }
+            select.append(" SELECT lr.shard_key, lr.logical_resource_id, '" + resourceType + "' AS resource_type, lr.logical_id, xlr.version_id, lr.last_updated, lr.parameter_hash ");
+            select.append("   FROM logical_resources AS lr, ");
+            select.append(resourceType).append("_logical_resources AS xlr ");
+            select.append("  WHERE lr.logical_resource_id = xlr.logical_resource_id ");
+            select.append("    AND xlr.logical_resource_id IN (").append(inlist).append(")");
+        }
+        
+        return select.toString();
+    }
+
+    @Override
+    protected void checkReady(List messages, List okToProcess, List notReady) throws FHIRPersistenceException {
+        // Get a list of all the resources for which we can see the current logical resource data.
+        // If the resource doesn't yet exist or its version meta doesn't the message
+        // then we add to the notReady list. If the resource version meta already
+        // exceeds the message, then we'll skip processing altogether because it
+        // means that there should be another message in the queue with more
+        // up-to-date parameters
+        Map messageMap = new HashMap<>();
+        Map> messagesByResourceType = new HashMap<>();
+        for (RemoteIndexMessage msg: messages) {
+            Long logicalResourceId = msg.getData().getLogicalResourceId();
+            messageMap.put(logicalResourceId, msg);
+
+            // split out the messages per resource type because we need to read from xx_logical_resources
+            List values = messagesByResourceType.computeIfAbsent(msg.getData().getResourceType(), k -> new ArrayList<>());
+            values.add(msg);
+        }
+
+        Set found = new HashSet<>();
+        final String checkReadyQuery = buildCheckReadyQuery(messagesByResourceType);
+        logger.fine(() -> "check ready query: " + checkReadyQuery);
+        try (PreparedStatement ps = connection.prepareStatement(checkReadyQuery)) {
+            ResultSet rs = ps.executeQuery();
+            // wrap the ResultSet in a reader for easier consumption
+            ResultSetReader rsReader = new ResultSetReader(rs);
+            while (rsReader.next()) {
+                LogicalResourceValue lrv = LogicalResourceValue.builder()
+                        .withShardKey(rsReader.getShort())
+                        .withLogicalResourceId(rsReader.getLong())
+                        .withResourceType(rsReader.getString())
+                        .withLogicalId(rsReader.getString())
+                        .withVersionId(rsReader.getInt())
+                        .withLastUpdated(rsReader.getTimestamp())
+                        .withParameterHash(rsReader.getString())
+                        .build();
+                RemoteIndexMessage m = messageMap.get(lrv.getLogicalResourceId());
+                if (m == null) {
+                    throw new IllegalStateException("query returned a logical resource which we didn't request");
+                }
+
+                // Check the values from the database to see if they match
+                // the information in the message.
+                if (m.getData().getVersionId() == lrv.getVersionId()) {
+                    // only process this message if the parameter hash and lastUpdated
+                    // times match - which is a good check that we're storing parameters
+                    // from the correct transaction. If these don't match, we can simply
+                    // say we found the data but don't need to process the message.
+                    final String lastUpdated = lrv.getLastUpdated().toString();
+                    if (lrv.getParameterHash().equals(m.getData().getParameterHash()) 
+                            && lastUpdated.equals(m.getData().getLastUpdated())) {
+                        okToProcess.add(m);
+                    }
+                    found.add(lrv.getLogicalResourceId()); // won't be marked as missing
+                } else if (m.getData().getVersionId() > lrv.getVersionId()) {
+                    // we can skip processing this record because the database has already
+                    // been updated with a newer version. Identify the record as having been
+                    // found so we don't keep waiting for it
+                    found.add(lrv.getLogicalResourceId());
+                }
+                // if the version in the database is prior to version in the message we
+                // received it means that the server transaction hasn't been committed...
+                // so we have to wait just as though it were missing altogether
+            }
+        } catch (SQLException x) {
+            logger.log(Level.SEVERE, "prepare failed: " + checkReadyQuery, x);
+            throw new FHIRPersistenceException("prepare query failed");
+        }
+
+        if (found.size() < messages.size()) {
+            // identify the missing records and add to the notReady list
+            for (RemoteIndexMessage m: messages) {
+                if (!found.contains(m.getData().getLogicalResourceId())) {
+                    notReady.add(m);
+                }
+            }
+        }
+    }
+}
\ No newline at end of file
diff --git a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/sharded/ShardedPostgresParameterBatch.java b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/sharded/ShardedPostgresParameterBatch.java
new file mode 100644
index 00000000000..54dbfe7252e
--- /dev/null
+++ b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/sharded/ShardedPostgresParameterBatch.java
@@ -0,0 +1,423 @@
+/*
+ * (C) Copyright IBM Corp. 2022
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+ 
+package com.ibm.fhir.remote.index.sharded;
+
+import java.math.BigDecimal;
+import java.sql.Connection;
+import java.sql.PreparedStatement;
+import java.sql.SQLException;
+import java.sql.Timestamp;
+import java.sql.Types;
+import java.util.Calendar;
+
+import com.ibm.fhir.database.utils.common.CalendarHelper;
+import com.ibm.fhir.database.utils.common.PreparedStatementHelper;
+
+/**
+ * Parameter batch statements configured for a given resource type
+ */
+public class ShardedPostgresParameterBatch {
+    private final Connection connection;
+    private final String resourceType;
+
+    private PreparedStatement strings;
+    private int stringCount;
+
+    private PreparedStatement numbers;
+    private int numberCount;
+
+    private PreparedStatement dates;
+    private int dateCount;
+
+    private PreparedStatement quantities;
+    private int quantityCount;
+
+    private PreparedStatement locations;
+    private int locationCount;
+
+    private PreparedStatement resourceTokenRefs;
+    private int resourceTokenRefCount;
+
+    private PreparedStatement tags;
+    private int tagCount;
+
+    private PreparedStatement profiles;
+    private int profileCount;
+
+    private PreparedStatement security;
+    private int securityCount;
+
+    private PreparedStatement refs;
+    private int refCount;
+
+    /**
+     * Public constructor
+     * @param c
+     * @param resourceType
+     */
+    public ShardedPostgresParameterBatch(Connection c, String resourceType) {
+        this.connection = c;
+        this.resourceType = resourceType;
+    }
+
+    /**
+     * Push the current batch
+     */
+    public void pushBatch() throws SQLException {
+        if (stringCount > 0) {
+            strings.executeBatch();
+            stringCount = 0;
+        }
+        if (numberCount > 0) {
+            numbers.executeBatch();
+            numberCount = 0;
+        }
+        if (dateCount > 0) {
+            dates.executeBatch();
+            dateCount = 0;
+        }
+        if (quantityCount > 0) {
+            quantities.executeBatch();
+            quantityCount = 0;
+        }
+        if (locationCount > 0) {
+            locations.executeBatch();
+            locationCount = 0;
+        }
+        if (resourceTokenRefCount > 0) {
+            resourceTokenRefs.executeBatch();
+            resourceTokenRefCount = 0;
+        }
+        if (tagCount > 0) {
+            tags.executeBatch();
+            tagCount = 0;
+        }
+        if (profileCount > 0) {
+            profiles.executeBatch();
+            profileCount = 0;
+        }
+        if (securityCount > 0) {
+            security.executeBatch();
+            securityCount = 0;
+        }
+        if (refCount > 0) {
+            refs.executeBatch();
+            refCount = 0;
+        }
+    }
+
+    /**
+     * Resets the state of the DAO by closing all statements and
+     * setting any batch counts to 0
+     */
+    public void close() {
+        if (strings != null) {
+            try {
+                strings.close();
+            } catch (SQLException x) {
+                // NOP
+            } finally {
+                strings = null;
+                stringCount = 0;
+            }
+        }
+
+        if (numbers != null) {
+            try {
+                numbers.close();
+            } catch (SQLException x) {
+                // NOP
+            } finally {
+                numbers = null;
+                numberCount = 0;
+            }
+        }
+
+        if (dates != null) {
+            try {
+                dates.close();
+            } catch (SQLException x) {
+                // NOP
+            } finally {
+                dates = null;
+                dateCount = 0;
+            }
+        }
+
+        if (quantities != null) {
+            try {
+                quantities.close();
+            } catch (SQLException x) {
+                // NOP
+            } finally {
+                quantities = null;
+                quantityCount = 0;
+            }
+        }
+
+        if (locations != null) {
+            try {
+                locations.close();
+            } catch (SQLException x) {
+                // NOP
+            } finally {
+                locations = null;
+                locationCount = 0;
+            }
+        }
+
+        if (resourceTokenRefs != null) {
+            try {
+                resourceTokenRefs.close();
+            } catch (SQLException x) {
+                // NOP
+            } finally {
+                resourceTokenRefs = null;
+                resourceTokenRefCount = 0;
+            }
+        }
+        if (tags != null) {
+            try {
+                tags.close();
+            } catch (SQLException x) {
+                // NOP
+            } finally {
+                tags = null;
+                tagCount = 0;
+            }            
+        }
+        if (profiles != null) {
+            try {
+                profiles.close();
+            } catch (SQLException x) {
+                // NOP
+            } finally {
+                profiles = null;
+                profileCount = 0;
+            }            
+        }
+        if (security != null) {
+            try {
+                security.close();
+            } catch (SQLException x) {
+                // NOP
+            } finally {
+                security = null;
+                securityCount = 0;
+            }
+        }
+        if (refs != null) {
+            try {
+                refs.close();
+            } catch (SQLException x) {
+                // NOP
+            } finally {
+                refs = null;
+                refCount = 0;
+            }
+        }
+    }
+
+    /**
+     * Set the compositeId on the given PreparedStatement, handling a value if necessary
+     * @param ps
+     * @param index
+     * @param compositeId
+     * @throws SQLException
+     */
+    private void setComposite(PreparedStatement ps, int index, Integer compositeId) throws SQLException {
+        if (compositeId != null) {
+            ps.setInt(index, compositeId);
+        } else {
+            ps.setNull(index, Types.INTEGER);
+        }
+    }
+    /**
+     * Utility method to set a string value and handle null
+     * @param ps
+     * @param index
+     * @param value
+     * @throws SQLException
+     */
+    private void setString(PreparedStatement ps, int index, String value) throws SQLException {
+        if (value == null) {
+            ps.setNull(index, Types.VARCHAR);
+        } else {
+            ps.setString(index, value);
+        }
+    }
+
+    public void addString(long logicalResourceId, int parameterNameId, String strValue, String strValueLower, Integer compositeId, short shardKey) throws SQLException {
+        if (strings == null) {
+            final String tablePrefix = resourceType.toLowerCase();
+            final String insertString = "INSERT INTO " + tablePrefix + "_str_values (parameter_name_id, str_value, str_value_lcase, logical_resource_id, composite_id, shard_key) VALUES (?,?,?,?,?,?)";
+            strings = connection.prepareStatement(insertString);
+        }
+
+        strings.setInt(1, parameterNameId);
+        strings.setString(2, strValue);
+        strings.setString(3, strValueLower);
+        strings.setLong(4, logicalResourceId);
+        setComposite(strings, 5, compositeId);
+        strings.setShort(6, shardKey);
+        strings.addBatch();
+        stringCount++;
+    }
+
+    public void addNumber(long logicalResourceId, int parameterNameId, BigDecimal value, BigDecimal valueLow, BigDecimal valueHigh, Integer compositeId, short shardKey) throws SQLException {
+        if (numbers == null) {
+            final String tablePrefix = resourceType.toLowerCase();
+            final String insertNumber = "INSERT INTO " + tablePrefix + "_number_values (parameter_name_id, number_value, number_value_low, number_value_high, logical_resource_id, composite_id, shard_key) VALUES (?,?,?,?,?,?,?)";
+            numbers = connection.prepareStatement(insertNumber);
+        }
+        numbers.setInt(1, parameterNameId);
+        numbers.setBigDecimal(2, value);
+        numbers.setBigDecimal(3, valueLow);
+        numbers.setBigDecimal(4, valueHigh);
+        numbers.setLong(5, logicalResourceId);
+        setComposite(numbers, 6, compositeId);
+        numbers.setShort(7, shardKey);
+        numbers.addBatch();
+        numberCount++;
+    }
+
+    public void addDate(long logicalResourceId, int parameterNameId, Timestamp dateStart, Timestamp dateEnd, Integer compositeId, short shardKey) throws SQLException {
+        if (dates == null) {
+            final String tablePrefix = resourceType.toLowerCase();
+            final String insertDate = "INSERT INTO " + tablePrefix + "_date_values (parameter_name_id, date_start, date_end, logical_resource_id, composite_id, shard_key) VALUES (?,?,?,?,?,?)";
+            dates = connection.prepareStatement(insertDate);
+        }
+
+        final Calendar UTC = CalendarHelper.getCalendarForUTC();
+        dates.setInt(1, parameterNameId);
+        dates.setTimestamp(2, dateStart, UTC);
+        dates.setTimestamp(3, dateEnd, UTC);
+        dates.setLong(4, logicalResourceId);
+        setComposite(dates, 5, compositeId);
+        dates.setShort(6, shardKey);
+        dates.addBatch();
+        dateCount++;
+    }
+
+    public void addQuantity(long logicalResourceId, int parameterNameId, Integer codeSystemId, String valueCode, BigDecimal valueNumber, BigDecimal valueNumberLow, BigDecimal valueNumberHigh, Integer compositeId, short shardKey) throws SQLException {
+        if (quantities == null) {
+            final String tablePrefix = resourceType.toLowerCase();
+            final String insertQuantity = "INSERT INTO " + tablePrefix + "_quantity_values (parameter_name_id, code_system_id, code, quantity_value, quantity_value_low, quantity_value_high, logical_resource_id, composite_id, shard_key) VALUES (?,?,?,?,?,?,?,?,?)";
+            quantities = connection.prepareStatement(insertQuantity);
+        }
+
+        quantities.setInt(1, parameterNameId);
+        quantities.setInt(2, codeSystemId);
+        quantities.setString(3, valueCode);
+        quantities.setBigDecimal(4, valueNumber);
+        quantities.setBigDecimal(5, valueNumberLow);
+        quantities.setBigDecimal(6, valueNumberHigh);
+        quantities.setLong(7, logicalResourceId);
+        setComposite(quantities, 8, compositeId);
+        quantities.setShort(9, shardKey);
+        quantities.addBatch();
+        quantityCount++;
+    }
+
+    public void addLocation(long logicalResourceId, int parameterNameId, Double lat, Double lng, Integer compositeId, short shardKey) throws SQLException {
+        if (locations == null) {
+            final String tablePrefix = resourceType.toLowerCase();
+            final String insertLocation = "INSERT INTO " + tablePrefix + "_latlng_values (parameter_name_id, latitude_value, longitude_value, logical_resource_id, composite_id, shard_key) VALUES (?,?,?,?,?,?)";
+            locations = connection.prepareStatement(insertLocation);
+        }
+
+        locations.setInt(1, parameterNameId);
+        locations.setDouble(2, lat);
+        locations.setDouble(3, lng);
+        locations.setLong(4, logicalResourceId);
+        setComposite(locations, 5, compositeId);
+        locations.setShort(6, shardKey);
+        locations.addBatch();
+        locationCount++;
+    }
+
+    public void addResourceTokenRef(long logicalResourceId, int parameterNameId, long commonTokenValueId, Integer refVersionId, Integer compositeId, short shardKey) throws SQLException {
+        if (resourceTokenRefs == null) {
+            final String tablePrefix = resourceType.toLowerCase();
+            final String tokenString = "INSERT INTO " + tablePrefix + "_resource_token_refs (parameter_name_id, common_token_value_id, ref_version_id, logical_resource_id, composite_id, shard_key) VALUES (?,?,?,?,?,?)";
+            resourceTokenRefs = connection.prepareStatement(tokenString);
+        }
+        resourceTokenRefs.setInt(1, parameterNameId);
+        resourceTokenRefs.setLong(2, commonTokenValueId);
+        setComposite(resourceTokenRefs, 3, refVersionId);
+        resourceTokenRefs.setLong(4, logicalResourceId);
+        setComposite(resourceTokenRefs, 5, compositeId);
+        resourceTokenRefs.setShort(6, shardKey);
+        resourceTokenRefs.addBatch();
+        resourceTokenRefCount++;
+    }
+
+    public void addTag(long logicalResourceId, long commonTokenValueId, short shardKey) throws SQLException {
+        if (tags == null) {
+            final String tablePrefix = resourceType.toLowerCase();
+            final String tokenString = "INSERT INTO " + tablePrefix + "_tags (common_token_value_id, logical_resource_id, shard_key) VALUES (?,?,?)";
+            tags = connection.prepareStatement(tokenString);
+        }
+        tags.setLong(1, commonTokenValueId);
+        tags.setLong(2, logicalResourceId);
+        tags.setShort(3, shardKey);
+        tags.addBatch();
+        tagCount++;
+    }
+
+    public void addProfile(long logicalResourceId, long canonicalId, String version, String fragment, short shardKey) throws SQLException {
+        if (profiles == null) {
+            final String tablePrefix = resourceType.toLowerCase();
+            final String tokenString = "INSERT INTO " + tablePrefix + "_profiles (canonical_id, logical_resource_id, shard_key, version, fragment) VALUES (?,?,?,?,?)";
+            profiles = connection.prepareStatement(tokenString);
+        }
+        profiles.setLong(1, canonicalId);
+        profiles.setLong(2, logicalResourceId);
+        profiles.setShort(3, shardKey);
+        setString(profiles, 4, version);
+        setString(profiles, 5, fragment);
+        profiles.addBatch();
+        profileCount++;
+    }
+
+    public void addSecurity(long logicalResourceId, long commonTokenValueId, short shardKey) throws SQLException {
+        if (tags == null) {
+            final String tablePrefix = resourceType.toLowerCase();
+            final String INS = "INSERT INTO " + tablePrefix + "_security (common_token_value_id, logical_resource_id, shard_key) VALUES (?,?,?)";
+            security = connection.prepareStatement(INS);
+        }
+        PreparedStatementHelper psh = new PreparedStatementHelper(security);
+        psh.setLong(commonTokenValueId)
+            .setLong(logicalResourceId)
+            .setShort(shardKey)
+            .addBatch();
+        securityCount++;
+    }
+
+    /**
+     * @param logicalResourceId
+     * @param parameterNameId
+     * @param refLogicalResourceId
+     * @param refVersionId
+     * @param shardKey
+     */
+    public void addReference(long logicalResourceId, int parameterNameId, long refLogicalResourceId, Integer refVersionId, short shardKey) throws SQLException {
+        if (refs == null) {
+            final String tablePrefix = resourceType.toLowerCase();
+            final String insertString = "INSERT INTO " + tablePrefix + "_ref_values (parameter_name_id, logical_resource_id, ref_logical_resource_id, ref_version_id, shard_key) VALUES (?,?,?,?,?)";
+            refs = connection.prepareStatement(insertString);
+        }
+        PreparedStatementHelper psh = new PreparedStatementHelper(security);
+        psh.setInt(parameterNameId)
+            .setLong(logicalResourceId)
+            .setLong(refLogicalResourceId)
+            .setInt(refVersionId)
+            .setShort(shardKey)
+            .addBatch();
+        refCount++;
+    }
+}
\ No newline at end of file
diff --git a/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/sharded/ShardedPostgresSystemParameterBatch.java b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/sharded/ShardedPostgresSystemParameterBatch.java
new file mode 100644
index 00000000000..7f03b9cdc95
--- /dev/null
+++ b/fhir-remote-index/src/main/java/com/ibm/fhir/remote/index/sharded/ShardedPostgresSystemParameterBatch.java
@@ -0,0 +1,221 @@
+/*
+ * (C) Copyright IBM Corp. 2022
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+ 
+package com.ibm.fhir.remote.index.sharded;
+
+import java.sql.Connection;
+import java.sql.PreparedStatement;
+import java.sql.SQLException;
+import java.sql.Timestamp;
+import java.sql.Types;
+import java.util.Calendar;
+
+import com.ibm.fhir.database.utils.common.CalendarHelper;
+import com.ibm.fhir.database.utils.common.PreparedStatementHelper;
+
+/**
+ * Batch insert statements for system-level parameters
+ * @implNote targets the distributed variant of the schema
+ * where each table includes a shard_key column
+ */
+public class ShardedPostgresSystemParameterBatch {
+    private final Connection connection;
+
+    private PreparedStatement systemStrings;
+    private int systemStringCount;
+
+    private PreparedStatement systemDates;
+    private int systemDateCount;
+
+    private PreparedStatement systemProfiles;
+    private int systemProfileCount;
+
+    private PreparedStatement systemTags;
+    private int systemTagCount;
+
+    private PreparedStatement systemSecurity;
+    private int systemSecurityCount;
+
+    /**
+     * Public constructor
+     * @param c
+     */
+    public ShardedPostgresSystemParameterBatch(Connection c) {
+        this.connection = c;
+    }
+
+    /**
+     * Push the current batch
+     */
+    public void pushBatch() throws SQLException {
+        if (systemStringCount > 0) {
+            systemStrings.executeBatch();
+            systemStringCount = 0;
+        }
+        if (systemDateCount > 0) {
+            systemDates.executeBatch();
+            systemDateCount = 0;
+        }
+        if (systemTagCount > 0) {
+            systemTags.executeBatch();
+            systemTagCount = 0;
+        }
+        if (systemProfileCount > 0) {
+            systemProfiles.executeBatch();
+            systemProfileCount = 0;
+        }
+        if (systemSecurityCount > 0) {
+            systemSecurity.executeBatch();
+            systemSecurityCount = 0;
+        }
+    }
+
+    /**
+     * Closes all the statements currently open
+     */
+    public void close() {
+
+        if (systemStrings != null) {
+            try {
+                systemStrings.close();
+            } catch (SQLException x) {
+                // NOP
+            } finally {
+                systemStrings = null;
+                systemStringCount = 0;
+            }
+        }
+
+        if (systemDates != null) {
+            try {
+                systemDates.close();
+            } catch (SQLException x) {
+                // NOP
+            } finally {
+                systemDates = null;
+                systemDateCount = 0;
+            }
+        }
+        if (systemTags != null) {
+            try {
+                systemTags.close();
+            } catch (SQLException x) {
+                // NOP
+            } finally {
+                systemTags = null;
+                systemTagCount = 0;
+            }
+        }
+        if (systemProfiles != null) {
+            try {
+                systemProfiles.close();
+            } catch (SQLException x) {
+                // NOP
+            } finally {
+                systemProfiles = null;
+                systemProfileCount = 0;
+            }
+        }
+        if (systemSecurity != null) {
+            try {
+                systemSecurity.close();
+            } catch (SQLException x) {
+                // NOP
+            } finally {
+                systemSecurity = null;
+                systemProfileCount = 0;
+            }
+        }
+    }
+
+    /**
+     * Set the compositeId on the given PreparedStatement, handling a value if necessary
+     * @param ps
+     * @param index
+     * @param compositeId
+     * @throws SQLException
+     */
+    private void setComposite(PreparedStatement ps, int index, Integer compositeId) throws SQLException {
+        if (compositeId != null) {
+            ps.setInt(index, compositeId);
+        } else {
+            ps.setNull(index, Types.INTEGER);
+        }
+    }
+
+    public void addString(long logicalResourceId, int parameterNameId, String strValue, String strValueLower, Integer compositeId, short shardKey) throws SQLException {
+            // System level string attributes
+            if (systemStrings == null) {
+                final String insertSystemString = "INSERT INTO str_values (parameter_name_id, str_value, str_value_lcase, logical_resource_id, shard_key) VALUES (?,?,?,?,?)";
+                systemStrings = connection.prepareStatement(insertSystemString);
+            }
+            systemStrings.setInt(1, parameterNameId);
+            systemStrings.setString(2, strValue);
+            systemStrings.setString(3, strValueLower);
+            systemStrings.setLong(4, logicalResourceId);
+            setComposite(systemStrings, 5, compositeId);
+            systemStrings.setShort(6, shardKey);
+            systemStrings.addBatch();
+            systemStringCount++;
+    }
+
+    public void addDate(long logicalResourceId, int parameterNameId, Timestamp dateStart, Timestamp dateEnd, Integer compositeId, short shardKey) throws SQLException {
+        if (systemDates == null) {
+            final String insertSystemDate = "INSERT INTO date_values (parameter_name_id, date_start, date_end, logical_resource_id, shard_key) VALUES (?,?,?,?,?)";
+            systemDates = connection.prepareStatement(insertSystemDate);
+        }
+        final Calendar UTC = CalendarHelper.getCalendarForUTC();
+        systemDates.setInt(1, parameterNameId);
+        systemDates.setTimestamp(2, dateStart, UTC);
+        systemDates.setTimestamp(3, dateEnd, UTC);
+        systemDates.setLong(4, logicalResourceId);
+        setComposite(systemDates, 5, compositeId);
+        systemDates.setShort(6, shardKey);
+        systemDates.addBatch();
+        systemDateCount++;
+    }
+
+    public void addTag(long logicalResourceId, long commonTokenValueId, short shardKey) throws SQLException {
+        if (systemTags == null) {
+            final String INS = "INSERT INTO logical_resource_tags(common_token_value_id, logical_resource_id, shard_key) VALUES (?,?,?)";
+            systemTags = connection.prepareStatement(INS);
+        }
+        PreparedStatementHelper psh = new PreparedStatementHelper(systemTags);
+        psh.setLong(commonTokenValueId)
+            .setLong(logicalResourceId)
+            .setShort(shardKey)
+            .addBatch();
+        systemTagCount++;
+    }
+
+    public void addProfile(long logicalResourceId, long canonicalId, String version, String fragment, short shardKey) throws SQLException {
+        if (systemProfiles == null) {
+            final String INS = "INSERT INTO logical_resource_profiles(canonical_id, logical_resource_id, shard_key, version, fragment) VALUES (?,?,?,?,?)";
+            systemProfiles = connection.prepareStatement(INS);
+        }
+        PreparedStatementHelper psh = new PreparedStatementHelper(systemProfiles);
+        psh.setLong(canonicalId)
+            .setLong(logicalResourceId)
+            .setShort(shardKey)
+            .setString(version)
+            .setString(fragment)
+            .addBatch();
+        systemProfileCount++;
+    }
+
+    public void addSecurity(long logicalResourceId, long commonTokenValueId, short shardKey) throws SQLException {
+        if (systemTags == null) {
+            final String INS = "INSERT INTO logical_resource_security(common_token_value_id, logical_resource_id, shard_key) VALUES (?,?,?)";
+            systemSecurity = connection.prepareStatement(INS);
+        }
+        PreparedStatementHelper psh = new PreparedStatementHelper(systemSecurity);
+        psh.setLong(commonTokenValueId)
+            .setLong(logicalResourceId)
+            .setShort(shardKey)
+            .addBatch();
+        systemSecurityCount++;
+    }
+}
\ No newline at end of file
diff --git a/fhir-remote-index/src/test/java/com/ibm/fhir/remote/index/DerbyFhirFactory.java b/fhir-remote-index/src/test/java/com/ibm/fhir/remote/index/DerbyFhirFactory.java
new file mode 100644
index 00000000000..602a328c118
--- /dev/null
+++ b/fhir-remote-index/src/test/java/com/ibm/fhir/remote/index/DerbyFhirFactory.java
@@ -0,0 +1,151 @@
+/*
+ * (C) Copyright IBM Corp. 2022
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package com.ibm.fhir.remote.index;
+
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.SQLException;
+import java.util.HashSet;
+import java.util.Properties;
+import java.util.Set;
+
+import com.ibm.fhir.database.utils.api.IConnectionProvider;
+import com.ibm.fhir.database.utils.api.IDatabaseTranslator;
+import com.ibm.fhir.database.utils.common.JdbcConnectionProvider;
+import com.ibm.fhir.database.utils.common.JdbcPropertyAdapter;
+import com.ibm.fhir.database.utils.derby.DerbyMaster;
+import com.ibm.fhir.database.utils.derby.DerbyPropertyAdapter;
+import com.ibm.fhir.database.utils.derby.DerbyTranslator;
+import com.ibm.fhir.persistence.exception.FHIRPersistenceException;
+import com.ibm.fhir.schema.derby.DerbyFhirDatabase;
+
+/**
+ * Initialize a FHIR database schema in Derby for use by the remote index tests
+ */
+public class DerbyFhirFactory {
+    // All tests use this same database, which we only have to bootstrap once
+    public static final String DB_NAME = "target/derby/fhirDB";
+
+    // The translator to help us out with Derby urls/syntax/exceptions
+    private static final IDatabaseTranslator DERBY_TRANSLATOR = new DerbyTranslator();
+
+    private Properties dbProps;
+
+    // When not null, restricts the set of resource types we create tables for
+    private Set resourceTypeNames;
+
+    /**
+     * Constructs a new DerbyInitializer using default database properties.
+     */
+    public DerbyFhirFactory(Set resourceTypeNames) {
+        this.dbProps = new Properties();
+        if (resourceTypeNames != null) {
+            this.resourceTypeNames = new HashSet<>(resourceTypeNames);
+        } else {
+            this.resourceTypeNames = null;
+        }
+    }
+
+    /**
+     * Constructs a new DerbyInitializer using the passed database properties.
+     */
+    public DerbyFhirFactory(Properties props, Set resourceTypeNames) {
+        this.dbProps = props;
+        if (resourceTypeNames != null) {
+            this.resourceTypeNames = new HashSet<>(resourceTypeNames);
+        } else {
+            this.resourceTypeNames = null;
+        }
+    }
+
+    /**
+     * Default bootstrap of the database. Does not drop/rebuild.
+     *
+     * @return a DerbyFhirDatabase instance that represents the create database if one was created or null if it already exists
+     * @throws FHIRPersistenceDBConnectException
+     * @throws SQLException
+     */
+    public DerbyFhirDatabase bootstrapDb() throws FHIRPersistenceException, SQLException {
+        return bootstrapDb(false);
+    }
+
+    /**
+     * Tests for the existence of fhirDB and creates the database if necessary, complete with tables and indices.
+     *
+     * @param reset
+     *            Whether to "reset" the database by deleting the existing one before attempting the create
+     * @return a DerbyFhirDatabase object if one was created or null if it already exists and {@code reset} is false
+     * @throws FHIRPersistenceDBConnectException
+     * @throws SQLException
+     */
+    public DerbyFhirDatabase bootstrapDb(boolean reset) throws FHIRPersistenceException, SQLException {
+        if (reset) {
+            // wipes the disk content of the database. Hopefully there aren't any
+            // open connections at this point
+            DerbyMaster.dropDatabase(DB_NAME);
+        }
+
+        // Inject the DB_NAME into the dbProps
+        DerbyPropertyAdapter adapter = new DerbyPropertyAdapter(dbProps);
+        adapter.setDatabase(DB_NAME);
+
+        // Only bootstrap the database if it is new
+        boolean exists;
+        try (Connection connection = getConnection()) {
+            exists = true;
+        } catch (SQLException x) {
+            exists = false;
+        }
+
+        if (exists) {
+            System.out.println("Existing database: skipping bootstrap");
+            return null;
+        } else {
+            System.out.println("Bootstrapping database");
+            if (resourceTypeNames != null) {
+                return new DerbyFhirDatabase(DB_NAME, resourceTypeNames);
+            } else {
+                return new DerbyFhirDatabase(DB_NAME);
+            }
+        }
+    }
+
+    /**
+     * Get the name of the schema holding all the FHIR resource tables.
+     */
+    protected String getDataSchemaName() {
+        return dbProps.getProperty("schemaName", "FHIRDATA");
+    }
+
+    /**
+     * Get a connection to an established database.
+     * Autocommit is disabled (of course).
+     */
+    public Connection getConnection() throws SQLException {
+        Connection connection = DriverManager.getConnection(DERBY_TRANSLATOR.getUrl(dbProps));
+        connection.setAutoCommit(false);
+        return connection;
+    }
+
+    /**
+     * Bootstrap the database if necessary, and get a connection provider for it
+     * @return an {@link IConnectionProvider} configured for the FHIR Derby database
+     * @param reset resets the database if true
+     * @throws SQLException 
+     * @throws FHIRPersistenceDBConnectException 
+     */
+    public IConnectionProvider getConnectionProvider(boolean reset) throws FHIRPersistenceException, SQLException {
+        bootstrapDb(reset);
+        JdbcPropertyAdapter propAdapter = new JdbcPropertyAdapter(this.dbProps);
+        
+        // make sure the schema name is correctly set in the properties
+        String fhirDataSchema = getDataSchemaName();
+        propAdapter.setDefaultSchema(fhirDataSchema);
+        
+        return new JdbcConnectionProvider(new DerbyTranslator(), propAdapter);
+    }
+}
diff --git a/fhir-remote-index/src/test/java/com/ibm/fhir/remote/index/RemoteIndexTest.java b/fhir-remote-index/src/test/java/com/ibm/fhir/remote/index/RemoteIndexTest.java
new file mode 100644
index 00000000000..cacd13a33d6
--- /dev/null
+++ b/fhir-remote-index/src/test/java/com/ibm/fhir/remote/index/RemoteIndexTest.java
@@ -0,0 +1,630 @@
+/*
+ * (C) Copyright IBM Corp. 2022
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+ 
+package com.ibm.fhir.remote.index;
+
+import static org.testng.Assert.assertEquals;
+
+import java.math.BigDecimal;
+import java.sql.Connection;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.sql.Timestamp;
+import java.time.Duration;
+import java.time.Instant;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Properties;
+import java.util.Set;
+import java.util.UUID;
+import java.util.logging.Logger;
+
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.Test;
+
+import com.ibm.fhir.database.utils.api.IConnectionProvider;
+import com.ibm.fhir.database.utils.api.IDatabaseTranslator;
+import com.ibm.fhir.database.utils.common.PreparedStatementHelper;
+import com.ibm.fhir.database.utils.common.ResultSetReader;
+import com.ibm.fhir.database.utils.derby.DerbyTranslator;
+import com.ibm.fhir.model.test.TestUtil;
+import com.ibm.fhir.persistence.exception.FHIRPersistenceException;
+import com.ibm.fhir.persistence.helper.RemoteIndexSupport;
+import com.ibm.fhir.persistence.index.RemoteIndexConstants;
+import com.ibm.fhir.persistence.index.RemoteIndexMessage;
+import com.ibm.fhir.persistence.index.SearchParametersTransportAdapter;
+import com.ibm.fhir.remote.index.cache.IdentityCacheImpl;
+import com.ibm.fhir.remote.index.database.CacheLoader;
+import com.ibm.fhir.remote.index.database.PlainDerbyMessageHandler;
+
+/**
+ * Unit test for remote index message handling and database processing
+ */
+public class RemoteIndexTest {
+    private static final Logger logger = Logger.getLogger(RemoteIndexTest.class.getName());
+    private Properties testProps;
+
+    private IConnectionProvider connectionProvider;
+    private String[] TEST_RESOURCE_TYPES = {"Patient", "Observation" };
+    private IdentityCacheImpl identityCache;
+    private static final String SCHEMA_NAME = "FHIRDATA";
+    private static final boolean WHOLE_SYSTEM = true;
+    private static final IDatabaseTranslator translator = new DerbyTranslator();
+    private static final String OBSERVATION = "Observation";
+
+    private final String OBSERVATION_LOGICAL_ID = UUID.randomUUID().toString();
+    private final int versionId = 1;
+    private final Instant lastUpdated = Instant.now();
+    private final String requestShard = null;
+    private final String parameterHash = "1Z+NWYZb739Ava9Pd/d7wt2xecKmC2FkfLlCCml0I5M=";
+    private final Instant ts1 = lastUpdated.plusMillis(1000);
+    private final Instant ts2 = lastUpdated.plusMillis(2000);
+    private final BigDecimal valueNumber = BigDecimal.valueOf(1.0);
+    private final BigDecimal valueNumberLow = BigDecimal.valueOf(0.5);
+    private final BigDecimal valueNumberHigh = BigDecimal.valueOf(1.5);
+    private final String valueSystem = "system1";
+    private final String valueCode = "code1";
+    private final String refResourceType = "Patient";
+    private final String refLogicalId = "pat1";
+    private final Integer refVersion = 2;
+    private final Integer compositeId = null;
+    private final String valueString = "str1";
+    private final String url = "http://some.profile/location";
+    private final String profileVersion = "1.0";
+    private final String instanceIdentifier = "a-unique-id-value-1";
+    
+    @BeforeClass
+    public void bootstrapDatabase() throws Exception {
+        final Set resourceTypeNames = Set.of(TEST_RESOURCE_TYPES);
+        this.testProps = TestUtil.readTestProperties("test-remote-index.properties");
+        DerbyFhirFactory derbyInit;
+        String dbDriverName = this.testProps.getProperty("dbDriverName");
+        if (dbDriverName == null || !dbDriverName.contains("derby")) {
+            throw new IllegalStateException("test properties missing derby driver configuration");
+        }
+
+        derbyInit = new DerbyFhirFactory(this.testProps, resourceTypeNames);
+        this.connectionProvider = derbyInit.getConnectionProvider(false);
+        Duration cacheDuration = Duration.ofDays(1);
+        this.identityCache = new IdentityCacheImpl(
+            100, cacheDuration,  // code systems
+            100, cacheDuration,  // common token values
+            100, cacheDuration,  // canonical values
+            100, cacheDuration); // logical resource idents
+
+        // Preload the cache so we have all the resource types available
+        try (Connection c = connectionProvider.getConnection()) {
+            CacheLoader cacheLoader = new CacheLoader(identityCache);
+            cacheLoader.apply(c);
+            c.commit();
+        }
+    }
+    
+    /**
+     * Get a list of messages to process
+     * @return
+     */
+    private List getMessages(long logicalResourceId) {
+        RemoteIndexMessage sent = new RemoteIndexMessage();
+        sent.setMessageVersion(RemoteIndexConstants.MESSAGE_VERSION);
+        sent.setInstanceIdentifier(instanceIdentifier);
+
+        // Create an Observation resource with a few parameters
+        SearchParametersTransportAdapter adapter = new SearchParametersTransportAdapter(OBSERVATION, OBSERVATION_LOGICAL_ID, logicalResourceId, 
+            versionId, lastUpdated, requestShard, parameterHash);
+        adapter.stringValue("string-param", valueString, compositeId, WHOLE_SYSTEM);
+        adapter.dateValue("date-param", ts1, ts2, null, WHOLE_SYSTEM);
+        adapter.numberValue("number-param", valueNumber, valueNumberLow, valueNumberHigh, null);
+        adapter.quantityValue("quantity-param", valueSystem, valueCode, valueNumber, valueNumberLow, valueNumberHigh, compositeId);
+        adapter.tokenValue("token-param", valueSystem, valueCode, compositeId);
+        adapter.locationValue("location-param", 0.1, 0.2, null);
+        adapter.referenceValue("reference-param", refResourceType, refLogicalId, refVersion, compositeId);
+        adapter.securityValue("security-param", valueSystem, valueCode, WHOLE_SYSTEM);
+        adapter.profileValue("profile-param", url, profileVersion, null, WHOLE_SYSTEM);
+        adapter.tagValue("tag-param", valueSystem, valueCode, WHOLE_SYSTEM);
+
+        sent.setData(adapter.build());
+        final String payload = RemoteIndexSupport.marshallToString(sent);     
+        final List result = new ArrayList<>();
+        result.add(payload);
+        return result;
+    }
+
+    @Test
+    public void testFill() throws Exception {
+        final long logicalResourceId;
+        try (Connection c = connectionProvider.getConnection()) {
+            logicalResourceId = addObservationLogicalResource(c, OBSERVATION_LOGICAL_ID);
+            c.commit();
+        }
+
+        try (Connection c = connectionProvider.getConnection()) {
+            try {
+                PlainDerbyMessageHandler handler = new PlainDerbyMessageHandler(instanceIdentifier, c, SCHEMA_NAME, identityCache, 1000L);
+                handler.process(getMessages(logicalResourceId));
+                checkData(c, logicalResourceId);
+                c.commit();
+            } catch (Throwable t) {
+                safeRollback(c);
+                throw t;
+            }
+        }
+    }
+
+    /**
+     * Try and rollback the transaction, squashing any exception
+     * @param c
+     */
+    private void safeRollback(Connection c) {
+        try {
+            c.rollback();
+        } catch (SQLException x) {
+            logger.warning("rollback failed: " + x.getMessage());
+        }
+    }
+    /**
+     * Inject the logical_resource_ident, logical_resources and observation_logical_resources
+     * record as we would normally see added by the FHIR server. We're not dealing with any
+     * concurrency here, so we just use 3 simple inserts.
+     * @param c
+     * @throws SQLException
+     */
+    private long addObservationLogicalResource(Connection c, String logicalId) throws SQLException {
+        final int resourceTypeId = identityCache.getResourceTypeId(OBSERVATION);
+        final String getNextLogicalId = translator.selectSequenceNextValue(SCHEMA_NAME, "fhir_sequence");
+        long logicalResourceId;
+        try (Statement s = c.createStatement()) {
+            ResultSet rs = s.executeQuery(getNextLogicalId);
+            if (rs.next()) {
+                logicalResourceId = rs.getLong(1);
+            } else {
+                throw new IllegalStateException("no row from '" + getNextLogicalId + "'");
+            }
+        }
+        
+        final String insertIdent = "INSERT INTO logical_resource_ident(logical_resource_id, resource_type_id, logical_id) VALUES (?, ?, ?)";
+        try (PreparedStatement ps = c.prepareStatement(insertIdent)) {
+            ps.setLong(1, logicalResourceId);
+            ps.setInt(2, resourceTypeId);
+            ps.setString(3, logicalId);
+            ps.executeUpdate();
+        }
+
+        final Timestamp lastUpdated = Timestamp.from(this.lastUpdated);
+        final String insertLogicalResource = "INSERT INTO logical_resources(logical_resource_id, resource_type_id, logical_id, last_updated, is_deleted, parameter_hash)"
+                + " VALUES (?,?,?,?,?,?)";
+        try (PreparedStatement ps = c.prepareStatement(insertLogicalResource)) {
+            PreparedStatementHelper psh = new PreparedStatementHelper(ps);
+            
+            psh.setLong(logicalResourceId)
+            .setInt(resourceTypeId)
+            .setString(logicalId)
+            .setTimestamp(lastUpdated)
+            .setString("N")
+            .setString(parameterHash);
+            ps.executeUpdate();
+        }
+
+        final String insertObservationLogicalResource = "INSERT INTO observation_logical_resources(logical_resource_id, logical_id, is_deleted, last_updated, version_id)"
+                + " VALUES (?,?,?,?,?)";
+        try (PreparedStatement ps = c.prepareStatement(insertObservationLogicalResource)) {
+            PreparedStatementHelper psh = new PreparedStatementHelper(ps);
+            
+            psh.setLong(logicalResourceId)
+            .setString(logicalId)
+            .setString("N")
+            .setTimestamp(lastUpdated)
+            .setInt(this.versionId);
+            ps.executeUpdate();
+        }
+
+        return logicalResourceId;
+    }
+    /**
+     * Check that the data in the processed messages now exists in
+     * the database
+     * @param c
+     * @throws Exception
+     */
+    private void checkData(Connection c, long logicalResourceId) throws Exception {
+        // check the resource level parameters
+        checkStringParam(c, OBSERVATION, logicalResourceId, valueString);
+        checkDateParam(c, OBSERVATION, logicalResourceId, ts1, ts2);
+        checkNumberParam(c, OBSERVATION, logicalResourceId, valueNumber, valueNumberLow, valueNumberHigh);
+        checkLocationParam(c, OBSERVATION, logicalResourceId, 0.1, 0.2);
+        checkProfileParam(c, OBSERVATION, logicalResourceId, url, profileVersion);
+        checkQuantityParam(c, OBSERVATION, logicalResourceId, valueSystem, valueCode, valueNumber, valueNumberLow, valueNumberHigh);
+        checkReferenceParam(c, OBSERVATION, logicalResourceId, refResourceType, refLogicalId);
+        checkTagParam(c, OBSERVATION, logicalResourceId, valueSystem, valueCode);
+        checkSecurityParam(c, OBSERVATION, logicalResourceId, valueSystem, valueCode);
+        checkTokenParam(c, OBSERVATION, logicalResourceId, valueSystem, valueCode);
+
+        // check the whole-system level parameters
+        checkStringSystemParam(c, OBSERVATION, logicalResourceId, valueString);
+        checkDateSystemParam(c, OBSERVATION, logicalResourceId, ts1, ts2);
+        checkProfileSystemParam(c, OBSERVATION, logicalResourceId, url, profileVersion);
+        checkTagSystemParam(c, OBSERVATION, logicalResourceId, valueSystem, valueCode);
+        checkSecuritySystemParam(c, OBSERVATION, logicalResourceId, valueSystem, valueCode);
+    }
+
+    /**
+     * @param c
+     * @param resourceType
+     * @param logicalResourceId
+     * @param valueSystem
+     * @param valueCode
+     * @param valueNumber
+     * @param valueNumberLow
+     * @param valueNumberHigh
+     */
+    private void checkQuantityParam(Connection c, String resourceType, long logicalResourceId, String valueSystem, String valueCode, BigDecimal valueNumber,
+        BigDecimal valueNumberLow, BigDecimal valueNumberHigh) throws Exception {
+        final String select = ""
+                + "SELECT c.code_system_name, p.code, p.quantity_value, p.quantity_value_low, p.quantity_value_high "
+                + "  FROM " + resourceType + "_quantity_values p "
+                + "  JOIN code_systems c ON c.code_system_id = p.code_system_id "
+                + " WHERE p.logical_resource_id = ?";
+        try (PreparedStatement ps = c.prepareStatement(select)) {
+            ps.setLong(1, logicalResourceId);
+            ResultSet rs = ps.executeQuery();
+            ResultSetReader rsr = new ResultSetReader(rs);
+            if (rsr.next()) {
+                assertEquals(rsr.getString(), valueSystem);
+                assertEquals(rsr.getString(), valueCode);
+                assertEquals(rsr.getBigDecimal(), valueNumber);
+                assertEquals(rsr.getBigDecimal(), valueNumberLow);
+                assertEquals(rsr.getBigDecimal(), valueNumberHigh);
+            } else {
+                throw new FHIRPersistenceException("missing value: " + select);
+            }
+    
+            if (rs.next()) {
+                // there can be only one
+                throw new FHIRPersistenceException("more than one quantity parameter");
+            }
+        }
+        
+    }
+
+    private void checkStringParam(Connection c, String resourceType, long logicalResourceId, String valueString) throws Exception {
+        final String select = "SELECT str_value FROM " + resourceType + "_str_values WHERE logical_resource_id = ?";
+        try (PreparedStatement ps = c.prepareStatement(select)) {
+            ps.setLong(1, logicalResourceId);
+            ResultSet rs = ps.executeQuery();
+            if (rs.next()) {
+                assertEquals(rs.getString(1), valueString);
+            } else {
+                throw new FHIRPersistenceException("missing value: " + select);
+            }
+
+            if (rs.next()) {
+                // there can be only one
+                throw new FHIRPersistenceException("more than one string parameter");
+            }
+        }
+    }
+
+    private void checkStringSystemParam(Connection c, String resourceType, long logicalResourceId, String valueString) throws Exception {
+        final String select = "SELECT str_value FROM str_values WHERE logical_resource_id = ?";
+        try (PreparedStatement ps = c.prepareStatement(select)) {
+            ps.setLong(1, logicalResourceId);
+            ResultSet rs = ps.executeQuery();
+            if (rs.next()) {
+                assertEquals(rs.getString(1), valueString);
+            } else {
+                throw new FHIRPersistenceException("missing value: " + select);
+            }
+
+            if (rs.next()) {
+                // there can be only one
+                throw new FHIRPersistenceException("more than one string parameter");
+            }
+        }
+    }
+
+    private void checkNumberParam(Connection c, String resourceType, long logicalResourceId, BigDecimal numberValue, 
+            BigDecimal numberValueLow, BigDecimal numberValueHigh) throws Exception {
+        final String select = "SELECT number_value, number_value_low, number_value_high FROM " + resourceType + "_number_values WHERE logical_resource_id = ?";
+        try (PreparedStatement ps = c.prepareStatement(select)) {
+            ps.setLong(1, logicalResourceId);
+            ResultSet rs = ps.executeQuery();
+            if (rs.next()) {
+                assertEquals(rs.getBigDecimal(1), numberValue);
+                assertEquals(rs.getBigDecimal(2), numberValueLow);
+                assertEquals(rs.getBigDecimal(3), numberValueHigh);
+            } else {
+                throw new FHIRPersistenceException("missing value: " + select);
+            }
+
+            if (rs.next()) {
+                // there can be only one
+                throw new FHIRPersistenceException("more than one number parameter");
+            }
+        }
+    }
+
+    private void checkDateParam(Connection c, String resourceType, long logicalResourceId, Instant dateStart, 
+        Instant dateEnd) throws Exception {
+        final String select = "SELECT date_start, date_end FROM " + resourceType + "_date_values WHERE logical_resource_id = ?";
+        try (PreparedStatement ps = c.prepareStatement(select)) {
+            ps.setLong(1, logicalResourceId);
+            ResultSet rs = ps.executeQuery();
+            ResultSetReader rsr = new ResultSetReader(rs);
+            if (rsr.next()) {
+                assertEquals(rsr.getTimestamp(), Timestamp.from(dateStart));
+                assertEquals(rsr.getTimestamp(), Timestamp.from(dateEnd));
+            } else {
+                throw new FHIRPersistenceException("missing value: " + select);
+            }
+    
+            if (rs.next()) {
+                // there can be only one
+                throw new FHIRPersistenceException("more than one date parameter");
+            }
+        }
+    }
+
+    private void checkDateSystemParam(Connection c, String resourceType, long logicalResourceId, Instant dateStart, 
+        Instant dateEnd) throws Exception {
+        final String select = "SELECT date_start, date_end FROM date_values WHERE logical_resource_id = ?";
+        try (PreparedStatement ps = c.prepareStatement(select)) {
+            ps.setLong(1, logicalResourceId);
+            ResultSet rs = ps.executeQuery();
+            ResultSetReader rsr = new ResultSetReader(rs);
+            if (rsr.next()) {
+                assertEquals(rsr.getTimestamp(), Timestamp.from(dateStart));
+                assertEquals(rsr.getTimestamp(), Timestamp.from(dateEnd));
+            } else {
+                throw new FHIRPersistenceException("missing date system value: " + select);
+            }
+    
+            if (rs.next()) {
+                // there can be only one
+                throw new FHIRPersistenceException("more than one date system parameter");
+            }
+        }
+    }
+    
+    private void checkLocationParam(Connection c, String resourceType, long logicalResourceId, double latitude, 
+            double longitude) throws Exception {
+        final String select = "SELECT latitude_value, longitude_value FROM " + resourceType + "_latlng_values WHERE logical_resource_id = ?";
+        try (PreparedStatement ps = c.prepareStatement(select)) {
+            ps.setLong(1, logicalResourceId);
+            ResultSet rs = ps.executeQuery();
+            ResultSetReader rsr = new ResultSetReader(rs);
+            if (rsr.next()) {
+                assertEquals(rsr.getDouble(), latitude);
+                assertEquals(rsr.getDouble(), longitude);
+            } else {
+                throw new FHIRPersistenceException("missing value: " + select);
+            }
+    
+            if (rs.next()) {
+                // there can be only one
+                throw new FHIRPersistenceException("more than one date parameter");
+            }
+        }
+    }
+
+    private void checkProfileParam(Connection c, String resourceType, long logicalResourceId, String profile, String version) throws Exception { 
+        final String select = ""
+                + "SELECT c.url, p.version FROM " + resourceType + "_profiles p"
+                + "  JOIN common_canonical_values c ON c.canonical_id = p.canonical_id "
+                + " WHERE logical_resource_id = ?";
+    
+        try (PreparedStatement ps = c.prepareStatement(select)) {
+            ps.setLong(1, logicalResourceId);
+            ResultSet rs = ps.executeQuery();
+            ResultSetReader rsr = new ResultSetReader(rs);
+            if (rsr.next()) {
+                assertEquals(rsr.getString(), profile);
+                assertEquals(rsr.getString(), version);
+            } else {
+                throw new FHIRPersistenceException("missing value: " + select);
+            }
+    
+            if (rs.next()) {
+                // there can be only one
+                throw new FHIRPersistenceException("more than one profile parameter");
+            }
+        }
+    }
+
+    private void checkProfileSystemParam(Connection c, String resourceType, long logicalResourceId, String profile, String version) throws Exception { 
+        final String select = ""
+                + "SELECT c.url, p.version FROM logical_resource_profiles p"
+                + "  JOIN common_canonical_values c ON c.canonical_id = p.canonical_id "
+                + " WHERE logical_resource_id = ?";
+    
+        try (PreparedStatement ps = c.prepareStatement(select)) {
+            ps.setLong(1, logicalResourceId);
+            ResultSet rs = ps.executeQuery();
+            ResultSetReader rsr = new ResultSetReader(rs);
+            if (rsr.next()) {
+                assertEquals(rsr.getString(), profile);
+                assertEquals(rsr.getString(), version);
+            } else {
+                throw new FHIRPersistenceException("missing profile system value: " + select);
+            }
+    
+            if (rs.next()) {
+                // there can be only one
+                throw new FHIRPersistenceException("more than one profile system parameter");
+            }
+        }
+    }
+
+    private void checkSecurityParam(Connection c, String resourceType, long logicalResourceId, String codeSystem, String tokenValue) throws Exception { 
+        final String select = ""
+                + "SELECT 1 FROM " + resourceType + "_security p"
+                + "  JOIN common_token_values c ON c.common_token_value_id = p.common_token_value_id "
+                + "  JOIN code_systems s ON s.code_system_id = c.code_system_id "
+                + " WHERE logical_resource_id = ? "
+                + "   AND s.code_system_name = ? "
+                + "   AND c.token_value = ? ";
+    
+        try (PreparedStatement ps = c.prepareStatement(select)) {
+            ps.setLong(1, logicalResourceId);
+            ps.setString(2, codeSystem);
+            ps.setString(3, tokenValue);
+            ResultSet rs = ps.executeQuery();
+            ResultSetReader rsr = new ResultSetReader(rs);
+            if (rsr.next()) {
+                // OK
+            } else {
+                throw new FHIRPersistenceException("missing security value: " + select);
+            }
+    
+            if (rs.next()) {
+                // there can be only one
+                throw new FHIRPersistenceException("more than one security parameter");
+            }
+        }
+    }
+
+    private void checkSecuritySystemParam(Connection c, String resourceType, long logicalResourceId, String codeSystem, String tokenValue) throws Exception { 
+        final String select = ""
+                + "SELECT 1 FROM logical_resource_security p"
+                + "  JOIN common_token_values c ON c.common_token_value_id = p.common_token_value_id "
+                + "  JOIN code_systems s ON s.code_system_id = c.code_system_id "
+                + " WHERE logical_resource_id = ? "
+                + "   AND s.code_system_name = ? "
+                + "   AND c.token_value = ? ";
+    
+        try (PreparedStatement ps = c.prepareStatement(select)) {
+            ps.setLong(1, logicalResourceId);
+            ps.setString(2, codeSystem);
+            ps.setString(3, tokenValue);
+            ResultSet rs = ps.executeQuery();
+            ResultSetReader rsr = new ResultSetReader(rs);
+            if (rsr.next()) {
+                // OK
+            } else {
+                throw new FHIRPersistenceException("missing security value: " + select);
+            }
+    
+            if (rs.next()) {
+                // there can be only one
+                throw new FHIRPersistenceException("more than one security parameter");
+            }
+        }
+    }
+
+    private void checkTagParam(Connection c, String resourceType, long logicalResourceId, String codeSystem, String tokenValue) throws Exception { 
+        final String select = ""
+                + "SELECT 1 FROM " + resourceType + "_tags p"
+                + "  JOIN common_token_values c ON c.common_token_value_id = p.common_token_value_id "
+                + "  JOIN code_systems s ON s.code_system_id = c.code_system_id "
+                + " WHERE logical_resource_id = ? "
+                + "   AND s.code_system_name = ? "
+                + "   AND c.token_value = ? ";
+    
+        try (PreparedStatement ps = c.prepareStatement(select)) {
+            ps.setLong(1, logicalResourceId);
+            ps.setString(2, codeSystem);
+            ps.setString(3, tokenValue);
+            ResultSet rs = ps.executeQuery();
+            ResultSetReader rsr = new ResultSetReader(rs);
+            if (rsr.next()) {
+                // OK
+            } else {
+                throw new FHIRPersistenceException("missing tag value: " + select);
+            }
+    
+            if (rs.next()) {
+                // there can be only one
+                throw new FHIRPersistenceException("more than one tag parameter");
+            }
+        }
+    }
+
+    private void checkTagSystemParam(Connection c, String resourceType, long logicalResourceId, String codeSystem, String tokenValue) throws Exception { 
+        final String select = ""
+                + "SELECT 1 FROM logical_resource_tags p"
+                + "  JOIN common_token_values c ON c.common_token_value_id = p.common_token_value_id "
+                + "  JOIN code_systems s ON s.code_system_id = c.code_system_id "
+                + " WHERE logical_resource_id = ? "
+                + "   AND s.code_system_name = ? "
+                + "   AND c.token_value = ? ";
+    
+        try (PreparedStatement ps = c.prepareStatement(select)) {
+            ps.setLong(1, logicalResourceId);
+            ps.setString(2, codeSystem);
+            ps.setString(3, tokenValue);
+            ResultSet rs = ps.executeQuery();
+            ResultSetReader rsr = new ResultSetReader(rs);
+            if (rsr.next()) {
+                // OK
+            } else {
+                throw new FHIRPersistenceException("missing tag value: " + select);
+            }
+    
+            if (rs.next()) {
+                // there can be only one
+                throw new FHIRPersistenceException("more than one tag parameter");
+            }
+        }
+    }
+
+    private void checkTokenParam(Connection c, String resourceType, long logicalResourceId, String codeSystem, String tokenValue) throws Exception { 
+        final String select = ""
+                + "SELECT 1 FROM " + resourceType + "_resource_token_refs p"
+                + "  JOIN common_token_values c ON c.common_token_value_id = p.common_token_value_id "
+                + "  JOIN code_systems s ON s.code_system_id = c.code_system_id "
+                + " WHERE logical_resource_id = ? "
+                + "   AND s.code_system_name = ? "
+                + "   AND c.token_value = ? ";
+    
+        try (PreparedStatement ps = c.prepareStatement(select)) {
+            ps.setLong(1, logicalResourceId);
+            ps.setString(2, codeSystem);
+            ps.setString(3, tokenValue);
+            ResultSet rs = ps.executeQuery();
+            ResultSetReader rsr = new ResultSetReader(rs);
+            if (rsr.next()) {
+                // OK
+            } else {
+                throw new FHIRPersistenceException("missing token value: " + select);
+            }
+    
+            if (rs.next()) {
+                // there can be only one
+                throw new FHIRPersistenceException("more than one token parameter");
+            }
+        }
+    }
+
+    private void checkReferenceParam(Connection c, String resourceType, long logicalResourceId, String refResourceType,
+        String refLogicalId) throws Exception {
+        final String select = ""
+                + "SELECT 1 FROM " + resourceType + "_ref_values p "
+                + "  JOIN logical_resource_ident i ON i.logical_resource_id = p.ref_logical_resource_id "
+                + "  JOIN resource_types rrt ON rrt.resource_type_id = i.resource_type_id "
+                + " WHERE p.logical_resource_id = ?"
+                + "   AND rrt.resource_type = ? "
+                + "   AND i.logical_id = ? ";
+        try (PreparedStatement ps = c.prepareStatement(select)) {
+            ps.setLong(1, logicalResourceId);
+            ps.setString(2, refResourceType);
+            ps.setString(3, refLogicalId);
+            ResultSet rs = ps.executeQuery();
+            ResultSetReader rsr = new ResultSetReader(rs);
+            if (rsr.next()) {
+                // ok
+            } else {
+                throw new FHIRPersistenceException("missing value: " + select);
+            }
+    
+            if (rs.next()) {
+                // there can be only one
+                throw new FHIRPersistenceException("more than one date parameter");
+            }
+        }
+    }
+
+}
\ No newline at end of file
diff --git a/fhir-remote-index/src/test/resources/test-remote-index.properties b/fhir-remote-index/src/test/resources/test-remote-index.properties
new file mode 100644
index 00000000000..1cd33b12b59
--- /dev/null
+++ b/fhir-remote-index/src/test/resources/test-remote-index.properties
@@ -0,0 +1,31 @@
+# Properties for TestNG tests in the fhir-remote-index project
+# Default datastore is a Derby database using the embedded Derby driver.
+# Note that the dbUser and dbPassword properties are not used for Derby.
+
+#Derby properties
+dbDriverName = org.apache.derby.jdbc.EmbeddedDriver
+dbUrl = jdbc:derby:target/derby/fhirDB
+schemaName = FHIRDATA
+
+#Derby network properties
+#dbDriverName = org.apache.derby.jdbc.ClientXADataSource
+#dbUrl=jdbc:derby://localhost:1527/fhirdb
+#schemaName = FHIRDATA
+
+#Db2 properties
+#dbDriverName = com.ibm.db2.jcc.DB2Driver
+#dbUrl = jdbc:db2://localhost:50000/fhirdb
+#user = db2inst1
+#password = change-password
+#schemaName = FHIRDATA
+
+#PostgreSql properties
+#dbDriverName = org.postgresql.Driver
+#dbUrl = jdbc:postgresql://localhost:5432/fhirdb
+#user = postgre
+#password = change-password
+#PostgreSql use lower case by default
+#schemaName = fhirdata
+
+#common properties
+updateCreateEnabled = true
diff --git a/fhir-search/src/main/java/com/ibm/fhir/search/SearchConstants.java b/fhir-search/src/main/java/com/ibm/fhir/search/SearchConstants.java
index 35c17fbddb4..538d929b54c 100644
--- a/fhir-search/src/main/java/com/ibm/fhir/search/SearchConstants.java
+++ b/fhir-search/src/main/java/com/ibm/fhir/search/SearchConstants.java
@@ -98,6 +98,9 @@ private SearchConstants() {
     // url
     public static final String URL = "url";
 
+    // uri
+    public static final String URI = "uri";
+
     // version
     public static final String VERSION = "version";
 
diff --git a/fhir-search/src/main/java/com/ibm/fhir/search/util/ReferenceUtil.java b/fhir-search/src/main/java/com/ibm/fhir/search/util/ReferenceUtil.java
index db3c9b0fa16..423a7604331 100644
--- a/fhir-search/src/main/java/com/ibm/fhir/search/util/ReferenceUtil.java
+++ b/fhir-search/src/main/java/com/ibm/fhir/search/util/ReferenceUtil.java
@@ -140,7 +140,7 @@ public static ReferenceValue createReferenceValueFrom(String refValue, String re
                         version = Integer.parseInt(tokens[3]);
                     }
                 }
-            } else if (value != null && value.startsWith(HTTP) || value.startsWith(HTTPS) || value.startsWith(URN)) {
+            } else if (isAbsolute(value)) {
                 // - Absolute reference. We only know the type if it is given by the type field
                 referenceType = ReferenceType.LITERAL_ABSOLUTE;
                 if (refType != null) {
@@ -165,6 +165,15 @@ public static ReferenceValue createReferenceValueFrom(String refValue, String re
         return new ReferenceValue(targetResourceType, value, referenceType, version);
     }
 
+    /**
+     * Does the given value appear to be an absolute reference?
+     * @param value
+     * @return
+     */
+    public static boolean isAbsolute(String value) {
+        return value != null && value.startsWith(HTTP) || value.startsWith(HTTPS) || value.startsWith(URN);
+    }
+
     /**
      * Extract the base URL from the bundle entry if one is given, otherwise
      * use the service base URL.
diff --git a/fhir-server-spi/src/main/java/com/ibm/fhir/server/spi/operation/FHIRResourceHelpers.java b/fhir-server-spi/src/main/java/com/ibm/fhir/server/spi/operation/FHIRResourceHelpers.java
index 9604f67d724..f5860263bec 100644
--- a/fhir-server-spi/src/main/java/com/ibm/fhir/server/spi/operation/FHIRResourceHelpers.java
+++ b/fhir-server-spi/src/main/java/com/ibm/fhir/server/spi/operation/FHIRResourceHelpers.java
@@ -490,11 +490,12 @@ Resource doInvoke(FHIROperationContext operationContext, String resourceTypeName
      * @param indexIds list of index IDs of resources to reindex, or null
      * @param resourceLogicalId resourceType (e.g. "Patient"), or resourceType/logicalId a specific resource (e.g. "Patient/abc123"), to reindex, or null;
      * this parameter is ignored if the indexIds parameter value is non-null
+     * @param force if true, ignore parameter hash and always replace the parameters
      * @return count of the number of resources reindexed by this call
      * @throws Exception
      */
     int doReindex(FHIROperationContext operationContext, OperationOutcome.Builder operationOutcomeResult, Instant tstamp, List indexIds,
-        String resourceLogicalId) throws Exception;
+        String resourceLogicalId, boolean force) throws Exception;
 
     /**
      * Invoke the FHIR Persistence erase operation for a specific instance of the erase.
diff --git a/fhir-server/pom.xml b/fhir-server/pom.xml
index 661c2bdb593..25862d99770 100644
--- a/fhir-server/pom.xml
+++ b/fhir-server/pom.xml
@@ -117,6 +117,10 @@
             org.apache.httpcomponents
             httpclient
         
+        
+            com.google.code.gson
+            gson
+        
         
             ${project.groupId}
             fhir-examples
diff --git a/fhir-server/src/main/java/com/ibm/fhir/server/filter/rest/FHIRRestServletFilter.java b/fhir-server/src/main/java/com/ibm/fhir/server/filter/rest/FHIRRestServletFilter.java
index a343ef84791..bf0389d6fb0 100644
--- a/fhir-server/src/main/java/com/ibm/fhir/server/filter/rest/FHIRRestServletFilter.java
+++ b/fhir-server/src/main/java/com/ibm/fhir/server/filter/rest/FHIRRestServletFilter.java
@@ -57,6 +57,7 @@ public class FHIRRestServletFilter extends HttpFilter {
     private static String tenantIdHeaderName = null;
     private static String datastoreIdHeaderName = null;
     private static String originalRequestUriHeaderName = null;
+    private static String shardKeyHeaderName = null;
     private static final String preferHeaderName = "Prefer";
     private static final String preferHandlingHeaderSectionName = "handling";
     private static final String preferReturnHeaderSectionName = "return";
@@ -72,6 +73,7 @@ public void doFilter(HttpServletRequest request, HttpServletResponse response, F
 
         long initialTime = System.nanoTime();
 
+        String shardKey = null;
         String tenantId = defaultTenantId;
         String dsId = FHIRConfiguration.DEFAULT_DATASTORE_ID;
         String requestUrl = getRequestURL(request);
@@ -93,6 +95,11 @@ public void doFilter(HttpServletRequest request, HttpServletResponse response, F
             dsId = t;
         }
 
+        t = request.getHeader(shardKeyHeaderName);
+        if (t != null) {
+            shardKey = t;
+        }
+
         t = getOriginalRequestURI(request);
         if (t != null) {
             originalRequestUri = t;
@@ -115,6 +122,9 @@ public void doFilter(HttpServletRequest request, HttpServletResponse response, F
             requestDescription.append(originalRequestUri);
         }
         requestDescription.append("]");
+        if (shardKey != null) {
+            requestDescription.append(" shardKey:[").append(shardKey).append("]");
+        }
         String encodedRequestDescription = Encode.forHtml(requestDescription.toString());
         log.info("Received request: " + encodedRequestDescription);
 
@@ -129,6 +139,10 @@ public void doFilter(HttpServletRequest request, HttpServletResponse response, F
             // Don't try using FHIRConfigHelper before setting the context!
             FHIRRequestContext.set(context);
 
+            if (shardKey != null) {
+                context.setRequestShardKey(shardKey);
+            }
+
             context.setOriginalRequestUri(originalRequestUri);
 
             // Set the request headers.
@@ -501,6 +515,10 @@ public void init(FilterConfig filterConfig) throws ServletException {
             defaultTenantId =
                     config.getStringProperty(FHIRConfiguration.PROPERTY_DEFAULT_TENANT_ID, FHIRConfiguration.DEFAULT_TENANT_ID);
             log.info("Configured default tenant-id value is: " +  defaultTenantId);
+
+            shardKeyHeaderName = config.getStringProperty(FHIRConfiguration.PROPERTY_SHARD_KEY_HEADER_NAME,
+                FHIRConfiguration.DEFAULT_SHARD_KEY_HEADER_NAME);
+            log.info("Configured shard-key header name is: '" +  shardKeyHeaderName + "'");
         } catch (Exception e) {
             throw new ServletException("Servlet filter initialization error.", e);
         }
diff --git a/fhir-server/src/main/java/com/ibm/fhir/server/index/kafka/FHIRRemoteIndexKafkaService.java b/fhir-server/src/main/java/com/ibm/fhir/server/index/kafka/FHIRRemoteIndexKafkaService.java
new file mode 100644
index 00000000000..2a2a2a21e84
--- /dev/null
+++ b/fhir-server/src/main/java/com/ibm/fhir/server/index/kafka/FHIRRemoteIndexKafkaService.java
@@ -0,0 +1,154 @@
+/*
+ * (C) Copyright IBM Corp. 2022
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+ 
+package com.ibm.fhir.server.index.kafka;
+
+import java.util.Properties;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Future;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+import org.apache.kafka.clients.producer.KafkaProducer;
+import org.apache.kafka.clients.producer.Producer;
+import org.apache.kafka.clients.producer.ProducerConfig;
+import org.apache.kafka.clients.producer.ProducerRecord;
+import org.apache.kafka.clients.producer.RecordMetadata;
+
+import com.ibm.fhir.config.FHIRRequestContext;
+import com.ibm.fhir.persistence.helper.RemoteIndexSupport;
+import com.ibm.fhir.persistence.index.FHIRRemoteIndexService;
+import com.ibm.fhir.persistence.index.IndexProviderResponse;
+import com.ibm.fhir.persistence.index.RemoteIndexConstants;
+import com.ibm.fhir.persistence.index.RemoteIndexData;
+import com.ibm.fhir.persistence.index.RemoteIndexMessage;
+import com.ibm.fhir.server.index.kafka.KafkaPropertyAdapter.Mode;
+
+
+/**
+ * Forwards parameter blocks to a partitioned Kafka topic. This allows us to
+ * skip the expensive parameter insert operations during ingestion and offload
+ * them to a separate process where we can process the operations more efficiently
+ * by using larger batches and different concurrency control mechanisms
+ */
+public class FHIRRemoteIndexKafkaService extends FHIRRemoteIndexService {
+    private static final Logger logger = Logger.getLogger(FHIRRemoteIndexKafkaService.class.getName());
+
+    private String topicName = null;
+    private Producer producer;
+    private KafkaPropertyAdapter.Mode mode;
+    private String instanceIdentifier;
+
+    /**
+     * Default constructor
+     */
+    public FHIRRemoteIndexKafkaService() {
+    }
+
+    /**
+     * Initialize the provider
+     * @param properties
+     */
+    public void init(KafkaPropertyAdapter properties) {
+        this.mode = properties.getMode();
+        this.topicName = properties.getTopicName();
+        this.instanceIdentifier = properties.getInstanceIdentifier();
+        Properties kafkaProps = new Properties();
+        properties.putPropertiesTo(kafkaProps);
+
+        if (logger.isLoggable(Level.FINER)) {
+            logger.finer("Kafka async index publisher is configured with the following properties:\n" + kafkaProps.toString());
+            logger.finer("Topic name: " + this.topicName);
+        }
+
+        String bootstrapServers = kafkaProps.getProperty(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG);
+        if (bootstrapServers == null) {
+            throw new IllegalStateException("The " + ProducerConfig.BOOTSTRAP_SERVERS_CONFIG + " property was missing from the index service Kafka connection properties.");
+        }
+
+        if (this.mode != Mode.LOGONLY) {
+            producer = new KafkaProducer<>(kafkaProps);
+        }
+    }
+
+    /**
+     * Performs any necessary "shutdown" logic to disconnect from the topic.
+     */
+    public void shutdown() {
+        logger.entering(this.getClass().getName(), "shutdown");
+
+        try {
+            if (logger.isLoggable(Level.FINE)) {
+                logger.fine("Shutting down Kafka index service for topic: '" + topicName + "'.");
+            }
+            if (producer != null) {
+                producer.close();
+            }
+        } finally {
+            logger.exiting(this.getClass().getName(), "shutdown");
+        }
+    }
+
+    @Override
+    public IndexProviderResponse submit(final RemoteIndexData data) {
+        // We rely on the default Kafka partitioner, which in our case will
+        // select a partition based on a hash of the key, which should be
+        // something like "Patient/a-patient-logical-id"
+        final String tenantId = FHIRRequestContext.get().getTenantId();
+        RemoteIndexMessage msg = new RemoteIndexMessage();
+        msg.setMessageVersion(RemoteIndexConstants.MESSAGE_VERSION);
+        msg.setInstanceIdentifier(this.instanceIdentifier);
+        msg.setTenantId(tenantId);
+        msg.setData(data.getSearchParameters());
+        final String message = RemoteIndexSupport.marshallToString(msg);
+
+        if (this.mode == Mode.ACTIVE) {
+            ProducerRecord record = new ProducerRecord(topicName, data.getPartitionKey(), message);
+            Future rmd = producer.send(record);
+            
+            // convert the future we get from the producer into a CompletableFuture
+            // and then map this to the response type we use (information hiding...
+            // the caller shouldn't be exposed to the fact that we're using Kafka)
+            CompletableFuture cf = backToThe(rmd)
+                    .thenApply(v -> {
+                        return null;
+                    });
+            return new IndexProviderResponse(data, cf);
+        } else {
+            // just log the message to help debug
+            logger.info("Remote index request: " + message);
+            return new IndexProviderResponse(data, CompletableFuture.completedFuture(null));
+        }
+    }
+
+    /**
+     * Convert a Future into a CompletableFuture
+     * @param 
+     * @param f
+     * @return
+     */
+    public static  CompletableFuture backToThe(Future f) {
+        // This composition means we don't call f.get() until the get
+        // is called on the CompletableFuture result
+        return CompletableFuture.completedFuture(null).thenCompose(noValue -> {
+            try {
+                logger.finest("Waiting for index service Kafka send to complete");
+                return CompletableFuture.completedFuture(f.get());
+            } catch (InterruptedException e) {
+                // the only time we should be interrupted is during shutdown, so no
+                // need to log here because it should be expected that we'll fail
+                return CompletableFuture.failedFuture(e);
+            } catch (ExecutionException e) {
+                // Log the issue right away so that it can be time-correlated with other issues
+                logger.log(Level.SEVERE, "Failed async submission to Kafka", e);
+                return CompletableFuture.failedFuture(e.getCause());
+            } finally {
+                logger.finest("completed");
+            }
+        });
+    }
+}
\ No newline at end of file
diff --git a/fhir-server/src/main/java/com/ibm/fhir/server/index/kafka/KafkaPropertyAdapter.java b/fhir-server/src/main/java/com/ibm/fhir/server/index/kafka/KafkaPropertyAdapter.java
new file mode 100644
index 00000000000..6db10342a61
--- /dev/null
+++ b/fhir-server/src/main/java/com/ibm/fhir/server/index/kafka/KafkaPropertyAdapter.java
@@ -0,0 +1,76 @@
+/*
+ * (C) Copyright IBM Corp. 2022
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+ 
+package com.ibm.fhir.server.index.kafka;
+
+import java.util.Properties;
+
+import org.apache.kafka.clients.producer.ProducerConfig;
+
+/**
+ * Wrapper around a {@link Properties} making them easier to consume
+ */
+public class KafkaPropertyAdapter {
+    private final Properties properties;
+    private final String topicName;
+    private final String instanceIdentifier;
+    private final Mode mode;
+    
+    public static enum Mode {
+        ACTIVE,
+        LOGONLY
+    }
+
+    /**
+     * Public constructor
+     * 
+     * @param instanceIdentifier
+     * @param topicName
+     * @param properties
+     * @param mode
+     */
+    public KafkaPropertyAdapter(String instanceIdentifier, String topicName, Properties properties, Mode mode) {
+        this.instanceIdentifier = instanceIdentifier;
+        this.topicName = topicName;
+        this.properties = properties;
+        this.mode = mode;
+    }
+
+    /**
+     * Fill the given kafkaProperties object with the configuration properties
+     * held by this adapter
+     * @param kafkaProps
+     */
+    public void putPropertiesTo(Properties kafkaProps) {
+        kafkaProps.put(ProducerConfig.CLIENT_ID_CONFIG, "fhir-server");
+        kafkaProps.putAll(this.properties);
+        // Make sure we always use these serializers, even if the property
+        // has been defined in this.properties
+        kafkaProps.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringSerializer");
+        kafkaProps.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringSerializer");
+    }
+
+    /**
+     * @return the topicName
+     */
+    public String getTopicName() {
+        return topicName;
+    }
+
+    /**
+     * @return the mode
+     */
+    public Mode getMode() {
+        return mode;
+    }
+
+    /**
+     * @return the instanceIdentifier
+     */
+    public String getInstanceIdentifier() {
+        return instanceIdentifier;
+    }
+}
diff --git a/fhir-server/src/main/java/com/ibm/fhir/server/listener/FHIRServletContextListener.java b/fhir-server/src/main/java/com/ibm/fhir/server/listener/FHIRServletContextListener.java
index f85ba74354e..98404ebd189 100644
--- a/fhir-server/src/main/java/com/ibm/fhir/server/listener/FHIRServletContextListener.java
+++ b/fhir-server/src/main/java/com/ibm/fhir/server/listener/FHIRServletContextListener.java
@@ -12,6 +12,9 @@
 import static com.ibm.fhir.config.FHIRConfiguration.PROPERTY_EXTENDED_CODEABLE_CONCEPT_VALIDATION;
 import static com.ibm.fhir.config.FHIRConfiguration.PROPERTY_KAFKA_CONNECTIONPROPS;
 import static com.ibm.fhir.config.FHIRConfiguration.PROPERTY_KAFKA_ENABLED;
+import static com.ibm.fhir.config.FHIRConfiguration.PROPERTY_KAFKA_INDEX_SERVICE_CONNECTIONPROPS;
+import static com.ibm.fhir.config.FHIRConfiguration.PROPERTY_KAFKA_INDEX_SERVICE_MODE;
+import static com.ibm.fhir.config.FHIRConfiguration.PROPERTY_KAFKA_INDEX_SERVICE_TOPICNAME;
 import static com.ibm.fhir.config.FHIRConfiguration.PROPERTY_KAFKA_TOPICNAME;
 import static com.ibm.fhir.config.FHIRConfiguration.PROPERTY_NATS_CHANNEL;
 import static com.ibm.fhir.config.FHIRConfiguration.PROPERTY_NATS_CLIENT;
@@ -23,6 +26,8 @@
 import static com.ibm.fhir.config.FHIRConfiguration.PROPERTY_NATS_TLS_ENABLED;
 import static com.ibm.fhir.config.FHIRConfiguration.PROPERTY_NATS_TRUSTSTORE;
 import static com.ibm.fhir.config.FHIRConfiguration.PROPERTY_NATS_TRUSTSTORE_PW;
+import static com.ibm.fhir.config.FHIRConfiguration.PROPERTY_REMOTE_INDEX_SERVICE_INSTANCEIDENTIFIER;
+import static com.ibm.fhir.config.FHIRConfiguration.PROPERTY_REMOTE_INDEX_SERVICE_TYPE;
 import static com.ibm.fhir.config.FHIRConfiguration.PROPERTY_SERVER_REGISTRY_RESOURCE_PROVIDER_ENABLED;
 import static com.ibm.fhir.config.FHIRConfiguration.PROPERTY_SERVER_RESOLVE_FUNCTION_ENABLED;
 import static com.ibm.fhir.config.FHIRConfiguration.PROPERTY_WEBSOCKET_ENABLED;
@@ -54,9 +59,13 @@
 import com.ibm.fhir.model.util.FHIRUtil;
 import com.ibm.fhir.model.util.ModelSupport;
 import com.ibm.fhir.path.function.registry.FHIRPathFunctionRegistry;
+import com.ibm.fhir.persistence.exception.FHIRPersistenceException;
 import com.ibm.fhir.persistence.helper.FHIRPersistenceHelper;
+import com.ibm.fhir.persistence.index.FHIRRemoteIndexService;
 import com.ibm.fhir.registry.FHIRRegistry;
 import com.ibm.fhir.search.util.SearchHelper;
+import com.ibm.fhir.server.index.kafka.FHIRRemoteIndexKafkaService;
+import com.ibm.fhir.server.index.kafka.KafkaPropertyAdapter;
 import com.ibm.fhir.server.notification.kafka.FHIRNotificationKafkaPublisher;
 import com.ibm.fhir.server.notification.websocket.FHIRNotificationServiceEndpointConfig;
 import com.ibm.fhir.server.notifications.nats.FHIRNotificationNATSPublisher;
@@ -81,12 +90,14 @@ public class FHIRServletContextListener implements ServletContextListener {
 
     private static final String ATTRNAME_WEBSOCKET_SERVERCONTAINER = "javax.websocket.server.ServerContainer";
     private static final String DEFAULT_KAFKA_TOPICNAME = "fhirNotifications";
+    private static final String DEFAULT_KAFKA_INDEX_SERVICE_TOPICNAME = "fhirIndex";
     private static final String DEFAULT_NATS_CHANNEL = "fhirNotifications";
     private static final String DEFAULT_NATS_CLUSTER = "nats-streaming";
     private static final String DEFAULT_NATS_CLIENT = "fhir-server";
     public static final String FHIR_SERVER_INIT_COMPLETE = "com.ibm.fhir.webappInitComplete";
     private static FHIRNotificationKafkaPublisher kafkaPublisher = null;
     private static FHIRNotificationNATSPublisher natsPublisher = null;
+    private static FHIRRemoteIndexService remoteIndexService = null;
 
     private List graphTermServiceProviders = new ArrayList<>();
     private List remoteTermServiceProviders = new ArrayList<>();
@@ -215,6 +226,40 @@ public void contextInitialized(ServletContextEvent event) {
                 log.info("Bypassing NATS notification init.");
             }
 
+            // If the Kafka async indexing service is enabled, set it up so that it's ready to go
+            // before we starting processing any requests.
+            String remoteIndexServiceType = fhirConfig.getStringProperty(PROPERTY_REMOTE_INDEX_SERVICE_TYPE, null);
+            if (remoteIndexServiceType != null) {
+                if ("kafka".equals(remoteIndexServiceType)) {
+                    String topicName = fhirConfig.getStringProperty(PROPERTY_KAFKA_INDEX_SERVICE_TOPICNAME, DEFAULT_KAFKA_INDEX_SERVICE_TOPICNAME);
+                    String instanceIdentifier = fhirConfig.getStringProperty(PROPERTY_REMOTE_INDEX_SERVICE_INSTANCEIDENTIFIER);
+                    String mode = fhirConfig.getStringProperty(PROPERTY_KAFKA_INDEX_SERVICE_MODE, "active");
+    
+                    // Gather up the Kafka connection properties for the async index service
+                    Properties kafkaProps = new Properties();
+                    PropertyGroup pg = fhirConfig.getPropertyGroup(PROPERTY_KAFKA_INDEX_SERVICE_CONNECTIONPROPS);
+                    if (pg != null) {
+                        List connectionProps = pg.getProperties();
+                        if (connectionProps != null) {
+                            for (PropertyEntry entry : connectionProps) {
+                                kafkaProps.setProperty(entry.getName(), entry.getValue().toString());
+                            }
+                        }
+                    }
+    
+                    log.info("Initializing Kafka async indexing service.");
+                    FHIRRemoteIndexKafkaService s = new FHIRRemoteIndexKafkaService();
+                    s.init(new KafkaPropertyAdapter(instanceIdentifier, topicName, kafkaProps, KafkaPropertyAdapter.Mode.valueOf(mode)));
+                    // Now the service is ready, we can publish it
+                    remoteIndexService = s;
+                    FHIRRemoteIndexService.setServiceInstance(remoteIndexService);
+                } else {
+                    throw new FHIRPersistenceException("Invalid value for remote index service property '" + PROPERTY_REMOTE_INDEX_SERVICE_TYPE + "'");
+                }
+            } else {
+                log.info("Bypassing Kafka async indexing service configuration.");
+            }
+
             Boolean checkReferenceTypes = fhirConfig.getBooleanProperty(PROPERTY_CHECK_REFERENCE_TYPES, Boolean.TRUE);
             FHIRModelConfig.setCheckReferenceTypes(checkReferenceTypes);
 
diff --git a/fhir-server/src/main/java/com/ibm/fhir/server/registry/ServerRegistryResourceProvider.java b/fhir-server/src/main/java/com/ibm/fhir/server/registry/ServerRegistryResourceProvider.java
index 44831fa00c3..78089451ff7 100644
--- a/fhir-server/src/main/java/com/ibm/fhir/server/registry/ServerRegistryResourceProvider.java
+++ b/fhir-server/src/main/java/com/ibm/fhir/server/registry/ServerRegistryResourceProvider.java
@@ -121,7 +121,7 @@ private List computeRegistryResources(Class getRegistryResources(Class doRead(String type, String id,
             getInterceptorMgr().fireBeforeReadEvent(event);
 
             FHIRPersistenceContext persistenceContext =
-                    FHIRPersistenceContextFactory.createPersistenceContext(event, searchContext);
+                    FHIRPersistenceContextFactory.createPersistenceContext(event, searchContext, requestContext.getRequestShardKey());
             result = persistence.read(persistenceContext, resourceType, id);
             event.setFhirResource(result.getResource());
 
@@ -1228,7 +1230,7 @@ public SingleResourceResult doVRead(String type, String id,
             getInterceptorMgr().fireBeforeVreadEvent(event);
 
             FHIRPersistenceContext persistenceContext =
-                    FHIRPersistenceContextFactory.createPersistenceContext(event, searchContext);
+                    FHIRPersistenceContextFactory.createPersistenceContext(event, searchContext, requestContext.getRequestShardKey());
             SingleResourceResult srr = persistence.vread(persistenceContext, resourceType, id, versionId);
 
             // The resource may be null if it doesn't exist or has been deleted
@@ -1378,7 +1380,7 @@ public Bundle doSearch(String type, String compartment, String compartmentId,
             getInterceptorMgr().fireBeforeSearchEvent(event);
 
             FHIRPersistenceContext persistenceContext =
-                    FHIRPersistenceContextFactory.createPersistenceContext(event, searchContext);
+                    FHIRPersistenceContextFactory.createPersistenceContext(event, searchContext, requestContext.getRequestShardKey());
             MultiResourceResult searchResult =
                     persistence.search(persistenceContext, resourceType);
 
@@ -2609,13 +2611,13 @@ private void setOperationContextProperties(FHIROperationContext operationContext
 
     @Override
     public int doReindex(FHIROperationContext operationContext, OperationOutcome.Builder operationOutcomeResult, Instant tstamp,
-            List indexIds, String resourceLogicalId) throws Exception {
+            List indexIds, String resourceLogicalId, boolean force) throws Exception {
         int result = 0;
         // Since the try logic is slightly different in the code paths, we want to dispatch to separate methods to simplify the logic.
         if (indexIds == null) {
-            result = doReindexSingle(operationOutcomeResult, tstamp, resourceLogicalId);
+            result = doReindexSingle(operationOutcomeResult, tstamp, resourceLogicalId, force);
         } else {
-            result = doReindexList(operationOutcomeResult, tstamp, indexIds);
+            result = doReindexList(operationOutcomeResult, tstamp, indexIds, force);
         }
         return result;
     }
@@ -2626,10 +2628,11 @@ public int doReindex(FHIROperationContext operationContext, OperationOutcome.Bui
      * @param operationOutcomeResult
      * @param tstamp
      * @param indexIds
+     * @param force
      * @return
      * @throws Exception
      */
-    public int doReindexList(OperationOutcome.Builder operationOutcomeResult, Instant tstamp, List indexIds) throws Exception {
+    public int doReindexList(OperationOutcome.Builder operationOutcomeResult, Instant tstamp, List indexIds, boolean force) throws Exception {
         // If the indexIds are empty or null, then it's not properly formed.
         if (indexIds == null || indexIds.isEmpty()) {
             throw new IllegalArgumentException("No indexIds sent to the $reindex list method");
@@ -2682,7 +2685,7 @@ public int doReindexList(OperationOutcome.Builder operationOutcomeResult, Instan
             txn.begin();
             try {
                 FHIRPersistenceContext persistenceContext = null;
-                result += persistence.reindex(persistenceContext, operationOutcomeResult, tstamp, subListIndexIds, null);
+                result += persistence.reindex(persistenceContext, operationOutcomeResult, tstamp, subListIndexIds, null, force);
             } catch (FHIRPersistenceDataAccessException x) {
                 // At this point, the transaction is marked for rollback
                 if (x.isTransactionRetryable() && ++attempt <= TX_ATTEMPTS) {
@@ -2720,10 +2723,11 @@ public int doReindexList(OperationOutcome.Builder operationOutcomeResult, Instan
      * @param operationOutcomeResult
      * @param tstamp
      * @param resourceLogicalId
+     * @param force
      * @return
      * @throws Exception
      */
-    public int doReindexSingle(OperationOutcome.Builder operationOutcomeResult, Instant tstamp, String resourceLogicalId) throws Exception {
+    public int doReindexSingle(OperationOutcome.Builder operationOutcomeResult, Instant tstamp, String resourceLogicalId, boolean force) throws Exception {
         int result = 0;
         // handle some retries in case of deadlock exceptions
         final int TX_ATTEMPTS = 5;
@@ -2733,7 +2737,7 @@ public int doReindexSingle(OperationOutcome.Builder operationOutcomeResult, Inst
             txn.begin();
             try {
                 FHIRPersistenceContext persistenceContext = null;
-                result = persistence.reindex(persistenceContext, operationOutcomeResult, tstamp, null, resourceLogicalId);
+                result = persistence.reindex(persistenceContext, operationOutcomeResult, tstamp, null, resourceLogicalId, force);
                 attempt = TX_ATTEMPTS; // end the retry loop
             } catch (FHIRPersistenceDataAccessException x) {
                 if (x.isTransactionRetryable() && attempt < TX_ATTEMPTS) {
@@ -3040,6 +3044,8 @@ public Bundle doHistory(MultivaluedMap queryParameters, String r
         FHIRPersistenceEvent event =
                 new FHIRPersistenceEvent(null, buildPersistenceEventProperties(resourceType == null ? "Resource" : resourceType, null, null, null, historyContext));
         getInterceptorMgr().fireBeforeHistoryEvent(event);
+        // Build a context
+        FHIRPersistenceContext context = FHIRPersistenceContextImpl.builder(event).withRequestShard(requestContext.getRequestShardKey()).build();
 
         // Start a new txn in the persistence layer if one is not already active.
         Integer count = historyContext.getCount();
@@ -3056,7 +3062,7 @@ public Bundle doHistory(MultivaluedMap queryParameters, String r
 
             if (resourceType != null) {
                 // Use the resource type on the path, ignoring any _type parameter
-                records = persistence.changes(count, since, before, historyContext.getChangeIdMarker(), Collections.singletonList(resourceType),
+                records = persistence.changes(context, count, since, before, historyContext.getChangeIdMarker(), Collections.singletonList(resourceType),
                         historyContext.isExcludeTransactionTimeoutWindow(), historyContext.getHistorySortOrder());
             } else if (historyContext.getResourceTypes().size() > 0) {
                 // New API allows us to filter using multiple resource type names, but first we
@@ -3064,12 +3070,12 @@ public Bundle doHistory(MultivaluedMap queryParameters, String r
                 for (String rt: historyContext.getResourceTypes()) {
                     validateInteraction(Interaction.HISTORY, rt);
                 }
-                records = persistence.changes(count, since, before, historyContext.getChangeIdMarker(), historyContext.getResourceTypes(),
+                records = persistence.changes(context, count, since, before, historyContext.getChangeIdMarker(), historyContext.getResourceTypes(),
                         historyContext.isExcludeTransactionTimeoutWindow(), historyContext.getHistorySortOrder());
             } else {
                 // no resource type filter
                 final List NULL_RESOURCE_TYPE_NAMES = null;
-                records = persistence.changes(count, since, before, historyContext.getChangeIdMarker(), NULL_RESOURCE_TYPE_NAMES,
+                records = persistence.changes(context, count, since, before, historyContext.getChangeIdMarker(), NULL_RESOURCE_TYPE_NAMES,
                         historyContext.isExcludeTransactionTimeoutWindow(), historyContext.getHistorySortOrder());
             }
 
@@ -3301,6 +3307,7 @@ public Bundle doHistory(MultivaluedMap queryParameters, String r
     @Override
     public ResourceEraseRecord doErase(FHIROperationContext operationContext, EraseDTO eraseDto) throws FHIROperationException {
         // @implNote doReindex has a nice pattern to handle some retries in case of deadlock exceptions
+        FHIRPersistenceContext context = null;
         final int TX_ATTEMPTS = 5;
         int attempt = 1;
         ResourceEraseRecord eraseRecord = new ResourceEraseRecord();
@@ -3309,7 +3316,7 @@ public ResourceEraseRecord doErase(FHIROperationContext operationContext, EraseD
             try {
                 txn = new FHIRTransactionHelper(getTransaction());
                 txn.begin();
-                eraseRecord = persistence.erase(eraseDto);
+                eraseRecord = persistence.erase(context, eraseDto);
                 attempt = TX_ATTEMPTS; // end the retry loop
             } catch (FHIRPersistenceDataAccessException x) {
                 if (x.isTransactionRetryable() && attempt < TX_ATTEMPTS) {
@@ -3333,11 +3340,13 @@ public ResourceEraseRecord doErase(FHIROperationContext operationContext, EraseD
     public List doRetrieveIndex(FHIROperationContext operationContext, String resourceTypeName, int count, Instant notModifiedAfter, Long afterIndexId) throws Exception {
         List indexIds = null;
 
+        FHIRPersistenceContext context = null;
+        
         FHIRTransactionHelper txn = null;
         try {
             txn = new FHIRTransactionHelper(getTransaction());
             txn.begin();
-            indexIds = persistence.retrieveIndex(count, notModifiedAfter, afterIndexId, resourceTypeName);
+            indexIds = persistence.retrieveIndex(context, count, notModifiedAfter, afterIndexId, resourceTypeName);
         } finally {
             if (txn != null) {
                 txn.end();
diff --git a/fhir-server/src/test/java/com/ibm/fhir/server/test/MockPersistenceImpl.java b/fhir-server/src/test/java/com/ibm/fhir/server/test/MockPersistenceImpl.java
index e37f9912460..6c798873207 100644
--- a/fhir-server/src/test/java/com/ibm/fhir/server/test/MockPersistenceImpl.java
+++ b/fhir-server/src/test/java/com/ibm/fhir/server/test/MockPersistenceImpl.java
@@ -144,7 +144,7 @@ public String generateResourceId() {
 
     @Override
     public int reindex(FHIRPersistenceContext context, Builder operationOutcomeResult, java.time.Instant tstamp, List indexIds,
-        String resourceLogicalId) throws FHIRPersistenceException {
+        String resourceLogicalId, boolean force) throws FHIRPersistenceException {
         return 0;
     }
 
@@ -162,7 +162,7 @@ public ResourcePayload fetchResourcePayloads(Class resourceT
     }
 
     @Override
-    public List changes(int resourceCount, java.time.Instant sinceLastModified, java.time.Instant beforeLastModified,
+    public List changes(FHIRPersistenceContext context, int resourceCount, java.time.Instant sinceLastModified, java.time.Instant beforeLastModified,
             Long changeIdMarker, List resourceTypeNames, boolean excludeTransactionTimeoutWindow, HistorySortOrder historySortOrder)
             throws FHIRPersistenceException {
         // NOP
@@ -170,7 +170,7 @@ public List changes(int resourceCount, java.time.Instan
     }
 
     @Override
-    public List retrieveIndex(int count, java.time.Instant notModifiedAfter, Long afterIndexId, String resourceTypeName) throws FHIRPersistenceException {
+    public List retrieveIndex(FHIRPersistenceContext context, int count, java.time.Instant notModifiedAfter, Long afterIndexId, String resourceTypeName) throws FHIRPersistenceException {
         // NOP
         return null;
     }
diff --git a/fhir-server/src/test/java/com/ibm/fhir/server/test/ServerResolveFunctionTest.java b/fhir-server/src/test/java/com/ibm/fhir/server/test/ServerResolveFunctionTest.java
index a6abf24e76d..b30492c3b65 100644
--- a/fhir-server/src/test/java/com/ibm/fhir/server/test/ServerResolveFunctionTest.java
+++ b/fhir-server/src/test/java/com/ibm/fhir/server/test/ServerResolveFunctionTest.java
@@ -418,7 +418,8 @@ public int reindex(
                 Builder operationOutcomeResult,
                 java.time.Instant tstamp,
                 List indexIds,
-                String resourceLogicalId) throws FHIRPersistenceException {
+                String resourceLogicalId,
+                boolean force) throws FHIRPersistenceException {
             throw new UnsupportedOperationException();
         }
 
@@ -433,6 +434,7 @@ public ResourcePayload fetchResourcePayloads(
 
         @Override
         public List changes(
+                FHIRPersistenceContext context,
                 int resourceCount,
                 java.time.Instant sinceLastModified,
                 java.time.Instant beforeLastModified,
@@ -474,7 +476,7 @@ private  SingleResourceResult createOrUpdate(T resource)
         }
 
         @Override
-        public List retrieveIndex(int count, java.time.Instant notModifiedAfter, Long afterIndexId, String resourceTypeName) throws FHIRPersistenceException {
+        public List retrieveIndex(FHIRPersistenceContext context, int count, java.time.Instant notModifiedAfter, Long afterIndexId, String resourceTypeName) throws FHIRPersistenceException {
             throw new UnsupportedOperationException();
         }
 
diff --git a/fhir-smart/src/test/java/com/ibm/fhir/smart/test/MockPersistenceImpl.java b/fhir-smart/src/test/java/com/ibm/fhir/smart/test/MockPersistenceImpl.java
index 507a2ef0cd7..cbca679ba0c 100644
--- a/fhir-smart/src/test/java/com/ibm/fhir/smart/test/MockPersistenceImpl.java
+++ b/fhir-smart/src/test/java/com/ibm/fhir/smart/test/MockPersistenceImpl.java
@@ -169,7 +169,7 @@ public OperationOutcome getHealth() throws FHIRPersistenceException {
 
     @Override
     public int reindex(FHIRPersistenceContext context, OperationOutcome.Builder oob, Instant tstamp, List indexIds,
-        String resourceLogicalId) throws FHIRPersistenceException {
+            String resourceLogicalId, boolean force) throws FHIRPersistenceException {
         return 0;
     }
 
@@ -185,13 +185,13 @@ public ResourcePayload fetchResourcePayloads(Class resourceT
     }
 
     @Override
-    public List changes(int resourceCount, Instant sinceLastModified, Instant beforeLastModified, Long afterResourceId, List resourceTypeNames,
+    public List changes(FHIRPersistenceContext context, int resourceCount, Instant sinceLastModified, Instant beforeLastModified, Long afterResourceId, List resourceTypeNames,
             boolean excludeTransactionTimeoutWindow, HistorySortOrder historySortOrder) throws FHIRPersistenceException {
         return null;
     }
 
     @Override
-    public List retrieveIndex(int count, java.time.Instant notModifiedAfter, Long afterIndexId, String resourceTypeName) throws FHIRPersistenceException {
+    public List retrieveIndex(FHIRPersistenceContext context, int count, java.time.Instant notModifiedAfter, Long afterIndexId, String resourceTypeName) throws FHIRPersistenceException {
         return null;
     }
 
diff --git a/operation/fhir-operation-erase/src/test/java/com/ibm/fhir/operation/erase/mock/MockFHIRResourceHelpers.java b/operation/fhir-operation-erase/src/test/java/com/ibm/fhir/operation/erase/mock/MockFHIRResourceHelpers.java
index e78252814f0..a9824d7a80d 100644
--- a/operation/fhir-operation-erase/src/test/java/com/ibm/fhir/operation/erase/mock/MockFHIRResourceHelpers.java
+++ b/operation/fhir-operation-erase/src/test/java/com/ibm/fhir/operation/erase/mock/MockFHIRResourceHelpers.java
@@ -74,7 +74,7 @@ public FHIRPersistenceTransaction getTransaction() throws Exception {
 
     @Override
     public int doReindex(FHIROperationContext operationContext, Builder operationOutcomeResult, Instant tstamp, List indexIds,
-        String resourceLogicalId) throws Exception {
+        String resourceLogicalId, boolean force) throws Exception {
         return 0;
     }
 
diff --git a/operation/fhir-operation-member-match/src/test/java/com/ibm/fhir/operation/davinci/hrex/test/MemberMatchTest.java b/operation/fhir-operation-member-match/src/test/java/com/ibm/fhir/operation/davinci/hrex/test/MemberMatchTest.java
index 52e1456c37e..9ee0ae3a2f8 100644
--- a/operation/fhir-operation-member-match/src/test/java/com/ibm/fhir/operation/davinci/hrex/test/MemberMatchTest.java
+++ b/operation/fhir-operation-member-match/src/test/java/com/ibm/fhir/operation/davinci/hrex/test/MemberMatchTest.java
@@ -1702,7 +1702,7 @@ public FHIRPersistenceTransaction getTransaction() throws Exception {
 
         @Override
         public int doReindex(FHIROperationContext operationContext, Builder operationOutcomeResult, Instant tstamp, List indexIds,
-            String resourceLogicalId) throws Exception {
+            String resourceLogicalId, boolean force) throws Exception {
             throw new AssertionError("Unused");
         }
 
diff --git a/operation/fhir-operation-reindex/src/main/java/com/ibm/fhir/operation/reindex/ReindexOperation.java b/operation/fhir-operation-reindex/src/main/java/com/ibm/fhir/operation/reindex/ReindexOperation.java
index 5806a177acb..d32f644a784 100644
--- a/operation/fhir-operation-reindex/src/main/java/com/ibm/fhir/operation/reindex/ReindexOperation.java
+++ b/operation/fhir-operation-reindex/src/main/java/com/ibm/fhir/operation/reindex/ReindexOperation.java
@@ -46,6 +46,7 @@ public class ReindexOperation extends AbstractOperation {
     private static final String PARAM_INDEX_IDS = "indexIds";
     private static final String PARAM_RESOURCE_COUNT = "resourceCount";
     private static final String PARAM_RESOURCE_LOGICAL_ID = "resourceLogicalId";
+    private static final String PARAM_FORCE = "force";
 
     // The max number of resources we allow to be processed by one request
     private static final int MAX_RESOURCE_COUNT = 1000;
@@ -84,6 +85,7 @@ protected Parameters doInvoke(FHIROperationContext operationContext, Class indexIds = null;
             int resourceCount = 10;
             String resourceLogicalId = null;
+            boolean force = false;
 
             boolean hasSpecificResourceType = false;
             if (resourceType != null) {
@@ -100,7 +102,8 @@ protected Parameters doInvoke(FHIROperationContext operationContext, Class MAX_RESOURCE_COUNT) {
@@ -137,7 +140,15 @@ protected Parameters doInvoke(FHIROperationContext operationContext, Class 0; i++) {
-                    processed = resourceHelper.doReindex(operationContext, result, tstamp, null, resourceLogicalId);
+                    processed = resourceHelper.doReindex(operationContext, result, tstamp, null, resourceLogicalId, force);
                     totalProcessed += processed;
                 }
             }
diff --git a/operation/fhir-operation-reindex/src/main/resources/reindex.json b/operation/fhir-operation-reindex/src/main/resources/reindex.json
index 45abc9f7ce3..51f78138910 100644
--- a/operation/fhir-operation-reindex/src/main/resources/reindex.json
+++ b/operation/fhir-operation-reindex/src/main/resources/reindex.json
@@ -52,6 +52,14 @@
             "max": "1",
             "documentation": "Reindex only the specified resource or resources of the given resource type when no id is provided. Format as Patient/abc123 or Patient. If indexIds is specified, this parameter is not used.",
             "type": "string"
+        },
+        {
+            "name": "force",
+            "use": "in",
+            "min": 0,
+            "max": "1",
+            "documentation": "When true, always replace the parameters even if the parameter hash matches. This is typically used when a schema migration step changes structure used to stored parameters in the database.",
+            "type": "boolean"
         }
     ]
-}
\ No newline at end of file
+}