diff --git a/README.md b/README.md index e4236be..ce8e069 100644 --- a/README.md +++ b/README.md @@ -27,11 +27,12 @@ This section describes how to download, install and run the application. git clone https://github.com/prebid/prebid-cache-java.git ``` -(2). Start Redis or Aerospike: -If you have installed [Redis](https://redis.io/docs/install/install-redis/) or [Aerospike](https://aerospike.com/docs/server/operations/install) locally, you may start them both (or separately, depends on your needs) via bash: +(2). Start Repo Locally: +If you have installed [Redis](https://redis.io/docs/install/install-redis/) or [Aerospike](https://aerospike.com/docs/server/operations/install) or [Apache Ignite](https://ignite.apache.org/docs/latest/installation/deb-rpm) locally, you may start them both (or separately, depends on your needs) via bash: ```bash sudo systemctl start redis sudo systemctl start aerospike +sudo systemctl start apache-ignite ``` Alternatively, you may start DB as Docker image. You should install [Docker Engine](https://docs.docker.com/engine/install/) if you don't have one. @@ -67,7 +68,29 @@ docker run -d --name aerospike -e "NAMESPACE=" -p : +``` +2. Run Apache Ignite container + - the `` should correspond to the pulled image version + - the `OPTION_LIBS=ignite-rest-http` corresponds to the library that enables REST API access to the server (i.e. to create a cache inside Apache Ignite) + - the `-p 10800:10800` exposes the default port `10800` that will be a connection port for the Prebid Cache + - the host will be the `localhost` +```bash +docker run -d \ + --name apache-ignite + -e "OPTION_LIBS=ignite-rest-http" \ + -p 8080:8080 -p 10800:10800 -p 11211:11211 -p 47100:47100 -p 47500:47500 \ + apacheignite/ignite: +``` +3. Create a cache via Rest Api +```bash +GET http://localhost:10800/ignite?cmd=getorcreate&cacheName= +``` + +(2.4) Make sure that the Aerospike, Redis and/or Apache Ignite is up and running ```bash docker ps ``` @@ -112,8 +135,37 @@ _VM Options:_ java -jar prebid-cache.jar -Dspring.profiles.active=prod -Dlog.dir=/app/prebid-cache-java/log/ ``` +_Note_ +The `Apache Ignite` requires additional VM parameters to be added to support Java 17+ +```bash +--add-opens=java.base/jdk.internal.access=ALL-UNNAMED +--add-opens=java.base/jdk.internal.misc=ALL-UNNAMED +--add-opens=java.base/sun.nio.ch=ALL-UNNAMED +--add-opens=java.base/sun.util.calendar=ALL-UNNAMED +--add-opens=java.management/com.sun.jmx.mbeanserver=ALL-UNNAMED +--add-opens=jdk.internal.jvmstat/sun.jvmstat.monitor=ALL-UNNAMED +--add-opens=java.base/sun.reflect.generics.reflectiveObjects=ALL-UNNAMED +--add-opens=jdk.management/com.sun.management.internal=ALL-UNNAMED +--add-opens=java.base/java.io=ALL-UNNAMED +--add-opens=java.base/java.nio=ALL-UNNAMED +--add-opens=java.base/java.net=ALL-UNNAMED +--add-opens=java.base/java.util=ALL-UNNAMED +--add-opens=java.base/java.util.concurrent=ALL-UNNAMED +--add-opens=java.base/java.util.concurrent.locks=ALL-UNNAMED +--add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED +--add-opens=java.base/java.lang=ALL-UNNAMED +--add-opens=java.base/java.lang.invoke=ALL-UNNAMED +--add-opens=java.base/java.math=ALL-UNNAMED +--add-opens=java.sql/java.sql=ALL-UNNAMED +--add-opens=java.base/java.lang.reflect=ALL-UNNAMED +--add-opens=java.base/java.time=ALL-UNNAMED +--add-opens=java.base/java.text=ALL-UNNAMED +--add-opens=java.management/sun.management=ALL-UNNAMED +--add-opens java.desktop/java.awt.font=ALL-UNNAMED +``` + ### _Cache Configuration_ -Prebid cache uses Aerospike as a default cache implementation but also supports Redis. For switching from Aerospike +Prebid cache uses Aerospike as a default cache implementation but also supports Redis and Apache Ignite. For switching from Aerospike to Redis replace next: _application.yml:_ @@ -125,7 +177,13 @@ with ```yaml spring.redis.host: value -``` +``` + +or + +```yaml + spring.ignite.host: value +``` For configuring single redis node, please use next properties: @@ -137,11 +195,20 @@ For configuring single redis node, please use next properties: port: value ``` +or +```yaml + spring: + ignite: + host: host + port: port + cache-name: cacheName +``` + It is possible to override the default YAML configuration by supplying a custom configuration. See example scenario(s) below. -###### Cluster config for Redis and Aerospike +###### Cluster config for Redis, Aerospike and Apache Ignite Redis cluster settings _application-default.yml:_ @@ -161,6 +228,12 @@ _application-default.yml:_ spring.aerospike.host: aerospike_host_1:port,aerospike_host_2:port,aerospike_host_3:port ``` +Apache Ignite cluster settings +_application-default.yml:_ +```yaml +spring.ignite.host: ignite_host_1:port,ignite_host_2:port,ignite_host_3:port +``` + ### _Optional: Bring Your Own (BYO) Cache Implementation_ Prebid Cache can support any cache implementation, although Redis is provided as default. Spring injects the cache repository bean instances during context initialization, or application startup. This section describes how to setup a custom cache repository. diff --git a/docs/differenceBetweenPBCGo-and-Java.md b/docs/differenceBetweenPBCGo-and-Java.md index 1db418a..c780773 100644 --- a/docs/differenceBetweenPBCGo-and-Java.md +++ b/docs/differenceBetweenPBCGo-and-Java.md @@ -52,5 +52,5 @@ and not the other for an interim period. This page tracks known differences that - PBC-Go for `JSON` value uses an array with JSON content and additional parameters; 2) PBC-Java has different implementations of cache storage with PBC-Go: -- PBC-Java has ability to store data in `Aerospike` and `Redis`; -- PBC-Go has ability to store data in `Cassandra` , `Memcache`, `Aerospike`; +- PBC-Java has ability to store data in `Aerospike`, `Redis`, `Apache Ignite`; +- PBC-Go has ability to store data in `Cassandra` , `Memcache`, `Aerospike`, `Apache Ignite`; diff --git a/pom.xml b/pom.xml index 1d14d59..337fa59 100644 --- a/pom.xml +++ b/pom.xml @@ -61,6 +61,7 @@ 3.0.0 0.10.2 6.2.0 + 2.16.0 3.5.1 3.3.1 @@ -215,6 +216,11 @@ aerospike-client ${aerospike-client.version} + + org.apache.ignite + ignite-core + ${ignite-client.version} + io.projectreactor.addons reactor-extra diff --git a/src/main/java/org/prebid/cache/config/RepositoryConfig.java b/src/main/java/org/prebid/cache/config/RepositoryConfig.java index 320cdbd..d5e810b 100644 --- a/src/main/java/org/prebid/cache/config/RepositoryConfig.java +++ b/src/main/java/org/prebid/cache/config/RepositoryConfig.java @@ -5,6 +5,7 @@ import com.aerospike.client.policy.Policy; import io.github.resilience4j.circuitbreaker.CircuitBreaker; import io.lettuce.core.api.reactive.RedisStringReactiveCommands; +import org.apache.ignite.client.ClientCache; import org.prebid.cache.model.PayloadWrapper; import org.prebid.cache.repository.CacheConfig; import org.prebid.cache.repository.CircuitBreakerSecuredReactiveRepository; @@ -12,6 +13,7 @@ import org.prebid.cache.repository.TimeOutCapableReactiveRepository; import org.prebid.cache.repository.aerospike.AerospikePropertyConfiguration; import org.prebid.cache.repository.aerospike.AerospikeRepositoryImpl; +import org.prebid.cache.repository.ignite.IgniteRepositoryImpl; import org.prebid.cache.repository.redis.RedisRepositoryImpl; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.context.annotation.Bean; @@ -41,6 +43,12 @@ ReactiveRepository aerospikeRepository(AerospikeProperty return new AerospikeRepositoryImpl(configuration, client, eventLoops, policy); } + @Bean + @ConditionalOnProperty(prefix = "spring.ignite", name = {"host"}) + ReactiveRepository igniteRepository(ClientCache igniteCache) { + return new IgniteRepositoryImpl(igniteCache); + } + @Bean @Primary ReactiveRepository circuitBreakerSecuredRepository( diff --git a/src/main/java/org/prebid/cache/repository/ignite/IgniteConfiguration.java b/src/main/java/org/prebid/cache/repository/ignite/IgniteConfiguration.java new file mode 100644 index 0000000..c977c4c --- /dev/null +++ b/src/main/java/org/prebid/cache/repository/ignite/IgniteConfiguration.java @@ -0,0 +1,73 @@ +package org.prebid.cache.repository.ignite; + +import jakarta.validation.constraints.NotNull; +import org.apache.commons.lang3.BooleanUtils; +import org.apache.ignite.Ignition; +import org.apache.ignite.client.ClientCache; +import org.apache.ignite.client.ClientConnectionException; +import org.apache.ignite.client.IgniteClient; +import org.apache.ignite.client.SslMode; +import org.apache.ignite.configuration.ClientConfiguration; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +import java.util.Arrays; + +import static java.util.Objects.requireNonNull; + +@Configuration +@EnableConfigurationProperties(IgniteConfigurationProperties.class) +@ConditionalOnProperty(prefix = "spring.ignite", name = {"host"}) +public class IgniteConfiguration { + + private static final int DEFAULT_PORT = 10800; + + @Bean + public ClientConfiguration clientConfiguration(IgniteConfigurationProperties properties) { + ClientConfiguration cfg = new ClientConfiguration(); + + final String host = properties.getHost(); + final int port = properties.getPort() == null ? DEFAULT_PORT : properties.getPort(); + + if (isCluster(host)) { + cfg.setAddresses(extractHosts(host)); + } else { + cfg.setAddresses(host + ":" + port); + } + + cfg.setSslMode(BooleanUtils.isTrue(properties.getSecure()) ? SslMode.REQUIRED : SslMode.DISABLED); + return cfg; + } + + public static boolean isCluster(@NotNull String hostList) { + return hostList.split(",").length > 1; + } + + private static String[] extractHosts(@NotNull String hostList) { + return Arrays.stream(hostList.split(",")) + .map(host -> { + String[] params = host.split(":"); + String hostname = requireNonNull(params[0]); + int port = DEFAULT_PORT; + if (params.length == 2) { + port = Integer.parseInt(params[1]); + } + return hostname + ":" + port; + }) + .toArray(String[]::new); + } + + @Bean(destroyMethod = "close") + public IgniteClient igniteClient(ClientConfiguration clientConfiguration) throws ClientConnectionException { + return Ignition.startClient(clientConfiguration); + } + + @Bean + public ClientCache igniteCache(IgniteClient igniteClient, + IgniteConfigurationProperties properties) { + + return igniteClient.cache(properties.getCacheName()); + } +} diff --git a/src/main/java/org/prebid/cache/repository/ignite/IgniteConfigurationProperties.java b/src/main/java/org/prebid/cache/repository/ignite/IgniteConfigurationProperties.java new file mode 100644 index 0000000..f3c307b --- /dev/null +++ b/src/main/java/org/prebid/cache/repository/ignite/IgniteConfigurationProperties.java @@ -0,0 +1,17 @@ +package org.prebid.cache.repository.ignite; + +import lombok.Data; +import org.springframework.boot.context.properties.ConfigurationProperties; + +@Data +@ConfigurationProperties(prefix = "spring.ignite") +public class IgniteConfigurationProperties { + + private String host; + + private Integer port; + + private String cacheName; + + private Boolean secure; +} diff --git a/src/main/java/org/prebid/cache/repository/ignite/IgniteRepositoryImpl.java b/src/main/java/org/prebid/cache/repository/ignite/IgniteRepositoryImpl.java new file mode 100644 index 0000000..4b2c79f --- /dev/null +++ b/src/main/java/org/prebid/cache/repository/ignite/IgniteRepositoryImpl.java @@ -0,0 +1,62 @@ +package org.prebid.cache.repository.ignite; + +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.exception.ExceptionUtils; +import org.apache.ignite.client.ClientCache; +import org.apache.ignite.client.ClientException; +import org.prebid.cache.exceptions.PayloadWrapperPropertyException; +import org.prebid.cache.exceptions.RepositoryException; +import org.prebid.cache.helpers.Json; +import org.prebid.cache.model.PayloadWrapper; +import org.prebid.cache.repository.ReactiveRepository; +import reactor.core.publisher.Mono; + +import javax.cache.expiry.CreatedExpiryPolicy; +import javax.cache.expiry.Duration; +import javax.cache.expiry.ExpiryPolicy; +import java.util.concurrent.TimeUnit; + +@Slf4j +public class IgniteRepositoryImpl implements ReactiveRepository { + + private final ClientCache cache; + + public IgniteRepositoryImpl(ClientCache cache) { + this.cache = cache; + } + + @Override + public Mono save(PayloadWrapper wrapper) { + ExpiryPolicy expiryPolicy; + String normalizedId; + + try { + normalizedId = wrapper.getNormalizedId(); + expiryPolicy = new CreatedExpiryPolicy(new Duration(TimeUnit.SECONDS, wrapper.getExpiry())); + } catch (PayloadWrapperPropertyException e) { + log.error("Exception occurred while extracting normalized id from payload: '{}', cause: '{}'", + ExceptionUtils.getMessage(e), ExceptionUtils.getMessage(e)); + return Mono.empty(); + } + + final ClientCache expiredCache = cache.withExpirePolicy(expiryPolicy); + return Mono.fromFuture(expiredCache.putIfAbsentAsync(normalizedId, Json.toJson(wrapper)).toCompletableFuture()) + .map(payload -> wrapper) + .onErrorResume(IgniteRepositoryImpl::handleError); + } + + @Override + public Mono findById(String id) { + return Mono.fromFuture(cache.getAsync(id).toCompletableFuture()) + .map(json -> Json.createPayloadFromJson(json, PayloadWrapper.class)) + .onErrorResume(IgniteRepositoryImpl::handleError); + } + + private static Mono handleError(Throwable throwable) { + if (throwable instanceof ClientException) { + return Mono.error(new RepositoryException(throwable.toString(), throwable)); + } + + return Mono.error(throwable); + } +} diff --git a/src/test/java/org/prebid/cache/repository/ignite/IgniteRepositoryImplTest.java b/src/test/java/org/prebid/cache/repository/ignite/IgniteRepositoryImplTest.java new file mode 100644 index 0000000..4da3fa4 --- /dev/null +++ b/src/test/java/org/prebid/cache/repository/ignite/IgniteRepositoryImplTest.java @@ -0,0 +1,189 @@ +package org.prebid.cache.repository.ignite; + +import org.apache.ignite.client.ClientCache; +import org.apache.ignite.client.ClientException; +import org.apache.ignite.internal.client.thin.IgniteClientFutureImpl; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.prebid.cache.exceptions.RepositoryException; +import org.prebid.cache.model.Payload; +import org.prebid.cache.model.PayloadWrapper; +import reactor.core.publisher.Mono; +import reactor.test.StepVerifier; + +import javax.cache.expiry.CreatedExpiryPolicy; +import javax.cache.expiry.Duration; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.TimeUnit; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.BDDMockito.given; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; + +@ExtendWith(MockitoExtension.class) +class IgniteRepositoryImplTest { + + @Mock + private ClientCache cache; + + private IgniteRepositoryImpl target; + + @BeforeEach + public void before() { + target = new IgniteRepositoryImpl(cache); + } + + @Test + public void findByIdShouldSuccessfullyReturnPayloadWrapper() { + // given + final String givenPayload = """ + { + "id": "key", + "prefix": "", + "payload": { + "type": "text", + "key": "key", + "value": "value" + }, + "expiry": 999 + } + """; + + given(cache.getAsync("key")).willReturn(IgniteClientFutureImpl.completedFuture(givenPayload)); + + // when + final Mono result = target.findById("key"); + + // then + final PayloadWrapper expectedPayload = PayloadWrapper.builder() + .id("key") + .prefix("") + .payload(Payload.of("text", "key", "value")) + .expiry(999L) + .build(); + + StepVerifier.create(result) + .consumeNextWith(actualPayload -> assertThat(actualPayload).isEqualTo(expectedPayload)) + .expectComplete() + .verify(); + } + + @Test + public void findByIdShouldFailWithClientException() { + // given + given(cache.getAsync("key")).willReturn(new IgniteClientFutureImpl<>( + CompletableFuture.failedFuture(new ClientException("something went wrong")))); + + // when + final Mono result = target.findById("key"); + + // then + StepVerifier.create(result) + .expectErrorSatisfies(throwable -> { + assertThat(throwable).isInstanceOf(RepositoryException.class); + assertThat(throwable.getMessage()).hasSameClassAs("something went wrong"); + }) + .verify(); + } + + @Test + public void saveShouldSuccessfullySavePayload() { + // given + final String givenPayload = """ + { + "id": "key", + "prefix": "", + "payload": { + "type": "text", + "key": "key", + "value": "value" + }, + "expiry": 999 + } + """.replaceAll("\\s+", ""); + + given(cache.withExpirePolicy(any())).willAnswer(ignored -> cache); + given(cache.putIfAbsentAsync("key", givenPayload)).willReturn(IgniteClientFutureImpl.completedFuture(true)); + + + // when + final PayloadWrapper givenPayloadWrapper = PayloadWrapper.builder() + .id("key") + .prefix("") + .payload(Payload.of("text", "key", "value")) + .expiry(999L) + .build(); + + final Mono result = target.save(givenPayloadWrapper); + + // then + StepVerifier.create(result) + .consumeNextWith(actualPayload -> assertThat(actualPayload).isEqualTo(givenPayloadWrapper)) + .expectComplete() + .verify(); + + final CreatedExpiryPolicy expectedPolicy = new CreatedExpiryPolicy(new Duration(TimeUnit.SECONDS, 999)); + verify(cache).withExpirePolicy(eq(expectedPolicy)); + } + + @Test + public void saveShouldFailWithClientException() { + // given + final String givenPayload = """ + { + "id": "key", + "prefix": "", + "payload": { + "type": "text", + "key": "key", + "value": "value" + }, + "expiry": 999 + } + """.replaceAll("\\s+", ""); + + given(cache.withExpirePolicy(any())).willAnswer(ignored -> cache); + given(cache.putIfAbsentAsync("key", givenPayload)).willReturn(new IgniteClientFutureImpl<>( + CompletableFuture.failedFuture(new ClientException("something went wrong")))); + + + // when + final PayloadWrapper givenPayloadWrapper = PayloadWrapper.builder() + .id("key") + .prefix("") + .payload(Payload.of("text", "key", "value")) + .expiry(999L) + .build(); + + final Mono result = target.save(givenPayloadWrapper); + + // then + StepVerifier.create(result) + .expectErrorSatisfies(throwable -> { + assertThat(throwable).isInstanceOf(RepositoryException.class); + assertThat(throwable.getMessage()).hasSameClassAs("something went wrong"); + }) + .verify(); + } + + @Test + public void saveShouldReturnEmptyResultWhenKeyAndPrefixAreAbsent() { + // when + final PayloadWrapper givenPayloadWrapper = PayloadWrapper.builder() + .payload(Payload.of("text", "key", "value")) + .expiry(999L) + .build(); + + final Mono result = target.save(givenPayloadWrapper); + + // then + StepVerifier.create(result).verifyComplete(); + verifyNoInteractions(cache); + } +} diff --git a/src/test/kotlin/org/prebid/cache/functional/ApacheIgniteCacheSpec.kt b/src/test/kotlin/org/prebid/cache/functional/ApacheIgniteCacheSpec.kt new file mode 100644 index 0000000..fc59dd5 --- /dev/null +++ b/src/test/kotlin/org/prebid/cache/functional/ApacheIgniteCacheSpec.kt @@ -0,0 +1,194 @@ +package org.prebid.cache.functional + +import io.kotest.assertions.assertSoftly +import io.kotest.assertions.throwables.shouldNotThrowAny +import io.kotest.assertions.throwables.shouldThrowExactly +import io.kotest.core.spec.style.ShouldSpec +import io.kotest.matchers.collections.shouldContain +import io.kotest.matchers.shouldBe +import io.kotest.matchers.string.shouldContain +import io.ktor.client.statement.bodyAsText +import java.util.UUID +import org.prebid.cache.functional.BaseSpec.Companion.prebidCacheConfig +import org.prebid.cache.functional.mapper.objectMapper +import org.prebid.cache.functional.model.request.PayloadTransfer +import org.prebid.cache.functional.model.request.RequestObject +import org.prebid.cache.functional.model.request.TransferValue +import org.prebid.cache.functional.model.response.ResponseObject +import org.prebid.cache.functional.service.ApiException +import org.prebid.cache.functional.testcontainers.ContainerDependencies +import org.prebid.cache.functional.util.getRandomString +import org.prebid.cache.functional.util.getRandomUuid +import org.springframework.http.HttpStatus.INTERNAL_SERVER_ERROR +import org.springframework.http.HttpStatus.NOT_FOUND + +class ApacheIgniteCacheSpec : ShouldSpec({ + + should("throw an exception when cache record is absent in Apache ignite repository") { + // given: Prebid cache config + val config = prebidCacheConfig.getBaseApacheIgniteConfig("true") + val cachePrefix = config["cache.prefix"] + + // when: GET cache endpoint with random UUID is called + val randomUuid = getRandomUuid() + val exception = shouldThrowExactly { BaseSpec.getPrebidCacheApi(config).getCache(randomUuid) } + + // then: Not Found exception is thrown + assertSoftly { + exception.statusCode shouldBe NOT_FOUND.value() + exception.responseBody shouldContain "\"message\":\"Resource Not Found: uuid $cachePrefix$randomUuid\"" + } + } + + should("throw an exception when cache is absent in Apache ignite repository due to expiration") { + // given: Prebid Cache with set min and max expiry as 0 + val config = prebidCacheConfig.getBaseApacheIgniteConfig("false") + + prebidCacheConfig.getCacheExpiryConfig("0", "0") + val prebidCacheApi = BaseSpec.getPrebidCacheApi(config) + + // and: Request object + val requestObject = RequestObject.getDefaultJsonRequestObject() + + // when: POST cache endpoint is called + val responseObject = prebidCacheApi.postCache(requestObject) + + // then: Internal Server Error exception is thrown + val exception = shouldThrowExactly { prebidCacheApi.getCache(responseObject.responses[0].uuid) } + assertSoftly { + exception.statusCode shouldBe NOT_FOUND.value() + exception.responseBody shouldContain "\"message\":\"Resource Not Found: uuid prebid_${responseObject.responses[0].uuid}" + } + + // cleanup + ContainerDependencies.prebidCacheContainerPool.stopPrebidCacheContainer(config) + } + + should("rethrow an exception from Apache ignite cache server when such happens") { + // given: Prebid Cache with not matched to Apache ignite server namespace + val config = prebidCacheConfig.getBaseApacheIgniteConfig("true", getRandomString()) + val prebidCacheApi = BaseSpec.getPrebidCacheApi(config) + + // and: Default request object + val requestObject = RequestObject.getDefaultJsonRequestObject() + + // when: POST cache endpoint is called + val exception = shouldThrowExactly { prebidCacheApi.postCache(requestObject) } + + // then: Internal Server Error exception is thrown + assertSoftly { + exception.statusCode shouldBe INTERNAL_SERVER_ERROR.value() + exception.responseBody shouldContain Regex("""Ignite failed to process request \[\d+]: Cache does not exist""") + } + + // cleanup + ContainerDependencies.prebidCacheContainerPool.stopPrebidCacheContainer(config) + } + + should("return back PBC UUID when allow_external_UUID=false and request object was successfully cached in Apache ignite") { + // given: Prebid Cache with disabled allow_external_UUID property + val prebidCacheApi = BaseSpec.getPrebidCacheApi(prebidCacheConfig.getBaseApacheIgniteConfig("false")) + + // and: Request object + val requestObject = RequestObject.getDefaultJsonRequestObject() + + // when: POST cache endpoint is called + val responseObject: ResponseObject = prebidCacheApi.postCache(requestObject) + + // then: Response contains 1 UUID generated by Prebid Cache + responseObject.responses.isEmpty() shouldBe false + responseObject.responses.size shouldBe 1 + shouldNotThrowAny { UUID.fromString(responseObject.responses[0].uuid) } + } + + should("return back two random UUIDs when allow_external_UUID=false and 2 payload transfers were successfully cached") { + // given: Prebid Cache with disabled allow_external_UUID property + val prebidCacheApi = BaseSpec.getPrebidCacheApi(prebidCacheConfig.getBaseApacheIgniteConfig("false")) + + // given: Request object + val xmlPayloadTransfer = PayloadTransfer.getDefaultXmlPayloadTransfer().apply { key = null } + val jsonPayloadTransfer = PayloadTransfer.getDefaultJsonPayloadTransfer().apply { key = null } + val requestObject = RequestObject.of(xmlPayloadTransfer, jsonPayloadTransfer) + + // when: POST cache endpoint is called + val responseObject: ResponseObject = prebidCacheApi.postCache(requestObject) + + // then: Response contains 2 UUIDs generated by Prebid Cache + responseObject.responses.isEmpty() shouldBe false + responseObject.responses.size shouldBe 2 + shouldNotThrowAny { UUID.fromString(responseObject.responses[0].uuid) } + shouldNotThrowAny { UUID.fromString(responseObject.responses[1].uuid) } + } + + should("return back a request UUID when allow_external_UUID=true and request object was successfully cached") { + // given: Prebid Cache with enabled allow_external_UUID property + val prebidCacheApi = BaseSpec.getPrebidCacheApi(prebidCacheConfig.getBaseApacheIgniteConfig("true")) + + // and: Request object with set payload transfer UUID key + val requestObject = RequestObject.getDefaultJsonRequestObject().apply { puts[0].key = getRandomUuid() } + + // when: POST cache endpoint is called + val responseObject: ResponseObject = prebidCacheApi.postCache(requestObject) + + // then: UUID from request is returned + responseObject.responses.isEmpty() shouldBe false + responseObject.responses.size shouldBe 1 + responseObject.responses[0].uuid shouldBe requestObject.puts[0].key + } + + should("return back two request UUIDs when allow_external_UUID=true and 2 payload transfers were successfully cached in Apache ignite") { + // given: Prebid Cache with allow_external_UUID=true property + val prebidCacheApi = BaseSpec.getPrebidCacheApi(prebidCacheConfig.getBaseApacheIgniteConfig("true")) + + // and: Request object with 2 payload transfers and set UUIDs is prepared + val xmlPayloadTransfer = PayloadTransfer.getDefaultXmlPayloadTransfer().apply { key = getRandomUuid() } + val jsonPayloadTransfer = PayloadTransfer.getDefaultJsonPayloadTransfer().apply { key = getRandomUuid() } + val requestObject = RequestObject.of(xmlPayloadTransfer, jsonPayloadTransfer) + + // when: POST cache endpoint is called + val responseObject: ResponseObject = prebidCacheApi.postCache(requestObject) + + // then: UUIDs from a request object are returned + responseObject.responses.isEmpty() shouldBe false + responseObject.responses.size shouldBe 2 + + val responseUuidList = listOf(responseObject.responses[0].uuid, responseObject.responses[1].uuid) + responseUuidList shouldContain requestObject.puts[0].key + responseUuidList shouldContain requestObject.puts[1].key + } + + should("shouldn't update existing cache record when request with already existing UUID is send") { + // given: Prebid Cache with allow_external_UUID=true property + val prebidCacheApi = BaseSpec.getPrebidCacheApi(prebidCacheConfig.getBaseApacheIgniteConfig("true")) + + // and: First request object + val uuid = getRandomUuid() + val xmlPayloadTransfer = PayloadTransfer.getDefaultXmlPayloadTransfer().apply { key = uuid } + val requestObject = RequestObject.of(xmlPayloadTransfer) + val requestTransferValue = objectMapper.readValue(requestObject.puts[0].value, TransferValue::class.java) + + // and: First request object is saved to Aerospike cache + prebidCacheApi.postCache(requestObject) + + // and: Second request object with already existing UUID is prepared + val jsonPayloadTransfer = PayloadTransfer.getDefaultJsonPayloadTransfer().apply { key = uuid } + val secondRequestObject = RequestObject.of(jsonPayloadTransfer) + + // when: POST cache endpoint is called for the second time + val responseObject = prebidCacheApi.postCache(secondRequestObject) + + // then: UUID from request is returned + responseObject.responses.isEmpty() shouldBe false + responseObject.responses.size shouldBe 1 + responseObject.responses[0].uuid shouldBe secondRequestObject.puts[0].key + + // and: Cache record was updated in Aerospike with a second request object payload + val getCacheResponse = prebidCacheApi.getCache(responseObject.responses[0].uuid) + val responseTransferValue = objectMapper.readValue(getCacheResponse.bodyAsText(), TransferValue::class.java) + + assertSoftly { + responseTransferValue.adm shouldBe requestTransferValue.adm + responseTransferValue.width shouldBe requestTransferValue.width + responseTransferValue.height shouldBe requestTransferValue.height + } + } +}) diff --git a/src/test/kotlin/org/prebid/cache/functional/BaseSpec.kt b/src/test/kotlin/org/prebid/cache/functional/BaseSpec.kt index 6fa3529..1d3d23c 100644 --- a/src/test/kotlin/org/prebid/cache/functional/BaseSpec.kt +++ b/src/test/kotlin/org/prebid/cache/functional/BaseSpec.kt @@ -9,7 +9,8 @@ abstract class BaseSpec { companion object { val prebidCacheConfig = PrebidCacheContainerConfig( ContainerDependencies.redisContainer.getContainerHost(), - ContainerDependencies.aerospikeContainer.getContainerHost() + ContainerDependencies.aerospikeContainer.getContainerHost(), + ContainerDependencies.apacheIgniteContainer.getContainerHost(), ) fun getPrebidCacheApi(config: Map = prebidCacheConfig.getBaseRedisConfig("false")): PrebidCacheApi { diff --git a/src/test/kotlin/org/prebid/cache/functional/testcontainers/ContainerDependencies.kt b/src/test/kotlin/org/prebid/cache/functional/testcontainers/ContainerDependencies.kt index 87e2f72..d4a4888 100644 --- a/src/test/kotlin/org/prebid/cache/functional/testcontainers/ContainerDependencies.kt +++ b/src/test/kotlin/org/prebid/cache/functional/testcontainers/ContainerDependencies.kt @@ -1,6 +1,7 @@ package org.prebid.cache.functional.testcontainers import org.prebid.cache.functional.testcontainers.container.AerospikeContainer +import org.prebid.cache.functional.testcontainers.container.ApacheIgniteContainer import org.prebid.cache.functional.testcontainers.container.RedisContainer import org.prebid.cache.functional.testcontainers.container.WebCacheContainer import org.testcontainers.containers.Network @@ -12,6 +13,7 @@ abstract class ContainerDependencies { companion object { private const val redisImageName = "redis:6.2.6-alpine" private const val aerospikeImageName = "aerospike:ce-5.7.0.11" + private const val apacheIgniteImageName = "apacheignite/ignite:2.16.0" private const val prebidCacheImageName = "prebid-cache:latest" private const val mockServerImageVersion = "mockserver/mockserver:5.15.0" @@ -20,19 +22,20 @@ abstract class ContainerDependencies { val network: Network = Network.newNetwork() val redisContainer: RedisContainer = RedisContainer(redisImageName).withNetwork(network) val aerospikeContainer: AerospikeContainer = AerospikeContainer(aerospikeImageName).withNetwork(network) + val apacheIgniteContainer: ApacheIgniteContainer = ApacheIgniteContainer(apacheIgniteImageName).withNetwork(network) val webCacheContainer: WebCacheContainer = WebCacheContainer(DockerImageName.parse(mockServerImageVersion)).withNetwork(network) as WebCacheContainer val prebidCacheContainerPool = PrebidCacheContainerPool(prebidCacheImageName) fun startCacheServerContainers() { if (launchContainers) { - Startables.deepStart(listOf(redisContainer, aerospikeContainer)).join() + Startables.deepStart(listOf(redisContainer, aerospikeContainer, apacheIgniteContainer)).join() } } fun stopCacheServerContainers() { if (launchContainers) { - listOf(redisContainer, aerospikeContainer).parallelStream().forEach { it.stop() } + listOf(redisContainer, aerospikeContainer, apacheIgniteContainer).parallelStream().forEach { it.stop() } } } } diff --git a/src/test/kotlin/org/prebid/cache/functional/testcontainers/PrebidCacheContainerConfig.kt b/src/test/kotlin/org/prebid/cache/functional/testcontainers/PrebidCacheContainerConfig.kt index 9fb389f..0694523 100644 --- a/src/test/kotlin/org/prebid/cache/functional/testcontainers/PrebidCacheContainerConfig.kt +++ b/src/test/kotlin/org/prebid/cache/functional/testcontainers/PrebidCacheContainerConfig.kt @@ -2,10 +2,14 @@ package org.prebid.cache.functional.testcontainers import org.prebid.cache.functional.testcontainers.container.AerospikeContainer import org.prebid.cache.functional.testcontainers.container.AerospikeContainer.Companion.NAMESPACE +import org.prebid.cache.functional.testcontainers.container.ApacheIgniteContainer +import org.prebid.cache.functional.testcontainers.container.ApacheIgniteContainer.Companion.CACHE_NAME import org.prebid.cache.functional.testcontainers.container.RedisContainer import org.prebid.cache.functional.testcontainers.container.WebCacheContainer.Companion.WEB_CACHE_PATH -class PrebidCacheContainerConfig(private val redisHost: String, private val aerospikeHost: String) { +class PrebidCacheContainerConfig(private val redisHost: String, + private val aerospikeHost: String, + private val apacheIgniteHost: String) { fun getBaseRedisConfig(allowExternalUuid: String): Map = getBaseConfig(allowExternalUuid) + getRedisConfig() @@ -13,6 +17,9 @@ class PrebidCacheContainerConfig(private val redisHost: String, private val aero fun getBaseAerospikeConfig(allowExternalUuid: String, aerospikeNamespace: String = NAMESPACE): Map = getBaseConfig(allowExternalUuid) + getAerospikeConfig(aerospikeNamespace) + fun getBaseApacheIgniteConfig(allowExternalUuid: String, ingineCacheName: String = CACHE_NAME): Map = + getBaseConfig(allowExternalUuid) + getApacheIgniteConfig(ingineCacheName) + fun getBaseModuleStorageConfig(applicationName: String, apiKey: String): Map = getBaseConfig("true") + getModuleStorageRedisConfig(apiKey, applicationName) + getRedisConfig() @@ -48,6 +55,13 @@ class PrebidCacheContainerConfig(private val redisHost: String, private val aero "spring.redis.timeout" to "300" ) + private fun getApacheIgniteConfig(cacheName: String): Map = + mapOf( + "spring.ignite.port" to ApacheIgniteContainer.PORT.toString(), + "spring.ignite.host" to apacheIgniteHost, + "spring.ignite.cache-name" to cacheName + ) + private fun getAerospikeConfig(aerospikeNamespace: String): Map = mapOf( "spring.aerospike.port" to AerospikeContainer.PORT.toString(), diff --git a/src/test/kotlin/org/prebid/cache/functional/testcontainers/container/ApacheIgniteContainer.kt b/src/test/kotlin/org/prebid/cache/functional/testcontainers/container/ApacheIgniteContainer.kt new file mode 100644 index 0000000..428be2e --- /dev/null +++ b/src/test/kotlin/org/prebid/cache/functional/testcontainers/container/ApacheIgniteContainer.kt @@ -0,0 +1,25 @@ +package org.prebid.cache.functional.testcontainers.container + +import org.testcontainers.containers.BindMode +import org.testcontainers.containers.GenericContainer + +class ApacheIgniteContainer(imageName: String) : GenericContainer(imageName) { + + fun getContainerHost(): String = networkAliases.first() + + init { + withClasspathResourceMapping( + "functional/$CONFIG_FILE_NAME", + "/$CONFIG_FILE_NAME", + BindMode.READ_ONLY + ) + withCommand("/opt/ignite/apache-ignite/bin/ignite.sh", "/$CONFIG_FILE_NAME") + + } + + companion object { + const val PORT = 10800 + const val CACHE_NAME = "prebid_cache_ignite" + const val CONFIG_FILE_NAME = "ignite-config.xml" + } +} diff --git a/src/test/kotlin/org/prebid/cache/functional/testcontainers/container/PrebidCacheContainer.kt b/src/test/kotlin/org/prebid/cache/functional/testcontainers/container/PrebidCacheContainer.kt index d4e13f4..eed9f27 100644 --- a/src/test/kotlin/org/prebid/cache/functional/testcontainers/container/PrebidCacheContainer.kt +++ b/src/test/kotlin/org/prebid/cache/functional/testcontainers/container/PrebidCacheContainer.kt @@ -10,7 +10,7 @@ class PrebidCacheContainer(imageName: String, config: Map) : withNetwork(ContainerDependencies.network) withExposedPorts(PORT, DEBUG_PORT) withFixedExposedPorts() - withDebug() + withJavaToolOptions() withConfig(config) } @@ -26,10 +26,37 @@ class PrebidCacheContainer(imageName: String, config: Map) : } } - private fun withDebug(): PrebidCacheContainer = withEnv( - "JAVA_TOOL_OPTIONS", - "-agentlib:jdwp=transport=dt_socket,address=*:$DEBUG_PORT,server=y,suspend=n" - ) + private fun withJavaToolOptions(): PrebidCacheContainer { + val debugOptions = "-agentlib:jdwp=transport=dt_socket,address=*:$DEBUG_PORT,server=y,suspend=n" + val opensOptions = listOf( + "--add-opens=java.base/jdk.internal.access=ALL-UNNAMED", + "--add-opens=java.base/jdk.internal.misc=ALL-UNNAMED", + "--add-opens=java.base/sun.nio.ch=ALL-UNNAMED", + "--add-opens=java.base/sun.util.calendar=ALL-UNNAMED", + "--add-opens=java.management/com.sun.jmx.mbeanserver=ALL-UNNAMED", + "--add-opens=jdk.internal.jvmstat/sun.jvmstat.monitor=ALL-UNNAMED", + "--add-opens=java.base/sun.reflect.generics.reflectiveObjects=ALL-UNNAMED", + "--add-opens=jdk.management/com.sun.management.internal=ALL-UNNAMED", + "--add-opens=java.base/java.io=ALL-UNNAMED", + "--add-opens=java.base/java.nio=ALL-UNNAMED", + "--add-opens=java.base/java.net=ALL-UNNAMED", + "--add-opens=java.base/java.util=ALL-UNNAMED", + "--add-opens=java.base/java.util.concurrent=ALL-UNNAMED", + "--add-opens=java.base/java.util.concurrent.locks=ALL-UNNAMED", + "--add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED", + "--add-opens=java.base/java.lang=ALL-UNNAMED", + "--add-opens=java.base/java.lang.invoke=ALL-UNNAMED", + "--add-opens=java.base/java.math=ALL-UNNAMED", + "--add-opens=java.sql/java.sql=ALL-UNNAMED", + "--add-opens=java.base/java.lang.reflect=ALL-UNNAMED", + "--add-opens=java.base/java.time=ALL-UNNAMED", + "--add-opens=java.base/java.text=ALL-UNNAMED", + "--add-opens=java.management/sun.management=ALL-UNNAMED", + "--add-opens=java.desktop/java.awt.font=ALL-UNNAMED" + ).joinToString(" ") + + return withEnv("JAVA_TOOL_OPTIONS", "$debugOptions $opensOptions") + } private fun normalizeProperties(config: Map): Map = config.mapKeys { normalizeProperty(it.key) } diff --git a/src/test/resources/functional/ignite-config.xml b/src/test/resources/functional/ignite-config.xml new file mode 100644 index 0000000..60fb30f --- /dev/null +++ b/src/test/resources/functional/ignite-config.xml @@ -0,0 +1,16 @@ + + + + + + + + + + + + + +