diff --git a/.env.test b/.env.test index bc1b9b3..b03d136 100644 --- a/.env.test +++ b/.env.test @@ -21,7 +21,7 @@ POSTGRES_IMAGE=postgres:17-alpine RABBITMQ_IMAGE=rabbitmq:4.3.0-management MINIO_IMAGE=minio/minio:RELEASE.2025-09-07T16-13-09Z -ELASTIC_IMAGE=docker.elastic.co/elasticsearch/elasticsearch:9.4.1 +ELASTIC_IMAGE=docker.elastic.co/elasticsearch/elasticsearch:9.1.3 # Container credentials (placeholders; Testcontainers regenerates) POSTGRES_DB=paperless diff --git a/PaperlessREST.Tests/Integration/SharedRestContainerFixture.cs b/PaperlessREST.Tests/Integration/SharedRestContainerFixture.cs index f3066b5..b7a7252 100644 --- a/PaperlessREST.Tests/Integration/SharedRestContainerFixture.cs +++ b/PaperlessREST.Tests/Integration/SharedRestContainerFixture.cs @@ -25,7 +25,7 @@ public SharedRestContainerFixture() string minioImage = Environment.GetEnvironmentVariable("MINIO_IMAGE") ?? "minio/minio:RELEASE.2025-09-07T16-13-09Z"; string elasticImage = Environment.GetEnvironmentVariable("ELASTIC_IMAGE") ?? - "docker.elastic.co/elasticsearch/elasticsearch:9.4.1"; + "docker.elastic.co/elasticsearch/elasticsearch:9.1.3"; _postgres = new PostgreSqlBuilder(postgresImage) .WithWaitStrategy(Wait.ForUnixContainer() diff --git a/PaperlessServices.Tests/Integration/SearchIndexConcurrencyTests.cs b/PaperlessServices.Tests/Integration/SearchIndexConcurrencyTests.cs index 67ed117..620f3e0 100644 --- a/PaperlessServices.Tests/Integration/SearchIndexConcurrencyTests.cs +++ b/PaperlessServices.Tests/Integration/SearchIndexConcurrencyTests.cs @@ -113,9 +113,13 @@ await sut.IndexDocumentAsync(documentId, "exists.pdf", "Already there", CountCreateIndexLogs(collector, indexName).Should().Be(0, "InitializeAsync must skip the create path when ExistsAsync returns true"); - // Document was still indexed successfully via the existing index. - GetResponse response = await fixture.WaitForDocumentAsync( - documentId.ToString(), TestContext.Current.CancellationToken); + // Document was still indexed successfully into the SUT's pre-created index. + // We query that index directly — fixture.WaitForDocumentAsync uses the client's + // DefaultIndex, which is the shared fixture index, not this test's per-test index. + GetResponse response = await ElasticClient.GetAsync( + documentId.ToString(), + g => g.Index(indexName), + TestContext.Current.CancellationToken); response.Found.Should().BeTrue("document should be indexed in the pre-created index"); } diff --git a/PaperlessServices.Tests/Integration/WorkerTestBase.cs b/PaperlessServices.Tests/Integration/WorkerTestBase.cs index 2efc966..aeafe91 100644 --- a/PaperlessServices.Tests/Integration/WorkerTestBase.cs +++ b/PaperlessServices.Tests/Integration/WorkerTestBase.cs @@ -22,7 +22,7 @@ public class SharedContainerFixture : IAsyncLifetime private const int MinioPort = 9000; // Default image versions - override via environment variables for CI flexibility - private const string DefaultElasticsearchImage = "docker.elastic.co/elasticsearch/elasticsearch:9.4.1"; + private const string DefaultElasticsearchImage = "docker.elastic.co/elasticsearch/elasticsearch:9.1.3"; private const string DefaultMinioImage = "minio/minio:RELEASE.2025-09-07T16-13-09Z"; private const string DefaultRabbitmqImage = "rabbitmq:4.3.0-management"; @@ -32,6 +32,10 @@ public class SharedContainerFixture : IAsyncLifetime Environment.GetEnvironmentVariable("ELASTIC_IMAGE") ?? DefaultElasticsearchImage) .WithEnvironment("discovery.type", "single-node") .WithEnvironment("xpack.security.enabled", "false") + // Required so Testcontainers' ElasticsearchConfiguration.TlsEnabled evaluates to false + // (it AND-s xpack.security.enabled with xpack.security.http.ssl.enabled). Without this, + // the built-in wait strategy probes HTTPS while ES listens on plain HTTP, and hangs. + .WithEnvironment("xpack.security.http.ssl.enabled", "false") .WithEnvironment("ES_JAVA_OPTS", "-Xms512m -Xmx512m") .Build(); @@ -110,14 +114,21 @@ await Task.WhenAll( public async ValueTask DisposeAsync() { - await _host.StopAsync(); - _host.Dispose(); + // _host is assigned in InitializeAsync. If init throws before that line + // (e.g. a container wait-strategy times out), _host is still null and a + // naive `_host.StopAsync()` here NREs — which masks the real init error + // in xUnit's collection-fixture cleanup report. Guard the host, and + // swallow per-container Dispose failures for the same reason. + if (_host is not null) + { + try { await _host.StopAsync(); } + catch { /* best-effort: don't mask the InitializeAsync exception */ } + _host.Dispose(); + } - await Task.WhenAll( - _rabbit.DisposeAsync().AsTask(), - _minio.DisposeAsync().AsTask(), - _elastic.DisposeAsync().AsTask() - ); + try { await _rabbit.DisposeAsync(); } catch { /* best-effort */ } + try { await _minio.DisposeAsync(); } catch { /* best-effort */ } + try { await _elastic.DisposeAsync(); } catch { /* best-effort */ } } public async Task UploadPdfAsync(string content) @@ -188,27 +199,63 @@ public async Task> WaitForSearchResultsAsync( TimeSpan? timeout = null, TimeSpan? pollInterval = null) { - timeout ??= TimeSpan.FromSeconds(10); + // 30s overall budget: GitHub-hosted runners are markedly slower than local + // dev machines and the first SearchAsync after index creation can spend + // several seconds priming query caches even after Refresh.True returns. + timeout ??= TimeSpan.FromSeconds(30); pollInterval ??= TimeSpan.FromMilliseconds(100); ElasticsearchClient client = Services.GetRequiredService(); - using CancellationTokenSource cts = new(timeout.Value); - using CancellationTokenSource linked = - CancellationTokenSource.CreateLinkedTokenSource(cts.Token, cancellationToken); + using CancellationTokenSource overallCts = new(timeout.Value); + using CancellationTokenSource overallLinked = + CancellationTokenSource.CreateLinkedTokenSource(overallCts.Token, cancellationToken); + + // Force an index-level refresh up front. SearchIndexService writes documents + // with Refresh.True (`?refresh=true`), which is supposed to guarantee + // immediate searchability — but on slow CI disks the per-document refresh + // is observed to not always propagate before the first SearchAsync. The + // explicit Indices.RefreshAsync here is defensive and idempotent: locally + // it's a no-op (everything's already refreshed), on CI it converts an + // invisible flake into a passing search. + try + { + await client.Indices.RefreshAsync( + r => r.Indices(client.ElasticsearchClientSettings.DefaultIndex), + overallLinked.Token); + } + catch (OperationCanceledException) when (overallLinked.Token.IsCancellationRequested) + { + // Fall through to the final attempt below. + } - while (!linked.Token.IsCancellationRequested) + while (!overallLinked.Token.IsCancellationRequested) { - SearchResponse response = await client.SearchAsync(configureSearch, linked.Token); + try + { + SearchResponse response = await client.SearchAsync(configureSearch, overallLinked.Token); - if (response.Documents.Count > 0) + if (response.Documents.Count > 0) + { + return response; + } + } + catch (OperationCanceledException) when (overallLinked.Token.IsCancellationRequested) { - return response; + break; } - await Task.Delay(pollInterval.Value, linked.Token); + try + { + await Task.Delay(pollInterval.Value, overallLinked.Token); + } + catch (OperationCanceledException) + { + break; + } } - // Final attempt + // Final attempt with the caller's token only so the assertion sees real + // "found nothing" data rather than a TaskCanceledException at the wait boundary. return await client.SearchAsync(configureSearch, cancellationToken); } diff --git a/PaperlessServices.Tests/xunit.runner.json b/PaperlessServices.Tests/xunit.runner.json index 5c0209e..0bc55b4 100644 --- a/PaperlessServices.Tests/xunit.runner.json +++ b/PaperlessServices.Tests/xunit.runner.json @@ -4,7 +4,7 @@ "methodDisplay": "classAndMethod", "methodDisplayOptions": "replaceUnderscoreWithSpace,useOperatorMonikers", "parallelizeTestCollections": true, - "parallelAlgorithm": "aggressive", + "parallelAlgorithm": "conservative", "maxParallelThreads": "4x", "stopOnFail": false, "failSkips": false, diff --git a/PaperlessServices/Features/OcrProcessing/Infrastructure/Search/SearchIndexService.cs b/PaperlessServices/Features/OcrProcessing/Infrastructure/Search/SearchIndexService.cs index 5f6fa04..106fb4b 100644 --- a/PaperlessServices/Features/OcrProcessing/Infrastructure/Search/SearchIndexService.cs +++ b/PaperlessServices/Features/OcrProcessing/Infrastructure/Search/SearchIndexService.cs @@ -50,7 +50,7 @@ public async Task IndexDocumentAsync(Guid id, string fileName, string content, D processedAt = now // storagePath deliberately excluded - internal detail not exposed in search results // summary will be added later by GenAI service - }, i => i.Id(id.ToString()).Refresh(Refresh.True), cancellationToken); + }, i => i.Index(options.Value.DefaultIndex).Id(id.ToString()).Refresh(Refresh.True), cancellationToken); LogIndexResult(id, response.IsValidResponse); }