From 8d5235deffca61a1f678021d4a35371531f18191 Mon Sep 17 00:00:00 2001 From: Simon Cropp Date: Mon, 1 Jun 2026 16:52:03 +1000 Subject: [PATCH 1/2] Allow conversion splitters for text extensions Allow conversion splitters for text extensions Previously RegisterStreamConverter threw for text extensions, and the splitter pipeline only handled stream targets, so a converter could not run against a text extension (eg html). - Drop the text-extension guard in RegisterStreamConverter - DoExtensionConversion now takes the Target itself and resolves the converter input from either a stream or a text (utf8) target, instead of reconstructing an illegal stream Target for text extensions - Scrub text targets before they are fed to a converter, so derived targets (eg rendered images) reflect the scrubbed content - Move converted-target scrubbing into DoExtensionConversion and pass already-converted targets through GetTargets untouched, so each text target is scrubbed exactly once (no double scrub of the re-emitted source) Add ExtensionConverterTests.TextSplitter covering a converter registered against a text extension. --- ...rTests.TextSplitter.verified.texttoconvert | 1 + ...onConverterTests.TextSplitter.verified.txt | 1 + .../Converters/ExtensionConverterTests.cs | 20 ++++++ ...rifyChecksTests.GetExtensions.verified.txt | 1 + src/Verify/Splitters/Settings_Extension.cs | 5 -- src/Verify/Verifier/InnerVerifier_Inner.cs | 70 +++++++++++-------- src/Verify/Verifier/InnerVerifier_Stream.cs | 39 +++++++++-- 7 files changed, 94 insertions(+), 43 deletions(-) create mode 100644 src/Verify.Tests/Converters/ExtensionConverterTests.TextSplitter.verified.texttoconvert create mode 100644 src/Verify.Tests/Converters/ExtensionConverterTests.TextSplitter.verified.txt diff --git a/src/Verify.Tests/Converters/ExtensionConverterTests.TextSplitter.verified.texttoconvert b/src/Verify.Tests/Converters/ExtensionConverterTests.TextSplitter.verified.texttoconvert new file mode 100644 index 0000000000..925839fe6a --- /dev/null +++ b/src/Verify.Tests/Converters/ExtensionConverterTests.TextSplitter.verified.texttoconvert @@ -0,0 +1 @@ +the source text \ No newline at end of file diff --git a/src/Verify.Tests/Converters/ExtensionConverterTests.TextSplitter.verified.txt b/src/Verify.Tests/Converters/ExtensionConverterTests.TextSplitter.verified.txt new file mode 100644 index 0000000000..f925c4ac50 --- /dev/null +++ b/src/Verify.Tests/Converters/ExtensionConverterTests.TextSplitter.verified.txt @@ -0,0 +1 @@ +derived from text \ No newline at end of file diff --git a/src/Verify.Tests/Converters/ExtensionConverterTests.cs b/src/Verify.Tests/Converters/ExtensionConverterTests.cs index 47615ae249..cb3366be01 100644 --- a/src/Verify.Tests/Converters/ExtensionConverterTests.cs +++ b/src/Verify.Tests/Converters/ExtensionConverterTests.cs @@ -1,5 +1,25 @@ public class ExtensionConverterTests { + [ModuleInitializer] + public static void TextSplitterInit() + { + FileExtensions.AddTextExtension("texttoconvert"); + VerifierSettings.RegisterStreamConverter( + "texttoconvert", + async (_, stream, _) => + new( + null, + [ + new("texttoconvert", await stream.ReadStringBuilderWithFixedLines()), + new("txt", "derived from text") + ])); + } + + // a conversion splitter registered against a text extension + [Fact] + public Task TextSplitter() => + Verify("the source text", "texttoconvert"); + [ModuleInitializer] public static void RecursiveInit() => VerifierSettings.RegisterStreamConverter( diff --git a/src/Verify.Tests/InnerVerifyChecksTests/VerifyChecksTests.GetExtensions.verified.txt b/src/Verify.Tests/InnerVerifyChecksTests/VerifyChecksTests.GetExtensions.verified.txt index fa9ccb15e2..3956841f05 100644 --- a/src/Verify.Tests/InnerVerifyChecksTests/VerifyChecksTests.GetExtensions.verified.txt +++ b/src/Verify.Tests/InnerVerifyChecksTests/VerifyChecksTests.GetExtensions.verified.txt @@ -6,6 +6,7 @@ nuspec, props, staticComparerExt, + texttoconvert, txt, xml ] \ No newline at end of file diff --git a/src/Verify/Splitters/Settings_Extension.cs b/src/Verify/Splitters/Settings_Extension.cs index 1577b61747..c166f70a26 100644 --- a/src/Verify/Splitters/Settings_Extension.cs +++ b/src/Verify/Splitters/Settings_Extension.cs @@ -23,11 +23,6 @@ public static void RegisterStreamConverter( { InnerVerifier.ThrowIfVerifyHasBeenRun(); Guards.AgainstBadExtension(extension); - if (FileExtensions.IsTextExtension(extension)) - { - throw new("RegisterStreamConverter is only supported for non-text extensions"); - } - extensionConverters[extension] = conversion; } } \ No newline at end of file diff --git a/src/Verify/Verifier/InnerVerifier_Inner.cs b/src/Verify/Verifier/InnerVerifier_Inner.cs index 3bfc93d4e0..9096affa84 100644 --- a/src/Verify/Verifier/InnerVerifier_Inner.cs +++ b/src/Verify/Verifier/InnerVerifier_Inner.cs @@ -46,46 +46,54 @@ async Task VerifyInner(object? root, Func? cleanup, IEnumera { List list = [..targets, ..VerifierSettings.GetFileAppenders(settings)]; var cleanup = () => Task.CompletedTask; - if (doExtensionConversion) - { - var result = new List(); - foreach (var target in list) - { - if (!target.PerformConversion || - !VerifierSettings.HasStreamConverter(target.Extension)) - { - result.Add(target); - continue; - } - - var (info, converted, itemCleanup) = await DoExtensionConversion(target.Extension, target.StreamData, null, target.Name); - cleanup += itemCleanup; - if (info != null) - { - result.Add( - new( - settings.TxtOrJson, - JsonFormatter.AsJson( - settings, - counter, - info))); - } - - result.AddRange(converted); - } - list = result; + // When doExtensionConversion is false the targets have already been run through + // conversion and scrubbing (the only caller is the post-conversion stream path), + // so pass them through untouched to avoid double scrubbing. + if (!doExtensionConversion) + { + return (list, cleanup); } + var result = new List(); foreach (var target in list) { - if (target.TryGetStringBuilder(out var builder)) + if (!target.PerformConversion || + !VerifierSettings.HasStreamConverter(target.Extension)) { - ApplyScrubbers.ApplyForExtension(target.Extension, builder, settings, counter); + Scrub(target); + result.Add(target); + continue; } + + var (info, converted, itemCleanup) = await DoExtensionConversion(target, null); + cleanup += itemCleanup; + if (info != null) + { + Target infoTarget = new( + settings.TxtOrJson, + JsonFormatter.AsJson( + settings, + counter, + info)); + Scrub(infoTarget); + result.Add(infoTarget); + } + + // converted targets are scrubbed within DoExtensionConversion + result.AddRange(converted); } - return (list, cleanup); + return (result, cleanup); + } + + // Scrubs a text target in place. Stream (binary) targets are left untouched. + void Scrub(in Target target) + { + if (target.TryGetStringBuilder(out var builder)) + { + ApplyScrubbers.ApplyForExtension(target.Extension, builder, settings, counter); + } } bool TryGetRootTarget(object? root,bool ignoreNullRoot, [NotNullWhen(true)] out Target? target) diff --git a/src/Verify/Verifier/InnerVerifier_Stream.cs b/src/Verify/Verifier/InnerVerifier_Stream.cs index 51344cbd87..1bf163e7a8 100644 --- a/src/Verify/Verifier/InnerVerifier_Stream.cs +++ b/src/Verify/Verifier/InnerVerifier_Stream.cs @@ -90,7 +90,8 @@ public async Task VerifyStream(Stream? stream, string extension, o { if (VerifierSettings.HasStreamConverter(extension)) { - var (newInfo, converted, cleanup) = await DoExtensionConversion(extension, stream, info, null); + var initial = await GetTarget(stream, extension); + var (newInfo, converted, cleanup) = await DoExtensionConversion(initial, info); return await VerifyInner(newInfo, cleanup, converted, false, true); } @@ -122,10 +123,15 @@ static async Task GetTarget(Stream stream, string extension) return new(extension, stream); } - //TODO: possibly pass in the target here - async Task<(object? info, List targets, Func cleanup)> DoExtensionConversion(string extension, Stream stream, object? info, string? name) + async Task<(object? info, List targets, Func cleanup)> DoExtensionConversion(Target initial, object? info) { - var cleanup = stream.DisposeAsyncEx; + var cleanup = () => Task.CompletedTask; + // the source stream of a stream target is owned here, so dispose it once consumed + if (initial.IsStream) + { + cleanup += initial.StreamData.DisposeAsyncEx; + } + var infos = new List(); if (info != null) { @@ -135,7 +141,7 @@ static async Task GetTarget(Stream stream, string extension) var targets = new List(); var queue = new Queue(); - queue.Enqueue(new(extension, stream, name)); + queue.Enqueue(initial); while (queue.Count > 0) { @@ -143,11 +149,29 @@ static async Task GetTarget(Stream stream, string extension) if (!VerifierSettings.TryGetStreamConverter(target.Extension, out var conversion)) { + // terminal target: scrub text before it is finalized + Scrub(target); targets.Add(target); continue; } - var targetStream = target.StreamData; + // scrub text before conversion so derived targets (eg rendered images) reflect the scrubbed content + Scrub(target); + + Stream targetStream; + if (target.IsStream) + { + targetStream = target.StreamData; + } + else + { + // a text target is fed to the converter as a utf8 stream + target.TryGetStringBuilder(out var builder); + var memory = new MemoryStream(Encoding.UTF8.GetBytes(builder!.ToString())); + cleanup += memory.DisposeAsyncEx; + targetStream = memory; + } + var result = await conversion(target.Name, targetStream, settings.Context); if (result.Cleanup != null) { @@ -163,7 +187,8 @@ static async Task GetTarget(Stream stream, string extension) foreach (var resultTarget in resultTargets) { - // if the same extension is returned. no need to re process + // if the same extension is returned. no need to re process. + // its content derives from the already scrubbed input, so it is not scrubbed again if (resultTarget.Extension == target.Extension) { targets.Add(resultTarget); From 7106f5831830188ad69df4617c07982c93791a44 Mon Sep 17 00:00:00 2001 From: Simon Cropp Date: Wed, 10 Jun 2026 06:57:21 +1000 Subject: [PATCH 2/2] . --- docs/converter.md | 39 +++++++++++++++++++ docs/mdsource/converter.source.md | 13 +++++++ .../Converters/ExtensionConverterTests.cs | 11 ++++++ src/Verify/Compare/Png/PngSsimComparer.cs | 3 +- 4 files changed, 65 insertions(+), 1 deletion(-) diff --git a/docs/converter.md b/docs/converter.md index af35c3ee2a..2efb49a29f 100644 --- a/docs/converter.md +++ b/docs/converter.md @@ -146,6 +146,45 @@ await VerifyFile("sample.tif"); +### Text extensions + +A stream converter can also be registered against a text extension. This is useful when a text document needs derived targets, for example rendering html to an image for visual verification. + +The text target is scrubbed before being passed to the converter, so any derived targets (for example a rendered image) reflect the scrubbed content. + +For a custom text extension, register it as text via `FileExtensions.AddTextExtension`. Built-in text extensions (for example `html` or `csv`) do not require this. + + + +```cs +// "texttoconvert" is a custom text extension, so register it as text first. +// For built-in text extensions (eg html or csv) this step is not required. +FileExtensions.AddTextExtension("texttoconvert"); + +// The input text is scrubbed before being passed to the converter, so any +// derived targets (eg a rendered image) reflect the scrubbed content. +VerifierSettings.RegisterStreamConverter( + "texttoconvert", + async (_, stream, _) => + new( + null, + [ + new("texttoconvert", await stream.ReadStringBuilderWithFixedLines()), + new("txt", "derived from text") + ])); +``` +snippet source | anchor + + + + +```cs +Verify("the source text", "texttoconvert"); +``` +snippet source | anchor + + + ### Cleanup If cleanup needs to occur after verification a callback can be passes to `ConversionResult`: diff --git a/docs/mdsource/converter.source.md b/docs/mdsource/converter.source.md index ba559678d3..033d4917b5 100644 --- a/docs/mdsource/converter.source.md +++ b/docs/mdsource/converter.source.md @@ -50,6 +50,19 @@ snippet: RegisterStreamConverterExtension snippet: FileConverterExtensionVerify +### Text extensions + +A stream converter can also be registered against a text extension. This is useful when a text document needs derived targets, for example rendering html to an image for visual verification. + +The text target is scrubbed before being passed to the converter, so any derived targets (for example a rendered image) reflect the scrubbed content. + +For a custom text extension, register it as text via `FileExtensions.AddTextExtension`. Built-in text extensions (for example `html` or `csv`) do not require this. + +snippet: RegisterStreamConverterTextExtension + +snippet: TextExtensionConverterVerify + + ### Cleanup If cleanup needs to occur after verification a callback can be passes to `ConversionResult`: diff --git a/src/Verify.Tests/Converters/ExtensionConverterTests.cs b/src/Verify.Tests/Converters/ExtensionConverterTests.cs index cb3366be01..f0c992b7b1 100644 --- a/src/Verify.Tests/Converters/ExtensionConverterTests.cs +++ b/src/Verify.Tests/Converters/ExtensionConverterTests.cs @@ -3,7 +3,14 @@ [ModuleInitializer] public static void TextSplitterInit() { + #region RegisterStreamConverterTextExtension + + // "texttoconvert" is a custom text extension, so register it as text first. + // For built-in text extensions (eg html or csv) this step is not required. FileExtensions.AddTextExtension("texttoconvert"); + + // The input text is scrubbed before being passed to the converter, so any + // derived targets (eg a rendered image) reflect the scrubbed content. VerifierSettings.RegisterStreamConverter( "texttoconvert", async (_, stream, _) => @@ -13,12 +20,16 @@ public static void TextSplitterInit() new("texttoconvert", await stream.ReadStringBuilderWithFixedLines()), new("txt", "derived from text") ])); + + #endregion } // a conversion splitter registered against a text extension [Fact] public Task TextSplitter() => + #region TextExtensionConverterVerify Verify("the source text", "texttoconvert"); + #endregion [ModuleInitializer] public static void RecursiveInit() => diff --git a/src/Verify/Compare/Png/PngSsimComparer.cs b/src/Verify/Compare/Png/PngSsimComparer.cs index 1086ba79bc..1630c67a07 100644 --- a/src/Verify/Compare/Png/PngSsimComparer.cs +++ b/src/Verify/Compare/Png/PngSsimComparer.cs @@ -13,7 +13,8 @@ internal static Task Compare(Stream received, Stream verified, do var receivedImage = PngDecoder.Decode(received); var verifiedImage = PngDecoder.Decode(verified); - if (receivedImage.Width != verifiedImage.Width || receivedImage.Height != verifiedImage.Height) + if (receivedImage.Width != verifiedImage.Width || + receivedImage.Height != verifiedImage.Height) { return Task.FromResult(CompareResult.NotEqual( $"PNG dimensions differ. Received: {receivedImage.Width}x{receivedImage.Height}, Verified: {verifiedImage.Width}x{verifiedImage.Height}"));