From 97a3deb29255149ea50887c381f440340a9aa9f6 Mon Sep 17 00:00:00 2001 From: ModerRAS Date: Fri, 15 May 2026 00:31:49 +0800 Subject: [PATCH 1/2] Slim update feed storage --- .github/workflows/push.yml | 99 ++++-- Docs/Build_and_Test_Guide.md | 4 +- README.md | 2 +- .../Model/Update/UpdateCatalog.cs | 5 + .../Model/Update/UpdateCatalogEntry.cs | 2 + .../Model/Update/UpdateManifest.cs | 1 + .../Model/Update/UpdatePackageFormats.cs | 7 + .../ModerUpdateIntegrationTests.cs | 14 +- .../Update/SelfUpdateBootstrapTests.cs | 68 +++- .../Service/Update/UpdateBuilderTests.cs | 334 ++++++++++++++++++ TelegramSearchBot.UpdateBuilder/Program.cs | 271 +++++++------- .../Update/SelfUpdateBootstrap.Windows.cs | 113 +++++- .../Moder.Update/Models/UpdateCatalogEntry.cs | 6 + .../src/Moder.Update/Models/UpdateManifest.cs | 3 + 14 files changed, 740 insertions(+), 189 deletions(-) create mode 100644 TelegramSearchBot.Common/Model/Update/UpdatePackageFormats.cs create mode 100644 TelegramSearchBot.Test/Service/Update/UpdateBuilderTests.cs diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index 5a5b53a9..a9479e0f 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -238,7 +238,7 @@ jobs: } catch { Write-Host "No existing catalog found on CDN, will create fresh catalog." } - - name: Fetch previous release standalone for step package comparison + - name: Fetch previous release standalone for cumulative planning shell: pwsh env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -248,7 +248,7 @@ jobs: $prevTag = $prevRelease[0].tagName $prevVersion = $prevTag -replace '^v', '' if ($prevVersion -eq "${{ needs.prepare.outputs.build-version }}") { - Write-Host "Previous release is the current build version; skipping step package generation." + Write-Host "Previous release is the current build version; skipping previous-version cumulative planning." exit 0 } Write-Host "Previous release: $prevTag (version: $prevVersion)" @@ -261,7 +261,7 @@ jobs: Add-Content -Path $env:GITHUB_ENV -Value "PREV_STANDALONE_DIR=artifacts\prev-standalone" -Encoding utf8 Write-Host "Extracted previous standalone from $($zipFile.Name)." } else { - Write-Host "No previous full zip found; step package will not be generated." + Write-Host "No previous full zip found; previous-version cumulative planning will be skipped." } } else { Write-Host "No previous release found; this is the first release." @@ -287,7 +287,7 @@ jobs: New-Item -ItemType Directory -Path artifacts\anchor-release -Force | Out-Null gh release download $anchorTag --repo ${{ github.repository }} --pattern "TelegramSearchBot-win-x64-full-*.zip" --dir artifacts\anchor-release if ($LASTEXITCODE -ne 0) { - throw "Failed to download cumulative update anchor $anchorTag; refusing to publish a full-only update feed." + throw "Failed to download cumulative update anchor $anchorTag; refusing to publish a cumulative-free update feed." } $zipFile = Get-ChildItem artifacts\anchor-release -Filter "TelegramSearchBot-win-x64-full-*.zip" | Select-Object -First 1 @@ -297,7 +297,7 @@ jobs: Add-Content -Path $env:GITHUB_ENV -Value "ANCHOR_STANDALONE_DIR=artifacts\anchor-standalone" -Encoding utf8 Write-Host "Extracted cumulative update anchor from $($zipFile.Name)." } else { - throw "No anchor full zip found; refusing to publish a full-only update feed." + throw "No anchor full zip found; refusing to publish a cumulative-free update feed." } - name: Fetch previous cumulative update package shell: pwsh @@ -315,7 +315,9 @@ jobs: $baseEntry = @($catalog.Entries | Where-Object { $_.TargetVersion -eq $env:PREV_VERSION ` -and $_.MinSourceVersion -eq $env:CUMULATIVE_UPDATE_ANCHOR_VERSION ` - -and $_.IsCumulative + -and $_.IsCumulative ` + -and $_.PackagePath ` + -and $_.PackagePath -like '*-cumulative.zst' } | Sort-Object CompressedSize | Select-Object -First 1) if ($baseEntry.Count -eq 0 -or -not $baseEntry[0].PackagePath) { @@ -325,7 +327,11 @@ jobs: New-Item -ItemType Directory -Path artifacts\base-cumulative -Force | Out-Null $basePackagePath = 'artifacts\base-cumulative\base-cumulative.zst' - $uri = "$($env:UPDATE_BASE_URL.TrimEnd('/'))/$($baseEntry[0].PackagePath.TrimStart('/'))" + $uri = if ($baseEntry[0].PackageUrl) { + $baseEntry[0].PackageUrl + } else { + "$($env:UPDATE_BASE_URL.TrimEnd('/'))/$($baseEntry[0].PackagePath.TrimStart('/'))" + } try { Invoke-WebRequest -UseBasicParsing -Uri $uri -OutFile $basePackagePath } catch { @@ -339,10 +345,6 @@ jobs: env: BUILD_VERSION: ${{ needs.prepare.outputs.build-version }} run: | - if ($env:BASE_CUMULATIVE_PACKAGE -and (Test-Path $env:BASE_CUMULATIVE_PACKAGE)) { - Write-Host "Base cumulative package is available; historical source packages are not needed." - exit 0 - } if (-not (Test-Path '.\artifacts\existing-catalog.json')) { Write-Host "No existing catalog found; cumulative source packages will not be used." exit 0 @@ -353,7 +355,7 @@ jobs: $catalog = Get-Content '.\artifacts\existing-catalog.json' -Raw | ConvertFrom-Json $entriesByVersion = @{} foreach ($entry in $catalog.Entries) { - if (-not $entry.PackagePath -or $entry.PackagePath -notlike '*-full.zst') { + if (-not $entry.PackagePath -or $entry.PackagePath -notlike '*-cumulative.zst') { continue } @@ -374,7 +376,7 @@ jobs: } if ($entriesByVersion.Count -eq 0) { - Write-Host "No cumulative source full packages found in existing catalog." + Write-Host "No cumulative source packages found in existing catalog." exit 0 } @@ -383,7 +385,11 @@ jobs: foreach ($key in ($entriesByVersion.Keys | Sort-Object { [Version]$_ })) { $entry = $entriesByVersion[$key] $packagePath = Join-Path $sourcePackageDir ("source-$($key.Replace('.', '-')).zst") - $uri = "$($env:UPDATE_BASE_URL.TrimEnd('/'))/$($entry.PackagePath.TrimStart('/'))" + $uri = if ($entry.PackageUrl) { + $entry.PackageUrl + } else { + "$($env:UPDATE_BASE_URL.TrimEnd('/'))/$($entry.PackagePath.TrimStart('/'))" + } try { Invoke-WebRequest -UseBasicParsing -Uri $uri -OutFile $packagePath Write-Host "Downloaded cumulative source package from CDN: $($entry.PackagePath)" @@ -395,16 +401,43 @@ jobs: if ((Get-ChildItem $sourcePackageDir -Filter '*.zst' -File | Measure-Object).Count -gt 0) { Add-Content -Path $env:GITHUB_ENV -Value "CUMULATIVE_SOURCE_PACKAGE_DIR=$sourcePackageDir" -Encoding utf8 } + - name: Package full release asset + shell: pwsh + env: + BUILD_VERSION: ${{ needs.prepare.outputs.build-version }} + RELEASE_TAG: ${{ needs.prepare.outputs.release-tag }} + run: | + $releaseDir = '.\artifacts\release' + if (Test-Path $releaseDir) { + Remove-Item $releaseDir -Recurse -Force + } + New-Item -ItemType Directory -Path $releaseDir | Out-Null + $packageName = "TelegramSearchBot-win-x64-full-$env:BUILD_VERSION.zip" + $packagePath = Join-Path $releaseDir $packageName + Compress-Archive -Path .\artifacts\standalone\* -DestinationPath $packagePath -CompressionLevel Optimal + $hashPath = "$packagePath.sha512" + $hash = (Get-FileHash $packagePath -Algorithm SHA512).Hash.ToLower() + Set-Content -Path $hashPath -Value "$hash $packageName" -Encoding ascii + $repository = '${{ github.repository }}' + $fullUrl = "https://github.com/$repository/releases/download/$env:RELEASE_TAG/$packageName" + Add-Content -Path $env:GITHUB_ENV -Value "FULL_PACKAGE_NAME=$packageName" -Encoding utf8 + Add-Content -Path $env:GITHUB_ENV -Value "FULL_PACKAGE_URL=$fullUrl" -Encoding utf8 + Add-Content -Path $env:GITHUB_ENV -Value "FULL_PACKAGE_CHECKSUM=$hash" -Encoding utf8 + Add-Content -Path $env:GITHUB_ENV -Value "FULL_PACKAGE_SIZE=$((Get-Item $packagePath).Length)" -Encoding utf8 - name: Build Moder.Update feed shell: pwsh env: BUILD_VERSION: ${{ needs.prepare.outputs.build-version }} + RELEASE_TAG: ${{ needs.prepare.outputs.release-tag }} run: | + $updateBaseUrl = $env:UPDATE_BASE_URL.TrimEnd('/') $builderArgs = @( '--source-dir', '.\artifacts\standalone', '--output-dir', '.\artifacts\update-feed', '--target-version', $env:BUILD_VERSION, - '--min-source-version', $env:LEGACY_BRIDGE_VERSION + '--min-source-version', $env:LEGACY_BRIDGE_VERSION, + '--updater-url', "$updateBaseUrl/moder_update_updater.exe", + '--package-base-url', $updateBaseUrl ) if (Test-Path '.\artifacts\existing-catalog.json') { $builderArgs += @('--existing-catalog', '.\artifacts\existing-catalog.json') @@ -412,7 +445,7 @@ jobs: } if ($env:PREV_VERSION -and (Test-Path $env:PREV_STANDALONE_DIR)) { $builderArgs += @('--prev-source-dir', $env:PREV_STANDALONE_DIR, '--prev-version', $env:PREV_VERSION) - Write-Host "Passing previous version $env:PREV_VERSION for step package generation." + Write-Host "Passing previous version $env:PREV_VERSION for cumulative planning." } if ($env:ANCHOR_VERSION -and (Test-Path $env:ANCHOR_STANDALONE_DIR)) { $builderArgs += @('--anchor-source-dir', $env:ANCHOR_STANDALONE_DIR, '--anchor-version', $env:ANCHOR_VERSION) @@ -424,6 +457,15 @@ jobs: } Write-Host "Passing cumulative update anchor $env:ANCHOR_VERSION." } + if ($env:FULL_PACKAGE_URL -and $env:FULL_PACKAGE_CHECKSUM) { + $builderArgs += @( + '--full-package-url', $env:FULL_PACKAGE_URL, + '--full-package-name', $env:FULL_PACKAGE_NAME, + '--full-package-checksum', $env:FULL_PACKAGE_CHECKSUM, + '--full-package-size', $env:FULL_PACKAGE_SIZE + ) + Write-Host "Passing GitHub full package URL $env:FULL_PACKAGE_URL." + } dotnet run --project .\TelegramSearchBot.UpdateBuilder\TelegramSearchBot.UpdateBuilder.csproj ` -c Release ` --no-build ` @@ -433,11 +475,12 @@ jobs: $catalogPath = '.\artifacts\update-feed\catalog.json' $catalog = Get-Content $catalogPath -Raw | ConvertFrom-Json $catalog | Add-Member -NotePropertyName UpdaterChecksum -NotePropertyValue ((Get-FileHash .\artifacts\update-feed\moder_update_updater.exe -Algorithm SHA512).Hash.ToLower()) -Force + $catalog | Add-Member -NotePropertyName UpdaterUrl -NotePropertyValue "$updateBaseUrl/moder_update_updater.exe" -Force $catalog | ConvertTo-Json -Depth 10 | Set-Content $catalogPath -Encoding utf8 $catalog = Get-Content $catalogPath -Raw | ConvertFrom-Json $entryTable = $catalog.Entries | Sort-Object TargetVersion, PackagePath | - Select-Object TargetVersion, MinSourceVersion, MaxSourceVersion, IsCumulative, FileCount, CompressedSize, PackagePath | + Select-Object TargetVersion, MinSourceVersion, MaxSourceVersion, IsCumulative, PackageFormat, FileCount, CompressedSize, PackagePath, PackageUrl | Format-Table -AutoSize | Out-String Write-Host "Update feed catalog entries:" @@ -453,22 +496,6 @@ jobs: throw "Cumulative update package missing for $env:ANCHOR_VERSION to $env:BUILD_VERSION." } } - - name: Package full release asset - shell: pwsh - env: - BUILD_VERSION: ${{ needs.prepare.outputs.build-version }} - run: | - $releaseDir = '.\artifacts\release' - if (Test-Path $releaseDir) { - Remove-Item $releaseDir -Recurse -Force - } - New-Item -ItemType Directory -Path $releaseDir | Out-Null - $packageName = "TelegramSearchBot-win-x64-full-$env:BUILD_VERSION.zip" - $packagePath = Join-Path $releaseDir $packageName - Compress-Archive -Path .\artifacts\standalone\* -DestinationPath $packagePath -CompressionLevel Optimal - $hashPath = "$packagePath.sha512" - $hash = (Get-FileHash $packagePath -Algorithm SHA512).Hash.ToLower() - Set-Content -Path $hashPath -Value "$hash $packageName" -Encoding ascii - name: Upload update feed artifact uses: actions/upload-artifact@v4 with: @@ -636,7 +663,7 @@ jobs: $catalog = Get-Content $CatalogPath -Raw | ConvertFrom-Json $reachablePaths = [System.Collections.Generic.HashSet[string]]::new([System.StringComparer]::OrdinalIgnoreCase) foreach ($entry in $catalog.Entries) { - if ($entry.PackagePath) { + if ($entry.PackagePath -and $entry.PackagePath -like 'packages/*') { $normalized = $entry.PackagePath -replace '^packages/', '' [void]$reachablePaths.Add("TelegramSearchBot/packages/$normalized") } @@ -672,7 +699,9 @@ jobs: } } - b2 sync .\artifacts\update-feed\packages b2://$env:B2_BUCKET/TelegramSearchBot/packages --quiet + if (Test-Path .\artifacts\update-feed\packages) { + b2 sync .\artifacts\update-feed\packages b2://$env:B2_BUCKET/TelegramSearchBot/packages --quiet + } b2 upload-file $env:B2_BUCKET .\artifacts\update-feed\moder_update_updater.exe TelegramSearchBot/moder_update_updater.exe b2 upload-file $env:B2_BUCKET .\artifacts\update-feed\catalog.json TelegramSearchBot/catalog.json diff --git a/Docs/Build_and_Test_Guide.md b/Docs/Build_and_Test_Guide.md index d8412b05..8c822545 100644 --- a/Docs/Build_and_Test_Guide.md +++ b/Docs/Build_and_Test_Guide.md @@ -258,10 +258,10 @@ jobs: 仓库内实际的 `push.yml` 现在维护两条发布线： - 保留根目录中的最终 ClickOnce bridge，供旧安装版本过渡到新更新链路。 -- 每次主分支发布都会生成 `catalog.json`、`packages/` 和 `moder_update_updater.exe`，供 `%LOCALAPPDATA%\TelegramSearchBot\app` 中的独立安装目录继续使用 Moder.Update 协议升级。 +- 每次主分支发布都会生成 `catalog.json`、必要的累计更新包和 `moder_update_updater.exe`，供 `%LOCALAPPDATA%\TelegramSearchBot\app` 中的独立安装目录继续使用 Moder.Update 协议升级。 - `push.yml` 会把校验、`telegram-bot-api` 构建、Moder.Update updater 构建、独立包构建和桥接包构建尽量拆成并行 job，最后再统一上传到 Backblaze B2 与 GitHub Releases。 - 同一分支上的发布运行会通过 workflow `concurrency` 串行化，避免并发发布互相覆盖 B2 包或 Release 产物。 -- 同一条发布流水线会在新文件上传成功后裁剪 Backblaze B2 上重复或过期的更新包版本，并把 `TelegramSearchBot-win-x64-full-.zip` 全量包上传到 GitHub Releases，便于手动分发和回滚。 +- 同一条发布流水线会在新文件上传成功后裁剪 Backblaze B2 上重复或过期的更新包版本。普通 `TelegramSearchBot-win-x64-full-.zip` 全量包只上传到 GitHub Releases，并以绝对 URL 写入 `catalog.json` 作为完整包 fallback；B2 不再保存 step 包或自定义 MUP full `.zst` 包。 ### 监控与日志 diff --git a/README.md b/README.md index cbc0c7ca..4fe038cf 100644 --- a/README.md +++ b/README.md @@ -96,7 +96,7 @@ - **自动更新**: - `EnableAutoUpdate`: 是否启用内置自更新流程(默认true) - `UpdateBaseUrl`: 更新目录根地址，默认使用 `https://clickonce.miaostay.com/TelegramSearchBot` - - **说明**: 首次安装仍通过 `Publish.html` 分发桥接版；之后程序会从 `catalog.json`、`packages/` 和 `moder_update_updater.exe` 拉取更新并升级独立安装目录。每次主分支发布也会同步上传一份 `TelegramSearchBot-win-x64-full-*.zip` 到 GitHub Releases，便于手动全量更新或回滚。 + - **说明**: 首次安装仍通过 `Publish.html` 分发桥接版；之后程序会从 `catalog.json`、累计更新包和 `moder_update_updater.exe` 拉取更新并升级独立安装目录。每次主分支发布也会同步上传一份普通 `TelegramSearchBot-win-x64-full-*.zip` 到 GitHub Releases，便于新用户手动部署、全量更新或回滚。 - **AI相关**: - `OllamaModelName`: 本地模型名称(默认"qwen2.5:72b-instruct-q2_K") diff --git a/TelegramSearchBot.Common/Model/Update/UpdateCatalog.cs b/TelegramSearchBot.Common/Model/Update/UpdateCatalog.cs index 4dcd630c..3220cc87 100644 --- a/TelegramSearchBot.Common/Model/Update/UpdateCatalog.cs +++ b/TelegramSearchBot.Common/Model/Update/UpdateCatalog.cs @@ -7,4 +7,9 @@ public sealed class UpdateCatalog public DateTime LastUpdated { get; init; } public string? MinRequiredVersion { get; init; } public string? UpdaterChecksum { get; init; } + public string? UpdaterUrl { get; init; } + public string? FullPackageUrl { get; init; } + public string? FullPackageName { get; init; } + public string? FullPackageChecksum { get; init; } + public long FullPackageSize { get; init; } } diff --git a/TelegramSearchBot.Common/Model/Update/UpdateCatalogEntry.cs b/TelegramSearchBot.Common/Model/Update/UpdateCatalogEntry.cs index fdd7128f..b3a00499 100644 --- a/TelegramSearchBot.Common/Model/Update/UpdateCatalogEntry.cs +++ b/TelegramSearchBot.Common/Model/Update/UpdateCatalogEntry.cs @@ -3,6 +3,8 @@ namespace TelegramSearchBot.Common.Model.Update; public sealed class UpdateCatalogEntry { public required string PackagePath { get; init; } + public string? PackageUrl { get; init; } + public string PackageFormat { get; init; } = UpdatePackageFormats.ModerUpdateZstd; public required string TargetVersion { get; init; } public required string MinSourceVersion { get; init; } public string? MaxSourceVersion { get; init; } diff --git a/TelegramSearchBot.Common/Model/Update/UpdateManifest.cs b/TelegramSearchBot.Common/Model/Update/UpdateManifest.cs index 3d4114c5..b647347f 100644 --- a/TelegramSearchBot.Common/Model/Update/UpdateManifest.cs +++ b/TelegramSearchBot.Common/Model/Update/UpdateManifest.cs @@ -9,6 +9,7 @@ public sealed class UpdateManifest public bool IsCumulative { get; init; } public int ChainDepth { get; init; } public required List Files { get; init; } + public List? SnapshotFiles { get; init; } public required string Checksum { get; init; } public DateTime CreatedAt { get; init; } } diff --git a/TelegramSearchBot.Common/Model/Update/UpdatePackageFormats.cs b/TelegramSearchBot.Common/Model/Update/UpdatePackageFormats.cs new file mode 100644 index 00000000..dde49d95 --- /dev/null +++ b/TelegramSearchBot.Common/Model/Update/UpdatePackageFormats.cs @@ -0,0 +1,7 @@ +namespace TelegramSearchBot.Common.Model.Update; + +public static class UpdatePackageFormats +{ + public const string ModerUpdateZstd = "moder-update-zst"; + public const string Zip = "zip"; +} diff --git a/TelegramSearchBot.Test/ModerUpdateIntegrationTests.cs b/TelegramSearchBot.Test/ModerUpdateIntegrationTests.cs index 9be5b28a..b315989a 100644 --- a/TelegramSearchBot.Test/ModerUpdateIntegrationTests.cs +++ b/TelegramSearchBot.Test/ModerUpdateIntegrationTests.cs @@ -353,15 +353,25 @@ public void CI_UpdateFeed_GeneratesCumulativeDeltaFromAnchor() { Assert.Contains("'--anchor-version'", pushYmlContent); Assert.Contains("'--base-cumulative-package'", pushYmlContent); Assert.Contains("'--cumulative-source-package-dir'", pushYmlContent); + Assert.Contains("'--full-package-url'", pushYmlContent); + Assert.Contains("'--full-package-checksum'", pushYmlContent); + Assert.Contains("'--updater-url'", pushYmlContent); + Assert.Contains("'--package-base-url'", pushYmlContent); Assert.Contains("Downloaded existing catalog.json from CDN", pushYmlContent); Assert.Contains("Downloaded base cumulative package from CDN", pushYmlContent); Assert.Contains("Downloaded cumulative source package from CDN", pushYmlContent); - Assert.Contains("historical source packages are not needed", pushYmlContent); Assert.Contains("Passing cumulative update anchor", pushYmlContent); + Assert.Contains("Passing GitHub full package URL", pushYmlContent); Assert.Contains("Update feed catalog entries", pushYmlContent); Assert.Contains("Cumulative update package missing", pushYmlContent); Assert.Contains("Failed to download cumulative update anchor", pushYmlContent); - Assert.Contains("refusing to publish a full-only update feed", pushYmlContent); + Assert.Contains("refusing to publish a cumulative-free update feed", pushYmlContent); + Assert.Contains("PackageUrl", pushYmlContent); + Assert.Contains("PackageFormat", pushYmlContent); + Assert.Contains("PackagePath -notlike '*-cumulative.zst'", pushYmlContent); + Assert.DoesNotContain("-or $entry.PackageUrl", pushYmlContent); + Assert.DoesNotContain("step package generation", pushYmlContent); + Assert.DoesNotContain("PackagePath -notlike '*-full.zst'", pushYmlContent); Assert.DoesNotContain("Fetch existing update catalog from B2", pushYmlContent); Assert.DoesNotContain("b2 download-file-by-name", pushYmlContent); Assert.DoesNotContain("Write-Host \"PREV_VERSION=$prevVersion\" | Out-File", pushYmlContent); diff --git a/TelegramSearchBot.Test/Service/Update/SelfUpdateBootstrapTests.cs b/TelegramSearchBot.Test/Service/Update/SelfUpdateBootstrapTests.cs index 4f815de0..4cedae10 100644 --- a/TelegramSearchBot.Test/Service/Update/SelfUpdateBootstrapTests.cs +++ b/TelegramSearchBot.Test/Service/Update/SelfUpdateBootstrapTests.cs @@ -4,6 +4,7 @@ using System.Diagnostics; using System.Formats.Tar; using System.IO; +using System.IO.Compression; using System.Net; using System.Net.Http; using System.Net.Http.Headers; @@ -321,11 +322,15 @@ private static UpdateCatalogEntry CreateEntry( bool isCumulative = false, bool isAnchor = false, int chainDepth = 0, - long compressedSize = 1024) + long compressedSize = 1024, + string? packageFormat = null, + string? packageUrl = null) { return new UpdateCatalogEntry { PackagePath = $"packages/v{minSourceVersion}_to_v{targetVersion}.tar.zst", + PackageUrl = packageUrl, + PackageFormat = packageFormat ?? UpdatePackageFormats.ModerUpdateZstd, TargetVersion = targetVersion, MinSourceVersion = minSourceVersion, MaxSourceVersion = maxSourceVersion, @@ -396,6 +401,23 @@ private static MemoryStream CreateTestPackage(Dictionary files) return packageStream; } + private static MemoryStream CreateTestZipPackage(Dictionary files) + { + var packageStream = new MemoryStream(); + using (var archive = new ZipArchive(packageStream, ZipArchiveMode.Create, leaveOpen: true)) + { + foreach (var file in files) + { + var entry = archive.CreateEntry(file.Key.Replace('\\', '/')); + using var writer = new StreamWriter(entry.Open(), Encoding.UTF8); + writer.Write(file.Value); + } + } + + packageStream.Position = 0; + return packageStream; + } + private static void WriteTarEntry(TarWriter tarWriter, string relativePath, byte[] content) { var entry = new PaxTarEntry(TarEntryType.RegularFile, relativePath.Replace('\\', '/')) @@ -557,7 +579,9 @@ public void PlanUpdatePath_PrefersCumulativeDeltaOverFullFallback() "2026.04.23.553", isCumulative: true, isAnchor: true, - compressedSize: 568_000_000), + compressedSize: 568_000_000, + packageFormat: UpdatePackageFormats.Zip, + packageUrl: "https://github.com/ModerRAS/TelegramSearchBot/releases/download/v2026.05.10.572/TelegramSearchBot-win-x64-full-2026.05.10.572.zip"), CreateEntry( "2026.05.10.572", "2026.04.25.561", @@ -573,6 +597,7 @@ public void PlanUpdatePath_PrefersCumulativeDeltaOverFullFallback() Assert.Single(result); Assert.Equal("2026.04.25.561", result[0].MinSourceVersion); + Assert.Null(result[0].PackageUrl); } [Fact] @@ -585,7 +610,9 @@ public void PlanUpdatePath_UsesFullFallbackWhenCurrentIsBeforeCumulativeAnchor() "2026.04.23.553", isCumulative: true, isAnchor: true, - compressedSize: 568_000_000), + compressedSize: 568_000_000, + packageFormat: UpdatePackageFormats.Zip, + packageUrl: "https://github.com/ModerRAS/TelegramSearchBot/releases/download/v2026.05.10.572/TelegramSearchBot-win-x64-full-2026.05.10.572.zip"), CreateEntry( "2026.05.10.572", "2026.04.25.561", @@ -601,6 +628,7 @@ public void PlanUpdatePath_UsesFullFallbackWhenCurrentIsBeforeCumulativeAnchor() Assert.Single(result); Assert.Equal("2026.04.23.553", result[0].MinSourceVersion); + Assert.Equal(UpdatePackageFormats.Zip, result[0].PackageFormat); } // ── CanApplyEntry ────────────────────────────────────────────── @@ -701,7 +729,8 @@ public void ExtractPackageToDirectory_ExtractsZstdTarPackage() try { - InvokePrivateStatic("ExtractPackageToDirectory", package, targetDirectory); + var entry = CreateEntry("2.0.0", "1.0.0"); + InvokePrivateStatic("ExtractPackageToDirectory", package, targetDirectory, entry); var extractedPath = Path.Combine(targetDirectory, "nested", "file.txt"); Assert.True(File.Exists(extractedPath)); @@ -716,6 +745,37 @@ public void ExtractPackageToDirectory_ExtractsZstdTarPackage() } } + [Fact] + public void ExtractPackageToDirectory_ExtractsZipPackage() + { + using var package = CreateTestZipPackage(new Dictionary + { + ["nested/file.txt"] = "hello from zip" + }); + var targetDirectory = Path.Combine(Path.GetTempPath(), "SelfUpdateBootstrapTests", Guid.NewGuid().ToString("N")); + + try + { + var entry = CreateEntry( + "2.0.0", + "1.0.0", + packageFormat: UpdatePackageFormats.Zip, + packageUrl: "https://github.com/ModerRAS/TelegramSearchBot/releases/download/v2.0.0/TelegramSearchBot-win-x64-full-2.0.0.zip"); + InvokePrivateStatic("ExtractPackageToDirectory", package, targetDirectory, entry); + + var extractedPath = Path.Combine(targetDirectory, "nested", "file.txt"); + Assert.True(File.Exists(extractedPath)); + Assert.Equal("hello from zip", File.ReadAllText(extractedPath)); + } + finally + { + if (Directory.Exists(targetDirectory)) + { + Directory.Delete(targetDirectory, recursive: true); + } + } + } + [Fact] public async Task DownloadFileInPartsAsync_RequestsRangesAndMergesFile() { diff --git a/TelegramSearchBot.Test/Service/Update/UpdateBuilderTests.cs b/TelegramSearchBot.Test/Service/Update/UpdateBuilderTests.cs new file mode 100644 index 00000000..49313521 --- /dev/null +++ b/TelegramSearchBot.Test/Service/Update/UpdateBuilderTests.cs @@ -0,0 +1,334 @@ +using System.Formats.Tar; +using System.Text.Json; +using TelegramSearchBot.Common.Model.Update; +using Xunit; +using ZstdSharp; + +namespace TelegramSearchBot.Test.Service.Update; + +public class UpdateBuilderTests : IDisposable +{ + private readonly string _testDirectory; + private static readonly string SolutionRoot = FindSolutionRoot(); + + public UpdateBuilderTests() + { + _testDirectory = Path.Combine(Path.GetTempPath(), "UpdateBuilderTests", Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(_testDirectory); + } + + [Fact] + public async Task BuildFeed_DoesNotGenerateStepOrFullZstdPackages() + { + var anchorDir = CreateVersionDirectory("anchor", new Dictionary + { + ["app.exe"] = "anchor", + ["changed.txt"] = "old" + }); + var prevDir = CreateVersionDirectory("prev", new Dictionary + { + ["app.exe"] = "anchor", + ["changed.txt"] = "intermediate" + }); + var currentDir = CreateVersionDirectory("current", new Dictionary + { + ["app.exe"] = "anchor", + ["changed.txt"] = "old" + }); + var outputDir = Path.Combine(_testDirectory, "feed"); + + await RunBuilderAsync( + "--source-dir", currentDir, + "--output-dir", outputDir, + "--target-version", "2.0.0", + "--min-source-version", "1.0.0", + "--prev-source-dir", prevDir, + "--prev-version", "1.5.0", + "--anchor-source-dir", anchorDir, + "--anchor-version", "1.0.0", + "--full-package-url", "https://github.com/ModerRAS/TelegramSearchBot/releases/download/v2.0.0/TelegramSearchBot-win-x64-full-2.0.0.zip", + "--full-package-name", "TelegramSearchBot-win-x64-full-2.0.0.zip", + "--full-package-checksum", new string('a', 128), + "--full-package-size", "12345", + "--updater-url", "https://clickonce.miaostay.com/TelegramSearchBot/moder_update_updater.exe", + "--package-base-url", "https://clickonce.miaostay.com/TelegramSearchBot"); + + var packageFiles = Directory.GetFiles(Path.Combine(outputDir, "packages"), "*.zst"); + Assert.Single(packageFiles); + Assert.EndsWith("-cumulative.zst", packageFiles[0]); + Assert.DoesNotContain(packageFiles, path => Path.GetFileName(path).Contains("-full", StringComparison.OrdinalIgnoreCase)); + Assert.DoesNotContain(packageFiles, path => Path.GetFileName(path).Contains("1-5-0-to-2-0-0", StringComparison.OrdinalIgnoreCase)); + + var catalog = ReadCatalog(outputDir); + var zipEntry = Assert.Single(catalog.Entries, entry => entry.PackageFormat == UpdatePackageFormats.Zip); + Assert.Equal("https://github.com/ModerRAS/TelegramSearchBot/releases/download/v2.0.0/TelegramSearchBot-win-x64-full-2.0.0.zip", zipEntry.PackageUrl); + Assert.Equal("TelegramSearchBot-win-x64-full-2.0.0.zip", zipEntry.PackagePath); + var cumulativeEntry = Assert.Single(catalog.Entries, entry => entry.PackagePath.EndsWith("-cumulative.zst", StringComparison.OrdinalIgnoreCase)); + Assert.Equal($"https://clickonce.miaostay.com/TelegramSearchBot/{cumulativeEntry.PackagePath}", cumulativeEntry.PackageUrl); + Assert.Equal("https://clickonce.miaostay.com/TelegramSearchBot/moder_update_updater.exe", catalog.UpdaterUrl); + Assert.Equal(zipEntry.PackageUrl, catalog.FullPackageUrl); + } + + [Fact] + public async Task BuildFeed_CumulativePackageKeepsRevertedTouchedFileAndSnapshot() + { + var anchorDir = CreateVersionDirectory("anchor", new Dictionary + { + ["reverted.txt"] = "anchor", + ["stable.txt"] = "same" + }); + var prevDir = CreateVersionDirectory("prev", new Dictionary + { + ["reverted.txt"] = "changed once", + ["stable.txt"] = "same" + }); + var currentDir = CreateVersionDirectory("current", new Dictionary + { + ["reverted.txt"] = "anchor", + ["stable.txt"] = "same" + }); + var outputDir = Path.Combine(_testDirectory, "feed-reverted"); + + await RunBuilderAsync( + "--source-dir", currentDir, + "--output-dir", outputDir, + "--target-version", "2.0.0", + "--min-source-version", "1.0.0", + "--prev-source-dir", prevDir, + "--prev-version", "1.5.0", + "--anchor-source-dir", anchorDir, + "--anchor-version", "1.0.0"); + + var packagePath = Assert.Single(Directory.GetFiles(Path.Combine(outputDir, "packages"), "*.zst")); + var manifest = ReadPackageManifest(packagePath); + Assert.Contains(manifest.Files, file => file.RelativePath == "reverted.txt"); + Assert.DoesNotContain(manifest.Files, file => file.RelativePath == "stable.txt"); + Assert.NotNull(manifest.SnapshotFiles); + Assert.Contains(manifest.SnapshotFiles!, file => file.RelativePath == "stable.txt"); + } + + [Fact] + public async Task BuildFeed_BaseCumulativeUsesPayloadAsTouchedSetNotSnapshot() + { + var anchorDir = CreateVersionDirectory("anchor-base", new Dictionary + { + ["changed.txt"] = "old", + ["snapshot-only.txt"] = "same" + }); + var firstCurrentDir = CreateVersionDirectory("first-current", new Dictionary + { + ["changed.txt"] = "new", + ["snapshot-only.txt"] = "same" + }); + var firstOutputDir = Path.Combine(_testDirectory, "first-feed"); + + await RunBuilderAsync( + "--source-dir", firstCurrentDir, + "--output-dir", firstOutputDir, + "--target-version", "1.5.0", + "--min-source-version", "1.0.0", + "--anchor-source-dir", anchorDir, + "--anchor-version", "1.0.0"); + + var basePackagePath = Assert.Single(Directory.GetFiles(Path.Combine(firstOutputDir, "packages"), "*.zst")); + var secondCurrentDir = CreateVersionDirectory("second-current", new Dictionary + { + ["changed.txt"] = "newer", + ["snapshot-only.txt"] = "same" + }); + var secondOutputDir = Path.Combine(_testDirectory, "second-feed"); + + await RunBuilderAsync( + "--source-dir", secondCurrentDir, + "--output-dir", secondOutputDir, + "--target-version", "2.0.0", + "--min-source-version", "1.0.0", + "--anchor-source-dir", anchorDir, + "--anchor-version", "1.0.0", + "--base-cumulative-package", basePackagePath); + + var secondPackagePath = Assert.Single(Directory.GetFiles(Path.Combine(secondOutputDir, "packages"), "*.zst")); + var manifest = ReadPackageManifest(secondPackagePath); + Assert.Contains(manifest.Files, file => file.RelativePath == "changed.txt"); + Assert.DoesNotContain(manifest.Files, file => file.RelativePath == "snapshot-only.txt"); + Assert.Contains(manifest.SnapshotFiles!, file => file.RelativePath == "snapshot-only.txt"); + } + + [Fact] + public async Task BuildFeed_DropsHistoricalStepAndFullMupEntriesFromCatalog() + { + var anchorDir = CreateVersionDirectory("anchor-prune", new Dictionary + { + ["app.exe"] = "old" + }); + var currentDir = CreateVersionDirectory("current-prune", new Dictionary + { + ["app.exe"] = "new" + }); + var existingCatalogPath = Path.Combine(_testDirectory, "existing-catalog.json"); + var existingCatalog = new UpdateCatalog + { + LatestVersion = "1.5.0", + LastUpdated = DateTime.UtcNow, + Entries = + [ + new UpdateCatalogEntry + { + PackagePath = "packages/update-1-0-0-to-1-5-0.zst", + TargetVersion = "1.5.0", + MinSourceVersion = "1.0.0", + MaxSourceVersion = "1.0.0", + PackageChecksum = new string('b', 128) + }, + new UpdateCatalogEntry + { + PackagePath = "packages/update-1-0-0-to-1-5-0-full.zst", + TargetVersion = "1.5.0", + MinSourceVersion = "1.0.0", + IsCumulative = true, + IsAnchor = true, + PackageChecksum = new string('c', 128) + }, + new UpdateCatalogEntry + { + PackagePath = "packages/update-1-0-0-to-1-5-0-cumulative.zst", + TargetVersion = "1.5.0", + MinSourceVersion = "1.0.0", + IsCumulative = true, + IsAnchor = true, + PackageChecksum = new string('d', 128) + } + ] + }; + File.WriteAllText(existingCatalogPath, JsonSerializer.Serialize(existingCatalog)); + var outputDir = Path.Combine(_testDirectory, "feed-prune"); + + await RunBuilderAsync( + "--source-dir", currentDir, + "--output-dir", outputDir, + "--target-version", "2.0.0", + "--min-source-version", "1.0.0", + "--existing-catalog", existingCatalogPath, + "--anchor-source-dir", anchorDir, + "--anchor-version", "1.0.0"); + + var catalog = ReadCatalog(outputDir); + Assert.DoesNotContain(catalog.Entries, entry => entry.PackagePath.EndsWith("-full.zst", StringComparison.OrdinalIgnoreCase)); + Assert.DoesNotContain(catalog.Entries, entry => !entry.IsCumulative && entry.PackagePath.EndsWith(".zst", StringComparison.OrdinalIgnoreCase)); + Assert.Contains(catalog.Entries, entry => entry.PackagePath.EndsWith("-cumulative.zst", StringComparison.OrdinalIgnoreCase)); + } + + private string CreateVersionDirectory(string name, Dictionary files) + { + var directory = Path.Combine(_testDirectory, name); + Directory.CreateDirectory(directory); + foreach (var file in files) + { + var path = Path.Combine(directory, file.Key); + Directory.CreateDirectory(Path.GetDirectoryName(path)!); + File.WriteAllText(path, file.Value); + } + + return directory; + } + + private static async Task RunBuilderAsync(params string[] arguments) + { + var processInfo = new System.Diagnostics.ProcessStartInfo + { + FileName = "dotnet", + WorkingDirectory = SolutionRoot, + UseShellExecute = false, + RedirectStandardOutput = true, + RedirectStandardError = true + }; + processInfo.ArgumentList.Add("run"); + processInfo.ArgumentList.Add("--project"); + processInfo.ArgumentList.Add(Path.Combine(SolutionRoot, "TelegramSearchBot.UpdateBuilder", "TelegramSearchBot.UpdateBuilder.csproj")); + processInfo.ArgumentList.Add("-c"); + processInfo.ArgumentList.Add("Release"); + processInfo.ArgumentList.Add("--"); + foreach (var argument in arguments) + { + processInfo.ArgumentList.Add(argument); + } + + using var process = System.Diagnostics.Process.Start(processInfo) + ?? throw new InvalidOperationException("Failed to start UpdateBuilder."); + var stdout = await process.StandardOutput.ReadToEndAsync(); + var stderr = await process.StandardError.ReadToEndAsync(); + await process.WaitForExitAsync(); + + Assert.True(process.ExitCode == 0, $"UpdateBuilder failed with exit code {process.ExitCode}.\nSTDOUT:\n{stdout}\nSTDERR:\n{stderr}"); + } + + private static UpdateCatalog ReadCatalog(string outputDir) + { + var catalogPath = Path.Combine(outputDir, "catalog.json"); + var json = File.ReadAllText(catalogPath); + return JsonSerializer.Deserialize(json, JsonOptions) + ?? throw new InvalidDataException("Failed to parse catalog.json."); + } + + private static UpdateManifest ReadPackageManifest(string packagePath) + { + using var packageStream = File.OpenRead(packagePath); + Span magic = stackalloc byte[4]; + Assert.Equal(4, packageStream.Read(magic)); + Assert.Equal(0x4D, magic[0]); + Assert.Equal(0x55, magic[1]); + Assert.Equal(0x50, magic[2]); + Assert.Equal(0x00, magic[3]); + using var decompressed = new DecompressionStream(packageStream, leaveOpen: false); + using var tarReader = new TarReader(decompressed, leaveOpen: true); + while (tarReader.GetNextEntry() is { } entry) + { + if (!string.Equals(entry.Name, "manifest.json", StringComparison.OrdinalIgnoreCase)) + { + continue; + } + + Assert.NotNull(entry.DataStream); + return JsonSerializer.Deserialize(entry.DataStream!, JsonOptions) + ?? throw new InvalidDataException("Failed to parse manifest.json."); + } + + throw new InvalidDataException("manifest.json not found."); + } + + private static string FindSolutionRoot() + { + var dir = AppContext.BaseDirectory; + while (!string.IsNullOrEmpty(dir)) + { + if (File.Exists(Path.Combine(dir, "TelegramSearchBot.sln"))) + { + return dir; + } + + dir = Directory.GetParent(dir)?.FullName; + } + + return Directory.GetCurrentDirectory(); + } + + public void Dispose() + { + try + { + if (Directory.Exists(_testDirectory)) + { + Directory.Delete(_testDirectory, recursive: true); + } + } + catch + { + // Best-effort test cleanup. + } + } + + private static readonly JsonSerializerOptions JsonOptions = new() + { + PropertyNameCaseInsensitive = true + }; +} diff --git a/TelegramSearchBot.UpdateBuilder/Program.cs b/TelegramSearchBot.UpdateBuilder/Program.cs index 81663fb4..53b088c6 100644 --- a/TelegramSearchBot.UpdateBuilder/Program.cs +++ b/TelegramSearchBot.UpdateBuilder/Program.cs @@ -61,45 +61,12 @@ if (!string.IsNullOrWhiteSpace(arguments.PrevSourceDir) && Directory.Exists(arguments.PrevSourceDir)) { prevFiles = LoadFiles(arguments.PrevSourceDir); - var prevVersion = arguments.PrevVersion!; - - var changedFiles = ComputeChangedFiles(prevFiles, currentFiles); - if (changedFiles.Count > 0) - { - var stepPackagePath = $"packages/update-{SanitizeVersion(prevVersion)}-to-{SanitizeVersion(currentVersion)}.zst"; - var (stepBytes, stepChecksum) = BuildPackageFile(changedFiles, prevVersion, currentVersion, - arguments.MinSourceVersion, isCumulative: false, isAnchor: false, chainDepth: 1); - - var stepFilePath = Path.Combine(outputDirectory, stepPackagePath.Replace('/', Path.DirectorySeparatorChar)); - Directory.CreateDirectory(Path.GetDirectoryName(stepFilePath)!); - await File.WriteAllBytesAsync(stepFilePath, stepBytes); - - catalogEntries.Add(new UpdateCatalogEntry - { - PackagePath = stepPackagePath.Replace('\\', '/'), - TargetVersion = currentVersion, - MinSourceVersion = prevVersion, - MaxSourceVersion = prevVersion, - IsCumulative = false, - IsAnchor = false, - ChainDepth = 1, - PackageChecksum = stepChecksum, - FileCount = changedFiles.Count, - CompressedSize = stepBytes.LongLength, - UncompressedSize = changedFiles.Sum(f => (long)f.Content.Length) - }); - - Console.WriteLine( - $"Step package written: {stepPackagePath} ({changedFiles.Count} changed files, {stepBytes.LongLength} bytes compressed)"); - } - else - { - Console.WriteLine("No changed files detected; skipping step package generation."); - } + Console.WriteLine( + $"Loaded previous source directory for cumulative planning: {arguments.PrevSourceDir} ({prevFiles.Count} files)"); } else { - Console.WriteLine("No --prev-source-dir provided; step package will not be generated this run."); + Console.WriteLine("No --prev-source-dir provided; previous-version touched-file planning will be skipped."); } if (!string.IsNullOrWhiteSpace(arguments.AnchorVersion) && !string.IsNullOrWhiteSpace(arguments.AnchorSourceDir)) @@ -107,36 +74,37 @@ var anchorDir = Path.GetFullPath(arguments.AnchorSourceDir); if (Directory.Exists(anchorDir)) { - var cumulativeFileSets = new List>(); + var touchedPaths = new HashSet(StringComparer.OrdinalIgnoreCase); if (!string.IsNullOrWhiteSpace(arguments.BaseCumulativePackage) && File.Exists(arguments.BaseCumulativePackage)) { - var baseCumulativeFiles = LoadPackageFiles(arguments.BaseCumulativePackage); - cumulativeFileSets.Add(baseCumulativeFiles); + AddFingerprintPaths(touchedPaths, LoadPackageManifestFiles(arguments.BaseCumulativePackage)); Console.WriteLine( - $"Loaded base cumulative package: {arguments.BaseCumulativePackage} ({baseCumulativeFiles.Count} files)"); + $"Loaded base cumulative package manifest: {arguments.BaseCumulativePackage} ({touchedPaths.Count} touched files)"); } foreach (var sourceManifest in LoadCumulativeSourceManifests(arguments.CumulativeSourcePackageDir)) { - cumulativeFileSets.Add(ComputeChangedFilesFromManifest(sourceManifest, currentFiles)); + AddContentPaths(touchedPaths, ComputeChangedFilesFromManifest(sourceManifest, currentFiles)); } var anchorFiles = LoadFiles(anchorDir); - cumulativeFileSets.Add(ComputeChangedFiles(anchorFiles, currentFiles)); + AddContentPaths(touchedPaths, ComputeChangedFiles(anchorFiles, currentFiles)); if (prevFiles is not null) { - cumulativeFileSets.Add(ComputeChangedFiles(prevFiles, currentFiles)); + AddContentPaths(touchedPaths, ComputeChangedFiles(prevFiles, currentFiles)); } - var changedFromAnchor = MergeChangedFiles(cumulativeFileSets.ToArray()); + var currentFilesByPath = currentFiles.ToDictionary(file => file.RelativePath, StringComparer.OrdinalIgnoreCase); + var changedFromAnchor = MaterializeTouchedFiles(touchedPaths, currentFilesByPath); if (changedFromAnchor.Count > 0) { var cumulativePath = $"packages/update-{SanitizeVersion(arguments.AnchorVersion)}-to-{SanitizeVersion(currentVersion)}-cumulative.zst"; var (cumBytes, cumChecksum) = BuildPackageFile(changedFromAnchor, arguments.AnchorVersion, - currentVersion, arguments.AnchorVersion, isCumulative: true, isAnchor: true, chainDepth: 0); + currentVersion, arguments.AnchorVersion, isCumulative: true, isAnchor: true, chainDepth: 0, + snapshotFiles: currentFiles); var cumFilePath = Path.Combine(outputDirectory, cumulativePath.Replace('/', Path.DirectorySeparatorChar)); Directory.CreateDirectory(Path.GetDirectoryName(cumFilePath)!); @@ -145,6 +113,7 @@ catalogEntries.Add(new UpdateCatalogEntry { PackagePath = cumulativePath.Replace('\\', '/'), + PackageUrl = BuildPackageUrl(arguments.PackageBaseUrl, cumulativePath), TargetVersion = currentVersion, MinSourceVersion = arguments.AnchorVersion, MaxSourceVersion = null, @@ -158,7 +127,7 @@ }); Console.WriteLine( - $"Cumulative package written: {cumulativePath} ({changedFromAnchor.Count} files from anchor {arguments.AnchorVersion}, {cumBytes.LongLength} bytes compressed)"); + $"Cumulative package written: {cumulativePath} ({changedFromAnchor.Count} touched files from anchor {arguments.AnchorVersion}, {currentFiles.Count} snapshot files, {cumBytes.LongLength} bytes compressed)"); } else { @@ -172,35 +141,42 @@ } } -var fallbackStepFrom = - arguments.MinSourceVersion ?? arguments.PrevVersion ?? "unknown"; -var fallbackChangedFiles = currentFiles; -var fallbackPackagePath = - $"packages/update-{SanitizeVersion(fallbackStepFrom)}-to-{SanitizeVersion(currentVersion)}-full.zst"; -var (fallbackBytes, fallbackChecksum) = BuildPackageFile(fallbackChangedFiles, fallbackStepFrom, - currentVersion, arguments.MinSourceVersion, isCumulative: true, isAnchor: true, chainDepth: 0); - -var fallbackFilePath = Path.Combine(outputDirectory, fallbackPackagePath.Replace('/', Path.DirectorySeparatorChar)); -Directory.CreateDirectory(Path.GetDirectoryName(fallbackFilePath)!); -await File.WriteAllBytesAsync(fallbackFilePath, fallbackBytes); +var fullPackageName = arguments.FullPackageName; +var fullPackageUrl = arguments.FullPackageUrl; +var fullPackageChecksum = arguments.FullPackageChecksum; +var fullPackageSize = arguments.FullPackageSize; +if (string.IsNullOrWhiteSpace(fullPackageName) && !string.IsNullOrWhiteSpace(fullPackageUrl)) +{ + fullPackageName = Path.GetFileName(Uri.TryCreate(fullPackageUrl, UriKind.Absolute, out var fullUri) + ? fullUri.AbsolutePath + : fullPackageUrl); +} -catalogEntries.Add(new UpdateCatalogEntry +if (!string.IsNullOrWhiteSpace(fullPackageUrl) && !string.IsNullOrWhiteSpace(fullPackageChecksum)) +{ + catalogEntries.Add(new UpdateCatalogEntry + { + PackagePath = fullPackageName ?? $"TelegramSearchBot-win-x64-full-{currentVersion}.zip", + PackageUrl = fullPackageUrl, + PackageFormat = UpdatePackageFormats.Zip, + TargetVersion = currentVersion, + MinSourceVersion = arguments.MinSourceVersion!, + MaxSourceVersion = null, + IsCumulative = true, + IsAnchor = true, + ChainDepth = 0, + PackageChecksum = fullPackageChecksum, + FileCount = currentFiles.Count, + CompressedSize = fullPackageSize, + UncompressedSize = currentFiles.Sum(f => (long)f.Content.Length) + }); + + Console.WriteLine($"Full package catalog entry added: {fullPackageUrl}"); +} +else { - PackagePath = fallbackPackagePath.Replace('\\', '/'), - TargetVersion = currentVersion, - MinSourceVersion = arguments.MinSourceVersion ?? fallbackStepFrom, - MaxSourceVersion = null, - IsCumulative = true, - IsAnchor = true, - ChainDepth = 0, - PackageChecksum = fallbackChecksum, - FileCount = fallbackChangedFiles.Count, - CompressedSize = fallbackBytes.LongLength, - UncompressedSize = fallbackChangedFiles.Sum(f => (long)f.Content.Length) -}); - -Console.WriteLine( - $"Fallback full package written: {fallbackPackagePath} ({fallbackChangedFiles.Count} files, {fallbackBytes.LongLength} bytes compressed)"); + Console.WriteLine("No --full-package-url/--full-package-checksum provided; catalog will not include a zip full fallback."); +} catalogEntries = PruneCatalogEntries(catalogEntries, currentVersion, arguments.MinSourceVersion, arguments.PrevVersion, arguments.AnchorVersion); @@ -210,7 +186,12 @@ LatestVersion = currentVersion, Entries = catalogEntries, LastUpdated = DateTime.UtcNow, - MinRequiredVersion = arguments.MinSourceVersion + MinRequiredVersion = arguments.MinSourceVersion, + UpdaterUrl = arguments.UpdaterUrl, + FullPackageUrl = fullPackageUrl, + FullPackageName = fullPackageName, + FullPackageChecksum = fullPackageChecksum, + FullPackageSize = fullPackageSize }; var catalogPath = Path.Combine(outputDirectory, "catalog.json"); @@ -234,7 +215,7 @@ static List PruneCatalogEntries( string? prevVersion, string? anchorVersion) { - const int MaxEntries = 20; + const int MaxHistoricalCumulativeEntries = 1; var latest = entries.Where(e => string.Equals(e.TargetVersion, latestVersion, StringComparison.OrdinalIgnoreCase)) @@ -242,17 +223,34 @@ static List PruneCatalogEntries( var rest = entries.Where(e => !string.Equals(e.TargetVersion, latestVersion, StringComparison.OrdinalIgnoreCase)) + .Where(IsRetainedHistoricalCumulativeEntry) .OrderByDescending(e => Version.TryParse(e.TargetVersion, out var v) ? v : new Version(0, 0)) - .ThenBy(e => e.IsCumulative) .ThenBy(e => e.ChainDepth) .ToList(); var result = latest; - result.AddRange(rest.Take(MaxEntries - latest.Count)); + result.AddRange(rest.Take(MaxHistoricalCumulativeEntries)); return result; } +static bool IsRetainedHistoricalCumulativeEntry(UpdateCatalogEntry entry) +{ + return entry.IsCumulative + && string.Equals(entry.PackageFormat, UpdatePackageFormats.ModerUpdateZstd, StringComparison.OrdinalIgnoreCase) + && entry.PackagePath.EndsWith("-cumulative.zst", StringComparison.OrdinalIgnoreCase); +} + +static string? BuildPackageUrl(string? packageBaseUrl, string packagePath) +{ + if (string.IsNullOrWhiteSpace(packageBaseUrl)) + { + return null; + } + + return $"{packageBaseUrl.TrimEnd('/')}/{packagePath.TrimStart('/')}"; +} + static (byte[] PackageBytes, string Checksum) BuildPackageFile( IReadOnlyList files, string fromVersion, @@ -260,7 +258,8 @@ static List PruneCatalogEntries( string? minSourceVersion, bool isCumulative, bool isAnchor, - int chainDepth) + int chainDepth, + IReadOnlyList? snapshotFiles = null) { var manifestFiles = files .Select(file => new UpdateFile @@ -279,6 +278,12 @@ static List PruneCatalogEntries( IsCumulative = isCumulative, ChainDepth = chainDepth, Files = manifestFiles, + SnapshotFiles = snapshotFiles?.Select(file => new UpdateFile + { + RelativePath = file.RelativePath, + NewChecksum = ComputeSha512(file.Content) + }) + .ToList(), Checksum = string.Empty, CreatedAt = DateTime.UtcNow }; @@ -335,41 +340,6 @@ static List LoadFiles(string sourceDirectory) .ToList(); } -static List LoadPackageFiles(string packagePath) -{ - using var packageStream = File.OpenRead(packagePath); - if (!ValidatePackageMagic(packageStream)) - { - throw new InvalidDataException($"Invalid Moder.Update package magic header: {packagePath}"); - } - - var files = new List(); - using var decompressed = new DecompressionStream(packageStream, leaveOpen: false); - using var tarReader = new TarReader(decompressed, leaveOpen: true); - while (tarReader.GetNextEntry() is { } entry) - { - if (string.Equals(entry.Name, "manifest.json", StringComparison.OrdinalIgnoreCase)) - { - continue; - } - - if (entry.EntryType != TarEntryType.RegularFile - && entry.EntryType != TarEntryType.V7RegularFile - || entry.DataStream is null) - { - continue; - } - - using var fileContent = new MemoryStream(); - entry.DataStream.CopyTo(fileContent); - files.Add(new UpdateFileContent(entry.Name.Replace('/', Path.DirectorySeparatorChar), fileContent.ToArray())); - } - - return files - .OrderBy(file => file.RelativePath, StringComparer.OrdinalIgnoreCase) - .ToList(); -} - static bool ValidatePackageMagic(Stream packageStream) { Span buffer = stackalloc byte[4]; @@ -391,7 +361,7 @@ static List> LoadCumulativeSourceManifests(string? p var manifests = new List>(); foreach (var packagePath in Directory.GetFiles(packageDirectory, "*.zst", SearchOption.TopDirectoryOnly)) { - var files = LoadPackageManifestFiles(packagePath); + var files = LoadPackageManifestFiles(packagePath, preferSnapshot: true); manifests.Add(files); Console.WriteLine($"Loaded cumulative source package manifest: {packagePath} ({files.Count} files)"); } @@ -399,7 +369,7 @@ static List> LoadCumulativeSourceManifests(string? p return manifests; } -static List LoadPackageManifestFiles(string packagePath) +static List LoadPackageManifestFiles(string packagePath, bool preferSnapshot = false) { using var packageStream = File.OpenRead(packagePath); if (!ValidatePackageMagic(packageStream)) @@ -423,7 +393,10 @@ static List LoadPackageManifestFiles(string packagePath) var manifest = JsonSerializer.Deserialize(entry.DataStream, JsonOptions) ?? throw new InvalidDataException($"Failed to parse package manifest: {packagePath}"); - return manifest.Files + var files = preferSnapshot + ? manifest.SnapshotFiles ?? manifest.Files + : manifest.Files; + return files .Select(file => new UpdateFileFingerprint(file.RelativePath, file.NewChecksum)) .OrderBy(file => file.RelativePath, StringComparer.OrdinalIgnoreCase) .ToList(); @@ -432,6 +405,33 @@ static List LoadPackageManifestFiles(string packagePath) throw new InvalidDataException($"Package manifest not found: {packagePath}"); } +static void AddContentPaths(HashSet paths, IEnumerable files) +{ + foreach (var file in files) + { + paths.Add(file.RelativePath); + } +} + +static void AddFingerprintPaths(HashSet paths, IEnumerable files) +{ + foreach (var file in files) + { + paths.Add(file.RelativePath); + } +} + +static List MaterializeTouchedFiles( + HashSet touchedPaths, + Dictionary currentFilesByPath) +{ + return touchedPaths + .Where(currentFilesByPath.ContainsKey) + .Select(path => currentFilesByPath[path]) + .OrderBy(file => file.RelativePath, StringComparer.OrdinalIgnoreCase) + .ToList(); +} + static List ComputeChangedFiles( List prevFiles, List currentFiles) @@ -479,22 +479,6 @@ static List ComputeChangedFilesFromManifest( return changed; } -static List MergeChangedFiles(params IEnumerable[] fileSets) -{ - var merged = new Dictionary(StringComparer.OrdinalIgnoreCase); - foreach (var fileSet in fileSets) - { - foreach (var file in fileSet) - { - merged[file.RelativePath] = file; - } - } - - return merged.Values - .OrderBy(file => file.RelativePath, StringComparer.OrdinalIgnoreCase) - .ToList(); -} - static string ComputeSha512(byte[] data) { var hash = SHA512.HashData(data); @@ -530,6 +514,12 @@ internal sealed class BuilderArguments public string? AnchorSourceDir { get; private init; } public string? BaseCumulativePackage { get; private init; } public string? CumulativeSourcePackageDir { get; private init; } + public string? FullPackageUrl { get; private init; } + public string? FullPackageName { get; private init; } + public string? FullPackageChecksum { get; private init; } + public long FullPackageSize { get; private init; } + public string? UpdaterUrl { get; private init; } + public string? PackageBaseUrl { get; private init; } public bool IsValid => !string.IsNullOrWhiteSpace(SourceDirectory) @@ -562,6 +552,13 @@ public static BuilderArguments Parse(string[] args) values.TryGetValue("--anchor-source-dir", out var anchorSourceDir); values.TryGetValue("--base-cumulative-package", out var baseCumulativePackage); values.TryGetValue("--cumulative-source-package-dir", out var cumulativeSourcePackageDir); + values.TryGetValue("--full-package-url", out var fullPackageUrl); + values.TryGetValue("--full-package-name", out var fullPackageName); + values.TryGetValue("--full-package-checksum", out var fullPackageChecksum); + values.TryGetValue("--full-package-size", out var fullPackageSizeText); + values.TryGetValue("--updater-url", out var updaterUrl); + values.TryGetValue("--package-base-url", out var packageBaseUrl); + _ = long.TryParse(fullPackageSizeText, out var fullPackageSize); return new BuilderArguments { @@ -576,7 +573,13 @@ public static BuilderArguments Parse(string[] args) AnchorVersion = anchorVersion, AnchorSourceDir = anchorSourceDir, BaseCumulativePackage = baseCumulativePackage, - CumulativeSourcePackageDir = cumulativeSourcePackageDir + CumulativeSourcePackageDir = cumulativeSourcePackageDir, + FullPackageUrl = fullPackageUrl, + FullPackageName = fullPackageName, + FullPackageChecksum = fullPackageChecksum, + FullPackageSize = fullPackageSize, + UpdaterUrl = updaterUrl, + PackageBaseUrl = packageBaseUrl }; } @@ -584,7 +587,7 @@ public static void PrintUsage() { Console.WriteLine("Usage:"); Console.WriteLine( - " TelegramSearchBot.UpdateBuilder --source-dir --output-dir --target-version --min-source-version [--prev-source-dir --prev-version ] [--existing-catalog ] [--anchor-version --anchor-source-dir ] [--base-cumulative-package ] [--cumulative-source-package-dir ]"); + " TelegramSearchBot.UpdateBuilder --source-dir --output-dir --target-version --min-source-version [--prev-source-dir --prev-version ] [--existing-catalog ] [--anchor-version --anchor-source-dir ] [--base-cumulative-package ] [--cumulative-source-package-dir ] [--full-package-url --full-package-name --full-package-checksum --full-package-size ] [--updater-url ] [--package-base-url ]"); } } diff --git a/TelegramSearchBot/Service/Update/SelfUpdateBootstrap.Windows.cs b/TelegramSearchBot/Service/Update/SelfUpdateBootstrap.Windows.cs index c49a50c9..5745d7ab 100644 --- a/TelegramSearchBot/Service/Update/SelfUpdateBootstrap.Windows.cs +++ b/TelegramSearchBot/Service/Update/SelfUpdateBootstrap.Windows.cs @@ -4,6 +4,7 @@ using System.Diagnostics; using System.Formats.Tar; using System.IO; +using System.IO.Compression; using System.Linq; using System.Net; using System.Net.Http; @@ -284,15 +285,15 @@ private static async Task PrepareUpdateChainAsync( foreach (var entry in updatePath) { Log.Information("下载更新包: {PackagePath} (from {MinVersion})", - entry.PackagePath, entry.MinSourceVersion); + entry.PackageUrl ?? entry.PackagePath, entry.MinSourceVersion); - var packagePath = Path.Combine(packageCacheDir, GetPackageCacheFileName(entry.PackagePath)); - await DownloadFileAsync(httpClient, entry.PackagePath, packagePath, cancellationToken); + var packagePath = Path.Combine(packageCacheDir, GetPackageCacheFileName(entry)); + await DownloadPackageAsync(httpClient, entry, packagePath, cancellationToken); await using var packageStream = File.OpenRead(packagePath); VerifyPackageChecksum(packageStream, entry); packageStream.Position = 0; - ExtractPackageToDirectory(packageStream, stagingDir); + ExtractPackageToDirectory(packageStream, stagingDir, entry); } var updaterPath = await DownloadUpdaterAsync(httpClient, cancellationToken); @@ -313,6 +314,7 @@ private static async Task FetchCatalogAsync(HttpClient httpClient var catalog = await JsonSerializer.DeserializeAsync(stream, JsonOptions, cancellationToken) ?? throw new InvalidDataException("Failed to deserialize update catalog."); UpdateCatalogCache.UpdaterChecksum = catalog.UpdaterChecksum; + UpdateCatalogCache.UpdaterUrl = catalog.UpdaterUrl; return catalog; } @@ -334,7 +336,7 @@ private static async Task DownloadUpdaterAsync(HttpClient httpClient, Ca { Directory.CreateDirectory(Path.GetDirectoryName(UpdaterCachePath)!); var downloadPath = UpdaterCachePath + ".download"; - await DownloadFileAsync(httpClient, UpdaterFileName, downloadPath, cancellationToken); + await DownloadFileAsync(httpClient, UpdateCatalogCache.UpdaterUrl ?? UpdaterFileName, downloadPath, cancellationToken); try { await using var updaterStream = File.OpenRead(downloadPath); @@ -350,14 +352,35 @@ private static async Task DownloadUpdaterAsync(HttpClient httpClient, Ca private static async Task DownloadFileAsync( HttpClient httpClient, - string relativePath, + string pathOrUrl, string targetPath, CancellationToken cancellationToken) { - var requestUri = $"{Env.UpdateBaseUrl.TrimEnd('/')}/{relativePath.TrimStart('/')}"; + var requestUri = ResolveDownloadUri(pathOrUrl); await DownloadFileFromUriAsync(httpClient, requestUri, targetPath, cancellationToken); } + private static Task DownloadPackageAsync( + HttpClient httpClient, + UpdateCatalogEntry entry, + string targetPath, + CancellationToken cancellationToken) + { + var pathOrUrl = string.IsNullOrWhiteSpace(entry.PackageUrl) + ? entry.PackagePath + : entry.PackageUrl; + return DownloadFileAsync(httpClient, pathOrUrl, targetPath, cancellationToken); + } + + private static string ResolveDownloadUri(string pathOrUrl) + { + if (Uri.TryCreate(pathOrUrl, UriKind.Absolute, out var absoluteUri)) { + return absoluteUri.ToString(); + } + + return $"{Env.UpdateBaseUrl.TrimEnd('/')}/{pathOrUrl.TrimStart('/')}"; + } + private static async Task DownloadFileFromUriAsync( HttpClient httpClient, string requestUri, @@ -609,7 +632,59 @@ private static void VerifyDownloadedLength(string path, long? expectedLength) } } - private static void ExtractPackageToDirectory(Stream packageStream, string targetDirectory) + private static void ExtractPackageToDirectory( + Stream packageStream, + string targetDirectory, + UpdateCatalogEntry updateEntry) + { + if (IsZipPackage(updateEntry)) { + ExtractZipPackageToDirectory(packageStream, targetDirectory); + return; + } + + ExtractModerPackageToDirectory(packageStream, targetDirectory); + } + + private static bool IsZipPackage(UpdateCatalogEntry updateEntry) + { + return string.Equals(updateEntry.PackageFormat, UpdatePackageFormats.Zip, StringComparison.OrdinalIgnoreCase) + || updateEntry.PackagePath.EndsWith(".zip", StringComparison.OrdinalIgnoreCase) + || (updateEntry.PackageUrl?.EndsWith(".zip", StringComparison.OrdinalIgnoreCase) ?? false); + } + + private static void ExtractZipPackageToDirectory(Stream packageStream, string targetDirectory) + { + Directory.CreateDirectory(targetDirectory); + var targetDirectoryPath = Path.GetFullPath(targetDirectory); + var directoryPrefix = targetDirectoryPath.TrimEnd(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar) + + Path.DirectorySeparatorChar; + + using var archive = new ZipArchive(packageStream, ZipArchiveMode.Read, leaveOpen: true); + foreach (var entry in archive.Entries) { + if (string.IsNullOrWhiteSpace(entry.Name)) { + var directoryPath = Path.GetFullPath(Path.Combine(targetDirectoryPath, NormalizeArchivePath(entry.FullName))); + EnsurePathWithinDirectory(directoryPath, directoryPrefix, entry.FullName); + Directory.CreateDirectory(directoryPath); + continue; + } + + var normalizedPath = NormalizeArchivePath(entry.FullName); + if (Path.IsPathRooted(normalizedPath)) { + throw new InvalidDataException($"Package entry '{entry.FullName}' is rooted and cannot be extracted."); + } + + var targetPath = Path.GetFullPath(Path.Combine(targetDirectoryPath, normalizedPath)); + EnsurePathWithinDirectory(targetPath, directoryPrefix, entry.FullName); + var directory = Path.GetDirectoryName(targetPath); + if (directory is not null) { + Directory.CreateDirectory(directory); + } + + entry.ExtractToFile(targetPath, overwrite: true); + } + } + + private static void ExtractModerPackageToDirectory(Stream packageStream, string targetDirectory) { Directory.CreateDirectory(targetDirectory); var targetDirectoryPath = Path.GetFullPath(targetDirectory); @@ -788,13 +863,28 @@ private static string NormalizeTarPath(string path) return path.Replace('/', Path.DirectorySeparatorChar); } + private static string NormalizeArchivePath(string path) + { + if (path.StartsWith("./", StringComparison.Ordinal)) { + path = path[2..]; + } + + return path.Replace('/', Path.DirectorySeparatorChar); + } + private static string SanitizeVersion(string version) => version.Replace('.', '_'); - private static string GetPackageCacheFileName(string relativePath) + private static string GetPackageCacheFileName(UpdateCatalogEntry entry) { - var fileName = Path.GetFileName(relativePath.Replace('/', Path.DirectorySeparatorChar)); + var pathOrUrl = string.IsNullOrWhiteSpace(entry.PackageUrl) + ? entry.PackagePath + : entry.PackageUrl; + var path = Uri.TryCreate(pathOrUrl, UriKind.Absolute, out var absoluteUri) + ? absoluteUri.AbsolutePath + : pathOrUrl; + var fileName = Path.GetFileName(path.Replace('/', Path.DirectorySeparatorChar)); if (string.IsNullOrWhiteSpace(fileName)) { - fileName = "package.zst"; + fileName = IsZipPackage(entry) ? "package.zip" : "package.zst"; } foreach (var invalidChar in Path.GetInvalidFileNameChars()) { @@ -917,6 +1007,7 @@ private enum UpdateCheckStatus private static class UpdateCatalogCache { public static string? UpdaterChecksum { get; set; } + public static string? UpdaterUrl { get; set; } } } #endif diff --git a/external/moder-update/src/Moder.Update/Models/UpdateCatalogEntry.cs b/external/moder-update/src/Moder.Update/Models/UpdateCatalogEntry.cs index bd05e92d..5b5df384 100644 --- a/external/moder-update/src/Moder.Update/Models/UpdateCatalogEntry.cs +++ b/external/moder-update/src/Moder.Update/Models/UpdateCatalogEntry.cs @@ -8,6 +8,12 @@ public class UpdateCatalogEntry /// Relative path to the compressed package (e.g. "packages/1.2.0.zst"). public required string PackagePath { get; init; } + /// Absolute URL to the package. When present, this takes precedence over PackagePath. + public string? PackageUrl { get; init; } + + /// Package payload format. Defaults to Moder.Update's custom zstd tar format. + public string PackageFormat { get; init; } = "moder-update-zst"; + /// Target version of this package. public required string TargetVersion { get; init; } diff --git a/external/moder-update/src/Moder.Update/Models/UpdateManifest.cs b/external/moder-update/src/Moder.Update/Models/UpdateManifest.cs index ef6d65a4..362abc35 100644 --- a/external/moder-update/src/Moder.Update/Models/UpdateManifest.cs +++ b/external/moder-update/src/Moder.Update/Models/UpdateManifest.cs @@ -29,6 +29,9 @@ public class UpdateManifest /// List of files to replace (full files, not diffs). public required List Files { get; init; } + /// Optional full target-version file snapshot for cumulative package planning. + public List? SnapshotFiles { get; init; } + /// SHA512 checksum of the entire package for integrity verification. public required string Checksum { get; init; } From e0651ba7fb73ed573e7073d216d063e0431d12da Mon Sep 17 00:00:00 2001 From: ModerRAS Date: Fri, 15 May 2026 00:47:06 +0800 Subject: [PATCH 2/2] Address update feed review feedback --- .github/workflows/push.yml | 4 +- .../Model/Update/UpdateCatalog.cs | 2 +- .../ModerUpdateIntegrationTests.cs | 2 +- .../Update/SelfUpdateBootstrapTests.cs | 35 ++++++++++++- .../Service/Update/UpdateBuilderTests.cs | 34 ++++++++++++- TelegramSearchBot.UpdateBuilder/Program.cs | 49 ++++++++++++++++--- .../Update/SelfUpdateBootstrap.Windows.cs | 10 +++- 7 files changed, 121 insertions(+), 15 deletions(-) diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index a9479e0f..34182d6d 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -287,7 +287,7 @@ jobs: New-Item -ItemType Directory -Path artifacts\anchor-release -Force | Out-Null gh release download $anchorTag --repo ${{ github.repository }} --pattern "TelegramSearchBot-win-x64-full-*.zip" --dir artifacts\anchor-release if ($LASTEXITCODE -ne 0) { - throw "Failed to download cumulative update anchor $anchorTag; refusing to publish a cumulative-free update feed." + throw "Failed to download cumulative update anchor $anchorTag; refusing to publish without cumulative package from anchor $anchorTag." } $zipFile = Get-ChildItem artifacts\anchor-release -Filter "TelegramSearchBot-win-x64-full-*.zip" | Select-Object -First 1 @@ -297,7 +297,7 @@ jobs: Add-Content -Path $env:GITHUB_ENV -Value "ANCHOR_STANDALONE_DIR=artifacts\anchor-standalone" -Encoding utf8 Write-Host "Extracted cumulative update anchor from $($zipFile.Name)." } else { - throw "No anchor full zip found; refusing to publish a cumulative-free update feed." + throw "No anchor full zip found; refusing to publish without cumulative package from anchor $anchorTag." } - name: Fetch previous cumulative update package shell: pwsh diff --git a/TelegramSearchBot.Common/Model/Update/UpdateCatalog.cs b/TelegramSearchBot.Common/Model/Update/UpdateCatalog.cs index 3220cc87..25b74f57 100644 --- a/TelegramSearchBot.Common/Model/Update/UpdateCatalog.cs +++ b/TelegramSearchBot.Common/Model/Update/UpdateCatalog.cs @@ -11,5 +11,5 @@ public sealed class UpdateCatalog public string? FullPackageUrl { get; init; } public string? FullPackageName { get; init; } public string? FullPackageChecksum { get; init; } - public long FullPackageSize { get; init; } + public long? FullPackageSize { get; init; } } diff --git a/TelegramSearchBot.Test/ModerUpdateIntegrationTests.cs b/TelegramSearchBot.Test/ModerUpdateIntegrationTests.cs index b315989a..26e74147 100644 --- a/TelegramSearchBot.Test/ModerUpdateIntegrationTests.cs +++ b/TelegramSearchBot.Test/ModerUpdateIntegrationTests.cs @@ -365,7 +365,7 @@ public void CI_UpdateFeed_GeneratesCumulativeDeltaFromAnchor() { Assert.Contains("Update feed catalog entries", pushYmlContent); Assert.Contains("Cumulative update package missing", pushYmlContent); Assert.Contains("Failed to download cumulative update anchor", pushYmlContent); - Assert.Contains("refusing to publish a cumulative-free update feed", pushYmlContent); + Assert.Contains("refusing to publish without cumulative package from anchor", pushYmlContent); Assert.Contains("PackageUrl", pushYmlContent); Assert.Contains("PackageFormat", pushYmlContent); Assert.Contains("PackagePath -notlike '*-cumulative.zst'", pushYmlContent); diff --git a/TelegramSearchBot.Test/Service/Update/SelfUpdateBootstrapTests.cs b/TelegramSearchBot.Test/Service/Update/SelfUpdateBootstrapTests.cs index 4cedae10..7dcf1873 100644 --- a/TelegramSearchBot.Test/Service/Update/SelfUpdateBootstrapTests.cs +++ b/TelegramSearchBot.Test/Service/Update/SelfUpdateBootstrapTests.cs @@ -324,11 +324,12 @@ private static UpdateCatalogEntry CreateEntry( int chainDepth = 0, long compressedSize = 1024, string? packageFormat = null, - string? packageUrl = null) + string? packageUrl = null, + string? packagePath = null) { return new UpdateCatalogEntry { - PackagePath = $"packages/v{minSourceVersion}_to_v{targetVersion}.tar.zst", + PackagePath = packagePath ?? $"packages/v{minSourceVersion}_to_v{targetVersion}.tar.zst", PackageUrl = packageUrl, PackageFormat = packageFormat ?? UpdatePackageFormats.ModerUpdateZstd, TargetVersion = targetVersion, @@ -776,6 +777,36 @@ public void ExtractPackageToDirectory_ExtractsZipPackage() } } + [Fact] + public void ExtractPackageToDirectory_DetectsZipPackageWithoutPackagePath() + { + using var package = CreateTestZipPackage(new Dictionary + { + ["file.txt"] = "zip without path" + }); + var targetDirectory = Path.Combine(Path.GetTempPath(), "SelfUpdateBootstrapTests", Guid.NewGuid().ToString("N")); + + try + { + var entry = CreateEntry( + "2.0.0", + "1.0.0", + packageFormat: UpdatePackageFormats.Zip, + packageUrl: "https://github.com/ModerRAS/TelegramSearchBot/releases/download/v2.0.0/full.zip", + packagePath: string.Empty); + InvokePrivateStatic("ExtractPackageToDirectory", package, targetDirectory, entry); + + Assert.Equal("zip without path", File.ReadAllText(Path.Combine(targetDirectory, "file.txt"))); + } + finally + { + if (Directory.Exists(targetDirectory)) + { + Directory.Delete(targetDirectory, recursive: true); + } + } + } + [Fact] public async Task DownloadFileInPartsAsync_RequestsRangesAndMergesFile() { diff --git a/TelegramSearchBot.Test/Service/Update/UpdateBuilderTests.cs b/TelegramSearchBot.Test/Service/Update/UpdateBuilderTests.cs index 49313521..dbd94c62 100644 --- a/TelegramSearchBot.Test/Service/Update/UpdateBuilderTests.cs +++ b/TelegramSearchBot.Test/Service/Update/UpdateBuilderTests.cs @@ -218,6 +218,29 @@ await RunBuilderAsync( Assert.Contains(catalog.Entries, entry => entry.PackagePath.EndsWith("-cumulative.zst", StringComparison.OrdinalIgnoreCase)); } + [Fact] + public async Task BuildFeed_RejectsInvalidFullPackageSize() + { + var currentDir = CreateVersionDirectory("current-invalid-size", new Dictionary + { + ["app.exe"] = "current" + }); + var outputDir = Path.Combine(_testDirectory, "feed-invalid-size"); + + var result = await RunBuilderForResultAsync( + "--source-dir", currentDir, + "--output-dir", outputDir, + "--target-version", "2.0.0", + "--min-source-version", "1.0.0", + "--full-package-url", "https://github.com/ModerRAS/TelegramSearchBot/releases/download/v2.0.0/full.zip", + "--full-package-name", "full.zip", + "--full-package-checksum", new string('a', 128), + "--full-package-size", "not-a-number"); + + Assert.NotEqual(0, result.ExitCode); + Assert.Contains("--full-package-size must be a non-negative integer", result.StandardError); + } + private string CreateVersionDirectory(string name, Dictionary files) { var directory = Path.Combine(_testDirectory, name); @@ -233,6 +256,13 @@ private string CreateVersionDirectory(string name, Dictionary fi } private static async Task RunBuilderAsync(params string[] arguments) + { + var result = await RunBuilderForResultAsync(arguments); + Assert.True(result.ExitCode == 0, + $"UpdateBuilder failed with exit code {result.ExitCode}.\nSTDOUT:\n{result.StandardOutput}\nSTDERR:\n{result.StandardError}"); + } + + private static async Task RunBuilderForResultAsync(params string[] arguments) { var processInfo = new System.Diagnostics.ProcessStartInfo { @@ -259,7 +289,7 @@ private static async Task RunBuilderAsync(params string[] arguments) var stderr = await process.StandardError.ReadToEndAsync(); await process.WaitForExitAsync(); - Assert.True(process.ExitCode == 0, $"UpdateBuilder failed with exit code {process.ExitCode}.\nSTDOUT:\n{stdout}\nSTDERR:\n{stderr}"); + return new ProcessResult(process.ExitCode, stdout, stderr); } private static UpdateCatalog ReadCatalog(string outputDir) @@ -327,6 +357,8 @@ public void Dispose() } } + private sealed record ProcessResult(int ExitCode, string StandardOutput, string StandardError); + private static readonly JsonSerializerOptions JsonOptions = new() { PropertyNameCaseInsensitive = true diff --git a/TelegramSearchBot.UpdateBuilder/Program.cs b/TelegramSearchBot.UpdateBuilder/Program.cs index 53b088c6..e5f54463 100644 --- a/TelegramSearchBot.UpdateBuilder/Program.cs +++ b/TelegramSearchBot.UpdateBuilder/Program.cs @@ -5,7 +5,18 @@ using TelegramSearchBot.Common.Model.Update; using ZstdSharp; -var arguments = BuilderArguments.Parse(args); +BuilderArguments arguments; +try +{ + arguments = BuilderArguments.Parse(args); +} +catch (ArgumentException ex) +{ + Console.Error.WriteLine(ex.Message); + BuilderArguments.PrintUsage(); + return 1; +} + if (!arguments.IsValid) { BuilderArguments.PrintUsage(); @@ -167,7 +178,7 @@ ChainDepth = 0, PackageChecksum = fullPackageChecksum, FileCount = currentFiles.Count, - CompressedSize = fullPackageSize, + CompressedSize = fullPackageSize.GetValueOrDefault(), UncompressedSize = currentFiles.Sum(f => (long)f.Content.Length) }); @@ -517,7 +528,7 @@ internal sealed class BuilderArguments public string? FullPackageUrl { get; private init; } public string? FullPackageName { get; private init; } public string? FullPackageChecksum { get; private init; } - public long FullPackageSize { get; private init; } + public long? FullPackageSize { get; private init; } public string? UpdaterUrl { get; private init; } public string? PackageBaseUrl { get; private init; } @@ -558,7 +569,21 @@ public static BuilderArguments Parse(string[] args) values.TryGetValue("--full-package-size", out var fullPackageSizeText); values.TryGetValue("--updater-url", out var updaterUrl); values.TryGetValue("--package-base-url", out var packageBaseUrl); - _ = long.TryParse(fullPackageSizeText, out var fullPackageSize); + long? fullPackageSize = null; + if (!string.IsNullOrWhiteSpace(fullPackageSizeText)) + { + if (!long.TryParse(fullPackageSizeText, out var parsedFullPackageSize) || parsedFullPackageSize < 0) + { + throw new ArgumentException("--full-package-size must be a non-negative integer."); + } + + fullPackageSize = parsedFullPackageSize; + } + + if (!string.IsNullOrWhiteSpace(fullPackageUrl) && fullPackageSize is null) + { + throw new ArgumentException("--full-package-size is required when --full-package-url is provided."); + } return new BuilderArguments { @@ -586,8 +611,20 @@ public static BuilderArguments Parse(string[] args) public static void PrintUsage() { Console.WriteLine("Usage:"); - Console.WriteLine( - " TelegramSearchBot.UpdateBuilder --source-dir --output-dir --target-version --min-source-version [--prev-source-dir --prev-version ] [--existing-catalog ] [--anchor-version --anchor-source-dir ] [--base-cumulative-package ] [--cumulative-source-package-dir ] [--full-package-url --full-package-name --full-package-checksum --full-package-size ] [--updater-url ] [--package-base-url ]"); + Console.WriteLine(" TelegramSearchBot.UpdateBuilder"); + Console.WriteLine(" --source-dir "); + Console.WriteLine(" --output-dir "); + Console.WriteLine(" --target-version "); + Console.WriteLine(" --min-source-version "); + Console.WriteLine(" [--prev-source-dir --prev-version ]"); + Console.WriteLine(" [--existing-catalog ]"); + Console.WriteLine(" [--anchor-version --anchor-source-dir ]"); + Console.WriteLine(" [--base-cumulative-package ]"); + Console.WriteLine(" [--cumulative-source-package-dir ]"); + Console.WriteLine(" [--full-package-url --full-package-name "); + Console.WriteLine(" --full-package-checksum --full-package-size ]"); + Console.WriteLine(" [--updater-url ]"); + Console.WriteLine(" [--package-base-url ]"); } } diff --git a/TelegramSearchBot/Service/Update/SelfUpdateBootstrap.Windows.cs b/TelegramSearchBot/Service/Update/SelfUpdateBootstrap.Windows.cs index 5745d7ab..e53f3bc1 100644 --- a/TelegramSearchBot/Service/Update/SelfUpdateBootstrap.Windows.cs +++ b/TelegramSearchBot/Service/Update/SelfUpdateBootstrap.Windows.cs @@ -648,8 +648,10 @@ private static void ExtractPackageToDirectory( private static bool IsZipPackage(UpdateCatalogEntry updateEntry) { return string.Equals(updateEntry.PackageFormat, UpdatePackageFormats.Zip, StringComparison.OrdinalIgnoreCase) - || updateEntry.PackagePath.EndsWith(".zip", StringComparison.OrdinalIgnoreCase) - || (updateEntry.PackageUrl?.EndsWith(".zip", StringComparison.OrdinalIgnoreCase) ?? false); + || (!string.IsNullOrWhiteSpace(updateEntry.PackagePath) + && updateEntry.PackagePath.EndsWith(".zip", StringComparison.OrdinalIgnoreCase)) + || (!string.IsNullOrWhiteSpace(updateEntry.PackageUrl) + && updateEntry.PackageUrl.EndsWith(".zip", StringComparison.OrdinalIgnoreCase)); } private static void ExtractZipPackageToDirectory(Stream packageStream, string targetDirectory) @@ -876,6 +878,10 @@ private static string NormalizeArchivePath(string path) private static string GetPackageCacheFileName(UpdateCatalogEntry entry) { + if (string.IsNullOrWhiteSpace(entry.PackageUrl) && string.IsNullOrWhiteSpace(entry.PackagePath)) { + throw new ArgumentException("Catalog entry must have either PackageUrl or PackagePath.", nameof(entry)); + } + var pathOrUrl = string.IsNullOrWhiteSpace(entry.PackageUrl) ? entry.PackagePath : entry.PackageUrl;