From 76f50f60931e85e9240715ebd1f345547cbae366 Mon Sep 17 00:00:00 2001 From: Roman Konecny Date: Fri, 12 Jul 2024 19:42:13 +0000 Subject: [PATCH 1/7] [release/8.0] Add missing delegating/override method in TlsStream Add missing delegating/override method in TlsStream Testing: I have validated that after those changes my local reproduction was no longer hitting this failure. --- src/libraries/Common/src/System/Net/TlsStream.cs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/libraries/Common/src/System/Net/TlsStream.cs b/src/libraries/Common/src/System/Net/TlsStream.cs index 503253099aac17..37fe2ff825fa56 100644 --- a/src/libraries/Common/src/System/Net/TlsStream.cs +++ b/src/libraries/Common/src/System/Net/TlsStream.cs @@ -68,6 +68,11 @@ public override Task WriteAsync(byte[] buffer, int offset, int count, Cancellati return _sslStream.WriteAsync(buffer, offset, count, cancellationToken); } + public override ValueTask WriteAsync(ReadOnlyMemory buffer, CancellationToken cancellationToken = default(CancellationToken)) + { + return _sslStream.WriteAsync(buffer, cancellationToken); + } + public override int Read(byte[] buffer, int offset, int size) { return _sslStream.Read(buffer, offset, size); From ecefd4461148ee13fd87ac00fbb9cc388582e749 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20S=C3=A1nchez=20L=C3=B3pez?= <1175054+carlossanlop@users.noreply.github.com> Date: Mon, 12 Aug 2024 16:35:04 -0700 Subject: [PATCH 2/7] Turn off OOB packages in the September Release that were turned on back in the August Release. --- .../src/System.Net.Http.WinHttpHandler.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libraries/System.Net.Http.WinHttpHandler/src/System.Net.Http.WinHttpHandler.csproj b/src/libraries/System.Net.Http.WinHttpHandler/src/System.Net.Http.WinHttpHandler.csproj index 4e4b50727b7fa8..8959c43540ca16 100644 --- a/src/libraries/System.Net.Http.WinHttpHandler/src/System.Net.Http.WinHttpHandler.csproj +++ b/src/libraries/System.Net.Http.WinHttpHandler/src/System.Net.Http.WinHttpHandler.csproj @@ -4,7 +4,7 @@ true true true - true + false 2 Provides a message handler for HttpClient based on the WinHTTP interface of Windows. While similar to HttpClientHandler, it provides developers more granular control over the application's HTTP communication than the HttpClientHandler. From e4ec9428d3262060fb5928b61632b35c144848ed Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 13 Aug 2024 10:50:48 -0700 Subject: [PATCH 3/7] [release/8.0] Disable W^X in Rosetta emulated x64 containers on macOS (#106353) * Disable W^X on x64 in rosetta based container The docker on macOS Arm64 uses Rosetta to run x64 containers. That has an effect on the double mapping. The Rosetta is unable to detect when an already executed code page is modified. So we cannot use double mapping on those containers. To detect that case, this change adds check that verifies that the double mapping works even when the code is modified. Close #102226 * Rework based on PR feedback * Check only for Rosetta * Enable the rosetta check for x86 too This will help WINE running 32 bit code under rosetta emulation on macOS. --------- Co-authored-by: Jan Vorlicek --- src/coreclr/minipal/CMakeLists.txt | 1 + src/coreclr/minipal/Unix/doublemapping.cpp | 33 ++++++----------- src/coreclr/minipal/Windows/doublemapping.cpp | 7 ++++ src/native/minipal/cpufeatures.c | 35 +++++++++++++++++++ src/native/minipal/cpufeatures.h | 1 + 5 files changed, 54 insertions(+), 23 deletions(-) diff --git a/src/coreclr/minipal/CMakeLists.txt b/src/coreclr/minipal/CMakeLists.txt index 3096237d2a2fe3..78a1726af3e815 100644 --- a/src/coreclr/minipal/CMakeLists.txt +++ b/src/coreclr/minipal/CMakeLists.txt @@ -1,4 +1,5 @@ include_directories(.) +include_directories(${CLR_SRC_NATIVE_DIR}) if (CLR_CMAKE_HOST_UNIX) add_subdirectory(Unix) else (CLR_CMAKE_HOST_UNIX) diff --git a/src/coreclr/minipal/Unix/doublemapping.cpp b/src/coreclr/minipal/Unix/doublemapping.cpp index cb65e5e284e2b3..67d516fb322a56 100644 --- a/src/coreclr/minipal/Unix/doublemapping.cpp +++ b/src/coreclr/minipal/Unix/doublemapping.cpp @@ -20,24 +20,13 @@ #define memfd_create(...) syscall(__NR_memfd_create, __VA_ARGS__) #endif // TARGET_LINUX && !MFD_CLOEXEC #include "minipal.h" +#include "minipal/cpufeatures.h" -#if defined(TARGET_OSX) && defined(TARGET_AMD64) -#include -#include +#ifdef TARGET_OSX -bool IsProcessTranslated() -{ - int ret = 0; - size_t size = sizeof(ret); - if (sysctlbyname("sysctl.proc_translated", &ret, &size, NULL, 0) == -1) - { - return false; - } - return ret == 1; -} -#endif // TARGET_OSX && TARGET_AMD64 +#include -#ifndef TARGET_OSX +#else // TARGET_OSX #ifdef TARGET_64BIT static const off_t MaxDoubleMappedSize = 2048ULL*1024*1024*1024; @@ -49,6 +38,12 @@ static const off_t MaxDoubleMappedSize = UINT_MAX; bool VMToOSInterface::CreateDoubleMemoryMapper(void** pHandle, size_t *pMaxExecutableCodeSize) { + if (minipal_detect_rosetta()) + { + // Rosetta doesn't support double mapping correctly + return false; + } + #ifndef TARGET_OSX #ifdef TARGET_FREEBSD @@ -78,14 +73,6 @@ bool VMToOSInterface::CreateDoubleMemoryMapper(void** pHandle, size_t *pMaxExecu *pHandle = (void*)(size_t)fd; #else // !TARGET_OSX -#ifdef TARGET_AMD64 - if (IsProcessTranslated()) - { - // Rosetta doesn't support double mapping correctly - return false; - } -#endif // TARGET_AMD64 - *pMaxExecutableCodeSize = SIZE_MAX; *pHandle = NULL; #endif // !TARGET_OSX diff --git a/src/coreclr/minipal/Windows/doublemapping.cpp b/src/coreclr/minipal/Windows/doublemapping.cpp index 0d7033b567056c..9e8ddfed8e964d 100644 --- a/src/coreclr/minipal/Windows/doublemapping.cpp +++ b/src/coreclr/minipal/Windows/doublemapping.cpp @@ -6,6 +6,7 @@ #include #include #include "minipal.h" +#include "minipal/cpufeatures.h" #define HIDWORD(_qw) ((ULONG)((_qw) >> 32)) #define LODWORD(_qw) ((ULONG)(_qw)) @@ -60,6 +61,12 @@ inline void *GetBotMemoryAddress(void) bool VMToOSInterface::CreateDoubleMemoryMapper(void **pHandle, size_t *pMaxExecutableCodeSize) { + if (minipal_detect_rosetta()) + { + // Rosetta doesn't support double mapping correctly. WINE on macOS ARM64 can be running under Rosetta. + return false; + } + *pMaxExecutableCodeSize = (size_t)MaxDoubleMappedSize; *pHandle = CreateFileMapping( INVALID_HANDLE_VALUE, // use paging file diff --git a/src/native/minipal/cpufeatures.c b/src/native/minipal/cpufeatures.c index be0aabe081c9fc..56ec81993c8521 100644 --- a/src/native/minipal/cpufeatures.c +++ b/src/native/minipal/cpufeatures.c @@ -4,6 +4,8 @@ #include #include #include +#include +#include #include "cpufeatures.h" #include "cpuid.h" @@ -427,3 +429,36 @@ int minipal_getcpufeatures(void) return result; } + +// Detect if the current process is running under the Apple Rosetta x64 emulator +bool minipal_detect_rosetta(void) +{ +#if defined(HOST_AMD64) || defined(HOST_X86) + // Check for CPU brand indicating emulation + int regs[4]; + char brand[49]; + + // Get the maximum value for extended function CPUID info + __cpuid(regs, (int)0x80000000); + if ((unsigned int)regs[0] < 0x80000004) + { + return false; // Extended CPUID not supported + } + + // Retrieve the CPU brand string + for (unsigned int i = 0x80000002; i <= 0x80000004; ++i) + { + __cpuid(regs, (int)i); + memcpy(brand + (i - 0x80000002) * sizeof(regs), regs, sizeof(regs)); + } + brand[sizeof(brand) - 1] = '\0'; + + // Check if CPU brand indicates emulation + if (strstr(brand, "VirtualApple") != NULL) + { + return true; + } +#endif // HOST_AMD64 || HOST_X86 + + return false; +} diff --git a/src/native/minipal/cpufeatures.h b/src/native/minipal/cpufeatures.h index 73d151f1e2d80f..a985cf0fc92379 100644 --- a/src/native/minipal/cpufeatures.h +++ b/src/native/minipal/cpufeatures.h @@ -72,6 +72,7 @@ extern "C" #endif // __cplusplus int minipal_getcpufeatures(void); +bool minipal_detect_rosetta(void); #ifdef __cplusplus } From 3bcfd7e445eb3cc2652de4bf6a63a07f98cb9420 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Thu, 15 Aug 2024 12:06:15 -0700 Subject: [PATCH 4/7] Update dependencies from https://github.com/dotnet/emsdk build 20240814.9 (#106438) Microsoft.SourceBuild.Intermediate.emsdk , Microsoft.NET.Workload.Emscripten.Current.Manifest-8.0.100 From Version 8.0.8-servicing.24367.1 -> To Version 8.0.9-servicing.24414.9 Co-authored-by: dotnet-maestro[bot] --- NuGet.config | 12 +----------- eng/Version.Details.xml | 8 ++++---- eng/Versions.props | 2 +- 3 files changed, 6 insertions(+), 16 deletions(-) diff --git a/NuGet.config b/NuGet.config index 961d774f91d772..855805e18be3db 100644 --- a/NuGet.config +++ b/NuGet.config @@ -9,17 +9,7 @@ - - - - - - - - - - - + diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 8c6dbdb2bf4f9a..cf95a485dc625d 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -90,13 +90,13 @@ 45dd3a73dd5b64b010c4251303b3664bb30df029 - + https://github.com/dotnet/emsdk - 2674f580a4b1d27322914df8488498065e71a3f2 + 885abbe904afd5987f463378522b82a2cc011cac - + https://github.com/dotnet/emsdk - 2674f580a4b1d27322914df8488498065e71a3f2 + 885abbe904afd5987f463378522b82a2cc011cac diff --git a/eng/Versions.props b/eng/Versions.props index 195b4176144e2d..d2b56a8f9649d6 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -242,7 +242,7 @@ Note: when the name is updated, make sure to update dependency name in eng/pipelines/common/xplat-setup.yml like - DarcDependenciesChanged.Microsoft_NET_Workload_Emscripten_Current_Manifest-8_0_100_Transport --> - 8.0.8 + 8.0.9 $(MicrosoftNETWorkloadEmscriptenCurrentManifest80100Version) 1.1.87-gba258badda From d40f32d2dc635e4b4c9d27dbe2673e185e6aa7ea Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Mon, 19 Aug 2024 14:33:00 -0700 Subject: [PATCH 5/7] [release/8.0] Update dependencies from dotnet/emsdk (#106502) * Update dependencies from https://github.com/dotnet/emsdk build 20240815.7 Microsoft.SourceBuild.Intermediate.emsdk , Microsoft.NET.Workload.Emscripten.Current.Manifest-8.0.100 From Version 8.0.9-servicing.24414.9 -> To Version 8.0.9-servicing.24415.7 * Update dependencies from https://github.com/dotnet/emsdk build 20240818.3 Microsoft.SourceBuild.Intermediate.emsdk , Microsoft.NET.Workload.Emscripten.Current.Manifest-8.0.100 From Version 8.0.9-servicing.24415.7 -> To Version 8.0.9-servicing.24418.3 --------- Co-authored-by: dotnet-maestro[bot] --- NuGet.config | 2 +- eng/Version.Details.xml | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/NuGet.config b/NuGet.config index 855805e18be3db..0d1eacbdec07b8 100644 --- a/NuGet.config +++ b/NuGet.config @@ -9,7 +9,7 @@ - + diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index cf95a485dc625d..f212f7b0eb31a2 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -92,11 +92,11 @@ https://github.com/dotnet/emsdk - 885abbe904afd5987f463378522b82a2cc011cac + d6672570f0fde2a4a3c7e65d60cec9f8406b039c - + https://github.com/dotnet/emsdk - 885abbe904afd5987f463378522b82a2cc011cac + d6672570f0fde2a4a3c7e65d60cec9f8406b039c From aa17c1b076d8a6c02243cb19514d8e8b2a61d8e1 Mon Sep 17 00:00:00 2001 From: vseanreesermsft <78103370+vseanreesermsft@users.noreply.github.com> Date: Thu, 29 Aug 2024 13:53:17 -0700 Subject: [PATCH 6/7] Update branding to 8.0.10 (#107151) --- eng/Versions.props | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eng/Versions.props b/eng/Versions.props index d2b56a8f9649d6..d5e5b5f3afc620 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -1,11 +1,11 @@ - 8.0.9 + 8.0.10 8 0 - 9 + 10 8.0.100 7.0.20 6.0.$([MSBuild]::Add($(PatchVersion),25)) From ca415e95171f3fa58bd142c40f9a14fa30308671 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Sat, 31 Aug 2024 00:49:30 -0500 Subject: [PATCH 7/7] [release/8.0] Update dependencies from dotnet/arcade (#107221) * Update dependencies from https://github.com/dotnet/arcade build 20240826.2 Microsoft.DotNet.Arcade.Sdk , Microsoft.DotNet.Build.Tasks.Archives , Microsoft.DotNet.Build.Tasks.Feed , Microsoft.DotNet.Build.Tasks.Installers , Microsoft.DotNet.Build.Tasks.Packaging , Microsoft.DotNet.Build.Tasks.TargetFramework , Microsoft.DotNet.Build.Tasks.Templating , Microsoft.DotNet.Build.Tasks.Workloads , Microsoft.DotNet.CodeAnalysis , Microsoft.DotNet.GenAPI , Microsoft.DotNet.GenFacades , Microsoft.DotNet.Helix.Sdk , Microsoft.DotNet.PackageTesting , Microsoft.DotNet.RemoteExecutor , Microsoft.DotNet.SharedFramework.Sdk , Microsoft.DotNet.VersionTools.Tasks , Microsoft.DotNet.XUnitConsoleRunner , Microsoft.DotNet.XUnitExtensions From Version 8.0.0-beta.24360.5 -> To Version 8.0.0-beta.24426.2 * Update Versions.props Update the workload testing sdk --------- Co-authored-by: dotnet-maestro[bot] Co-authored-by: Larry Ewing --- eng/Version.Details.xml | 72 +++++++++---------- eng/Versions.props | 34 ++++----- eng/common/sdl/NuGet.config | 4 +- eng/common/sdl/execute-all-sdl-tools.ps1 | 4 +- eng/common/sdl/init-sdl.ps1 | 8 --- eng/common/sdl/sdl.ps1 | 4 +- .../job/publish-build-assets.yml | 2 +- .../post-build/post-build.yml | 2 +- .../templates-official/steps/execute-sdl.yml | 2 - .../steps/get-federated-access-token.yml | 14 +++- .../templates/job/publish-build-assets.yml | 2 +- .../templates/post-build/post-build.yml | 2 +- eng/common/templates/steps/execute-sdl.yml | 7 +- .../steps/get-federated-access-token.yml | 14 +++- .../templates/steps/telemetry-start.yml | 2 +- global.json | 10 +-- 16 files changed, 99 insertions(+), 84 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index f212f7b0eb31a2..e4f498d6eacf9a 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -111,9 +111,9 @@ - + https://github.com/dotnet/arcade - c9efa535175049eb9cba06cae1f8c3d5dbe768a9 + 80264e60280e2815e7d65871081ccac04a32445c @@ -121,69 +121,69 @@ 73f0850939d96131c28cf6ea6ee5aacb4da0083a - + https://github.com/dotnet/arcade - c9efa535175049eb9cba06cae1f8c3d5dbe768a9 + 80264e60280e2815e7d65871081ccac04a32445c - + https://github.com/dotnet/arcade - c9efa535175049eb9cba06cae1f8c3d5dbe768a9 + 80264e60280e2815e7d65871081ccac04a32445c - + https://github.com/dotnet/arcade - c9efa535175049eb9cba06cae1f8c3d5dbe768a9 + 80264e60280e2815e7d65871081ccac04a32445c - + https://github.com/dotnet/arcade - c9efa535175049eb9cba06cae1f8c3d5dbe768a9 + 80264e60280e2815e7d65871081ccac04a32445c - + https://github.com/dotnet/arcade - c9efa535175049eb9cba06cae1f8c3d5dbe768a9 + 80264e60280e2815e7d65871081ccac04a32445c - + https://github.com/dotnet/arcade - c9efa535175049eb9cba06cae1f8c3d5dbe768a9 + 80264e60280e2815e7d65871081ccac04a32445c - + https://github.com/dotnet/arcade - c9efa535175049eb9cba06cae1f8c3d5dbe768a9 + 80264e60280e2815e7d65871081ccac04a32445c - + https://github.com/dotnet/arcade - c9efa535175049eb9cba06cae1f8c3d5dbe768a9 + 80264e60280e2815e7d65871081ccac04a32445c - + https://github.com/dotnet/arcade - c9efa535175049eb9cba06cae1f8c3d5dbe768a9 + 80264e60280e2815e7d65871081ccac04a32445c - + https://github.com/dotnet/arcade - c9efa535175049eb9cba06cae1f8c3d5dbe768a9 + 80264e60280e2815e7d65871081ccac04a32445c - + https://github.com/dotnet/arcade - c9efa535175049eb9cba06cae1f8c3d5dbe768a9 + 80264e60280e2815e7d65871081ccac04a32445c - + https://github.com/dotnet/arcade - c9efa535175049eb9cba06cae1f8c3d5dbe768a9 + 80264e60280e2815e7d65871081ccac04a32445c - + https://github.com/dotnet/arcade - c9efa535175049eb9cba06cae1f8c3d5dbe768a9 + 80264e60280e2815e7d65871081ccac04a32445c - + https://github.com/dotnet/arcade - c9efa535175049eb9cba06cae1f8c3d5dbe768a9 + 80264e60280e2815e7d65871081ccac04a32445c - + https://github.com/dotnet/arcade - c9efa535175049eb9cba06cae1f8c3d5dbe768a9 + 80264e60280e2815e7d65871081ccac04a32445c - + https://github.com/dotnet/arcade - c9efa535175049eb9cba06cae1f8c3d5dbe768a9 + 80264e60280e2815e7d65871081ccac04a32445c https://github.com/dotnet/runtime-assets @@ -334,9 +334,9 @@ https://github.com/dotnet/xharness aacfb6328fdef17e572617bbb551431bb9cb1ff2 - + https://github.com/dotnet/arcade - c9efa535175049eb9cba06cae1f8c3d5dbe768a9 + 80264e60280e2815e7d65871081ccac04a32445c https://dev.azure.com/dnceng/internal/_git/dotnet-optimization diff --git a/eng/Versions.props b/eng/Versions.props index d5e5b5f3afc620..82a77441ceec7a 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -87,21 +87,21 @@ 8.0.100 - 8.0.0-beta.24360.5 - 8.0.0-beta.24360.5 - 8.0.0-beta.24360.5 - 8.0.0-beta.24360.5 - 8.0.0-beta.24360.5 - 2.5.1-beta.24360.5 - 8.0.0-beta.24360.5 - 8.0.0-beta.24360.5 - 8.0.0-beta.24360.5 - 8.0.0-beta.24360.5 - 8.0.0-beta.24360.5 - 8.0.0-beta.24360.5 - 8.0.0-beta.24360.5 - 8.0.0-beta.24360.5 - 8.0.0-beta.24360.5 + 8.0.0-beta.24426.2 + 8.0.0-beta.24426.2 + 8.0.0-beta.24426.2 + 8.0.0-beta.24426.2 + 8.0.0-beta.24426.2 + 2.5.1-beta.24426.2 + 8.0.0-beta.24426.2 + 8.0.0-beta.24426.2 + 8.0.0-beta.24426.2 + 8.0.0-beta.24426.2 + 8.0.0-beta.24426.2 + 8.0.0-beta.24426.2 + 8.0.0-beta.24426.2 + 8.0.0-beta.24426.2 + 8.0.0-beta.24426.2 6.0.0-preview.1.102 @@ -258,8 +258,8 @@ 3.1.7 1.0.406601 - - 8.0.101 + + 8.0.108 $(MicrosoftDotnetSdkInternalVersion) diff --git a/eng/common/sdl/NuGet.config b/eng/common/sdl/NuGet.config index 3849bdb3cf51e8..5bfbb02ef043c8 100644 --- a/eng/common/sdl/NuGet.config +++ b/eng/common/sdl/NuGet.config @@ -5,11 +5,11 @@ - + - + diff --git a/eng/common/sdl/execute-all-sdl-tools.ps1 b/eng/common/sdl/execute-all-sdl-tools.ps1 index 4715d75e974d3c..81ded5b7f477a0 100644 --- a/eng/common/sdl/execute-all-sdl-tools.ps1 +++ b/eng/common/sdl/execute-all-sdl-tools.ps1 @@ -6,7 +6,6 @@ Param( [string] $BranchName=$env:BUILD_SOURCEBRANCH, # Optional: name of branch or version of gdn settings; defaults to master [string] $SourceDirectory=$env:BUILD_SOURCESDIRECTORY, # Required: the directory where source files are located [string] $ArtifactsDirectory = (Join-Path $env:BUILD_ARTIFACTSTAGINGDIRECTORY ('artifacts')), # Required: the directory where build artifacts are located - [string] $AzureDevOpsAccessToken, # Required: access token for dnceng; should be provided via KeyVault # Optional: list of SDL tools to run on source code. See 'configure-sdl-tool.ps1' for tools list # format. @@ -75,7 +74,7 @@ try { } Exec-BlockVerbosely { - & $(Join-Path $PSScriptRoot 'init-sdl.ps1') -GuardianCliLocation $guardianCliLocation -Repository $RepoName -BranchName $BranchName -WorkingDirectory $workingDirectory -AzureDevOpsAccessToken $AzureDevOpsAccessToken -GuardianLoggerLevel $GuardianLoggerLevel + & $(Join-Path $PSScriptRoot 'init-sdl.ps1') -GuardianCliLocation $guardianCliLocation -Repository $RepoName -BranchName $BranchName -WorkingDirectory $workingDirectory -GuardianLoggerLevel $GuardianLoggerLevel } $gdnFolder = Join-Path $workingDirectory '.gdn' @@ -104,7 +103,6 @@ try { -TargetDirectory $targetDirectory ` -GdnFolder $gdnFolder ` -ToolsList $tools ` - -AzureDevOpsAccessToken $AzureDevOpsAccessToken ` -GuardianLoggerLevel $GuardianLoggerLevel ` -CrScanAdditionalRunConfigParams $CrScanAdditionalRunConfigParams ` -PoliCheckAdditionalRunConfigParams $PoliCheckAdditionalRunConfigParams ` diff --git a/eng/common/sdl/init-sdl.ps1 b/eng/common/sdl/init-sdl.ps1 index 3ac1d92b37006f..588ff8e22fbe36 100644 --- a/eng/common/sdl/init-sdl.ps1 +++ b/eng/common/sdl/init-sdl.ps1 @@ -3,7 +3,6 @@ Param( [string] $Repository, [string] $BranchName='master', [string] $WorkingDirectory, - [string] $AzureDevOpsAccessToken, [string] $GuardianLoggerLevel='Standard' ) @@ -21,14 +20,7 @@ $ci = $true # Don't display the console progress UI - it's a huge perf hit $ProgressPreference = 'SilentlyContinue' -# Construct basic auth from AzDO access token; construct URI to the repository's gdn folder stored in that repository; construct location of zip file -$encodedPat = [Convert]::ToBase64String([System.Text.Encoding]::ASCII.GetBytes(":$AzureDevOpsAccessToken")) -$escapedRepository = [Uri]::EscapeDataString("/$Repository/$BranchName/.gdn") -$uri = "https://dev.azure.com/dnceng/internal/_apis/git/repositories/sdl-tool-cfg/Items?path=$escapedRepository&versionDescriptor[versionOptions]=0&`$format=zip&api-version=5.0" -$zipFile = "$WorkingDirectory/gdn.zip" - Add-Type -AssemblyName System.IO.Compression.FileSystem -$gdnFolder = (Join-Path $WorkingDirectory '.gdn') try { # if the folder does not exist, we'll do a guardian init and push it to the remote repository diff --git a/eng/common/sdl/sdl.ps1 b/eng/common/sdl/sdl.ps1 index 648c5068d7d60f..7fe603fe995deb 100644 --- a/eng/common/sdl/sdl.ps1 +++ b/eng/common/sdl/sdl.ps1 @@ -4,6 +4,8 @@ function Install-Gdn { [Parameter(Mandatory=$true)] [string]$Path, + [string]$Source = "https://pkgs.dev.azure.com/dnceng/_packaging/Guardian1ESPTUpstreamOrgFeed/nuget/v3/index.json", + # If omitted, install the latest version of Guardian, otherwise install that specific version. [string]$Version ) @@ -19,7 +21,7 @@ function Install-Gdn { $ci = $true . $PSScriptRoot\..\tools.ps1 - $argumentList = @("install", "Microsoft.Guardian.Cli", "-Source https://securitytools.pkgs.visualstudio.com/_packaging/Guardian/nuget/v3/index.json", "-OutputDirectory $Path", "-NonInteractive", "-NoCache") + $argumentList = @("install", "Microsoft.Guardian.Cli.win-x64", "-Source $Source", "-OutputDirectory $Path", "-NonInteractive", "-NoCache") if ($Version) { $argumentList += "-Version $Version" diff --git a/eng/common/templates-official/job/publish-build-assets.yml b/eng/common/templates-official/job/publish-build-assets.yml index ba3e7df8158719..0117328800c870 100644 --- a/eng/common/templates-official/job/publish-build-assets.yml +++ b/eng/common/templates-official/job/publish-build-assets.yml @@ -149,7 +149,7 @@ jobs: scriptPath: $(Build.SourcesDirectory)/eng/common/post-build/publish-using-darc.ps1 arguments: -BuildId $(BARBuildId) -PublishingInfraVersion 3 - -AzdoToken '$(publishing-dnceng-devdiv-code-r-build-re)' + -AzdoToken '$(System.AccessToken)' -WaitPublishingFinish true -ArtifactsPublishingAdditionalParameters '${{ parameters.artifactsPublishingAdditionalParameters }}' -SymbolPublishingAdditionalParameters '${{ parameters.symbolPublishingAdditionalParameters }}' diff --git a/eng/common/templates-official/post-build/post-build.yml b/eng/common/templates-official/post-build/post-build.yml index 0dfa387e7b789d..b81b8770b34687 100644 --- a/eng/common/templates-official/post-build/post-build.yml +++ b/eng/common/templates-official/post-build/post-build.yml @@ -281,7 +281,7 @@ stages: scriptPath: $(Build.SourcesDirectory)/eng/common/post-build/publish-using-darc.ps1 arguments: -BuildId $(BARBuildId) -PublishingInfraVersion ${{ parameters.publishingInfraVersion }} - -AzdoToken '$(publishing-dnceng-devdiv-code-r-build-re)' + -AzdoToken '$(System.AccessToken)' -WaitPublishingFinish true -ArtifactsPublishingAdditionalParameters '${{ parameters.artifactsPublishingAdditionalParameters }}' -SymbolPublishingAdditionalParameters '${{ parameters.symbolPublishingAdditionalParameters }}' diff --git a/eng/common/templates-official/steps/execute-sdl.yml b/eng/common/templates-official/steps/execute-sdl.yml index 07426fde05d824..301d5c591ebd13 100644 --- a/eng/common/templates-official/steps/execute-sdl.yml +++ b/eng/common/templates-official/steps/execute-sdl.yml @@ -9,8 +9,6 @@ parameters: steps: - task: NuGetAuthenticate@1 - inputs: - nuGetServiceConnections: GuardianConnect - task: NuGetToolInstaller@1 displayName: 'Install NuGet.exe' diff --git a/eng/common/templates-official/steps/get-federated-access-token.yml b/eng/common/templates-official/steps/get-federated-access-token.yml index e3786cef6dfd77..55e33bd38f71fb 100644 --- a/eng/common/templates-official/steps/get-federated-access-token.yml +++ b/eng/common/templates-official/steps/get-federated-access-token.yml @@ -3,6 +3,12 @@ parameters: type: string - name: outputVariableName type: string +- name: stepName + type: string + default: 'getFederatedAccessToken' +- name: condition + type: string + default: '' # Resource to get a token for. Common values include: # - '499b84ac-1321-427f-aa17-267ca6975798' for Azure DevOps # - 'https://storage.azure.com/' for storage @@ -10,10 +16,16 @@ parameters: - name: resource type: string default: '499b84ac-1321-427f-aa17-267ca6975798' +- name: isStepOutputVariable + type: boolean + default: false steps: - task: AzureCLI@2 displayName: 'Getting federated access token for feeds' + name: ${{ parameters.stepName }} + ${{ if ne(parameters.condition, '') }}: + condition: ${{ parameters.condition }} inputs: azureSubscription: ${{ parameters.federatedServiceConnection }} scriptType: 'pscore' @@ -25,4 +37,4 @@ steps: exit 1 } Write-Host "Setting '${{ parameters.outputVariableName }}' with the access token value" - Write-Host "##vso[task.setvariable variable=${{ parameters.outputVariableName }};issecret=true]$accessToken" + Write-Host "##vso[task.setvariable variable=${{ parameters.outputVariableName }};issecret=true;isOutput=${{ parameters.isStepOutputVariable }}]$accessToken" \ No newline at end of file diff --git a/eng/common/templates/job/publish-build-assets.yml b/eng/common/templates/job/publish-build-assets.yml index 57a41f0a3e1322..cc2b346ba8baf0 100644 --- a/eng/common/templates/job/publish-build-assets.yml +++ b/eng/common/templates/job/publish-build-assets.yml @@ -145,7 +145,7 @@ jobs: scriptPath: $(Build.SourcesDirectory)/eng/common/post-build/publish-using-darc.ps1 arguments: -BuildId $(BARBuildId) -PublishingInfraVersion 3 - -AzdoToken '$(publishing-dnceng-devdiv-code-r-build-re)' + -AzdoToken '$(System.AccessToken)' -WaitPublishingFinish true -ArtifactsPublishingAdditionalParameters '${{ parameters.artifactsPublishingAdditionalParameters }}' -SymbolPublishingAdditionalParameters '${{ parameters.symbolPublishingAdditionalParameters }}' diff --git a/eng/common/templates/post-build/post-build.yml b/eng/common/templates/post-build/post-build.yml index 2db4933468fd8e..c3b6a3012fee18 100644 --- a/eng/common/templates/post-build/post-build.yml +++ b/eng/common/templates/post-build/post-build.yml @@ -277,7 +277,7 @@ stages: scriptPath: $(Build.SourcesDirectory)/eng/common/post-build/publish-using-darc.ps1 arguments: -BuildId $(BARBuildId) -PublishingInfraVersion ${{ parameters.publishingInfraVersion }} - -AzdoToken '$(publishing-dnceng-devdiv-code-r-build-re)' + -AzdoToken '$(System.AccessToken)' -WaitPublishingFinish true -ArtifactsPublishingAdditionalParameters '${{ parameters.artifactsPublishingAdditionalParameters }}' -SymbolPublishingAdditionalParameters '${{ parameters.symbolPublishingAdditionalParameters }}' diff --git a/eng/common/templates/steps/execute-sdl.yml b/eng/common/templates/steps/execute-sdl.yml index 07426fde05d824..fe0ebf8c904eb7 100644 --- a/eng/common/templates/steps/execute-sdl.yml +++ b/eng/common/templates/steps/execute-sdl.yml @@ -9,8 +9,6 @@ parameters: steps: - task: NuGetAuthenticate@1 - inputs: - nuGetServiceConnections: GuardianConnect - task: NuGetToolInstaller@1 displayName: 'Install NuGet.exe' @@ -36,16 +34,19 @@ steps: displayName: Execute SDL (Overridden) continueOnError: ${{ parameters.sdlContinueOnError }} condition: ${{ parameters.condition }} + env: + GUARDIAN_DEFAULT_PACKAGE_SOURCE_SECRET: $(System.AccessToken) - ${{ if eq(parameters.overrideParameters, '') }}: - powershell: ${{ parameters.executeAllSdlToolsScript }} -GuardianCliLocation $(GuardianCliLocation) -NugetPackageDirectory $(Build.SourcesDirectory)\.packages - -AzureDevOpsAccessToken $(dn-bot-dotnet-build-rw-code-rw) ${{ parameters.additionalParameters }} displayName: Execute SDL continueOnError: ${{ parameters.sdlContinueOnError }} condition: ${{ parameters.condition }} + env: + GUARDIAN_DEFAULT_PACKAGE_SOURCE_SECRET: $(System.AccessToken) - ${{ if ne(parameters.publishGuardianDirectoryToPipeline, 'false') }}: # We want to publish the Guardian results and configuration for easy diagnosis. However, the diff --git a/eng/common/templates/steps/get-federated-access-token.yml b/eng/common/templates/steps/get-federated-access-token.yml index c8c49cc0e8f0fc..55e33bd38f71fb 100644 --- a/eng/common/templates/steps/get-federated-access-token.yml +++ b/eng/common/templates/steps/get-federated-access-token.yml @@ -3,6 +3,12 @@ parameters: type: string - name: outputVariableName type: string +- name: stepName + type: string + default: 'getFederatedAccessToken' +- name: condition + type: string + default: '' # Resource to get a token for. Common values include: # - '499b84ac-1321-427f-aa17-267ca6975798' for Azure DevOps # - 'https://storage.azure.com/' for storage @@ -10,10 +16,16 @@ parameters: - name: resource type: string default: '499b84ac-1321-427f-aa17-267ca6975798' +- name: isStepOutputVariable + type: boolean + default: false steps: - task: AzureCLI@2 displayName: 'Getting federated access token for feeds' + name: ${{ parameters.stepName }} + ${{ if ne(parameters.condition, '') }}: + condition: ${{ parameters.condition }} inputs: azureSubscription: ${{ parameters.federatedServiceConnection }} scriptType: 'pscore' @@ -25,4 +37,4 @@ steps: exit 1 } Write-Host "Setting '${{ parameters.outputVariableName }}' with the access token value" - Write-Host "##vso[task.setvariable variable=${{ parameters.outputVariableName }};issecret=true]$accessToken" \ No newline at end of file + Write-Host "##vso[task.setvariable variable=${{ parameters.outputVariableName }};issecret=true;isOutput=${{ parameters.isStepOutputVariable }}]$accessToken" \ No newline at end of file diff --git a/eng/common/templates/steps/telemetry-start.yml b/eng/common/templates/steps/telemetry-start.yml index 32c01ef0b553b4..6abbcb33a67147 100644 --- a/eng/common/templates/steps/telemetry-start.yml +++ b/eng/common/templates/steps/telemetry-start.yml @@ -8,7 +8,7 @@ parameters: steps: - ${{ if and(eq(parameters.runAsPublic, 'false'), not(eq(variables['System.TeamProject'], 'public'))) }}: - - task: AzureKeyVault@1 + - task: AzureKeyVault@2 inputs: azureSubscription: 'HelixProd_KeyVault' KeyVaultName: HelixProdKV diff --git a/global.json b/global.json index f678222c9fc032..5c4c290ae74fa4 100644 --- a/global.json +++ b/global.json @@ -1,16 +1,16 @@ { "sdk": { - "version": "8.0.101", + "version": "8.0.108", "allowPrerelease": true, "rollForward": "major" }, "tools": { - "dotnet": "8.0.101" + "dotnet": "8.0.108" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.24360.5", - "Microsoft.DotNet.Helix.Sdk": "8.0.0-beta.24360.5", - "Microsoft.DotNet.SharedFramework.Sdk": "8.0.0-beta.24360.5", + "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.24426.2", + "Microsoft.DotNet.Helix.Sdk": "8.0.0-beta.24426.2", + "Microsoft.DotNet.SharedFramework.Sdk": "8.0.0-beta.24426.2", "Microsoft.Build.NoTargets": "3.7.0", "Microsoft.Build.Traversal": "3.4.0", "Microsoft.NET.Sdk.IL": "8.0.0-rc.1.23406.6"