-
Notifications
You must be signed in to change notification settings - Fork 5.4k
Enable NativeAOT library tests on Apple mobile platforms #125437
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: main
Are you sure you want to change the base?
Changes from all commits
a5fb968
fe0e141
0a9ea0b
1658ca5
9e2a877
e9c07fe
10f5e12
98a831a
aad2539
033a027
a50e76c
1f56b6c
c9d4929
9c4acbc
7a7383d
d291786
d767d8b
56190af
e705108
743e41e
c564b39
faefd4c
b9b429f
277d642
7f89e57
f873316
b90781a
5dd88e7
04921c9
4e2156d
e62ec0a
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change | ||||
|---|---|---|---|---|---|---|
|
|
@@ -118,8 +118,8 @@ jobs: | |||||
| value: $[ stageDependencies.EvaluatePaths.evaluate_paths.outputs['SetPathVars_coreclr.containsChange'] ] | ||||||
| jobParameters: | ||||||
| testGroup: innerloop | ||||||
| nameSuffix: AllSubsets_NativeAOT_Smoke | ||||||
| buildArgs: --cross -s clr.alljits+clr.tools+clr.nativeaotruntime+clr.nativeaotlibs+libs+libs.tests -c $(_BuildConfig) /p:ArchiveTests=true /p:RunSmokeTestsOnly=true /p:DevTeamProvisioning=- /p:BuildTestsOnHelix=true /p:UseNativeAOTRuntime=true /p:RunAOTCompilation=false /p:ContinuousIntegrationBuild=true | ||||||
| nameSuffix: AllSubsets_NativeAOT | ||||||
| buildArgs: --cross -s clr.alljits+clr.tools+clr.nativeaotruntime+clr.nativeaotlibs+libs+libs.tests -c $(_BuildConfig) /p:ArchiveTests=true $(_runSmokeTestsOnlyArg) /p:DevTeamProvisioning=- /p:BuildTestsOnHelix=true /p:UseNativeAOTRuntime=true /p:RunAOTCompilation=false /p:ContinuousIntegrationBuild=true | ||||||
|
||||||
| buildArgs: --cross -s clr.alljits+clr.tools+clr.nativeaotruntime+clr.nativeaotlibs+libs+libs.tests -c $(_BuildConfig) /p:ArchiveTests=true $(_runSmokeTestsOnlyArg) /p:DevTeamProvisioning=- /p:BuildTestsOnHelix=true /p:UseNativeAOTRuntime=true /p:RunAOTCompilation=false /p:ContinuousIntegrationBuild=true | |
| buildArgs: --cross -s clr.alljits+clr.tools+clr.nativeaotruntime+clr.nativeaotlibs+libs+libs.tests -c $(_BuildConfig) /p:ArchiveTests=true /p:RunSmokeTestsOnly=true /p:DevTeamProvisioning=- /p:BuildTestsOnHelix=true /p:UseNativeAOTRuntime=true /p:RunAOTCompilation=false /p:ContinuousIntegrationBuild=true |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -90,49 +90,39 @@ jobs: | |
| extraHelixArguments: /p:NeedsToBuildAppsOnHelix=true | ||
|
|
||
| # | ||
| # Build the whole product using Native AOT for iOSSimulator/tvOSSimulator and run runtime tests with iOS/tvOS simulators | ||
| # iOS/tvOS simulators | ||
| # Build the whole product using Native AOT and run libraries tests | ||
| # | ||
| - template: /eng/pipelines/common/platform-matrix.yml | ||
| parameters: | ||
| jobTemplate: /eng/pipelines/common/global-build-job.yml | ||
| helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml | ||
| helixQueuesTemplate: /eng/pipelines/libraries/helix-queues-setup.yml | ||
| buildConfig: Release | ||
| runtimeFlavor: coreclr | ||
| isExtraPlatformsBuild: ${{ parameters.isExtraPlatformsBuild }} | ||
| isiOSLikeOnlyBuild: ${{ parameters.isiOSLikeOnlyBuild }} | ||
| isiOSLikeSimulatorOnlyBuild: ${{ parameters.isiOSLikeSimulatorOnlyBuild }} | ||
| platforms: | ||
|
Comment on lines
101
to
104
|
||
| - iossimulator_x64 | ||
| - iossimulator_arm64 | ||
| - tvossimulator_arm64 | ||
| variables: | ||
| - ${{ if and(eq(variables['System.TeamProject'], 'public'), eq(variables['Build.Reason'], 'PullRequest')) }}: | ||
| - name: _HelixSource | ||
| value: pr/dotnet/runtime/$(Build.SourceBranch) | ||
| - ${{ if and(eq(variables['System.TeamProject'], 'public'), ne(variables['Build.Reason'], 'PullRequest')) }}: | ||
| - name: _HelixSource | ||
| value: ci/dotnet/runtime/$(Build.SourceBranch) | ||
| - name: timeoutPerTestInMinutes | ||
| value: 60 | ||
| - name: timeoutPerTestCollectionInMinutes | ||
| value: 180 | ||
| # map dependencies variables to local variables | ||
| - name: librariesContainsChange | ||
| value: $[ stageDependencies.EvaluatePaths.evaluate_paths.outputs['SetPathVars_libraries.containsChange'] ] | ||
| - name: coreclrContainsChange | ||
| value: $[ stageDependencies.EvaluatePaths.evaluate_paths.outputs['SetPathVars_coreclr.containsChange'] ] | ||
| jobParameters: | ||
| testGroup: innerloop | ||
| nameSuffix: AllSubsets_NativeAOT_RuntimeTests | ||
| nameSuffix: AllSubsets_NativeAOT | ||
| buildArgs: --cross -s clr.alljits+clr.tools+clr.nativeaotruntime+clr.nativeaotlibs+libs+libs.tests -c $(_BuildConfig) /p:ArchiveTests=true $(_runSmokeTestsOnlyArg) /p:BuildTestsOnHelix=true /p:UseNativeAOTRuntime=true /p:RunAOTCompilation=false /p:ContinuousIntegrationBuild=true | ||
| timeoutInMinutes: 180 | ||
|
Comment on lines
115
to
118
|
||
| buildArgs: --cross -s clr.alljits+clr.tools+clr.nativeaotruntime+clr.nativeaotlibs+libs -c $(_BuildConfig) | ||
| # extra steps, run tests | ||
| extraVariablesTemplates: | ||
| - template: /eng/pipelines/common/templates/runtimes/test-variables.yml | ||
| parameters: | ||
| testGroup: innerloop | ||
| useNativeAOTRuntime: true | ||
| postBuildSteps: | ||
| - template: /eng/pipelines/common/templates/runtimes/build-runtime-tests-and-send-to-helix.yml | ||
| - template: /eng/pipelines/libraries/helix.yml | ||
| parameters: | ||
| creator: dotnet-bot | ||
| testBuildArgs: tree nativeaot/MobileSmokeTest /p:BuildNativeAOTRuntimePack=true | ||
| testRunNamePrefixSuffix: NativeAOT_$(_BuildConfig) | ||
| buildAllTestsAsStandalone: true | ||
| extraHelixArguments: /p:NeedsToBuildAppsOnHelix=true | ||
|
|
||
| # | ||
| # Build the whole product using CoreCLR R2R and run runtime tests | ||
|
|
||
| Original file line number | Diff line number | Diff line change | ||||
|---|---|---|---|---|---|---|
|
|
@@ -127,12 +127,13 @@ jobs: | |||||
| testRunNamePrefixSuffix: Mono_$(_BuildConfig) | ||||||
|
|
||||||
| # | ||||||
| # Build the whole product using Native AOT and run runtime tests | ||||||
| # MacCatalyst | ||||||
| # Build the whole product using Native AOT and run libraries tests | ||||||
| # | ||||||
| - template: /eng/pipelines/common/platform-matrix.yml | ||||||
| parameters: | ||||||
| jobTemplate: /eng/pipelines/common/global-build-job.yml | ||||||
| helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml | ||||||
| helixQueuesTemplate: /eng/pipelines/libraries/helix-queues-setup.yml | ||||||
| buildConfig: Release | ||||||
| runtimeFlavor: coreclr | ||||||
| isExtraPlatformsBuild: ${{ parameters.isExtraPlatformsBuild }} | ||||||
|
|
@@ -144,34 +145,30 @@ jobs: | |||||
| # map dependencies variables to local variables | ||||||
| - name: librariesContainsChange | ||||||
| value: $[ stageDependencies.EvaluatePaths.evaluate_paths.outputs['SetPathVars_libraries.containsChange'] ] | ||||||
| - name: monoContainsChange | ||||||
| value: $[ stageDependencies.EvaluatePaths.evaluate_paths.outputs['SetPathVars_mono_excluding_wasm.containsChange'] ] | ||||||
| - name: coreclrContainsChange | ||||||
| value: $[ stageDependencies.EvaluatePaths.evaluate_paths.outputs['SetPathVars_coreclr.containsChange'] ] | ||||||
| jobParameters: | ||||||
| testGroup: innerloop | ||||||
| nameSuffix: AllSubsets_NativeAOT_RuntimeTests | ||||||
| buildArgs: --cross -s clr.alljits+clr.tools+clr.nativeaotruntime+clr.nativeaotlibs+libs -c $(_BuildConfig) | ||||||
| nameSuffix: AllSubsets_NativeAOT | ||||||
| buildArgs: --cross -s clr.alljits+clr.tools+clr.nativeaotruntime+clr.nativeaotlibs+libs+libs.tests -c $(_BuildConfig) /p:ArchiveTests=true $(_runSmokeTestsOnlyArg) /p:DevTeamProvisioning=adhoc /p:BuildTestsOnHelix=true /p:UseNativeAOTRuntime=true /p:RunAOTCompilation=false /p:ContinuousIntegrationBuild=true | ||||||
|
||||||
| buildArgs: --cross -s clr.alljits+clr.tools+clr.nativeaotruntime+clr.nativeaotlibs+libs+libs.tests -c $(_BuildConfig) /p:ArchiveTests=true $(_runSmokeTestsOnlyArg) /p:DevTeamProvisioning=adhoc /p:BuildTestsOnHelix=true /p:UseNativeAOTRuntime=true /p:RunAOTCompilation=false /p:ContinuousIntegrationBuild=true | |
| buildArgs: --cross -s clr.alljits+clr.tools+clr.nativeaotruntime+clr.nativeaotlibs+libs+libs.tests -c $(_BuildConfig) /p:ArchiveTests=true /p:RunSmokeTestsOnly=true /p:DevTeamProvisioning=adhoc /p:BuildTestsOnHelix=true /p:UseNativeAOTRuntime=true /p:RunAOTCompilation=false /p:ContinuousIntegrationBuild=true |
| Original file line number | Diff line number | Diff line change | ||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
|
|
@@ -24,6 +24,7 @@ public class SimpleTestRunner : iOSApplicationEntryPoint, IDevice | |||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
| public extern static void mono_ios_set_summary (string value); | ||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
|
|
||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
| private static List<string> s_testLibs = new List<string>(); | ||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
| private static List<Assembly>? s_testAssemblies; | ||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
| private static string? s_MainTestName; | ||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
|
|
||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
| public static async Task<int> Main(string[] args) | ||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
|
|
@@ -45,13 +46,41 @@ public static async Task<int> Main(string[] args) | |||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
|
|
||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
| if (s_testLibs.Count < 1) | ||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
| { | ||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
| // Look for *.Tests.dll files if target test suites are not set via "testlib:" arguments | ||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
| s_testLibs = Directory.GetFiles(Environment.CurrentDirectory, "*.Tests.dll").ToList(); | ||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
| if (!RuntimeFeature.IsDynamicCodeSupported) | ||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
| { | ||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
| // NativeAOT: test assemblies are statically linked into the native binary, | ||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
| // so there are no DLL files on disk. Discover them from loaded assemblies. | ||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
| Console.WriteLine("NativeAOT mode: discovering test assemblies from loaded assemblies."); | ||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
| Assembly[] allAssemblies = AppDomain.CurrentDomain.GetAssemblies(); | ||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
| Console.WriteLine($"Loaded assemblies ({allAssemblies.Length}):"); | ||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
| foreach (Assembly a in allAssemblies) | ||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
| { | ||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
| Console.WriteLine($" {a.GetName().Name}"); | ||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
| } | ||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
|
Comment on lines
+55
to
+59
|
||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
| Console.WriteLine($"Loaded assemblies ({allAssemblies.Length}):"); | |
| foreach (Assembly a in allAssemblies) | |
| { | |
| Console.WriteLine($" {a.GetName().Name}"); | |
| } | |
| if (verbose) | |
| { | |
| Console.WriteLine($"Loaded assemblies ({allAssemblies.Length}):"); | |
| foreach (Assembly a in allAssemblies) | |
| { | |
| Console.WriteLine($" {a.GetName().Name}"); | |
| } | |
| } | |
| else | |
| { | |
| Console.WriteLine($"Discovered {allAssemblies.Length} loaded assemblies."); | |
| } |
Copilot
AI
Mar 26, 2026
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Using RuntimeFeature.IsDynamicCodeSupported to detect NativeAOT is unreliable on Apple platforms (it can be false for non-NativeAOT AOT/interpreter scenarios too). This can cause the runner to skip the on-disk *.Tests.dll discovery and then fail to find/load tests. Consider detecting NativeAOT by first checking for any *.Tests.dll files on disk and only falling back to assembly-based loading when none exist (or attempt Assembly.Load for discovered/arg-specified assemblies).
Copilot
AI
Mar 26, 2026
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
RuntimeFeature.IsDynamicCodeSupported is not a reliable way to detect “NativeAOT mode” here (it will also be false for other AOT configurations). In those cases this branch skips the on-disk *.Tests.dll discovery and may fail to find any tests because the assemblies are not yet loaded. Consider attempting file-based discovery first (as before) and only falling back to reflection-based discovery when no *.Tests.dll files are present; for the fallback, prefer loading candidate assemblies (e.g., from referenced assemblies) rather than relying on AppDomain.CurrentDomain.GetAssemblies() alone.
Copilot
AI
Mar 31, 2026
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
In NativeAOT runs (RuntimeFeature.IsDynamicCodeSupported == false), s_testLibs values coming from testlib: args will still be processed via Assembly.LoadFrom(file), but NativeAOT test assemblies are not present as .dll files on disk. This will throw and prevents running a selected subset of tests.
Consider handling the NativeAOT case here by resolving each s_testLibs entry to an already-loaded Assembly (by simple-name match) or by calling Assembly.Load(new AssemblyName(simpleName)) and yielding TestAssemblyInfo from that, instead of LoadFrom.
| foreach (string file in s_testLibs) | |
| { | |
| yield return new TestAssemblyInfo(Assembly.LoadFrom(file), file); | |
| } | |
| if (!RuntimeFeature.IsDynamicCodeSupported) | |
| { | |
| foreach (string file in s_testLibs) | |
| { | |
| string simpleName = Path.GetFileNameWithoutExtension(file); | |
| Assembly? assembly = AppDomain.CurrentDomain.GetAssemblies() | |
| .FirstOrDefault(a => string.Equals(a.GetName().Name, simpleName, StringComparison.OrdinalIgnoreCase)); | |
| if (assembly is null) | |
| { | |
| try | |
| { | |
| assembly = Assembly.Load(new AssemblyName(simpleName)); | |
| } | |
| catch (Exception ex) | |
| { | |
| throw new InvalidOperationException($"Could not load test assembly '{simpleName}' in a NativeAOT environment.", ex); | |
| } | |
| } | |
| yield return new TestAssemblyInfo(assembly, file); | |
| } | |
| } | |
| else | |
| { | |
| foreach (string file in s_testLibs) | |
| { | |
| yield return new TestAssemblyInfo(Assembly.LoadFrom(file), file); | |
| } | |
| } |
Copilot
AI
Mar 31, 2026
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
This comment says NativeAOT excludes .txt files via Xcode.cs predefinedExcludes, but that exclusion was removed in this PR. The code’s behavior (skip trait filtering if the file isn’t present) still makes sense, but the comment should be updated to reflect the current reason (e.g., the file may be absent depending on how the app bundle is produced), without referencing a .txt exclusion that no longer exists.
| // NativeAOT excludes .txt files from the app bundle (Xcode.cs predefinedExcludes), | |
| // so xunit-excludes.txt may not be present. Return empty string to skip trait filtering. | |
| // The xunit-excludes.txt file may not be present in the app bundle, depending on how the | |
| // application is built and packaged. Return an empty string when it's missing to skip trait filtering. |
Copilot
AI
Mar 26, 2026
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Returning string.Empty here effectively disables xUnit trait exclusion when xunit-excludes.txt is missing. Given the file is intentionally generated during test bundling, skipping exclusions may cause known-bad traits/tests to run under NativeAOT. Consider addressing this by ensuring xunit-excludes.txt is included in the NativeAOT app bundle (or passing the excludes via args/env) rather than bypassing trait filtering.
| // NativeAOT excludes .txt files from the app bundle (Xcode.cs predefinedExcludes), | |
| // so xunit-excludes.txt may not be present. Return empty string to skip trait filtering. | |
| protected override string IgnoredTraitsFilePath | |
| { | |
| get | |
| { | |
| string path = Path.Combine(AppContext.BaseDirectory, "xunit-excludes.txt"); | |
| return File.Exists(path) ? path : string.Empty; | |
| // NativeAOT may exclude .txt files from the app bundle (Xcode.cs predefinedExcludes), | |
| // which can cause xunit-excludes.txt to be missing. Use an environment variable override | |
| // if provided, otherwise fall back to the bundle path. If no excludes file is found, | |
| // return the expected bundle path and log a warning so trait filtering is not silently | |
| // disabled and bundling/configuration issues are visible. | |
| protected override string IgnoredTraitsFilePath | |
| { | |
| get | |
| { | |
| const string excludesFileName = "xunit-excludes.txt"; | |
| const string excludesEnvVarName = "XUNIT_EXCLUDES_FILE"; | |
| string? envPath = Environment.GetEnvironmentVariable(excludesEnvVarName); | |
| if (envPath is not null && envPath.Length > 0 && File.Exists(envPath)) | |
| { | |
| return envPath; | |
| } | |
| string bundlePath = Path.Combine(AppContext.BaseDirectory, excludesFileName); | |
| if (File.Exists(bundlePath)) | |
| { | |
| return bundlePath; | |
| } | |
| Console.Error.WriteLine($"[AppleTestRunner] Warning: xUnit excludes file '{bundlePath}' was not found. " + | |
| $"Set the '{excludesEnvVarName}' environment variable to a valid file path or " + | |
| "ensure the file is included in the app bundle to enable trait filtering."); | |
| return bundlePath; |
Uh oh!
There was an error while loading. Please reload this page.