From 3db296396df05dd675da6ae5ac0edfd743101096 Mon Sep 17 00:00:00 2001 From: TIHan Date: Thu, 22 Aug 2024 16:46:01 -0700 Subject: [PATCH 01/35] Added aspnet superpmi collection job --- eng/pipelines/coreclr/superpmi-collect.yml | 14 ++++++++++++++ .../coreclr/templates/run-superpmi-collect-job.yml | 8 ++++++-- src/coreclr/scripts/superpmi_aspnet.py | 11 ++++++++--- 3 files changed, 28 insertions(+), 5 deletions(-) diff --git a/eng/pipelines/coreclr/superpmi-collect.yml b/eng/pipelines/coreclr/superpmi-collect.yml index ea5b57d5f51df2..4ed8cd162381a1 100644 --- a/eng/pipelines/coreclr/superpmi-collect.yml +++ b/eng/pipelines/coreclr/superpmi-collect.yml @@ -380,3 +380,17 @@ extends: unifiedArtifactsName: BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_Checked helixArtifactsName: LibrariesTestArtifacts_$(osGroup)$(osSubgroup)_$(archType)_Checked unifiedBuildConfigOverride: checked + + - template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/coreclr/templates/superpmi-collect-job.yml + buildConfig: checked + platforms: + - windows_x64 + helixQueueGroup: ci + helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml + jobParameters: + testGroup: outerloop + liveLibrariesBuildConfig: Release + collectionType: run + collectionName: aspnet diff --git a/eng/pipelines/coreclr/templates/run-superpmi-collect-job.yml b/eng/pipelines/coreclr/templates/run-superpmi-collect-job.yml index 3ef32f30b2d415..3aa6be1807614f 100644 --- a/eng/pipelines/coreclr/templates/run-superpmi-collect-job.yml +++ b/eng/pipelines/coreclr/templates/run-superpmi-collect-job.yml @@ -116,8 +116,12 @@ jobs: displayName: Enable python venv condition: always() - - script: $(PythonScript) $(Build.SourcesDirectory)/src/coreclr/scripts/superpmi_collect_setup.py -payload_directory $(PayloadLocation) -source_directory $(Build.SourcesDirectory) -core_root_directory $(Core_Root_Dir) -arch $(archType) -platform $(osGroup) -mch_file_tag $(MchFileTag) -input_directory $(InputDirectory) -collection_name $(CollectionName) -collection_type $(CollectionType) -max_size 25 # size in MB - displayName: ${{ format('SuperPMI setup ({0})', parameters.osGroup) }} + - ${{ if eq(parameters.collectionName, 'aspnet') }}: + - script: $(PythonScript) $(Build.SourcesDirectory)/src/coreclr/scripts/superpmi_aspnet.py -source_directory $(Build.SourcesDirectory) -output_mch_path $(MchFilesLocation) -arch $(archType) + displayName: ${{ format('SuperPMI ASPNET setup ({0})', parameters.osGroup) }} + - ${{ else }}: + - script: $(PythonScript) $(Build.SourcesDirectory)/src/coreclr/scripts/superpmi_collect_setup.py -payload_directory $(PayloadLocation) -source_directory $(Build.SourcesDirectory) -core_root_directory $(Core_Root_Dir) -arch $(archType) -platform $(osGroup) -mch_file_tag $(MchFileTag) -input_directory $(InputDirectory) -collection_name $(CollectionName) -collection_type $(CollectionType) -max_size 25 # size in MB + displayName: ${{ format('SuperPMI setup ({0})', parameters.osGroup) }} # Create required directories for merged mch collection and superpmi logs - ${{ if ne(parameters.osGroup, 'windows') }}: diff --git a/src/coreclr/scripts/superpmi_aspnet.py b/src/coreclr/scripts/superpmi_aspnet.py index 8125184104ea47..4ac51b98cb80e8 100644 --- a/src/coreclr/scripts/superpmi_aspnet.py +++ b/src/coreclr/scripts/superpmi_aspnet.py @@ -132,6 +132,8 @@ def build_and_run(coreclr_args): with tempfile.TemporaryDirectory() as temp_location: + temp_location = "C:\\work\\tmp\\tmp" + print ("Executing in " + temp_location) os.chdir(temp_location) @@ -156,8 +158,9 @@ def build_and_run(coreclr_args): ## could probably just pass a URL and avoid this - run_command( - ["git", "clone", "--quiet", "--depth", "1", "https://github.com/aspnet/benchmarks"], temp_location, _exit_on_fail=True) + if not path.isdir(path.join(temp_location, 'benchmarks')): + run_command( + ["git", "clone", "--quiet", "--depth", "1", "https://github.com/aspnet/benchmarks"], temp_location, _exit_on_fail=True) crank_app = path.join(temp_location, "crank") mcs_path = determine_mcs_tool_path(coreclr_args) @@ -247,7 +250,9 @@ def build_and_run(coreclr_args): print("") description = ["--description", configName + "-" + scenario + "-" + "-".join(runtime_options)] - subprocess.run([crank_app] + crank_arguments + description + runtime_arguments, cwd=temp_location) + crank_app_args = [crank_app] + crank_arguments + description + runtime_arguments + print(' '.join(crank_app_args)) + subprocess.run(crank_app_args, cwd=temp_location) # merge command = [mcs_path, "-merge", "temp.mch", "*.mc", "-dedup", "-thin"] From e8fb009cb8d0d62eb853ca5b7bf6d7fb34c5c730 Mon Sep 17 00:00:00 2001 From: TIHan Date: Thu, 22 Aug 2024 17:01:42 -0700 Subject: [PATCH 02/35] Added ability to change temp location --- src/coreclr/scripts/superpmi_aspnet.py | 292 +++++++++++++------------ 1 file changed, 152 insertions(+), 140 deletions(-) diff --git a/src/coreclr/scripts/superpmi_aspnet.py b/src/coreclr/scripts/superpmi_aspnet.py index 4ac51b98cb80e8..20bb019758210d 100644 --- a/src/coreclr/scripts/superpmi_aspnet.py +++ b/src/coreclr/scripts/superpmi_aspnet.py @@ -31,6 +31,7 @@ parser.add_argument("-source_directory", help="path to source directory") parser.add_argument("-output_mch_path", help="Absolute path to the mch file to produce") parser.add_argument("-arch", help="Architecture") +parser.add_argument("-temp_location", required=False, help="Location to temporarily download ASPNET benchmarks and crank") def setup_args(args): """ Setup the args for SuperPMI to use. @@ -59,6 +60,13 @@ def setup_args(args): "arch", lambda arch: arch.lower() in ["x64", "arm64"], "Unable to set arch") + + if args.temp_location: + coreclr_args.temp_location = args.temp_location + coreclr_args.temp_is_explicit = True + else: + coreclr_args.temp_location = tempfile.TemporaryDirectory().name + coreclr_args.temp_is_explicit = False return coreclr_args @@ -130,158 +138,162 @@ def build_and_run(coreclr_args): dotnet_install_script_name = "dotnet-install.cmd" if is_windows else "dotnet-install.sh" dotnet_install_script_path = path.join(source_directory, "eng", "common", dotnet_install_script_name) - with tempfile.TemporaryDirectory() as temp_location: + temp_location = coreclr_args.temp_location + + if not coreclr_args.temp_is_explicit: + os.mkdir(temp_location) + + print ("Executing in " + temp_location) + os.chdir(temp_location) - temp_location = "C:\\work\\tmp\\tmp" + # install dotnet 8.0 + run_command([dotnet_install_script_path, "-Version", "8.0.0"], temp_location, _exit_on_fail=True) + os.environ['DOTNET_MULTILEVEL_LOOKUP'] = '0' + os.environ['DOTNET_SKIP_FIRST_TIME_EXPERIENCE'] = '1' + dotnet_path = path.join(source_directory, ".dotnet") + dotnet_exe = path.join(dotnet_path, "dotnet.exe") if is_windows else path.join(dotnet_path, "dotnet") + # run_command([dotnet_exe, "--info"], temp_location, _exit_on_fail=True) + os.environ['DOTNET_ROOT'] = dotnet_path - print ("Executing in " + temp_location) - os.chdir(temp_location) + ## install crank as local tool + run_command( + [dotnet_exe, "tool", "install", "Microsoft.Crank.Controller", "--version", "0.2.0-*", "--tool-path", temp_location], _exit_on_fail=True) - # install dotnet 8.0 - run_command([dotnet_install_script_path, "-Version", "8.0.0"], temp_location, _exit_on_fail=True) - os.environ['DOTNET_MULTILEVEL_LOOKUP'] = '0' - os.environ['DOTNET_SKIP_FIRST_TIME_EXPERIENCE'] = '1' - dotnet_path = path.join(source_directory, ".dotnet") - dotnet_exe = path.join(dotnet_path, "dotnet.exe") if is_windows else path.join(dotnet_path, "dotnet") - # run_command([dotnet_exe, "--info"], temp_location, _exit_on_fail=True) - os.environ['DOTNET_ROOT'] = dotnet_path + ## ideally just do sparse clone, but this doesn't work locally + ## git clone --filter=blob:none --no-checkout https://github.com/aspnet/benchmarks + ## cd benchmarks + ## git sparse-checkout init --cone + ## git sparse-checkout set scenarios - ## install crank as local tool + ## could probably just pass a URL and avoid this + + if not path.isdir(path.join(temp_location, 'benchmarks')): run_command( - [dotnet_exe, "tool", "install", "Microsoft.Crank.Controller", "--version", "0.2.0-*", "--tool-path", temp_location], _exit_on_fail=True) - - ## ideally just do sparse clone, but this doesn't work locally - ## git clone --filter=blob:none --no-checkout https://github.com/aspnet/benchmarks - ## cd benchmarks - ## git sparse-checkout init --cone - ## git sparse-checkout set scenarios - - ## could probably just pass a URL and avoid this - - if not path.isdir(path.join(temp_location, 'benchmarks')): - run_command( - ["git", "clone", "--quiet", "--depth", "1", "https://github.com/aspnet/benchmarks"], temp_location, _exit_on_fail=True) - - crank_app = path.join(temp_location, "crank") - mcs_path = determine_mcs_tool_path(coreclr_args) - superpmi_path = determine_superpmi_tool_path(coreclr_args) - - # todo: add grpc/signalr, perhaps - - configname_scenario_list = [ - ("platform", "plaintext"), - ("json", "json"), - ("plaintext", "mvc"), - ("database", "fortunes_dapper"), - ("database", "fortunes_ef_mvc_https"), - ("database", "updates"), - ("proxy", "proxy-yarp"), - ("staticfiles", "static"), - ("websocket", "websocket"), - ("orchard", "about-sqlite"), - ("signalr", "signalr"), - ("grpc", "grpcaspnetcoreserver-grpcnetclient"), - ("efcore", "NavigationsQuery"), - ("efcore", "Funcletization") - ] - - # configname_scenario_list = [("quic", "read-write")] - - # note tricks to get one element tuples - - runtime_options_list = [ - ("Dummy=0",), - ("TieredCompilation=0", ), - ("TieredPGO=0",), - ("TieredPGO=1", "ReadyToRun=0"), - ("ReadyToRun=0", "OSR_HitLimit=0", "TC_OnStackReplacement_InitialCounter=10"), - ("TC_PartialCompilation=1",) - ] - - # runtime_options_list = [("Dummy=0", )] - - mch_file = path.join(coreclr_args.output_mch_path, "aspnet.run." + target_os + "." + target_arch + ".checked.mch") - benchmark_machine = determine_benchmark_machine(coreclr_args) - - jitname = determine_native_name(coreclr_args, "clrjit", target_os) - coreclrname = determine_native_name(coreclr_args, "coreclr", target_os) - spminame = determine_native_name(coreclr_args, "superpmi-shim-collector", target_os) - corelibname = "System.Private.CoreLib.dll" - - jitpath = path.join(".", jitname) - jitlib = path.join(checked_root, jitname) - coreclr = path.join(release_root, coreclrname) - corelib = path.join(release_root, corelibname) - spmilib = path.join(checked_root, spminame) - - for (configName, scenario) in configname_scenario_list: - configYml = configName + ".benchmarks.yml" - configFile = path.join(temp_location, "benchmarks", "scenarios", configYml) - - crank_arguments = ["--config", configFile, - "--profile", benchmark_machine, - "--scenario", scenario, - "--application.framework", "net9.0", - "--application.channel", "edge", - "--application.sdkVersion", "latest", - "--application.environmentVariables", "DOTNET_JitName=" + spminame, - "--application.environmentVariables", "SuperPMIShimLogPath=.", - "--application.environmentVariables", "SuperPMIShimPath=" + jitpath, - "--application.environmentVariables", "DOTNET_EnableExtraSuperPmiQueries=1", - "--application.options.downloadFiles", "*.mc", - "--application.options.displayOutput", "true", + ["git", "clone", "--quiet", "--depth", "1", "https://github.com/aspnet/benchmarks"], temp_location, _exit_on_fail=True) + + crank_app = path.join(temp_location, "crank") + mcs_path = determine_mcs_tool_path(coreclr_args) + superpmi_path = determine_superpmi_tool_path(coreclr_args) + + # todo: add grpc/signalr, perhaps + + configname_scenario_list = [ + ("platform", "plaintext"), + ("json", "json"), + ("plaintext", "mvc"), + ("database", "fortunes_dapper"), + ("database", "fortunes_ef_mvc_https"), + ("database", "updates"), + ("proxy", "proxy-yarp"), + ("staticfiles", "static"), + ("websocket", "websocket"), + ("orchard", "about-sqlite"), + ("signalr", "signalr"), + ("grpc", "grpcaspnetcoreserver-grpcnetclient"), + ("efcore", "NavigationsQuery"), + ("efcore", "Funcletization") + ] + + # configname_scenario_list = [("quic", "read-write")] + + # note tricks to get one element tuples + + runtime_options_list = [ + ("Dummy=0",), + ("TieredCompilation=0", ), + ("TieredPGO=0",), + ("TieredPGO=1", "ReadyToRun=0"), + ("ReadyToRun=0", "OSR_HitLimit=0", "TC_OnStackReplacement_InitialCounter=10"), + ("TC_PartialCompilation=1",) + ] + + # runtime_options_list = [("Dummy=0", )] + + mch_file = path.join(coreclr_args.output_mch_path, "aspnet.run." + target_os + "." + target_arch + ".checked.mch") + benchmark_machine = determine_benchmark_machine(coreclr_args) + + jitname = determine_native_name(coreclr_args, "clrjit", target_os) + coreclrname = determine_native_name(coreclr_args, "coreclr", target_os) + spminame = determine_native_name(coreclr_args, "superpmi-shim-collector", target_os) + corelibname = "System.Private.CoreLib.dll" + + jitpath = path.join(".", jitname) + jitlib = path.join(checked_root, jitname) + coreclr = path.join(release_root, coreclrname) + corelib = path.join(release_root, corelibname) + spmilib = path.join(checked_root, spminame) + + for (configName, scenario) in configname_scenario_list: + configYml = configName + ".benchmarks.yml" + configFile = path.join(temp_location, "benchmarks", "scenarios", configYml) + + crank_arguments = ["--config", configFile, + "--profile", benchmark_machine, + "--scenario", scenario, + "--application.framework", "net9.0", + "--application.channel", "edge", + "--application.sdkVersion", "latest", + "--application.environmentVariables", "DOTNET_JitName=" + spminame, + "--application.environmentVariables", "SuperPMIShimLogPath=.", + "--application.environmentVariables", "SuperPMIShimPath=" + jitpath, + "--application.environmentVariables", "DOTNET_EnableExtraSuperPmiQueries=1", + "--application.options.downloadFiles", "*.mc", + "--application.options.displayOutput", "true", # "--application.options.dumpType", "full", # "--application.options.fetch", "true", - "--application.options.outputFiles", spmilib, - "--application.options.outputFiles", jitlib, - "--application.options.outputFiles", coreclr, - "--application.options.outputFiles", corelib] - - for runtime_options in runtime_options_list: - runtime_arguments = [] - for runtime_option in runtime_options: - runtime_arguments.append("--application.environmentVariables") - runtime_arguments.append("DOTNET_" + runtime_option) - - print("") - print("================================") - print("Config: " + configName + " scenario: " + scenario + " options: " + " ".join(runtime_options)) - print("================================") - print("") - - description = ["--description", configName + "-" + scenario + "-" + "-".join(runtime_options)] - crank_app_args = [crank_app] + crank_arguments + description + runtime_arguments - print(' '.join(crank_app_args)) - subprocess.run(crank_app_args, cwd=temp_location) - - # merge - command = [mcs_path, "-merge", "temp.mch", "*.mc", "-dedup", "-thin"] - run_command(command, temp_location) - - # clean - command = [superpmi_path, "-v", "ewmi", "-f", "fail.mcl", jitlib, "temp.mch"] + "--application.options.outputFiles", spmilib, + "--application.options.outputFiles", jitlib, + "--application.options.outputFiles", coreclr, + "--application.options.outputFiles", corelib] + + for runtime_options in runtime_options_list: + runtime_arguments = [] + for runtime_option in runtime_options: + runtime_arguments.append("--application.environmentVariables") + runtime_arguments.append("DOTNET_" + runtime_option) + + print("") + print("================================") + print("Config: " + configName + " scenario: " + scenario + " options: " + " ".join(runtime_options)) + print("================================") + print("") + + description = ["--description", configName + "-" + scenario + "-" + "-".join(runtime_options)] + crank_app_args = [crank_app] + crank_arguments + description + runtime_arguments + print(' '.join(crank_app_args)) + subprocess.run(crank_app_args, cwd=temp_location) + + # merge + command = [mcs_path, "-merge", "temp.mch", "*.mc", "-dedup", "-thin"] + run_command(command, temp_location) + + # clean + command = [superpmi_path, "-v", "ewmi", "-f", "fail.mcl", jitlib, "temp.mch"] + run_command(command, temp_location) + + # strip + if is_nonzero_length_file("fail.mcl"): + print("Replay had failures, cleaning..."); + fail_file = path.join(coreclr_args.output_mch_path, "fail.mcl"); + command = [mcs_path, "-strip", "fail.mcl", "temp.mch", mch_file] run_command(command, temp_location) + else: + print("Replay was clean..."); + shutil.copy2("temp.mch", mch_file) - # strip - if is_nonzero_length_file("fail.mcl"): - print("Replay had failures, cleaning..."); - fail_file = path.join(coreclr_args.output_mch_path, "fail.mcl"); - command = [mcs_path, "-strip", "fail.mcl", "temp.mch", mch_file] - run_command(command, temp_location) - else: - print("Replay was clean..."); - shutil.copy2("temp.mch", mch_file) + # index + command = [mcs_path, "-toc", mch_file] + run_command(command, temp_location) - # index - command = [mcs_path, "-toc", mch_file] - run_command(command, temp_location) + # overall summary + print("Merged summary for " + mch_file) + command = [mcs_path, "-jitflags", mch_file] + run_command(command, temp_location) - # overall summary - print("Merged summary for " + mch_file) - command = [mcs_path, "-jitflags", mch_file] - run_command(command, temp_location) + os.chdir(source_directory ) - os.chdir(source_directory ) + if not coreclr_args.temp_is_explicit: + shutil.rmtree(temp_location, ignore_errors=True) def main(main_args): """ Main entry point From 37fd84881c8d2929094dd6db7c9e65604b266e5e Mon Sep 17 00:00:00 2001 From: TIHan Date: Thu, 22 Aug 2024 17:04:07 -0700 Subject: [PATCH 03/35] Set temp location --- eng/pipelines/coreclr/templates/run-superpmi-collect-job.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/pipelines/coreclr/templates/run-superpmi-collect-job.yml b/eng/pipelines/coreclr/templates/run-superpmi-collect-job.yml index 3aa6be1807614f..e2239d11202af8 100644 --- a/eng/pipelines/coreclr/templates/run-superpmi-collect-job.yml +++ b/eng/pipelines/coreclr/templates/run-superpmi-collect-job.yml @@ -117,7 +117,7 @@ jobs: condition: always() - ${{ if eq(parameters.collectionName, 'aspnet') }}: - - script: $(PythonScript) $(Build.SourcesDirectory)/src/coreclr/scripts/superpmi_aspnet.py -source_directory $(Build.SourcesDirectory) -output_mch_path $(MchFilesLocation) -arch $(archType) + - script: $(PythonScript) $(Build.SourcesDirectory)/src/coreclr/scripts/superpmi_aspnet.py -source_directory $(Build.SourcesDirectory) -output_mch_path $(MchFilesLocation) -arch $(archType) -temp_location $(PayloadLocation) displayName: ${{ format('SuperPMI ASPNET setup ({0})', parameters.osGroup) }} - ${{ else }}: - script: $(PythonScript) $(Build.SourcesDirectory)/src/coreclr/scripts/superpmi_collect_setup.py -payload_directory $(PayloadLocation) -source_directory $(Build.SourcesDirectory) -core_root_directory $(Core_Root_Dir) -arch $(archType) -platform $(osGroup) -mch_file_tag $(MchFileTag) -input_directory $(InputDirectory) -collection_name $(CollectionName) -collection_type $(CollectionType) -max_size 25 # size in MB From f41baa3625fe4d869f213c517021db7759a027dc Mon Sep 17 00:00:00 2001 From: TIHan Date: Thu, 22 Aug 2024 17:46:03 -0700 Subject: [PATCH 04/35] move job --- eng/pipelines/coreclr/superpmi-collect.yml | 28 +++++++++++----------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/eng/pipelines/coreclr/superpmi-collect.yml b/eng/pipelines/coreclr/superpmi-collect.yml index 4ed8cd162381a1..27a1bf89a4da33 100644 --- a/eng/pipelines/coreclr/superpmi-collect.yml +++ b/eng/pipelines/coreclr/superpmi-collect.yml @@ -239,6 +239,20 @@ extends: collectionType: run collectionName: benchmarks + - template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/coreclr/templates/superpmi-collect-job.yml + buildConfig: checked + platforms: + - windows_x64 + helixQueueGroup: ci + helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml + jobParameters: + testGroup: outerloop + liveLibrariesBuildConfig: Release + collectionType: run + collectionName: aspnet + - template: /eng/pipelines/common/platform-matrix.yml parameters: jobTemplate: /eng/pipelines/coreclr/templates/superpmi-collect-job.yml @@ -380,17 +394,3 @@ extends: unifiedArtifactsName: BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_Checked helixArtifactsName: LibrariesTestArtifacts_$(osGroup)$(osSubgroup)_$(archType)_Checked unifiedBuildConfigOverride: checked - - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/coreclr/templates/superpmi-collect-job.yml - buildConfig: checked - platforms: - - windows_x64 - helixQueueGroup: ci - helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml - jobParameters: - testGroup: outerloop - liveLibrariesBuildConfig: Release - collectionType: run - collectionName: aspnet From 0f84e8bc5d0439cb42bf62b526f8e88b8a983965 Mon Sep 17 00:00:00 2001 From: TIHan Date: Thu, 22 Aug 2024 17:49:28 -0700 Subject: [PATCH 05/35] Temporarily comment out jobs --- eng/pipelines/coreclr/superpmi-collect.yml | 424 ++++++++++----------- 1 file changed, 212 insertions(+), 212 deletions(-) diff --git a/eng/pipelines/coreclr/superpmi-collect.yml b/eng/pipelines/coreclr/superpmi-collect.yml index 27a1bf89a4da33..9b08e48c062181 100644 --- a/eng/pipelines/coreclr/superpmi-collect.yml +++ b/eng/pipelines/coreclr/superpmi-collect.yml @@ -159,85 +159,85 @@ extends: jobParameters: testGroup: outerloop - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/coreclr/templates/superpmi-collect-job.yml - buildConfig: checked - platforms: - - osx_arm64 - - linux_arm - - linux_arm64 - - linux_x64 - - windows_x64 - - windows_x86 - - windows_arm64 - helixQueueGroup: ci - helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml - jobParameters: - testGroup: outerloop - liveLibrariesBuildConfig: Release - collectionType: pmi - collectionName: libraries + # - template: /eng/pipelines/common/platform-matrix.yml + # parameters: + # jobTemplate: /eng/pipelines/coreclr/templates/superpmi-collect-job.yml + # buildConfig: checked + # platforms: + # - osx_arm64 + # - linux_arm + # - linux_arm64 + # - linux_x64 + # - windows_x64 + # - windows_x86 + # - windows_arm64 + # helixQueueGroup: ci + # helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml + # jobParameters: + # testGroup: outerloop + # liveLibrariesBuildConfig: Release + # collectionType: pmi + # collectionName: libraries - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/coreclr/templates/superpmi-collect-job.yml - buildConfig: checked - platforms: - - osx_arm64 - - linux_arm - - linux_arm64 - - linux_x64 - - windows_x64 - - windows_x86 - - windows_arm64 - helixQueueGroup: ci - helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml - jobParameters: - testGroup: outerloop - liveLibrariesBuildConfig: Release - collectionType: crossgen2 - collectionName: libraries + # - template: /eng/pipelines/common/platform-matrix.yml + # parameters: + # jobTemplate: /eng/pipelines/coreclr/templates/superpmi-collect-job.yml + # buildConfig: checked + # platforms: + # - osx_arm64 + # - linux_arm + # - linux_arm64 + # - linux_x64 + # - windows_x64 + # - windows_x86 + # - windows_arm64 + # helixQueueGroup: ci + # helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml + # jobParameters: + # testGroup: outerloop + # liveLibrariesBuildConfig: Release + # collectionType: crossgen2 + # collectionName: libraries - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/coreclr/templates/superpmi-collect-job.yml - buildConfig: checked - platforms: - - osx_arm64 - - linux_arm - - linux_arm64 - - linux_x64 - - windows_x64 - - windows_x86 - - windows_arm64 - helixQueueGroup: ci - helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml - jobParameters: - testGroup: outerloop - liveLibrariesBuildConfig: Release - collectionType: run - collectionName: realworld + # - template: /eng/pipelines/common/platform-matrix.yml + # parameters: + # jobTemplate: /eng/pipelines/coreclr/templates/superpmi-collect-job.yml + # buildConfig: checked + # platforms: + # - osx_arm64 + # - linux_arm + # - linux_arm64 + # - linux_x64 + # - windows_x64 + # - windows_x86 + # - windows_arm64 + # helixQueueGroup: ci + # helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml + # jobParameters: + # testGroup: outerloop + # liveLibrariesBuildConfig: Release + # collectionType: run + # collectionName: realworld - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/coreclr/templates/superpmi-collect-job.yml - buildConfig: checked - platforms: - - osx_arm64 - - linux_arm - - linux_arm64 - - linux_x64 - - windows_x64 - - windows_x86 - - windows_arm64 - helixQueueGroup: ci - helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml - jobParameters: - testGroup: outerloop - liveLibrariesBuildConfig: Release - collectionType: run - collectionName: benchmarks + # - template: /eng/pipelines/common/platform-matrix.yml + # parameters: + # jobTemplate: /eng/pipelines/coreclr/templates/superpmi-collect-job.yml + # buildConfig: checked + # platforms: + # - osx_arm64 + # - linux_arm + # - linux_arm64 + # - linux_x64 + # - windows_x64 + # - windows_x86 + # - windows_arm64 + # helixQueueGroup: ci + # helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml + # jobParameters: + # testGroup: outerloop + # liveLibrariesBuildConfig: Release + # collectionType: run + # collectionName: benchmarks - template: /eng/pipelines/common/platform-matrix.yml parameters: @@ -253,144 +253,144 @@ extends: collectionType: run collectionName: aspnet - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/coreclr/templates/superpmi-collect-job.yml - buildConfig: checked - platforms: - - osx_arm64 - - linux_arm - - linux_arm64 - - linux_x64 - - windows_x64 - - windows_x86 - - windows_arm64 - helixQueueGroup: ci - helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml - jobParameters: - testGroup: outerloop - liveLibrariesBuildConfig: Release - collectionType: run_tiered - collectionName: benchmarks + # - template: /eng/pipelines/common/platform-matrix.yml + # parameters: + # jobTemplate: /eng/pipelines/coreclr/templates/superpmi-collect-job.yml + # buildConfig: checked + # platforms: + # - osx_arm64 + # - linux_arm + # - linux_arm64 + # - linux_x64 + # - windows_x64 + # - windows_x86 + # - windows_arm64 + # helixQueueGroup: ci + # helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml + # jobParameters: + # testGroup: outerloop + # liveLibrariesBuildConfig: Release + # collectionType: run_tiered + # collectionName: benchmarks - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/coreclr/templates/superpmi-collect-job.yml - buildConfig: checked - platforms: - - osx_arm64 - - linux_arm - - linux_arm64 - - linux_x64 - - windows_x64 - - windows_x86 - - windows_arm64 - helixQueueGroup: ci - helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml - jobParameters: - testGroup: outerloop - liveLibrariesBuildConfig: Release - collectionType: run_pgo - collectionName: benchmarks + # - template: /eng/pipelines/common/platform-matrix.yml + # parameters: + # jobTemplate: /eng/pipelines/coreclr/templates/superpmi-collect-job.yml + # buildConfig: checked + # platforms: + # - osx_arm64 + # - linux_arm + # - linux_arm64 + # - linux_x64 + # - windows_x64 + # - windows_x86 + # - windows_arm64 + # helixQueueGroup: ci + # helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml + # jobParameters: + # testGroup: outerloop + # liveLibrariesBuildConfig: Release + # collectionType: run_pgo + # collectionName: benchmarks - # - # Collection of coreclr test run - # - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/common/templates/runtimes/run-test-job.yml - buildConfig: checked - platforms: - - osx_arm64 - - linux_arm - - linux_arm64 - - linux_x64 - - windows_x64 - - windows_x86 - - windows_arm64 - helixQueueGroup: superpmi - helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml - jobParameters: - testGroup: outerloop - liveLibrariesBuildConfig: Release - SuperPmiCollect: true - unifiedArtifactsName: BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig) + # # + # # Collection of coreclr test run + # # + # - template: /eng/pipelines/common/platform-matrix.yml + # parameters: + # jobTemplate: /eng/pipelines/common/templates/runtimes/run-test-job.yml + # buildConfig: checked + # platforms: + # - osx_arm64 + # - linux_arm + # - linux_arm64 + # - linux_x64 + # - windows_x64 + # - windows_x86 + # - windows_arm64 + # helixQueueGroup: superpmi + # helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml + # jobParameters: + # testGroup: outerloop + # liveLibrariesBuildConfig: Release + # SuperPmiCollect: true + # unifiedArtifactsName: BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig) - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/coreclr/templates/superpmi-collect-job.yml - buildConfig: checked - platforms: - - linux_arm64 - - linux_x64 - - windows_x64 - - windows_arm64 - helixQueueGroup: ci - helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml - jobParameters: - testGroup: outerloop - liveLibrariesBuildConfig: Release - collectionType: nativeaot - collectionName: smoke_tests + # - template: /eng/pipelines/common/platform-matrix.yml + # parameters: + # jobTemplate: /eng/pipelines/coreclr/templates/superpmi-collect-job.yml + # buildConfig: checked + # platforms: + # - linux_arm64 + # - linux_x64 + # - windows_x64 + # - windows_arm64 + # helixQueueGroup: ci + # helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml + # jobParameters: + # testGroup: outerloop + # liveLibrariesBuildConfig: Release + # collectionType: nativeaot + # collectionName: smoke_tests - # - # Collection of libraries test run: normal - # Libraries Test Run using Release libraries, and Checked CoreCLR - # - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/libraries/run-test-job.yml - buildConfig: Release - platforms: - - osx_arm64 - - linux_arm - - linux_arm64 - - linux_x64 - - windows_x64 - - windows_x86 - - windows_arm64 - helixQueueGroup: superpmi - helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml - jobParameters: - testScope: innerloop - liveRuntimeBuildConfig: Checked - dependsOnTestBuildConfiguration: Release - dependsOnTestArchitecture: x64 - scenarios: - - normal - SuperPmiCollect: true - SuperPmiCollectionName: libraries_tests - unifiedArtifactsName: BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_Checked - helixArtifactsName: LibrariesTestArtifacts_$(osGroup)$(osSubgroup)_$(archType)_Checked - unifiedBuildConfigOverride: checked + # # + # # Collection of libraries test run: normal + # # Libraries Test Run using Release libraries, and Checked CoreCLR + # # + # - template: /eng/pipelines/common/platform-matrix.yml + # parameters: + # jobTemplate: /eng/pipelines/libraries/run-test-job.yml + # buildConfig: Release + # platforms: + # - osx_arm64 + # - linux_arm + # - linux_arm64 + # - linux_x64 + # - windows_x64 + # - windows_x86 + # - windows_arm64 + # helixQueueGroup: superpmi + # helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml + # jobParameters: + # testScope: innerloop + # liveRuntimeBuildConfig: Checked + # dependsOnTestBuildConfiguration: Release + # dependsOnTestArchitecture: x64 + # scenarios: + # - normal + # SuperPmiCollect: true + # SuperPmiCollectionName: libraries_tests + # unifiedArtifactsName: BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_Checked + # helixArtifactsName: LibrariesTestArtifacts_$(osGroup)$(osSubgroup)_$(archType)_Checked + # unifiedBuildConfigOverride: checked - # - # Collection of libraries test run: no_tiered_compilation - # Libraries Test Run using Release libraries, and Checked CoreCLR - # - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/libraries/run-test-job.yml - buildConfig: Release - platforms: - - osx_arm64 - - linux_arm - - linux_arm64 - - linux_x64 - - windows_x64 - - windows_x86 - - windows_arm64 - helixQueueGroup: superpmi - helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml - jobParameters: - testScope: innerloop - liveRuntimeBuildConfig: Checked - dependsOnTestBuildConfiguration: Release - dependsOnTestArchitecture: x64 - scenarios: - - no_tiered_compilation - SuperPmiCollect: true - SuperPmiCollectionName: libraries_tests_no_tiered_compilation - unifiedArtifactsName: BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_Checked - helixArtifactsName: LibrariesTestArtifacts_$(osGroup)$(osSubgroup)_$(archType)_Checked - unifiedBuildConfigOverride: checked + # # + # # Collection of libraries test run: no_tiered_compilation + # # Libraries Test Run using Release libraries, and Checked CoreCLR + # # + # - template: /eng/pipelines/common/platform-matrix.yml + # parameters: + # jobTemplate: /eng/pipelines/libraries/run-test-job.yml + # buildConfig: Release + # platforms: + # - osx_arm64 + # - linux_arm + # - linux_arm64 + # - linux_x64 + # - windows_x64 + # - windows_x86 + # - windows_arm64 + # helixQueueGroup: superpmi + # helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml + # jobParameters: + # testScope: innerloop + # liveRuntimeBuildConfig: Checked + # dependsOnTestBuildConfiguration: Release + # dependsOnTestArchitecture: x64 + # scenarios: + # - no_tiered_compilation + # SuperPmiCollect: true + # SuperPmiCollectionName: libraries_tests_no_tiered_compilation + # unifiedArtifactsName: BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_Checked + # helixArtifactsName: LibrariesTestArtifacts_$(osGroup)$(osSubgroup)_$(archType)_Checked + # unifiedBuildConfigOverride: checked From 6b8f6bdbb6b754a735a6c5030cb6c513169548f3 Mon Sep 17 00:00:00 2001 From: TIHan Date: Thu, 22 Aug 2024 17:52:01 -0700 Subject: [PATCH 06/35] Temporarily comment out jobs --- eng/pipelines/coreclr/superpmi-collect.yml | 160 ++++++++++----------- 1 file changed, 80 insertions(+), 80 deletions(-) diff --git a/eng/pipelines/coreclr/superpmi-collect.yml b/eng/pipelines/coreclr/superpmi-collect.yml index 9b08e48c062181..dbae6d64275174 100644 --- a/eng/pipelines/coreclr/superpmi-collect.yml +++ b/eng/pipelines/coreclr/superpmi-collect.yml @@ -39,7 +39,7 @@ extends: buildConfig: checked platforms: - windows_x64 - - linux_x64 + # - linux_x64 jobParameters: testGroup: outerloop buildArgs: -s clr+libs+libs.tests -rc $(_BuildConfig) -c Release /p:ArchiveTests=true @@ -68,86 +68,86 @@ extends: testGroup: outerloop disableComponentGovernance: true # No shipping artifacts produced by this pipeline - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/common/global-build-job.yml - buildConfig: checked - platforms: - - windows_x86 - - windows_arm64 - - osx_arm64 - jobParameters: - testGroup: outerloop - buildArgs: -s clr+libs+libs.tests -rc $(_BuildConfig) -c Release /p:ArchiveTests=true - timeoutInMinutes: 120 - postBuildSteps: - # Build CLR assets for x64 as well as the target as we need an x64 mcs - - template: /eng/pipelines/common/templates/global-build-step.yml - parameters: - buildArgs: -s clr.spmi -c $(_BuildConfig) - archParameter: -arch x64 - - template: /eng/pipelines/coreclr/templates/build-native-test-assets-step.yml - - template: /eng/pipelines/common/upload-artifact-step.yml - parameters: - rootFolder: $(Build.SourcesDirectory)/artifacts/bin - includeRootFolder: false - archiveType: $(archiveType) - archiveExtension: $(archiveExtension) - tarCompression: $(tarCompression) - artifactName: BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig) - - template: /eng/pipelines/common/upload-artifact-step.yml - parameters: - rootFolder: $(Build.SourcesDirectory)/artifacts/helix - includeRootFolder: false - archiveType: $(archiveType) - archiveExtension: $(archiveExtension) - tarCompression: $(tarCompression) - artifactName: LibrariesTestArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig) - extraVariablesTemplates: - - template: /eng/pipelines/common/templates/runtimes/native-test-assets-variables.yml - parameters: - testGroup: outerloop - disableComponentGovernance: true # No shipping artifacts produced by this pipeline + # - template: /eng/pipelines/common/platform-matrix.yml + # parameters: + # jobTemplate: /eng/pipelines/common/global-build-job.yml + # buildConfig: checked + # platforms: + # - windows_x86 + # - windows_arm64 + # - osx_arm64 + # jobParameters: + # testGroup: outerloop + # buildArgs: -s clr+libs+libs.tests -rc $(_BuildConfig) -c Release /p:ArchiveTests=true + # timeoutInMinutes: 120 + # postBuildSteps: + # # Build CLR assets for x64 as well as the target as we need an x64 mcs + # - template: /eng/pipelines/common/templates/global-build-step.yml + # parameters: + # buildArgs: -s clr.spmi -c $(_BuildConfig) + # archParameter: -arch x64 + # - template: /eng/pipelines/coreclr/templates/build-native-test-assets-step.yml + # - template: /eng/pipelines/common/upload-artifact-step.yml + # parameters: + # rootFolder: $(Build.SourcesDirectory)/artifacts/bin + # includeRootFolder: false + # archiveType: $(archiveType) + # archiveExtension: $(archiveExtension) + # tarCompression: $(tarCompression) + # artifactName: BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig) + # - template: /eng/pipelines/common/upload-artifact-step.yml + # parameters: + # rootFolder: $(Build.SourcesDirectory)/artifacts/helix + # includeRootFolder: false + # archiveType: $(archiveType) + # archiveExtension: $(archiveExtension) + # tarCompression: $(tarCompression) + # artifactName: LibrariesTestArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig) + # extraVariablesTemplates: + # - template: /eng/pipelines/common/templates/runtimes/native-test-assets-variables.yml + # parameters: + # testGroup: outerloop + # disableComponentGovernance: true # No shipping artifacts produced by this pipeline - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/common/global-build-job.yml - buildConfig: checked - platforms: - - linux_arm - - linux_arm64 - jobParameters: - testGroup: outerloop - buildArgs: -s clr+libs+libs.tests -rc $(_BuildConfig) -c Release /p:ArchiveTests=true - timeoutInMinutes: 120 - postBuildSteps: - # Build CLR assets for x64 as well as the target as we need an x64 mcs - - template: /eng/pipelines/common/templates/global-build-step.yml - parameters: - buildArgs: -s clr.spmi -c $(_BuildConfig) - archParameter: -arch x64 - container: linux_x64 - - template: /eng/pipelines/coreclr/templates/build-native-test-assets-step.yml - - template: /eng/pipelines/common/upload-artifact-step.yml - parameters: - rootFolder: $(Build.SourcesDirectory)/artifacts/bin - includeRootFolder: false - archiveType: $(archiveType) - archiveExtension: $(archiveExtension) - tarCompression: $(tarCompression) - artifactName: BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig) - - template: /eng/pipelines/common/upload-artifact-step.yml - parameters: - rootFolder: $(Build.SourcesDirectory)/artifacts/helix - includeRootFolder: false - archiveType: $(archiveType) - archiveExtension: $(archiveExtension) - tarCompression: $(tarCompression) - artifactName: LibrariesTestArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig) - extraVariablesTemplates: - - template: /eng/pipelines/common/templates/runtimes/native-test-assets-variables.yml - parameters: - testGroup: outerloop + # - template: /eng/pipelines/common/platform-matrix.yml + # parameters: + # jobTemplate: /eng/pipelines/common/global-build-job.yml + # buildConfig: checked + # platforms: + # - linux_arm + # - linux_arm64 + # jobParameters: + # testGroup: outerloop + # buildArgs: -s clr+libs+libs.tests -rc $(_BuildConfig) -c Release /p:ArchiveTests=true + # timeoutInMinutes: 120 + # postBuildSteps: + # # Build CLR assets for x64 as well as the target as we need an x64 mcs + # - template: /eng/pipelines/common/templates/global-build-step.yml + # parameters: + # buildArgs: -s clr.spmi -c $(_BuildConfig) + # archParameter: -arch x64 + # container: linux_x64 + # - template: /eng/pipelines/coreclr/templates/build-native-test-assets-step.yml + # - template: /eng/pipelines/common/upload-artifact-step.yml + # parameters: + # rootFolder: $(Build.SourcesDirectory)/artifacts/bin + # includeRootFolder: false + # archiveType: $(archiveType) + # archiveExtension: $(archiveExtension) + # tarCompression: $(tarCompression) + # artifactName: BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig) + # - template: /eng/pipelines/common/upload-artifact-step.yml + # parameters: + # rootFolder: $(Build.SourcesDirectory)/artifacts/helix + # includeRootFolder: false + # archiveType: $(archiveType) + # archiveExtension: $(archiveExtension) + # tarCompression: $(tarCompression) + # artifactName: LibrariesTestArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig) + # extraVariablesTemplates: + # - template: /eng/pipelines/common/templates/runtimes/native-test-assets-variables.yml + # parameters: + # testGroup: outerloop disableComponentGovernance: true # No shipping artifacts produced by this pipeline - template: /eng/pipelines/common/platform-matrix.yml From bb70d0a6e249d33d385efdad69f3cf87847e757d Mon Sep 17 00:00:00 2001 From: TIHan Date: Thu, 22 Aug 2024 17:53:03 -0700 Subject: [PATCH 07/35] Temporarily comment out jobs --- eng/pipelines/coreclr/superpmi-collect.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/pipelines/coreclr/superpmi-collect.yml b/eng/pipelines/coreclr/superpmi-collect.yml index dbae6d64275174..cb495315764927 100644 --- a/eng/pipelines/coreclr/superpmi-collect.yml +++ b/eng/pipelines/coreclr/superpmi-collect.yml @@ -148,7 +148,7 @@ extends: # - template: /eng/pipelines/common/templates/runtimes/native-test-assets-variables.yml # parameters: # testGroup: outerloop - disableComponentGovernance: true # No shipping artifacts produced by this pipeline + # disableComponentGovernance: true # No shipping artifacts produced by this pipeline - template: /eng/pipelines/common/platform-matrix.yml parameters: From f68127925fadf8acfd4b19ddf019352215e12dba Mon Sep 17 00:00:00 2001 From: TIHan Date: Thu, 22 Aug 2024 19:30:45 -0700 Subject: [PATCH 08/35] Added --local --- .../templates/run-superpmi-collect-job.yml | 11 +++++----- src/coreclr/scripts/superpmi-collect.proj | 22 +++++++++++++++++++ src/coreclr/scripts/superpmi_aspnet.py | 18 +++++++++++++-- src/coreclr/scripts/superpmi_collect_setup.py | 3 ++- 4 files changed, 45 insertions(+), 9 deletions(-) diff --git a/eng/pipelines/coreclr/templates/run-superpmi-collect-job.yml b/eng/pipelines/coreclr/templates/run-superpmi-collect-job.yml index e2239d11202af8..590abf2147b976 100644 --- a/eng/pipelines/coreclr/templates/run-superpmi-collect-job.yml +++ b/eng/pipelines/coreclr/templates/run-superpmi-collect-job.yml @@ -103,6 +103,9 @@ jobs: - ${{ if eq(parameters.collectionName, 'smoke_tests') }}: - name: InputDirectory value: '$(Build.SourcesDirectory)/artifacts/tests/coreclr/obj/${{ parameters.osGroup }}.${{ parameters.archType }}.$(buildConfigUpper)/Managed/nativeaot/SmokeTests' + - ${{ if eq(parameters.collectionName, 'aspnet') }}: + - name: InputDirectory + value: '$(Core_Root_Dir)' workspace: clean: all @@ -116,12 +119,8 @@ jobs: displayName: Enable python venv condition: always() - - ${{ if eq(parameters.collectionName, 'aspnet') }}: - - script: $(PythonScript) $(Build.SourcesDirectory)/src/coreclr/scripts/superpmi_aspnet.py -source_directory $(Build.SourcesDirectory) -output_mch_path $(MchFilesLocation) -arch $(archType) -temp_location $(PayloadLocation) - displayName: ${{ format('SuperPMI ASPNET setup ({0})', parameters.osGroup) }} - - ${{ else }}: - - script: $(PythonScript) $(Build.SourcesDirectory)/src/coreclr/scripts/superpmi_collect_setup.py -payload_directory $(PayloadLocation) -source_directory $(Build.SourcesDirectory) -core_root_directory $(Core_Root_Dir) -arch $(archType) -platform $(osGroup) -mch_file_tag $(MchFileTag) -input_directory $(InputDirectory) -collection_name $(CollectionName) -collection_type $(CollectionType) -max_size 25 # size in MB - displayName: ${{ format('SuperPMI setup ({0})', parameters.osGroup) }} + - script: $(PythonScript) $(Build.SourcesDirectory)/src/coreclr/scripts/superpmi_collect_setup.py -payload_directory $(PayloadLocation) -source_directory $(Build.SourcesDirectory) -core_root_directory $(Core_Root_Dir) -arch $(archType) -platform $(osGroup) -mch_file_tag $(MchFileTag) -input_directory $(InputDirectory) -collection_name $(CollectionName) -collection_type $(CollectionType) -max_size 25 # size in MB + displayName: ${{ format('SuperPMI setup ({0})', parameters.osGroup) }} # Create required directories for merged mch collection and superpmi logs - ${{ if ne(parameters.osGroup, 'windows') }}: diff --git a/src/coreclr/scripts/superpmi-collect.proj b/src/coreclr/scripts/superpmi-collect.proj index 2b15fac5698134..17593d028360ad 100644 --- a/src/coreclr/scripts/superpmi-collect.proj +++ b/src/coreclr/scripts/superpmi-collect.proj @@ -84,6 +84,13 @@ $HELIX_WORKITEM_PAYLOAD/performance + + %HELIX_WORKITEM_PAYLOAD% + + + $HELIX_WORKITEM_PAYLOAD + + @@ -145,6 +152,11 @@ 3:00 + + $(Python) $(AspNetDirectory)$(FileSeparatorChar)src$(FileSeparatorChar)coreclr$(FileSeparatorChar)scripts$(FileSeparatorChar)superpmi_aspnet.py -source_directory $(AspNetDirectory) -arch $(Architecture) + 3:00 + + false false @@ -282,6 +294,16 @@ + + + $(CollectionName).$(CollectionType).%(HelixWorkItem.Index).$(MchFileTag) + $(WorkItemDirectory) + $(WorkItemCommand) -output_mch_path $(OutputMchPath)$(FileSeparatorChar)%(OutputFileName).mch + $(WorkItemTimeout) + %(OutputFileName).mch;%(OutputFileName).mch.mct;%(OutputFileName).log + + + $(CollectionName).$(CollectionType).%(HelixWorkItem.Index).$(MchFileTag) diff --git a/src/coreclr/scripts/superpmi_aspnet.py b/src/coreclr/scripts/superpmi_aspnet.py index 20bb019758210d..404961f56e8794 100644 --- a/src/coreclr/scripts/superpmi_aspnet.py +++ b/src/coreclr/scripts/superpmi_aspnet.py @@ -32,6 +32,7 @@ parser.add_argument("-output_mch_path", help="Absolute path to the mch file to produce") parser.add_argument("-arch", help="Architecture") parser.add_argument("-temp_location", required=False, help="Location to temporarily download ASPNET benchmarks and crank") +parser.add_argument("--local", action="store_true", default=False) def setup_args(args): """ Setup the args for SuperPMI to use. @@ -68,6 +69,8 @@ def setup_args(args): coreclr_args.temp_location = tempfile.TemporaryDirectory().name coreclr_args.temp_is_explicit = False + coreclr_args.local = args.local + return coreclr_args @@ -142,6 +145,8 @@ def build_and_run(coreclr_args): if not coreclr_args.temp_is_explicit: os.mkdir(temp_location) + elif not path.isdir(temp_location): + os.mkdir(temp_location) print ("Executing in " + temp_location) os.chdir(temp_location) @@ -153,11 +158,15 @@ def build_and_run(coreclr_args): dotnet_path = path.join(source_directory, ".dotnet") dotnet_exe = path.join(dotnet_path, "dotnet.exe") if is_windows else path.join(dotnet_path, "dotnet") # run_command([dotnet_exe, "--info"], temp_location, _exit_on_fail=True) - os.environ['DOTNET_ROOT'] = dotnet_path + #os.environ['DOTNET_ROOT'] = dotnet_path ## install crank as local tool run_command( [dotnet_exe, "tool", "install", "Microsoft.Crank.Controller", "--version", "0.2.0-*", "--tool-path", temp_location], _exit_on_fail=True) + + if coreclr_args.local: + run_command( + [dotnet_exe, "tool", "install", "Microsoft.Crank.Agent", "--version", "0.2.0-*", "--tool-path", temp_location], _exit_on_fail=True) ## ideally just do sparse clone, but this doesn't work locally ## git clone --filter=blob:none --no-checkout https://github.com/aspnet/benchmarks @@ -172,6 +181,7 @@ def build_and_run(coreclr_args): ["git", "clone", "--quiet", "--depth", "1", "https://github.com/aspnet/benchmarks"], temp_location, _exit_on_fail=True) crank_app = path.join(temp_location, "crank") + crank_agent_app = path.join(temp_location, "crank-agent") mcs_path = determine_mcs_tool_path(coreclr_args) superpmi_path = determine_superpmi_tool_path(coreclr_args) @@ -210,7 +220,11 @@ def build_and_run(coreclr_args): # runtime_options_list = [("Dummy=0", )] mch_file = path.join(coreclr_args.output_mch_path, "aspnet.run." + target_os + "." + target_arch + ".checked.mch") - benchmark_machine = determine_benchmark_machine(coreclr_args) + + if coreclr_args.local: + benchmark_machine = "local" + else: + benchmark_machine = determine_benchmark_machine(coreclr_args) jitname = determine_native_name(coreclr_args, "clrjit", target_os) coreclrname = determine_native_name(coreclr_args, "coreclr", target_os) diff --git a/src/coreclr/scripts/superpmi_collect_setup.py b/src/coreclr/scripts/superpmi_collect_setup.py index 40c3a768e68283..ec26a639af3a44 100644 --- a/src/coreclr/scripts/superpmi_collect_setup.py +++ b/src/coreclr/scripts/superpmi_collect_setup.py @@ -405,7 +405,6 @@ def setup_benchmark(workitem_directory, arch): get_python_name() + [dotnet_install_script, "install", "--channels", "9.0", "--architecture", arch, "--install-dir", dotnet_directory, "--verbose"]) - def get_python_name(): """Gets the python name @@ -490,6 +489,8 @@ def main(main_args): if coreclr_args.collection_name == "benchmarks" or coreclr_args.collection_name == "realworld": # Setup benchmarks setup_benchmark(workitem_payload_directory, arch) + elif coreclr_args.collection_name == "aspnet": + () else: # Setup for pmi/crossgen2/nativeaot runs From 2e858692f95b9a1f48813ed13e23c4f387a92214 Mon Sep 17 00:00:00 2001 From: TIHan Date: Thu, 22 Aug 2024 19:31:55 -0700 Subject: [PATCH 09/35] Added --local --- .../coreclr/templates/run-superpmi-collect-job.yml | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/eng/pipelines/coreclr/templates/run-superpmi-collect-job.yml b/eng/pipelines/coreclr/templates/run-superpmi-collect-job.yml index 590abf2147b976..ef0e0bd70b8724 100644 --- a/eng/pipelines/coreclr/templates/run-superpmi-collect-job.yml +++ b/eng/pipelines/coreclr/templates/run-superpmi-collect-job.yml @@ -103,9 +103,6 @@ jobs: - ${{ if eq(parameters.collectionName, 'smoke_tests') }}: - name: InputDirectory value: '$(Build.SourcesDirectory)/artifacts/tests/coreclr/obj/${{ parameters.osGroup }}.${{ parameters.archType }}.$(buildConfigUpper)/Managed/nativeaot/SmokeTests' - - ${{ if eq(parameters.collectionName, 'aspnet') }}: - - name: InputDirectory - value: '$(Core_Root_Dir)' workspace: clean: all @@ -119,8 +116,12 @@ jobs: displayName: Enable python venv condition: always() - - script: $(PythonScript) $(Build.SourcesDirectory)/src/coreclr/scripts/superpmi_collect_setup.py -payload_directory $(PayloadLocation) -source_directory $(Build.SourcesDirectory) -core_root_directory $(Core_Root_Dir) -arch $(archType) -platform $(osGroup) -mch_file_tag $(MchFileTag) -input_directory $(InputDirectory) -collection_name $(CollectionName) -collection_type $(CollectionType) -max_size 25 # size in MB - displayName: ${{ format('SuperPMI setup ({0})', parameters.osGroup) }} + - ${{ if eq(parameters.collectionName, 'aspnet') }}: + - script: $(PythonScript) $(Build.SourcesDirectory)/src/coreclr/scripts/superpmi_aspnet.py -source_directory $(Build.SourcesDirectory) -output_mch_path $(MchFilesLocation) -arch $(archType) -temp_location $(PayloadLocation) --local + displayName: ${{ format('SuperPMI ASPNET setup ({0})', parameters.osGroup) }} + - ${{ else }}: + - script: $(PythonScript) $(Build.SourcesDirectory)/src/coreclr/scripts/superpmi_collect_setup.py -payload_directory $(PayloadLocation) -source_directory $(Build.SourcesDirectory) -core_root_directory $(Core_Root_Dir) -arch $(archType) -platform $(osGroup) -mch_file_tag $(MchFileTag) -input_directory $(InputDirectory) -collection_name $(CollectionName) -collection_type $(CollectionType) -max_size 25 # size in MB + displayName: ${{ format('SuperPMI setup ({0})', parameters.osGroup) }} # Create required directories for merged mch collection and superpmi logs - ${{ if ne(parameters.osGroup, 'windows') }}: From c4846e7255c0b9f5e4e74599a614bd315dd3de4a Mon Sep 17 00:00:00 2001 From: TIHan Date: Thu, 22 Aug 2024 19:59:52 -0700 Subject: [PATCH 10/35] Do not use local for now --- eng/pipelines/coreclr/templates/run-superpmi-collect-job.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/pipelines/coreclr/templates/run-superpmi-collect-job.yml b/eng/pipelines/coreclr/templates/run-superpmi-collect-job.yml index ef0e0bd70b8724..e2239d11202af8 100644 --- a/eng/pipelines/coreclr/templates/run-superpmi-collect-job.yml +++ b/eng/pipelines/coreclr/templates/run-superpmi-collect-job.yml @@ -117,7 +117,7 @@ jobs: condition: always() - ${{ if eq(parameters.collectionName, 'aspnet') }}: - - script: $(PythonScript) $(Build.SourcesDirectory)/src/coreclr/scripts/superpmi_aspnet.py -source_directory $(Build.SourcesDirectory) -output_mch_path $(MchFilesLocation) -arch $(archType) -temp_location $(PayloadLocation) --local + - script: $(PythonScript) $(Build.SourcesDirectory)/src/coreclr/scripts/superpmi_aspnet.py -source_directory $(Build.SourcesDirectory) -output_mch_path $(MchFilesLocation) -arch $(archType) -temp_location $(PayloadLocation) displayName: ${{ format('SuperPMI ASPNET setup ({0})', parameters.osGroup) }} - ${{ else }}: - script: $(PythonScript) $(Build.SourcesDirectory)/src/coreclr/scripts/superpmi_collect_setup.py -payload_directory $(PayloadLocation) -source_directory $(Build.SourcesDirectory) -core_root_directory $(Core_Root_Dir) -arch $(archType) -platform $(osGroup) -mch_file_tag $(MchFileTag) -input_directory $(InputDirectory) -collection_name $(CollectionName) -collection_type $(CollectionType) -max_size 25 # size in MB From 4b993ae88d91e4580b158781a6aed3fc578c18c0 Mon Sep 17 00:00:00 2001 From: TIHan Date: Fri, 23 Aug 2024 12:43:19 -0700 Subject: [PATCH 11/35] Able to launch crank-agent locally --- .../templates/run-superpmi-collect-job.yml | 2 +- src/coreclr/scripts/superpmi_aspnet.py | 18 ++++++++---------- 2 files changed, 9 insertions(+), 11 deletions(-) diff --git a/eng/pipelines/coreclr/templates/run-superpmi-collect-job.yml b/eng/pipelines/coreclr/templates/run-superpmi-collect-job.yml index e2239d11202af8..ef0e0bd70b8724 100644 --- a/eng/pipelines/coreclr/templates/run-superpmi-collect-job.yml +++ b/eng/pipelines/coreclr/templates/run-superpmi-collect-job.yml @@ -117,7 +117,7 @@ jobs: condition: always() - ${{ if eq(parameters.collectionName, 'aspnet') }}: - - script: $(PythonScript) $(Build.SourcesDirectory)/src/coreclr/scripts/superpmi_aspnet.py -source_directory $(Build.SourcesDirectory) -output_mch_path $(MchFilesLocation) -arch $(archType) -temp_location $(PayloadLocation) + - script: $(PythonScript) $(Build.SourcesDirectory)/src/coreclr/scripts/superpmi_aspnet.py -source_directory $(Build.SourcesDirectory) -output_mch_path $(MchFilesLocation) -arch $(archType) -temp_location $(PayloadLocation) --local displayName: ${{ format('SuperPMI ASPNET setup ({0})', parameters.osGroup) }} - ${{ else }}: - script: $(PythonScript) $(Build.SourcesDirectory)/src/coreclr/scripts/superpmi_collect_setup.py -payload_directory $(PayloadLocation) -source_directory $(Build.SourcesDirectory) -core_root_directory $(Core_Root_Dir) -arch $(archType) -platform $(osGroup) -mch_file_tag $(MchFileTag) -input_directory $(InputDirectory) -collection_name $(CollectionName) -collection_type $(CollectionType) -max_size 25 # size in MB diff --git a/src/coreclr/scripts/superpmi_aspnet.py b/src/coreclr/scripts/superpmi_aspnet.py index 404961f56e8794..70adae473b10f3 100644 --- a/src/coreclr/scripts/superpmi_aspnet.py +++ b/src/coreclr/scripts/superpmi_aspnet.py @@ -23,6 +23,7 @@ from coreclr_arguments import * from superpmi import TempDir, determine_mcs_tool_path, determine_superpmi_tool_path, is_nonzero_length_file from jitutil import run_command +from subprocess import Popen, DEVNULL # Start of parser object creation. is_windows = platform.system() == "Windows" @@ -137,10 +138,6 @@ def build_and_run(coreclr_args): checked_root = path.join(source_directory, "artifacts", "bin", "coreclr", target_os + "." + coreclr_args.arch + ".Checked") release_root = path.join(source_directory, "artifacts", "bin", "coreclr", target_os + "." + coreclr_args.arch + ".Release") - # We'll use repo script to install dotnet - dotnet_install_script_name = "dotnet-install.cmd" if is_windows else "dotnet-install.sh" - dotnet_install_script_path = path.join(source_directory, "eng", "common", dotnet_install_script_name) - temp_location = coreclr_args.temp_location if not coreclr_args.temp_is_explicit: @@ -151,14 +148,8 @@ def build_and_run(coreclr_args): print ("Executing in " + temp_location) os.chdir(temp_location) - # install dotnet 8.0 - run_command([dotnet_install_script_path, "-Version", "8.0.0"], temp_location, _exit_on_fail=True) - os.environ['DOTNET_MULTILEVEL_LOOKUP'] = '0' - os.environ['DOTNET_SKIP_FIRST_TIME_EXPERIENCE'] = '1' dotnet_path = path.join(source_directory, ".dotnet") dotnet_exe = path.join(dotnet_path, "dotnet.exe") if is_windows else path.join(dotnet_path, "dotnet") - # run_command([dotnet_exe, "--info"], temp_location, _exit_on_fail=True) - #os.environ['DOTNET_ROOT'] = dotnet_path ## install crank as local tool run_command( @@ -185,6 +176,10 @@ def build_and_run(coreclr_args): mcs_path = determine_mcs_tool_path(coreclr_args) superpmi_path = determine_superpmi_tool_path(coreclr_args) + crank_agent_p = None + if coreclr_args.local: + crank_agent_p = Popen(crank_agent_app, stdout=DEVNULL) + # todo: add grpc/signalr, perhaps configname_scenario_list = [ @@ -277,6 +272,9 @@ def build_and_run(coreclr_args): print(' '.join(crank_app_args)) subprocess.run(crank_app_args, cwd=temp_location) + if crank_agent_p is not None: + crank_agent_p.terminate() + # merge command = [mcs_path, "-merge", "temp.mch", "*.mc", "-dedup", "-thin"] run_command(command, temp_location) From 9fd5c590f8167a9e962fef7f05c771d99eca135f Mon Sep 17 00:00:00 2001 From: TIHan Date: Fri, 23 Aug 2024 12:54:14 -0700 Subject: [PATCH 12/35] More fixes --- src/coreclr/scripts/superpmi_aspnet.py | 135 +++++++++++++------------ 1 file changed, 68 insertions(+), 67 deletions(-) diff --git a/src/coreclr/scripts/superpmi_aspnet.py b/src/coreclr/scripts/superpmi_aspnet.py index 70adae473b10f3..681400293f92f4 100644 --- a/src/coreclr/scripts/superpmi_aspnet.py +++ b/src/coreclr/scripts/superpmi_aspnet.py @@ -136,7 +136,7 @@ def build_and_run(coreclr_args): target_os = coreclr_args.host_os checked_root = path.join(source_directory, "artifacts", "bin", "coreclr", target_os + "." + coreclr_args.arch + ".Checked") - release_root = path.join(source_directory, "artifacts", "bin", "coreclr", target_os + "." + coreclr_args.arch + ".Release") + # release_root = path.join(source_directory, "artifacts", "bin", "coreclr", target_os + "." + coreclr_args.arch + ".Release") temp_location = coreclr_args.temp_location @@ -176,27 +176,23 @@ def build_and_run(coreclr_args): mcs_path = determine_mcs_tool_path(coreclr_args) superpmi_path = determine_superpmi_tool_path(coreclr_args) - crank_agent_p = None - if coreclr_args.local: - crank_agent_p = Popen(crank_agent_app, stdout=DEVNULL) - # todo: add grpc/signalr, perhaps configname_scenario_list = [ ("platform", "plaintext"), - ("json", "json"), - ("plaintext", "mvc"), - ("database", "fortunes_dapper"), - ("database", "fortunes_ef_mvc_https"), - ("database", "updates"), - ("proxy", "proxy-yarp"), - ("staticfiles", "static"), - ("websocket", "websocket"), - ("orchard", "about-sqlite"), - ("signalr", "signalr"), - ("grpc", "grpcaspnetcoreserver-grpcnetclient"), - ("efcore", "NavigationsQuery"), - ("efcore", "Funcletization") + # ("json", "json"), + # ("plaintext", "mvc"), + # ("database", "fortunes_dapper"), + # ("database", "fortunes_ef_mvc_https"), + # ("database", "updates"), + # ("proxy", "proxy-yarp"), + # ("staticfiles", "static"), + # ("websocket", "websocket"), + # ("orchard", "about-sqlite"), + # ("signalr", "signalr"), + # ("grpc", "grpcaspnetcoreserver-grpcnetclient"), + # ("efcore", "NavigationsQuery"), + # ("efcore", "Funcletization") ] # configname_scenario_list = [("quic", "read-write")] @@ -205,11 +201,11 @@ def build_and_run(coreclr_args): runtime_options_list = [ ("Dummy=0",), - ("TieredCompilation=0", ), - ("TieredPGO=0",), - ("TieredPGO=1", "ReadyToRun=0"), - ("ReadyToRun=0", "OSR_HitLimit=0", "TC_OnStackReplacement_InitialCounter=10"), - ("TC_PartialCompilation=1",) + # ("TieredCompilation=0", ), + # ("TieredPGO=0",), + # ("TieredPGO=1", "ReadyToRun=0"), + # ("ReadyToRun=0", "OSR_HitLimit=0", "TC_OnStackReplacement_InitialCounter=10"), + # ("TC_PartialCompilation=1",) ] # runtime_options_list = [("Dummy=0", )] @@ -228,52 +224,57 @@ def build_and_run(coreclr_args): jitpath = path.join(".", jitname) jitlib = path.join(checked_root, jitname) - coreclr = path.join(release_root, coreclrname) - corelib = path.join(release_root, corelibname) + coreclr = path.join(checked_root, coreclrname) + corelib = path.join(checked_root, corelibname) spmilib = path.join(checked_root, spminame) - for (configName, scenario) in configname_scenario_list: - configYml = configName + ".benchmarks.yml" - configFile = path.join(temp_location, "benchmarks", "scenarios", configYml) - - crank_arguments = ["--config", configFile, - "--profile", benchmark_machine, - "--scenario", scenario, - "--application.framework", "net9.0", - "--application.channel", "edge", - "--application.sdkVersion", "latest", - "--application.environmentVariables", "DOTNET_JitName=" + spminame, - "--application.environmentVariables", "SuperPMIShimLogPath=.", - "--application.environmentVariables", "SuperPMIShimPath=" + jitpath, - "--application.environmentVariables", "DOTNET_EnableExtraSuperPmiQueries=1", - "--application.options.downloadFiles", "*.mc", - "--application.options.displayOutput", "true", -# "--application.options.dumpType", "full", -# "--application.options.fetch", "true", - "--application.options.outputFiles", spmilib, - "--application.options.outputFiles", jitlib, - "--application.options.outputFiles", coreclr, - "--application.options.outputFiles", corelib] - - for runtime_options in runtime_options_list: - runtime_arguments = [] - for runtime_option in runtime_options: - runtime_arguments.append("--application.environmentVariables") - runtime_arguments.append("DOTNET_" + runtime_option) - - print("") - print("================================") - print("Config: " + configName + " scenario: " + scenario + " options: " + " ".join(runtime_options)) - print("================================") - print("") - - description = ["--description", configName + "-" + scenario + "-" + "-".join(runtime_options)] - crank_app_args = [crank_app] + crank_arguments + description + runtime_arguments - print(' '.join(crank_app_args)) - subprocess.run(crank_app_args, cwd=temp_location) - - if crank_agent_p is not None: - crank_agent_p.terminate() + crank_agent_p = None + if coreclr_args.local: + crank_agent_p = Popen(crank_agent_app, stdout=DEVNULL) + + try: + for (configName, scenario) in configname_scenario_list: + configYml = configName + ".benchmarks.yml" + configFile = path.join(temp_location, "benchmarks", "scenarios", configYml) + + crank_arguments = ["--config", configFile, + "--profile", benchmark_machine, + "--scenario", scenario, + "--application.framework", "net9.0", + "--application.channel", "edge", + "--application.sdkVersion", "latest", + "--application.environmentVariables", "DOTNET_JitName=" + spminame, + "--application.environmentVariables", "SuperPMIShimLogPath=.", + "--application.environmentVariables", "SuperPMIShimPath=" + jitpath, + "--application.environmentVariables", "DOTNET_EnableExtraSuperPmiQueries=1", + "--application.options.downloadFiles", "*.mc", + "--application.options.displayOutput", "true", + # "--application.options.dumpType", "full", + # "--application.options.fetch", "true", + "--application.options.outputFiles", spmilib, + "--application.options.outputFiles", jitlib, + "--application.options.outputFiles", coreclr, + "--application.options.outputFiles", corelib] + + for runtime_options in runtime_options_list: + runtime_arguments = [] + for runtime_option in runtime_options: + runtime_arguments.append("--application.environmentVariables") + runtime_arguments.append("DOTNET_" + runtime_option) + + print("") + print("================================") + print("Config: " + configName + " scenario: " + scenario + " options: " + " ".join(runtime_options)) + print("================================") + print("") + + description = ["--description", configName + "-" + scenario + "-" + "-".join(runtime_options)] + crank_app_args = [crank_app] + crank_arguments + description + runtime_arguments + print(' '.join(crank_app_args)) + subprocess.run(crank_app_args, cwd=temp_location) + finally: + if crank_agent_p is not None: + crank_agent_p.terminate() # merge command = [mcs_path, "-merge", "temp.mch", "*.mc", "-dedup", "-thin"] From 9e663dc528c8598e1d2a3788855f4d01b3365108 Mon Sep 17 00:00:00 2001 From: TIHan Date: Fri, 23 Aug 2024 14:56:02 -0700 Subject: [PATCH 13/35] Trying to use helix --- .../templates/run-superpmi-collect-job.yml | 5 +--- src/coreclr/scripts/superpmi-collect.proj | 9 +------ src/coreclr/scripts/superpmi_aspnet.py | 27 ++++++++----------- 3 files changed, 13 insertions(+), 28 deletions(-) diff --git a/eng/pipelines/coreclr/templates/run-superpmi-collect-job.yml b/eng/pipelines/coreclr/templates/run-superpmi-collect-job.yml index ef0e0bd70b8724..73de56491ded8c 100644 --- a/eng/pipelines/coreclr/templates/run-superpmi-collect-job.yml +++ b/eng/pipelines/coreclr/templates/run-superpmi-collect-job.yml @@ -116,10 +116,7 @@ jobs: displayName: Enable python venv condition: always() - - ${{ if eq(parameters.collectionName, 'aspnet') }}: - - script: $(PythonScript) $(Build.SourcesDirectory)/src/coreclr/scripts/superpmi_aspnet.py -source_directory $(Build.SourcesDirectory) -output_mch_path $(MchFilesLocation) -arch $(archType) -temp_location $(PayloadLocation) --local - displayName: ${{ format('SuperPMI ASPNET setup ({0})', parameters.osGroup) }} - - ${{ else }}: + - ${{ if ne(parameters.collectionName, 'aspnet') }}: - script: $(PythonScript) $(Build.SourcesDirectory)/src/coreclr/scripts/superpmi_collect_setup.py -payload_directory $(PayloadLocation) -source_directory $(Build.SourcesDirectory) -core_root_directory $(Core_Root_Dir) -arch $(archType) -platform $(osGroup) -mch_file_tag $(MchFileTag) -input_directory $(InputDirectory) -collection_name $(CollectionName) -collection_type $(CollectionType) -max_size 25 # size in MB displayName: ${{ format('SuperPMI setup ({0})', parameters.osGroup) }} diff --git a/src/coreclr/scripts/superpmi-collect.proj b/src/coreclr/scripts/superpmi-collect.proj index 17593d028360ad..638c27b8644f3e 100644 --- a/src/coreclr/scripts/superpmi-collect.proj +++ b/src/coreclr/scripts/superpmi-collect.proj @@ -84,13 +84,6 @@ $HELIX_WORKITEM_PAYLOAD/performance - - %HELIX_WORKITEM_PAYLOAD% - - - $HELIX_WORKITEM_PAYLOAD - - @@ -153,7 +146,7 @@ - $(Python) $(AspNetDirectory)$(FileSeparatorChar)src$(FileSeparatorChar)coreclr$(FileSeparatorChar)scripts$(FileSeparatorChar)superpmi_aspnet.py -source_directory $(AspNetDirectory) -arch $(Architecture) + $(Python) $(SuperPMIDirectory)$(FileSeparatorChar)superpmi_aspnet.py -core_root_directory $(SuperPMIDirectory) -arch $(Architecture) -temp_location $(SuperPMIDirectory)$(FileSeparatorChar)tmp --local 3:00 diff --git a/src/coreclr/scripts/superpmi_aspnet.py b/src/coreclr/scripts/superpmi_aspnet.py index 681400293f92f4..c12c5206dec931 100644 --- a/src/coreclr/scripts/superpmi_aspnet.py +++ b/src/coreclr/scripts/superpmi_aspnet.py @@ -29,7 +29,7 @@ is_windows = platform.system() == "Windows" parser = argparse.ArgumentParser(description="description") -parser.add_argument("-source_directory", help="path to source directory") +parser.add_argument("-core_root_directory", required=True, help="Path to Core_Root directory") parser.add_argument("-output_mch_path", help="Absolute path to the mch file to produce") parser.add_argument("-arch", help="Architecture") parser.add_argument("-temp_location", required=False, help="Location to temporarily download ASPNET benchmarks and crank") @@ -49,9 +49,9 @@ def setup_args(args): require_built_test_dir=False, default_build_type="Checked") coreclr_args.verify(args, - "source_directory", - lambda source_directory: os.path.isdir(source_directory), - "source_directory doesn't exist") + "core_root_directory", + lambda core_root_directory: os.path.isdir(core_root_directory), + "core_root_directory doesn't exist") coreclr_args.verify(args, "output_mch_path", @@ -131,13 +131,10 @@ def build_and_run(coreclr_args): coreclr_args (CoreClrArguments): Arguments use to drive output_mch_name (string): Name of output mch file name """ - source_directory = coreclr_args.source_directory + core_root_directory = coreclr_args.core_root_directory target_arch = coreclr_args.arch target_os = coreclr_args.host_os - checked_root = path.join(source_directory, "artifacts", "bin", "coreclr", target_os + "." + coreclr_args.arch + ".Checked") - # release_root = path.join(source_directory, "artifacts", "bin", "coreclr", target_os + "." + coreclr_args.arch + ".Release") - temp_location = coreclr_args.temp_location if not coreclr_args.temp_is_explicit: @@ -148,8 +145,8 @@ def build_and_run(coreclr_args): print ("Executing in " + temp_location) os.chdir(temp_location) - dotnet_path = path.join(source_directory, ".dotnet") - dotnet_exe = path.join(dotnet_path, "dotnet.exe") if is_windows else path.join(dotnet_path, "dotnet") + dotnet_exe = "dotnet.exe" if is_windows else "dotnet" + run_command([dotnet_exe, "--info"], temp_location, _exit_on_fail=True) ## install crank as local tool run_command( @@ -223,10 +220,10 @@ def build_and_run(coreclr_args): corelibname = "System.Private.CoreLib.dll" jitpath = path.join(".", jitname) - jitlib = path.join(checked_root, jitname) - coreclr = path.join(checked_root, coreclrname) - corelib = path.join(checked_root, corelibname) - spmilib = path.join(checked_root, spminame) + jitlib = path.join(core_root_directory, jitname) + coreclr = path.join(core_root_directory, coreclrname) + corelib = path.join(core_root_directory, corelibname) + spmilib = path.join(core_root_directory, spminame) crank_agent_p = None if coreclr_args.local: @@ -303,8 +300,6 @@ def build_and_run(coreclr_args): command = [mcs_path, "-jitflags", mch_file] run_command(command, temp_location) - os.chdir(source_directory ) - if not coreclr_args.temp_is_explicit: shutil.rmtree(temp_location, ignore_errors=True) From acfc97419fcdb7359976d78ee7945d3124049f1c Mon Sep 17 00:00:00 2001 From: TIHan Date: Fri, 23 Aug 2024 16:38:13 -0700 Subject: [PATCH 14/35] Fix proj --- src/coreclr/scripts/superpmi-collect.proj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/coreclr/scripts/superpmi-collect.proj b/src/coreclr/scripts/superpmi-collect.proj index 638c27b8644f3e..b9e48f5c5d272e 100644 --- a/src/coreclr/scripts/superpmi-collect.proj +++ b/src/coreclr/scripts/superpmi-collect.proj @@ -198,7 +198,7 @@ - + From 9db20cf5d27c84eecd9351d2a2e6f1f3cfd78248 Mon Sep 17 00:00:00 2001 From: TIHan Date: Thu, 29 Aug 2024 12:32:04 -0700 Subject: [PATCH 15/35] Temporarily set up benchmark --- src/coreclr/scripts/superpmi_collect_setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/coreclr/scripts/superpmi_collect_setup.py b/src/coreclr/scripts/superpmi_collect_setup.py index 8eea4c03ee1a02..7bf27119bb94db 100644 --- a/src/coreclr/scripts/superpmi_collect_setup.py +++ b/src/coreclr/scripts/superpmi_collect_setup.py @@ -506,7 +506,7 @@ def main(main_args): # Setup benchmarks setup_benchmark(workitem_payload_directory, arch) elif coreclr_args.collection_name == "aspnet": - () + setup_benchmark(workitem_payload_directory, arch) else: # Setup for pmi/crossgen2/nativeaot runs From a2ec5e4eacc85dc871ff8ee4b134482e54ade314 Mon Sep 17 00:00:00 2001 From: TIHan Date: Thu, 29 Aug 2024 17:02:15 -0700 Subject: [PATCH 16/35] Trying to fix it --- eng/pipelines/coreclr/templates/run-superpmi-collect-job.yml | 5 ++--- src/coreclr/scripts/superpmi_collect_setup.py | 2 +- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/eng/pipelines/coreclr/templates/run-superpmi-collect-job.yml b/eng/pipelines/coreclr/templates/run-superpmi-collect-job.yml index 5c89f790ca2c5e..9bf2ed9d5bceee 100644 --- a/eng/pipelines/coreclr/templates/run-superpmi-collect-job.yml +++ b/eng/pipelines/coreclr/templates/run-superpmi-collect-job.yml @@ -122,9 +122,8 @@ jobs: displayName: Enable python venv condition: always() - - ${{ if ne(parameters.collectionName, 'aspnet') }}: - - script: $(PythonScript) $(Build.SourcesDirectory)/src/coreclr/scripts/superpmi_collect_setup.py -payload_directory $(PayloadLocation) -source_directory $(Build.SourcesDirectory) -core_root_directory $(Core_Root_Dir) -arch $(archType) -platform $(osGroup) -mch_file_tag $(MchFileTag) -input_directory $(InputDirectory) -collection_name $(CollectionName) -collection_type $(CollectionType) $(PublicQueuesCLIArg) -max_size 25 # size in MB - displayName: ${{ format('SuperPMI setup ({0})', parameters.osGroup) }} + - script: $(PythonScript) $(Build.SourcesDirectory)/src/coreclr/scripts/superpmi_collect_setup.py -payload_directory $(PayloadLocation) -source_directory $(Build.SourcesDirectory) -core_root_directory $(Core_Root_Dir) -arch $(archType) -platform $(osGroup) -mch_file_tag $(MchFileTag) -input_directory $(InputDirectory) -collection_name $(CollectionName) -collection_type $(CollectionType) $(PublicQueuesCLIArg) -max_size 25 # size in MB + displayName: ${{ format('SuperPMI setup ({0})', parameters.osGroup) }} # Create required directories for merged mch collection and superpmi logs - ${{ if ne(parameters.osGroup, 'windows') }}: diff --git a/src/coreclr/scripts/superpmi_collect_setup.py b/src/coreclr/scripts/superpmi_collect_setup.py index 7bf27119bb94db..8eea4c03ee1a02 100644 --- a/src/coreclr/scripts/superpmi_collect_setup.py +++ b/src/coreclr/scripts/superpmi_collect_setup.py @@ -506,7 +506,7 @@ def main(main_args): # Setup benchmarks setup_benchmark(workitem_payload_directory, arch) elif coreclr_args.collection_name == "aspnet": - setup_benchmark(workitem_payload_directory, arch) + () else: # Setup for pmi/crossgen2/nativeaot runs From 3a1bc38038025c0f9a72e464849765ea3cedc7cb Mon Sep 17 00:00:00 2001 From: TIHan Date: Thu, 29 Aug 2024 19:30:51 -0700 Subject: [PATCH 17/35] Trying to fix it --- src/coreclr/scripts/superpmi_collect_setup.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/coreclr/scripts/superpmi_collect_setup.py b/src/coreclr/scripts/superpmi_collect_setup.py index 8eea4c03ee1a02..dd8b7284575f9e 100644 --- a/src/coreclr/scripts/superpmi_collect_setup.py +++ b/src/coreclr/scripts/superpmi_collect_setup.py @@ -506,7 +506,8 @@ def main(main_args): # Setup benchmarks setup_benchmark(workitem_payload_directory, arch) elif coreclr_args.collection_name == "aspnet": - () + # TODO: Don't do this. + setup_benchmark(workitem_payload_directory, arch) else: # Setup for pmi/crossgen2/nativeaot runs From 1fd05f0e44bfc8461c082f246c1f2046bf2bfffe Mon Sep 17 00:00:00 2001 From: TIHan Date: Thu, 29 Aug 2024 21:53:10 -0700 Subject: [PATCH 18/35] Trying to fix it --- src/coreclr/scripts/superpmi_collect_setup.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/coreclr/scripts/superpmi_collect_setup.py b/src/coreclr/scripts/superpmi_collect_setup.py index dd8b7284575f9e..606d9f6b109157 100644 --- a/src/coreclr/scripts/superpmi_collect_setup.py +++ b/src/coreclr/scripts/superpmi_collect_setup.py @@ -505,9 +505,6 @@ def main(main_args): if coreclr_args.collection_name == "benchmarks" or coreclr_args.collection_name == "realworld": # Setup benchmarks setup_benchmark(workitem_payload_directory, arch) - elif coreclr_args.collection_name == "aspnet": - # TODO: Don't do this. - setup_benchmark(workitem_payload_directory, arch) else: # Setup for pmi/crossgen2/nativeaot runs @@ -554,7 +551,7 @@ def make_readable(folder_name): # We need the PMI tool if we're doing a PMI collection. We could download a cached copy from Azure DevOps JIT blob # storage, but instead we clone and build jitutils to build pmi.dll. - if coreclr_args.collection_type == "pmi": + if coreclr_args.collection_type == "pmi" or coreclr_args.collection_type == "aspnet": try: with TempDir() as jitutils_directory: run_command( From 6ced8a3b5be3b33cfbea08c5d53c98ad223fa549 Mon Sep 17 00:00:00 2001 From: TIHan Date: Fri, 30 Aug 2024 09:58:16 -0700 Subject: [PATCH 19/35] Trying to properly set up aspnet --- src/coreclr/scripts/superpmi-collect.proj | 38 +++++++++++++++---- src/coreclr/scripts/superpmi_collect_setup.py | 21 +++++++++- 2 files changed, 50 insertions(+), 9 deletions(-) diff --git a/src/coreclr/scripts/superpmi-collect.proj b/src/coreclr/scripts/superpmi-collect.proj index bb34b1d72a34e9..9a26d218e0b383 100644 --- a/src/coreclr/scripts/superpmi-collect.proj +++ b/src/coreclr/scripts/superpmi-collect.proj @@ -84,6 +84,15 @@ $HELIX_WORKITEM_PAYLOAD/performance + + + + %HELIX_WORKITEM_PAYLOAD%\aspnet + + + $HELIX_WORKITEM_PAYLOAD/aspnet + + @@ -146,7 +155,7 @@ - $(Python) $(SuperPMIDirectory)$(FileSeparatorChar)superpmi_aspnet.py -core_root_directory $(SuperPMIDirectory) -arch $(Architecture) -temp_location $(SuperPMIDirectory)$(FileSeparatorChar)tmp --local + $(Python) $(SuperPMIDirectory)$(FileSeparatorChar)superpmi_aspnet.py -core_root_directory $(SuperPMIDirectory) -arch $(Architecture) -temp_location $(AspNetDirectory) --local 3:00 @@ -266,7 +275,20 @@ - + + + 1 + src\Benchmarks\Benchmarks.csproj + Benchmarks.dll + + + 1 + src/Benchmarks/Benchmarks.csproj + Benchmarks.dll + + + + @@ -288,23 +310,23 @@ - + $(CollectionName).$(CollectionType).%(HelixWorkItem.Index).$(MchFileTag) $(WorkItemDirectory) - $(WorkItemCommand) -output_mch_path $(OutputMchPath)$(FileSeparatorChar)%(OutputFileName).mch + $(WorkItemCommand) -partition_count 1 -partition_index 0 -benchmark_path %(HelixWorkItem.BenchmarkPath) -benchmark_binary %(HelixWorkItem.BenchmarkBinary) -output_mch_path $(OutputMchPath)$(FileSeparatorChar)%(OutputFileName).mch -log_file $(OutputMchPath)$(FileSeparatorChar)%(OutputFileName).log $(WorkItemTimeout) %(OutputFileName).mch;%(OutputFileName).mch.mct;%(OutputFileName).log - + - + $(CollectionName).$(CollectionType).%(HelixWorkItem.Index).$(MchFileTag) $(WorkItemDirectory) - $(WorkItemCommand) -partition_count 1 -partition_index 0 -benchmark_path %(HelixWorkItem.BenchmarkPath) -benchmark_binary %(HelixWorkItem.BenchmarkBinary) -output_mch_path $(OutputMchPath)$(FileSeparatorChar)%(OutputFileName).mch -log_file $(OutputMchPath)$(FileSeparatorChar)%(OutputFileName).log + $(WorkItemCommand) -output_mch_path $(OutputMchPath)$(FileSeparatorChar)%(OutputFileName).mch $(WorkItemTimeout) %(OutputFileName).mch;%(OutputFileName).mch.mct;%(OutputFileName).log - + diff --git a/src/coreclr/scripts/superpmi_collect_setup.py b/src/coreclr/scripts/superpmi_collect_setup.py index 606d9f6b109157..6c31d286960d2c 100644 --- a/src/coreclr/scripts/superpmi_collect_setup.py +++ b/src/coreclr/scripts/superpmi_collect_setup.py @@ -410,6 +410,23 @@ def setup_benchmark(workitem_directory, arch): run_command( get_python_name() + [dotnet_install_script, "install", "--channels", "9.0", "--architecture", arch, "--install-dir", dotnet_directory, "--verbose"]) + +def setup_aspnet(workitem_directory, arch): + """ Perform setup of aspnet + + Args: + workitem_directory (string): Path to work + arch (string): Architecture for which dotnet will be installed + """ + aspnet_directory = os.path.join(workitem_directory, "aspnet") + + run_command( + ["git", "clone", "--quiet", "--depth", "1", "https://github.com/aspnet/benchmarks", aspnet_directory]) + + try: + shutil.rmtree(os.path.join(aspnet_directory, ".git")) + except Exception as ex: + print("Warning: failed to remove directory \"%s\": %s", os.path.join(aspnet_directory, ".git"), ex) def get_python_name(): """Gets the python name @@ -505,6 +522,8 @@ def main(main_args): if coreclr_args.collection_name == "benchmarks" or coreclr_args.collection_name == "realworld": # Setup benchmarks setup_benchmark(workitem_payload_directory, arch) + elif coreclr_args.collection_name == "aspnet": + setup_aspnet(workitem_payload_directory, arch) else: # Setup for pmi/crossgen2/nativeaot runs @@ -551,7 +570,7 @@ def make_readable(folder_name): # We need the PMI tool if we're doing a PMI collection. We could download a cached copy from Azure DevOps JIT blob # storage, but instead we clone and build jitutils to build pmi.dll. - if coreclr_args.collection_type == "pmi" or coreclr_args.collection_type == "aspnet": + if coreclr_args.collection_type == "pmi": try: with TempDir() as jitutils_directory: run_command( From 6ab212baf96a55ffac44e41da3640009d953730f Mon Sep 17 00:00:00 2001 From: TIHan Date: Fri, 30 Aug 2024 11:51:22 -0700 Subject: [PATCH 20/35] Trying to fix it --- src/coreclr/scripts/superpmi_aspnet.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/coreclr/scripts/superpmi_aspnet.py b/src/coreclr/scripts/superpmi_aspnet.py index c12c5206dec931..5c0bfe9ecd1f9e 100644 --- a/src/coreclr/scripts/superpmi_aspnet.py +++ b/src/coreclr/scripts/superpmi_aspnet.py @@ -45,7 +45,7 @@ def setup_args(args): args (CoreclrArguments) """ - coreclr_args = CoreclrArguments(args, require_built_core_root=True, require_built_product_dir=True, + coreclr_args = CoreclrArguments(args, require_built_core_root=False, require_built_product_dir=False, require_built_test_dir=False, default_build_type="Checked") coreclr_args.verify(args, From 8e554e5c3909f5169551cdfc5c0b4d641c27fddd Mon Sep 17 00:00:00 2001 From: TIHan Date: Fri, 30 Aug 2024 14:25:56 -0700 Subject: [PATCH 21/35] Trying to fix it --- src/coreclr/scripts/superpmi_aspnet.py | 3 +++ src/coreclr/scripts/superpmi_collect_setup.py | 15 +++++++++++++++ 2 files changed, 18 insertions(+) diff --git a/src/coreclr/scripts/superpmi_aspnet.py b/src/coreclr/scripts/superpmi_aspnet.py index 5c0bfe9ecd1f9e..f85b426ec50158 100644 --- a/src/coreclr/scripts/superpmi_aspnet.py +++ b/src/coreclr/scripts/superpmi_aspnet.py @@ -146,6 +146,9 @@ def build_and_run(coreclr_args): os.chdir(temp_location) dotnet_exe = "dotnet.exe" if is_windows else "dotnet" + dotnet_directory = os.path.join(temp_location, "tools", "dotnet", target_arch) + if os.path.isdir(dotnet_directory): + dotnet_exe = os.path.join(dotnet_directory, "dotnet") run_command([dotnet_exe, "--info"], temp_location, _exit_on_fail=True) ## install crank as local tool diff --git a/src/coreclr/scripts/superpmi_collect_setup.py b/src/coreclr/scripts/superpmi_collect_setup.py index 6c31d286960d2c..70f42f460426a2 100644 --- a/src/coreclr/scripts/superpmi_collect_setup.py +++ b/src/coreclr/scripts/superpmi_collect_setup.py @@ -428,6 +428,21 @@ def setup_aspnet(workitem_directory, arch): except Exception as ex: print("Warning: failed to remove directory \"%s\": %s", os.path.join(aspnet_directory, ".git"), ex) + with ChangeDir(aspnet_directory): + dotnet_directory = os.path.join(aspnet_directory, "tools", "dotnet", arch) + dotnet_install_script = os.path.join(aspnet_directory, "scripts", "dotnet.py") + + if not os.path.isfile(dotnet_install_script): + print("Missing " + dotnet_install_script) + return + + # Sometimes the dotnet version installed by the script is latest and expect certain versions of SDK that + # have not published yet. As a result, we hit errors of "dotnet restore". As a workaround, hard code the + # working version until we move to ".NET 8" in the script. + run_command( + get_python_name() + [dotnet_install_script, "install", "--channels", "9.0", "--architecture", arch, "--install-dir", + dotnet_directory, "--verbose"]) + def get_python_name(): """Gets the python name From eb47bfa9eb3cd256660bd1177f9ca24550134102 Mon Sep 17 00:00:00 2001 From: TIHan Date: Fri, 30 Aug 2024 14:42:27 -0700 Subject: [PATCH 22/35] Trying to fix it --- src/coreclr/scripts/superpmi_collect_setup.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/coreclr/scripts/superpmi_collect_setup.py b/src/coreclr/scripts/superpmi_collect_setup.py index 70f42f460426a2..7c0b44a7807efc 100644 --- a/src/coreclr/scripts/superpmi_collect_setup.py +++ b/src/coreclr/scripts/superpmi_collect_setup.py @@ -428,9 +428,15 @@ def setup_aspnet(workitem_directory, arch): except Exception as ex: print("Warning: failed to remove directory \"%s\": %s", os.path.join(aspnet_directory, ".git"), ex) - with ChangeDir(aspnet_directory): + # We use the performance repo to get access to the dotnet install script. + performance_directory = os.path.join(workitem_directory, "performance") + + run_command( + ["git", "clone", "--quiet", "--depth", "1", "https://github.com/dotnet/performance", performance_directory]) + + with ChangeDir(performance_directory): dotnet_directory = os.path.join(aspnet_directory, "tools", "dotnet", arch) - dotnet_install_script = os.path.join(aspnet_directory, "scripts", "dotnet.py") + dotnet_install_script = os.path.join(performance_directory, "scripts", "dotnet.py") if not os.path.isfile(dotnet_install_script): print("Missing " + dotnet_install_script) @@ -442,6 +448,8 @@ def setup_aspnet(workitem_directory, arch): run_command( get_python_name() + [dotnet_install_script, "install", "--channels", "9.0", "--architecture", arch, "--install-dir", dotnet_directory, "--verbose"]) + + shutil.rmtree(performance_directory, ignore_errors=True) def get_python_name(): """Gets the python name From a34d067398972245b54e6db2eb68fa80a4f80b51 Mon Sep 17 00:00:00 2001 From: TIHan Date: Fri, 30 Aug 2024 16:29:13 -0700 Subject: [PATCH 23/35] Trying to fix it --- src/coreclr/scripts/superpmi_collect_setup.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/coreclr/scripts/superpmi_collect_setup.py b/src/coreclr/scripts/superpmi_collect_setup.py index 7c0b44a7807efc..950d05a327d692 100644 --- a/src/coreclr/scripts/superpmi_collect_setup.py +++ b/src/coreclr/scripts/superpmi_collect_setup.py @@ -420,13 +420,15 @@ def setup_aspnet(workitem_directory, arch): """ aspnet_directory = os.path.join(workitem_directory, "aspnet") + aspnet_benchmarks_directory = os.path.join(aspnet_directory, "benchmarks") + run_command( - ["git", "clone", "--quiet", "--depth", "1", "https://github.com/aspnet/benchmarks", aspnet_directory]) + ["git", "clone", "--quiet", "--depth", "1", "https://github.com/aspnet/benchmarks", aspnet_benchmarks_directory]) try: - shutil.rmtree(os.path.join(aspnet_directory, ".git")) + shutil.rmtree(os.path.join(aspnet_benchmarks_directory, ".git")) except Exception as ex: - print("Warning: failed to remove directory \"%s\": %s", os.path.join(aspnet_directory, ".git"), ex) + print("Warning: failed to remove directory \"%s\": %s", os.path.join(aspnet_benchmarks_directory, ".git"), ex) # We use the performance repo to get access to the dotnet install script. performance_directory = os.path.join(workitem_directory, "performance") From 48ca4e32978575386834d304fdf16c74e5085173 Mon Sep 17 00:00:00 2001 From: TIHan Date: Fri, 30 Aug 2024 19:01:32 -0700 Subject: [PATCH 24/35] Trying to fix it --- src/coreclr/scripts/superpmi_aspnet.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/coreclr/scripts/superpmi_aspnet.py b/src/coreclr/scripts/superpmi_aspnet.py index f85b426ec50158..d885139f0fa33b 100644 --- a/src/coreclr/scripts/superpmi_aspnet.py +++ b/src/coreclr/scripts/superpmi_aspnet.py @@ -131,6 +131,7 @@ def build_and_run(coreclr_args): coreclr_args (CoreClrArguments): Arguments use to drive output_mch_name (string): Name of output mch file name """ + coreclr_args.core_root = coreclr_args.core_root_directory core_root_directory = coreclr_args.core_root_directory target_arch = coreclr_args.arch target_os = coreclr_args.host_os From 4214f8d2200222f06d7f2d11a81e070e9d0ecc9d Mon Sep 17 00:00:00 2001 From: TIHan Date: Fri, 30 Aug 2024 21:16:49 -0700 Subject: [PATCH 25/35] Trying to fix it --- src/coreclr/scripts/superpmi_aspnet.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/coreclr/scripts/superpmi_aspnet.py b/src/coreclr/scripts/superpmi_aspnet.py index d885139f0fa33b..9df66bf1d583ad 100644 --- a/src/coreclr/scripts/superpmi_aspnet.py +++ b/src/coreclr/scripts/superpmi_aspnet.py @@ -231,7 +231,7 @@ def build_and_run(coreclr_args): crank_agent_p = None if coreclr_args.local: - crank_agent_p = Popen(crank_agent_app, stdout=DEVNULL) + crank_agent_p = Popen(crank_agent_app) try: for (configName, scenario) in configname_scenario_list: From fed0762c62e8de000b442bc5ff3ba7dd0b794404 Mon Sep 17 00:00:00 2001 From: TIHan Date: Sat, 31 Aug 2024 12:06:22 -0700 Subject: [PATCH 26/35] Trying to properly set up aspnet --- src/coreclr/scripts/superpmi_aspnet.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/coreclr/scripts/superpmi_aspnet.py b/src/coreclr/scripts/superpmi_aspnet.py index 9df66bf1d583ad..e8b7055a0fdfd8 100644 --- a/src/coreclr/scripts/superpmi_aspnet.py +++ b/src/coreclr/scripts/superpmi_aspnet.py @@ -18,6 +18,7 @@ import zipfile import stat import tempfile +import time from os import path from coreclr_arguments import * @@ -231,7 +232,9 @@ def build_and_run(coreclr_args): crank_agent_p = None if coreclr_args.local: + print(f"Launching crank agent: {crank_agent_app}") crank_agent_p = Popen(crank_agent_app) + time.sleep(10) try: for (configName, scenario) in configname_scenario_list: From e5f0d13eb8163083be152328753698e51e7ea19a Mon Sep 17 00:00:00 2001 From: TIHan Date: Sat, 31 Aug 2024 12:22:55 -0700 Subject: [PATCH 27/35] Trying to properly set up aspnet --- src/coreclr/scripts/superpmi_aspnet.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/coreclr/scripts/superpmi_aspnet.py b/src/coreclr/scripts/superpmi_aspnet.py index e8b7055a0fdfd8..a0d0a21d7dc373 100644 --- a/src/coreclr/scripts/superpmi_aspnet.py +++ b/src/coreclr/scripts/superpmi_aspnet.py @@ -233,7 +233,7 @@ def build_and_run(coreclr_args): crank_agent_p = None if coreclr_args.local: print(f"Launching crank agent: {crank_agent_app}") - crank_agent_p = Popen(crank_agent_app) + crank_agent_p = Popen(crank_agent_app, stdout=subprocess.STDOUT, stderr=subprocess.STDOUT) time.sleep(10) try: @@ -275,7 +275,7 @@ def build_and_run(coreclr_args): description = ["--description", configName + "-" + scenario + "-" + "-".join(runtime_options)] crank_app_args = [crank_app] + crank_arguments + description + runtime_arguments print(' '.join(crank_app_args)) - subprocess.run(crank_app_args, cwd=temp_location) + subprocess.run(crank_app_args, cwd=temp_location, stdout=subprocess.STDOUT, stderr=subprocess.STDOUT) finally: if crank_agent_p is not None: crank_agent_p.terminate() From 7dd83785738ea60e45e4f79d3dc19bba91ad783b Mon Sep 17 00:00:00 2001 From: TIHan Date: Sat, 31 Aug 2024 13:16:27 -0700 Subject: [PATCH 28/35] Trying to fix it --- src/coreclr/scripts/jitutil.py | 1 - src/coreclr/scripts/superpmi_aspnet.py | 8 ++++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/src/coreclr/scripts/jitutil.py b/src/coreclr/scripts/jitutil.py index b397cecdc53b86..a7fa1f851d450d 100644 --- a/src/coreclr/scripts/jitutil.py +++ b/src/coreclr/scripts/jitutil.py @@ -179,7 +179,6 @@ def run_command(command_to_run, _cwd=None, _exit_on_fail=False, _output_file=Non sys.exit(1) return command_stdout, command_stderr, return_code - def copy_directory(src_path, dst_path, verbose_output=False, verbose_copy=False, verbose_skip=False, match_func=lambda path: True): """Copies directory in 'src_path' to 'dst_path' maintaining the directory structure. https://docs.python.org/3.5/library/shutil.html#shutil.copytree can't diff --git a/src/coreclr/scripts/superpmi_aspnet.py b/src/coreclr/scripts/superpmi_aspnet.py index a0d0a21d7dc373..e78741149c5468 100644 --- a/src/coreclr/scripts/superpmi_aspnet.py +++ b/src/coreclr/scripts/superpmi_aspnet.py @@ -24,7 +24,6 @@ from coreclr_arguments import * from superpmi import TempDir, determine_mcs_tool_path, determine_superpmi_tool_path, is_nonzero_length_file from jitutil import run_command -from subprocess import Popen, DEVNULL # Start of parser object creation. is_windows = platform.system() == "Windows" @@ -233,8 +232,8 @@ def build_and_run(coreclr_args): crank_agent_p = None if coreclr_args.local: print(f"Launching crank agent: {crank_agent_app}") - crank_agent_p = Popen(crank_agent_app, stdout=subprocess.STDOUT, stderr=subprocess.STDOUT) - time.sleep(10) + crank_agent_p = subprocess.Popen(crank_agent_app) + time.sleep(2) try: for (configName, scenario) in configname_scenario_list: @@ -275,7 +274,8 @@ def build_and_run(coreclr_args): description = ["--description", configName + "-" + scenario + "-" + "-".join(runtime_options)] crank_app_args = [crank_app] + crank_arguments + description + runtime_arguments print(' '.join(crank_app_args)) - subprocess.run(crank_app_args, cwd=temp_location, stdout=subprocess.STDOUT, stderr=subprocess.STDOUT) + run_command(crank_app_args, temp_location) + print("Crank finished...") finally: if crank_agent_p is not None: crank_agent_p.terminate() From 69a567a292260f94fa7662eb5998e08888ee9bbb Mon Sep 17 00:00:00 2001 From: TIHan Date: Sat, 31 Aug 2024 14:05:45 -0700 Subject: [PATCH 29/35] Trying to fix it --- src/coreclr/scripts/superpmi_aspnet.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/coreclr/scripts/superpmi_aspnet.py b/src/coreclr/scripts/superpmi_aspnet.py index e78741149c5468..206a0fce757da4 100644 --- a/src/coreclr/scripts/superpmi_aspnet.py +++ b/src/coreclr/scripts/superpmi_aspnet.py @@ -19,6 +19,8 @@ import stat import tempfile import time +import threading +import multiprocessing from os import path from coreclr_arguments import * @@ -232,7 +234,9 @@ def build_and_run(coreclr_args): crank_agent_p = None if coreclr_args.local: print(f"Launching crank agent: {crank_agent_app}") - crank_agent_p = subprocess.Popen(crank_agent_app) + crank_agent_p = subprocess.Popen(crank_agent_app, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT) time.sleep(2) try: @@ -278,6 +282,10 @@ def build_and_run(coreclr_args): print("Crank finished...") finally: if crank_agent_p is not None: + for line in iter(crank_agent_p.stdout.readline, ""): + if not line: + break + print(line) crank_agent_p.terminate() # merge From d1107910b174766403147689337795cd10a9d8f4 Mon Sep 17 00:00:00 2001 From: TIHan Date: Sat, 31 Aug 2024 17:18:54 -0700 Subject: [PATCH 30/35] Trying to fix it --- src/coreclr/scripts/superpmi_aspnet.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/coreclr/scripts/superpmi_aspnet.py b/src/coreclr/scripts/superpmi_aspnet.py index 206a0fce757da4..1bf756914a62ca 100644 --- a/src/coreclr/scripts/superpmi_aspnet.py +++ b/src/coreclr/scripts/superpmi_aspnet.py @@ -236,7 +236,7 @@ def build_and_run(coreclr_args): print(f"Launching crank agent: {crank_agent_app}") crank_agent_p = subprocess.Popen(crank_agent_app, stdout=subprocess.PIPE, - stderr=subprocess.STDOUT) + stderr=subprocess.PIPE) time.sleep(2) try: @@ -286,6 +286,10 @@ def build_and_run(coreclr_args): if not line: break print(line) + for line in iter(crank_agent_p.stderr.readline, ""): + if not line: + break + print(line) crank_agent_p.terminate() # merge From 66072520f4913802ba114428ae8426d2c2fb418f Mon Sep 17 00:00:00 2001 From: TIHan Date: Mon, 2 Sep 2024 14:59:23 -0700 Subject: [PATCH 31/35] Trying to fix it --- src/coreclr/scripts/superpmi_aspnet.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/coreclr/scripts/superpmi_aspnet.py b/src/coreclr/scripts/superpmi_aspnet.py index 1bf756914a62ca..4e33d37afd3acf 100644 --- a/src/coreclr/scripts/superpmi_aspnet.py +++ b/src/coreclr/scripts/superpmi_aspnet.py @@ -176,6 +176,10 @@ def build_and_run(coreclr_args): crank_app = path.join(temp_location, "crank") crank_agent_app = path.join(temp_location, "crank-agent") + crank_env = os.environ.copy() + if os.path.isdir(dotnet_directory): + crank_env['DOTNET_ROOT'] = dotnet_directory + crank_env['DOTNET_MULTILEVEL_LOOKUP'] = '1' mcs_path = determine_mcs_tool_path(coreclr_args) superpmi_path = determine_superpmi_tool_path(coreclr_args) @@ -236,7 +240,7 @@ def build_and_run(coreclr_args): print(f"Launching crank agent: {crank_agent_app}") crank_agent_p = subprocess.Popen(crank_agent_app, stdout=subprocess.PIPE, - stderr=subprocess.PIPE) + stderr=subprocess.PIPE,env=crank_env) time.sleep(2) try: @@ -278,14 +282,16 @@ def build_and_run(coreclr_args): description = ["--description", configName + "-" + scenario + "-" + "-".join(runtime_options)] crank_app_args = [crank_app] + crank_arguments + description + runtime_arguments print(' '.join(crank_app_args)) - run_command(crank_app_args, temp_location) + run_command(crank_app_args, temp_location, _env=crank_env) print("Crank finished...") finally: if crank_agent_p is not None: + print('Crank agent stdout:\n') for line in iter(crank_agent_p.stdout.readline, ""): if not line: break print(line) + print('Crank agent stderr:\n') for line in iter(crank_agent_p.stderr.readline, ""): if not line: break From 0a6ab02b2dc49be4bd80694413d645b1f96c4532 Mon Sep 17 00:00:00 2001 From: TIHan Date: Mon, 2 Sep 2024 15:01:25 -0700 Subject: [PATCH 32/35] Trying to fix it --- src/coreclr/scripts/superpmi_aspnet.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/coreclr/scripts/superpmi_aspnet.py b/src/coreclr/scripts/superpmi_aspnet.py index 4e33d37afd3acf..b94a6662decbcc 100644 --- a/src/coreclr/scripts/superpmi_aspnet.py +++ b/src/coreclr/scripts/superpmi_aspnet.py @@ -176,10 +176,10 @@ def build_and_run(coreclr_args): crank_app = path.join(temp_location, "crank") crank_agent_app = path.join(temp_location, "crank-agent") - crank_env = os.environ.copy() if os.path.isdir(dotnet_directory): - crank_env['DOTNET_ROOT'] = dotnet_directory - crank_env['DOTNET_MULTILEVEL_LOOKUP'] = '1' + os.environ['DOTNET_ROOT'] = dotnet_directory + os.environ['DOTNET_MULTILEVEL_LOOKUP'] = '1' + crank_env = os.environ.copy() mcs_path = determine_mcs_tool_path(coreclr_args) superpmi_path = determine_superpmi_tool_path(coreclr_args) From 34c3b3a70bb633a86df376c1c7f260745dfd09b4 Mon Sep 17 00:00:00 2001 From: TIHan Date: Mon, 2 Sep 2024 18:00:52 -0700 Subject: [PATCH 33/35] Trying to fix it --- src/coreclr/scripts/superpmi_collect_setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/coreclr/scripts/superpmi_collect_setup.py b/src/coreclr/scripts/superpmi_collect_setup.py index 950d05a327d692..c3f15589916e63 100644 --- a/src/coreclr/scripts/superpmi_collect_setup.py +++ b/src/coreclr/scripts/superpmi_collect_setup.py @@ -448,7 +448,7 @@ def setup_aspnet(workitem_directory, arch): # have not published yet. As a result, we hit errors of "dotnet restore". As a workaround, hard code the # working version until we move to ".NET 8" in the script. run_command( - get_python_name() + [dotnet_install_script, "install", "--channels", "9.0", "--architecture", arch, "--install-dir", + get_python_name() + [dotnet_install_script, "install", "--channels", "8.0", "--architecture", arch, "--install-dir", dotnet_directory, "--verbose"]) shutil.rmtree(performance_directory, ignore_errors=True) From dd58bc4a11ac328cfc6a5d3a4da056a3b426f25e Mon Sep 17 00:00:00 2001 From: TIHan Date: Tue, 3 Sep 2024 12:46:55 -0700 Subject: [PATCH 34/35] Trying to fix it --- src/coreclr/scripts/superpmi_aspnet.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/coreclr/scripts/superpmi_aspnet.py b/src/coreclr/scripts/superpmi_aspnet.py index b94a6662decbcc..ff83fefbe34747 100644 --- a/src/coreclr/scripts/superpmi_aspnet.py +++ b/src/coreclr/scripts/superpmi_aspnet.py @@ -238,9 +238,9 @@ def build_and_run(coreclr_args): crank_agent_p = None if coreclr_args.local: print(f"Launching crank agent: {crank_agent_app}") - crank_agent_p = subprocess.Popen(crank_agent_app, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE,env=crank_env) + # crank_agent_p = subprocess.Popen(crank_agent_app, + # stdout=subprocess.PIPE, + # stderr=subprocess.PIPE,env=crank_env) time.sleep(2) try: From ceae9a209acad462ba85515e9d26e72a7a7aaa79 Mon Sep 17 00:00:00 2001 From: TIHan Date: Tue, 3 Sep 2024 15:08:14 -0700 Subject: [PATCH 35/35] Trying to fix it --- src/coreclr/scripts/superpmi_aspnet.py | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/src/coreclr/scripts/superpmi_aspnet.py b/src/coreclr/scripts/superpmi_aspnet.py index ff83fefbe34747..44a7c2a21f4ffc 100644 --- a/src/coreclr/scripts/superpmi_aspnet.py +++ b/src/coreclr/scripts/superpmi_aspnet.py @@ -238,9 +238,9 @@ def build_and_run(coreclr_args): crank_agent_p = None if coreclr_args.local: print(f"Launching crank agent: {crank_agent_app}") - # crank_agent_p = subprocess.Popen(crank_agent_app, - # stdout=subprocess.PIPE, - # stderr=subprocess.PIPE,env=crank_env) + crank_agent_p = subprocess.Popen(crank_agent_app, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE,env=crank_env) time.sleep(2) try: @@ -286,16 +286,16 @@ def build_and_run(coreclr_args): print("Crank finished...") finally: if crank_agent_p is not None: - print('Crank agent stdout:\n') - for line in iter(crank_agent_p.stdout.readline, ""): - if not line: - break - print(line) - print('Crank agent stderr:\n') - for line in iter(crank_agent_p.stderr.readline, ""): - if not line: - break - print(line) + # print('Crank agent stdout:\n') + # for line in iter(crank_agent_p.stdout.readline, ""): + # if not line: + # break + # print(line) + # print('Crank agent stderr:\n') + # for line in iter(crank_agent_p.stderr.readline, ""): + # if not line: + # break + # print(line) crank_agent_p.terminate() # merge