From 6ce0f423f1a949f4b62bd8cdb910fcf22a77bc32 Mon Sep 17 00:00:00 2001 From: David Abramov Date: Mon, 9 Feb 2026 12:07:17 -0800 Subject: [PATCH 1/6] Updating dispatcher to include options for aclf-seg and nersc-seg forge flows --- orchestration/flows/bl832/dispatcher.py | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/orchestration/flows/bl832/dispatcher.py b/orchestration/flows/bl832/dispatcher.py index d28a02eb..69865946 100644 --- a/orchestration/flows/bl832/dispatcher.py +++ b/orchestration/flows/bl832/dispatcher.py @@ -17,6 +17,9 @@ class FlowParameterMapper: "alcf_recon_flow/alcf_recon_flow": [ "file_path", "config"], + "alcf_forge_recon_segment_flow/alcf_forge_recon_segment_flow": [ + "file_path", + "config"], # From move.py "new_832_file_flow/new_file_832": [ "file_path", @@ -67,6 +70,7 @@ class DecisionFlowInputModel(BaseModel): @task(name="setup_decision_settings") def setup_decision_settings( alcf_recon: bool, + alcf_forge_recon_segment: bool, nersc_recon: bool, nersc_petiole_segment: bool, nersc_moon_segment: bool, @@ -76,6 +80,7 @@ def setup_decision_settings( This task is used to define the settings for the decision making process of the BL832 beamline. :param alcf_recon: Boolean indicating whether to run the ALCF reconstruction flow. + :param alcf_forge_recon_segment: Boolean indicating whether to run the ALCF Forge reconstruction segment flow. :param nersc_recon: Boolean indicating whether to run the NERSC reconstruction flow. :param nersc_petiole_segment: Boolean indicating whether to run the NERSC petiole segmentation flow. :param nersc_moon_segment: Boolean indicating whether to run the NERSC moon segmentation flow. @@ -85,6 +90,7 @@ def setup_decision_settings( logger = get_run_logger() try: logger.info(f"Setting up decision settings: alcf_recon={alcf_recon}, " + f"alcf_forge_recon_segment={alcf_forge_recon_segment}, " f"nersc_recon={nersc_recon}, " f"nersc_petiole_segment={nersc_petiole_segment}, " f"nersc_moon_segment={nersc_moon_segment}, " @@ -92,6 +98,7 @@ def setup_decision_settings( # Define which flows to run based on the input settings settings = { "alcf_recon_flow/alcf_recon_flow": alcf_recon, + "alcf_forge_recon_segment_flow/alcf_forge_recon_segment_flow": alcf_forge_recon_segment, "nersc_recon_flow/nersc_recon_flow": nersc_recon, "nersc_petiole_segment_flow/nersc_petiole_segment_flow": nersc_petiole_segment, "nersc_moon_segment_flow/nersc_moon_segment_flow": nersc_moon_segment, @@ -168,6 +175,13 @@ async def dispatcher( alcf_params = FlowParameterMapper.get_flow_parameters("alcf_recon_flow/alcf_recon_flow", available_params) tasks.append(run_recon_flow_async("alcf_recon_flow/alcf_recon_flow", alcf_params)) + if decision_settings.get("alcf_forge_recon_segment_flow/alcf_forge_recon_segment_flow"): + alcf_forge_params = FlowParameterMapper.get_flow_parameters( + "alcf_forge_recon_segment_flow/alcf_forge_recon_segment_flow", + available_params + ) + tasks.append(run_recon_flow_async("alcf_forge_recon_segment_flow/alcf_forge_recon_segment_flow", alcf_forge_params)) + if decision_settings.get("nersc_recon_flow/nersc_recon_flow"): nersc_params = FlowParameterMapper.get_flow_parameters("nersc_recon_flow/nersc_recon_flow", available_params) tasks.append(run_recon_flow_async("nersc_recon_flow/nersc_recon_flow", nersc_params)) @@ -208,8 +222,11 @@ async def dispatcher( # Setup decision settings based on input parameters setup_decision_settings( alcf_recon=True, + alcf_forge_recon_segment=False, nersc_recon=True, - nersc_recon_multinode=True, + nersc_petiole_segment=False, + nersc_moon_segment=False, + nersc_forge_recon_segment=False, new_file_832=True ) # Run the main decision flow with the specified parameters From a35eccc7dd63ea7f68ab0408ee188aa72854454e Mon Sep 17 00:00:00 2001 From: David Abramov Date: Tue, 10 Feb 2026 10:28:36 -0800 Subject: [PATCH 2/6] Making multinode recon and multinode recon+seg on nersc separate options --- orchestration/flows/bl832/dispatcher.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/orchestration/flows/bl832/dispatcher.py b/orchestration/flows/bl832/dispatcher.py index 69865946..054478a4 100644 --- a/orchestration/flows/bl832/dispatcher.py +++ b/orchestration/flows/bl832/dispatcher.py @@ -200,6 +200,11 @@ async def dispatcher( ) tasks.append(run_recon_flow_async("nersc_moon_segment_flow/nersc_moon_segment_flow", moon_params)) + if decision_settings.get("nersc_forge_recon_segment_flow/nersc_forge_recon_segment_flow"): + nersc_forge_recon_segment_params = FlowParameterMapper.get_flow_parameters( + "nersc_forge_recon_segment_flow/nersc_forge_recon_segment_flow", available_params) + tasks.append(run_recon_flow_async( + "nersc_forge_recon_segment_flow/nersc_forge_recon_segment_flow", nersc_forge_recon_segment_params)) # Run ALCF and NERSC flows in parallel, if any if tasks: try: From c7fa1075a87399c463a3bbefbf1001a4bb79568d Mon Sep 17 00:00:00 2001 From: David Abramov Date: Fri, 20 Feb 2026 11:01:11 -0800 Subject: [PATCH 3/6] updating dispatcher for the multiseg flow at alcf --- orchestration/flows/bl832/dispatcher.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/orchestration/flows/bl832/dispatcher.py b/orchestration/flows/bl832/dispatcher.py index 054478a4..20c87723 100644 --- a/orchestration/flows/bl832/dispatcher.py +++ b/orchestration/flows/bl832/dispatcher.py @@ -20,6 +20,9 @@ class FlowParameterMapper: "alcf_forge_recon_segment_flow/alcf_forge_recon_segment_flow": [ "file_path", "config"], + "alcf_forge_recon_multisegment_flow/alcf_forge_recon_multisegment_flow": [ + "file_path", + "config"], # From move.py "new_832_file_flow/new_file_832": [ "file_path", @@ -71,6 +74,7 @@ class DecisionFlowInputModel(BaseModel): def setup_decision_settings( alcf_recon: bool, alcf_forge_recon_segment: bool, + alcf_forge_recon_multisegment: bool, nersc_recon: bool, nersc_petiole_segment: bool, nersc_moon_segment: bool, @@ -81,6 +85,7 @@ def setup_decision_settings( :param alcf_recon: Boolean indicating whether to run the ALCF reconstruction flow. :param alcf_forge_recon_segment: Boolean indicating whether to run the ALCF Forge reconstruction segment flow. + :param alcf_forge_recon_multisegment: Boolean indicating whether to run the ALCF Forge reconstruction multisegment flow. :param nersc_recon: Boolean indicating whether to run the NERSC reconstruction flow. :param nersc_petiole_segment: Boolean indicating whether to run the NERSC petiole segmentation flow. :param nersc_moon_segment: Boolean indicating whether to run the NERSC moon segmentation flow. @@ -91,6 +96,7 @@ def setup_decision_settings( try: logger.info(f"Setting up decision settings: alcf_recon={alcf_recon}, " f"alcf_forge_recon_segment={alcf_forge_recon_segment}, " + f"alcf_forge_recon_multisegment={alcf_forge_recon_multisegment}, " f"nersc_recon={nersc_recon}, " f"nersc_petiole_segment={nersc_petiole_segment}, " f"nersc_moon_segment={nersc_moon_segment}, " @@ -99,6 +105,7 @@ def setup_decision_settings( settings = { "alcf_recon_flow/alcf_recon_flow": alcf_recon, "alcf_forge_recon_segment_flow/alcf_forge_recon_segment_flow": alcf_forge_recon_segment, + "alcf_forge_recon_multisegment_flow/alcf_forge_recon_multisegment_flow": alcf_forge_recon_multisegment, "nersc_recon_flow/nersc_recon_flow": nersc_recon, "nersc_petiole_segment_flow/nersc_petiole_segment_flow": nersc_petiole_segment, "nersc_moon_segment_flow/nersc_moon_segment_flow": nersc_moon_segment, @@ -182,6 +189,14 @@ async def dispatcher( ) tasks.append(run_recon_flow_async("alcf_forge_recon_segment_flow/alcf_forge_recon_segment_flow", alcf_forge_params)) + if decision_settings.get("alcf_forge_recon_multisegment_flow/alcf_forge_recon_multisegment_flow"): + alcf_forge_params = FlowParameterMapper.get_flow_parameters( + "alcf_forge_recon_multisegment_flow/alcf_forge_recon_multisegment_flow", + available_params + ) + tasks.append(run_recon_flow_async("alcf_forge_recon_multisegment_flow/alcf_forge_recon_multisegment_flow", + alcf_forge_params)) + if decision_settings.get("nersc_recon_flow/nersc_recon_flow"): nersc_params = FlowParameterMapper.get_flow_parameters("nersc_recon_flow/nersc_recon_flow", available_params) tasks.append(run_recon_flow_async("nersc_recon_flow/nersc_recon_flow", nersc_params)) @@ -228,6 +243,7 @@ async def dispatcher( setup_decision_settings( alcf_recon=True, alcf_forge_recon_segment=False, + alcf_forge_recon_multisegment=False, nersc_recon=True, nersc_petiole_segment=False, nersc_moon_segment=False, From d84661cbebe2cd4759c7d95b62bb73af880f56fc Mon Sep 17 00:00:00 2001 From: David Abramov Date: Fri, 20 Feb 2026 11:30:30 -0800 Subject: [PATCH 4/6] updating dispatcher for the multiseg flow at nersc --- orchestration/flows/bl832/dispatcher.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/orchestration/flows/bl832/dispatcher.py b/orchestration/flows/bl832/dispatcher.py index 20c87723..919b37e9 100644 --- a/orchestration/flows/bl832/dispatcher.py +++ b/orchestration/flows/bl832/dispatcher.py @@ -38,6 +38,10 @@ class FlowParameterMapper: "num_nodes", "config"], "nersc_moon_segment_flow/nersc_moon_segment_flow": [ + "file_path", + "num_nodes", + "config"], + "nersc_forge_recon_multisegment_flow/nersc_forge_recon_multisegment_flow": [ "file_path", "num_nodes", "config"] @@ -220,6 +224,13 @@ async def dispatcher( "nersc_forge_recon_segment_flow/nersc_forge_recon_segment_flow", available_params) tasks.append(run_recon_flow_async( "nersc_forge_recon_segment_flow/nersc_forge_recon_segment_flow", nersc_forge_recon_segment_params)) + + if decision_settings.get("nersc_forge_recon_multisegment_flow/nersc_forge_recon_multisegment_flow"): + nersc_forge_recon_multisegment_params = FlowParameterMapper.get_flow_parameters( + "nersc_forge_recon_multisegment_flow/nersc_forge_recon_multisegment_flow", available_params) + tasks.append(run_recon_flow_async( + "nersc_forge_recon_multisegment_flow/nersc_forge_recon_multisegment_flow", nersc_forge_recon_multisegment_params)) + # Run ALCF and NERSC flows in parallel, if any if tasks: try: @@ -248,6 +259,7 @@ async def dispatcher( nersc_petiole_segment=False, nersc_moon_segment=False, nersc_forge_recon_segment=False, + nersc_forge_recon_multisegment=False, new_file_832=True ) # Run the main decision flow with the specified parameters From 59c79689baf37ebfaa41fb3a8f91597c86352012 Mon Sep 17 00:00:00 2001 From: David Abramov Date: Fri, 17 Apr 2026 13:30:15 -0700 Subject: [PATCH 5/6] updating dispatcher.py with recent changes for petiole/moon segmentation, removing old flows --- orchestration/flows/bl832/dispatcher.py | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/orchestration/flows/bl832/dispatcher.py b/orchestration/flows/bl832/dispatcher.py index 919b37e9..aaf00941 100644 --- a/orchestration/flows/bl832/dispatcher.py +++ b/orchestration/flows/bl832/dispatcher.py @@ -38,10 +38,6 @@ class FlowParameterMapper: "num_nodes", "config"], "nersc_moon_segment_flow/nersc_moon_segment_flow": [ - "file_path", - "num_nodes", - "config"], - "nersc_forge_recon_multisegment_flow/nersc_forge_recon_multisegment_flow": [ "file_path", "num_nodes", "config"] @@ -219,18 +215,6 @@ async def dispatcher( ) tasks.append(run_recon_flow_async("nersc_moon_segment_flow/nersc_moon_segment_flow", moon_params)) - if decision_settings.get("nersc_forge_recon_segment_flow/nersc_forge_recon_segment_flow"): - nersc_forge_recon_segment_params = FlowParameterMapper.get_flow_parameters( - "nersc_forge_recon_segment_flow/nersc_forge_recon_segment_flow", available_params) - tasks.append(run_recon_flow_async( - "nersc_forge_recon_segment_flow/nersc_forge_recon_segment_flow", nersc_forge_recon_segment_params)) - - if decision_settings.get("nersc_forge_recon_multisegment_flow/nersc_forge_recon_multisegment_flow"): - nersc_forge_recon_multisegment_params = FlowParameterMapper.get_flow_parameters( - "nersc_forge_recon_multisegment_flow/nersc_forge_recon_multisegment_flow", available_params) - tasks.append(run_recon_flow_async( - "nersc_forge_recon_multisegment_flow/nersc_forge_recon_multisegment_flow", nersc_forge_recon_multisegment_params)) - # Run ALCF and NERSC flows in parallel, if any if tasks: try: From bf8e1bfef0821238de045be6c2f2867126942a65 Mon Sep 17 00:00:00 2001 From: David Abramov Date: Thu, 30 Apr 2026 13:13:52 -0700 Subject: [PATCH 6/6] removing commented code, and rebasing --- orchestration/flows/bl832/dispatcher.py | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/orchestration/flows/bl832/dispatcher.py b/orchestration/flows/bl832/dispatcher.py index aaf00941..915a02cd 100644 --- a/orchestration/flows/bl832/dispatcher.py +++ b/orchestration/flows/bl832/dispatcher.py @@ -246,17 +246,6 @@ async def dispatcher( nersc_forge_recon_multisegment=False, new_file_832=True ) - # Run the main decision flow with the specified parameters - # asyncio.run(dispatcher( - # config={}, # PYTEST, ALCF, NERSC - # is_export_control=False, # ALCF & MOVE - # folder_name="folder", # ALCF - # file_name="file", # ALCF - # file_path="/path/to/file", # MOVE - # send_to_alcf=True, # ALCF - # send_to_nersc=True, # MOVE - # ) - # ) except Exception as e: logger = get_run_logger() logger.error(f"Failed to execute main flow: {e}")