diff --git a/config.yml b/config.yml index e9936adc..0677e516 100644 --- a/config.yml +++ b/config.yml @@ -90,6 +90,18 @@ globus: uuid: df82346e-9a15-11ea-b3c4-0ae144191ee3 name: nersc832 + bl832_beegfs_raw: + root_path: /beamline_staging/bl832/raw/ + uri: computeDTN.als.lbl.gov + uuid: d33b5d6e-1603-414e-93cb-bcb732b7914a + name: bl832_beegfs_raw + + bl832_beegfs_processed: + root_path: /beamline_staging/bl832/processed/ + uri: computeDTN.als.lbl.gov + uuid: d33b5d6e-1603-414e-93cb-bcb732b7914a + name: bl832_beegfs_processed + nersc7012: root_path: /global/cfs/cdirs/als/gsharing/data_mover/7012 uri: nersc.gov diff --git a/orchestration/_tests/test_globus_flow.py b/orchestration/_tests/test_globus_flow.py index e0823b27..97066d09 100644 --- a/orchestration/_tests/test_globus_flow.py +++ b/orchestration/_tests/test_globus_flow.py @@ -233,7 +233,7 @@ def test_alcf_recon_flow(mocker: MockFixture): result = alcf_recon_flow(file_path=file_path, config=mock_config) assert result is True, "Flow should return True if HPC + Tiff->Zarr + transfers all succeed" - assert mock_transfer_controller.copy.call_count == 3, "Should do 3 transfers in success path" + assert mock_transfer_controller.copy.call_count == 5, "Should do 5 transfers in success path" mock_hpc_reconstruct.assert_called_once() mock_hpc_multires.assert_called_once() mock_schedule_pruning.assert_called_once() diff --git a/orchestration/flows/bl832/alcf.py b/orchestration/flows/bl832/alcf.py index 0b6bcb1c..713efd7a 100644 --- a/orchestration/flows/bl832/alcf.py +++ b/orchestration/flows/bl832/alcf.py @@ -434,6 +434,15 @@ def alcf_recon_flow( destination=config.data832_scratch ) + logger.info(f"Transferring {file_name} from {config.alcf832_scratch} " + f"at ALCF to {config.compute_dtn_832_scratch}") + + transfer_controller.copy( + file_path=scratch_path_tiff, + source=config.alcf832_scratch, + destination=config.compute_dtn_832_scratch + ) + if alcf_multi_res_success: # Transfer B: Send reconstructed data (zarr) to data832 logger.info(f"Transferring {file_name} from {config.alcf832_scratch} " @@ -444,6 +453,14 @@ def alcf_recon_flow( destination=config.data832_scratch ) + logger.info(f"Transferring {file_name} from {config.alcf832_scratch} " + f"at ALCF to {config.compute_dtn_832_scratch}") + transfer_controller.copy( + file_path=scratch_path_zarr, + source=config.alcf832_scratch, + destination=config.compute_dtn_832_scratch + ) + # Place holder in case we want to transfer to NERSC for long term storage nersc_transfer_success = False diff --git a/orchestration/flows/bl832/config.py b/orchestration/flows/bl832/config.py index ff19a9c3..2deca530 100644 --- a/orchestration/flows/bl832/config.py +++ b/orchestration/flows/bl832/config.py @@ -22,5 +22,7 @@ def __init__(self) -> None: self.nersc832_alsdev_recon_scripts = self.endpoints["nersc832_alsdev_recon_scripts"] self.alcf832_raw = self.endpoints["alcf832_raw"] self.alcf832_scratch = self.endpoints["alcf832_scratch"] + self.bl832_beegfs_raw = self.endpoints["bl832_beegfs_raw"] + self.bl832_beegfs_processed = self.endpoints["bl832_beegfs_processed"] self.scicat = config["scicat"] self.ghcr_images832 = config["ghcr_images832"] diff --git a/orchestration/flows/bl832/nersc.py b/orchestration/flows/bl832/nersc.py index 05680ed0..109136db 100644 --- a/orchestration/flows/bl832/nersc.py +++ b/orchestration/flows/bl832/nersc.py @@ -500,6 +500,20 @@ def nersc_recon_flow( destination=config.data832_scratch ) + logger.info("Copy from NERSC /global/cfs/cdirs/als/data_mover/8.3.2/scratch to compute-dtn.als.lbl.gov") + + transfer_controller.copy( + file_path=tiff_file_path, + source=config.nersc832_alsdev_pscratch_scratch, + destination=config.bl832_beegfs_processed + ) + + transfer_controller.copy( + file_path=zarr_file_path, + source=config.nersc832_alsdev_pscratch_scratch, + destination=config.bl832_beegfs_processed + ) + logger.info("Scheduling pruning tasks.") schedule_pruning( config=config,