From 14921c830b3bf760598f2e4ee8bf7545408a08a5 Mon Sep 17 00:00:00 2001 From: Sergei Kliavinek Date: Fri, 7 Mar 2025 13:26:36 -0800 Subject: [PATCH 1/2] Initial commit for the automated pipeline --- pyproject.toml | 7 +- setup.py | 1 + src/perturbopy/auto_pipeline/__init__.py | 1 + src/perturbopy/auto_pipeline/ap_utils.py | 90 +++++++++++++++++++ .../auto_pipeline.py} | 35 +++----- 5 files changed, 110 insertions(+), 24 deletions(-) create mode 100644 src/perturbopy/auto_pipeline/__init__.py create mode 100644 src/perturbopy/auto_pipeline/ap_utils.py rename src/perturbopy/{test_utils/legacy/legacy.py => auto_pipeline/auto_pipeline.py} (90%) diff --git a/pyproject.toml b/pyproject.toml index 3425baf..1ef91b2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -27,7 +27,10 @@ interactive = [ [tool.setuptools] package-dir = {"" = "src"} -packages = ["perturbopy"] + +[tool.setuptools.packages.find] +where = ["src"] [project.scripts] -input_generation = "perturbopy.generate_input:input_generation" \ No newline at end of file +input_generation = "perturbopy.generate_input:input_generation" +auto_pipeline = "perturbopy.auto_pipeline.auto_pipeline:run_epr_calculation" \ No newline at end of file diff --git a/setup.py b/setup.py index 94facf8..9c68ab3 100644 --- a/setup.py +++ b/setup.py @@ -29,6 +29,7 @@ entry_points={ 'console_scripts': [ 'input_generation=perturbopy.generate_input:input_generation', + 'auto_pipeline = perturbopy.auto_pipeline.auto_pipeline:run_epr_calculation', ], }, ) diff --git a/src/perturbopy/auto_pipeline/__init__.py b/src/perturbopy/auto_pipeline/__init__.py new file mode 100644 index 0000000..05d8fce --- /dev/null +++ b/src/perturbopy/auto_pipeline/__init__.py @@ -0,0 +1 @@ +"The automated pipeline for the preliminary QE steps and qe2pert.x calculations." \ No newline at end of file diff --git a/src/perturbopy/auto_pipeline/ap_utils.py b/src/perturbopy/auto_pipeline/ap_utils.py new file mode 100644 index 0000000..269f227 --- /dev/null +++ b/src/perturbopy/auto_pipeline/ap_utils.py @@ -0,0 +1,90 @@ +import os +import shutil + + +def make_computational_folder(perturbo_inputs_dir_path, config_machine, rm_preexist_dir=True): + """ + Check if the COMP_FOLD variable is written in the config_machine file. + If not - use default location "/COMP_FOLD". + After it, this programm make the copy from folder with input files from inputs folder. + + + Parameters + ---------- + source_folder : str + path of source directory + + perturbo_inputs_dir_path : str + folder with all input files for the test + + config_machine : dict + dictionary with computational information, which we'll use in this set of computations. + + rm_preexist_dir : bool + whether to remove dir if it preexists + + Returns + ------- + comp_folder : str + string containing the path to generate directory in which + outputs for computation will be generated + + """ + # Read the perturbo_run variable from the environment + perturbo_folder_dir_prefix = "COMP_FOLD" + try: + perturbo_folder_dir_prefix = config_machine['COMP_FOLD'] + except KeyError: + print(f'COMP_FOLD not set in the config_machine. using default location - {perturbo_folder_dir_prefix}') + + # if folder already exist and we don't want to change it - simply pass rest of the function + perturbo_folder_dir = perturbo_folder_dir_prefix + if not rm_preexist_dir: + return perturbo_folder_dir + + # else - copy over input files to scratch dir + src = perturbo_inputs_dir_path + dst = perturbo_folder_dir + if os.path.isdir(dst): + print(f'\n directory {dst} exists. Removing this directory ...\n') + shutil.rmtree(dst) + copy_folder_with_softlinks(src, dst) + else: + copy_folder_with_softlinks(src, dst) + + return os.path.abspath(perturbo_folder_dir) + + +def copy_folder_with_softlinks(src, dst): + """ + Copy the src directory to the dst directory. + + Parameters + ---------- + src : str + folder with computational info, which is copied + + dst : str + folder, where the files will be copied + + Returns + ------- + None + """ + os.makedirs(dst) + + # Get the list of files in the source folder + for root, dirs, files in os.walk(src): + + # Determine the relative path of the current directory + relative_path = os.path.relpath(root, src) + + # Create the corresponding subdirectory in the destination folder + dst_subfolder = os.path.join(dst, relative_path) + if not os.path.exists(dst_subfolder): + os.makedirs(dst_subfolder) + + for file_name in files: + src_file_path = os.path.join(root, file_name) + dst_file_path = os.path.join(dst_subfolder, file_name) + shutil.copy2(src_file_path, dst_file_path) diff --git a/src/perturbopy/test_utils/legacy/legacy.py b/src/perturbopy/auto_pipeline/auto_pipeline.py similarity index 90% rename from src/perturbopy/test_utils/legacy/legacy.py rename to src/perturbopy/auto_pipeline/auto_pipeline.py index 3ab2a9e..9680ea0 100644 --- a/src/perturbopy/test_utils/legacy/legacy.py +++ b/src/perturbopy/auto_pipeline/auto_pipeline.py @@ -1,17 +1,14 @@ # Legacy code, it may be possible to use it again in the future, so don't delete it yet. # from test_driver.py -import numpy as np import os import sys -import shlex -import shutil import subprocess +import argparse from perturbopy.io_utils.io import open_yaml from perturbopy.test_utils.run_test.env_utils import run_from_config_machine -from perturbopy.test_utils.run_test.env_utils import perturbo_scratch_dir_config, move_qe2pert_files -from perturbopy.test_utils.run_test.run_utils import print_test_info, setup_default_tol from perturbopy.test_utils.run_test.run_utils import ph_collection, define_nq_num +from perturbopy.auto_pipeline.ap_utils import make_computational_folder def preliminary_commands(config_machine, step): @@ -288,7 +285,7 @@ def run_qe2pert(source_folder, work_path, config_machine, prefix, input_name='qe os.chdir(source_folder) -def run_epr_calculation(epr_name, config_machine, source_folder): +def run_epr_calculation(): """ Run one test: #. Run scf calculation @@ -299,8 +296,6 @@ def run_epr_calculation(epr_name, config_machine, source_folder): Parameters ---------- - epr_name : str - name of computed epr_name file config_machine : str name of file with computational information, which we'll use in this set of computations. Should be in folder {source_folder}/config_machine. @@ -311,23 +306,21 @@ def run_epr_calculation(epr_name, config_machine, source_folder): ----- None """ + parser = argparse.ArgumentParser() + parser.add_argument('--config_machine', type=str, help='config_machine file', default='config_machine.yml', required=False) + parser.add_argument('--source_folder', type=str, help='source folder', required=True) + args = parser.parse_args() + config_machine = args.config_machine + source_folder = args.source_folder + source_folder = os.path.abspath(source_folder) # suffixes of paths needed to find driver/utils/references - inputs_path_suffix = f'epr_computation/{epr_name}' config_machine = open_yaml(os.path.join(source_folder, f'config_machine/{config_machine}')) # determine needed paths - inputs_dir_path = os.path.join(source_folder, inputs_path_suffix) - work_path = perturbo_scratch_dir_config(source_folder, inputs_dir_path, epr_name, config_machine, test_case='epr_calculation') + inputs_dir_path = source_folder + work_path = make_computational_folder(inputs_dir_path, config_machine) - # open input yaml-files with supplementary info - # and computational commands - input_yaml = open_yaml(os.path.join(source_folder, 'test_listing.yml')) - - # print the test information before the run - print_test_info(epr_name, input_yaml, test_type='qe2pert') - - # define the prefix - we'll need to have it in the later computations - prefix = input_yaml[epr_name]['prefix'] + prefix = config_machine['prefix'] # run scf run_scf(source_folder, work_path, config_machine) @@ -343,5 +336,3 @@ def run_epr_calculation(epr_name, config_machine, source_folder): # run qe2pert run_qe2pert(source_folder, work_path, config_machine, prefix) - - return From 61da6c98bcca8d483a1971eb149813811b020d55 Mon Sep 17 00:00:00 2001 From: Sergei Kliavinek Date: Mon, 10 Mar 2025 10:42:22 -0700 Subject: [PATCH 2/2] Generalize pipeline for different wannier90 inputs --- src/perturbopy/auto_pipeline/auto_pipeline.py | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/src/perturbopy/auto_pipeline/auto_pipeline.py b/src/perturbopy/auto_pipeline/auto_pipeline.py index 9680ea0..a3b180c 100644 --- a/src/perturbopy/auto_pipeline/auto_pipeline.py +++ b/src/perturbopy/auto_pipeline/auto_pipeline.py @@ -263,18 +263,14 @@ def run_qe2pert(source_folder, work_path, config_machine, prefix, input_name='qe os.symlink(softlink, f'tmp/{prefix}.save') # link rest files - softlink = f'../pw-ph-wann/wann/{prefix}_u.mat' print(f'\n = Link rest files = :\n {softlink};') sys.stdout.flush() - os.symlink(softlink, f'{prefix}_u.mat') - softlink = f'../pw-ph-wann/wann/{prefix}_u_dis.mat' - print(f'\n {softlink};') - sys.stdout.flush() - os.symlink(softlink, f'{prefix}_u_dis.mat') - softlink = f'../pw-ph-wann/wann/{prefix}_centres.xyz' - print(f'\n {softlink};') - sys.stdout.flush() - os.symlink(softlink, f'{prefix}_centres.xyz') + for file in os.listdir(f'{work_path}/pw-ph-wann/wann/'): + if file.startswith(prefix): + softlink = f'../pw-ph-wann/wann/{file}' + print(f'\n {softlink};') + sys.stdout.flush() + os.symlink(softlink, file) # run qe2pert print(f' = Running qe2pert = :\n {run}')