Automated pipeline

pyproject.toml

@@ -27,7 +27,10 @@ interactive = [
 
 [tool.setuptools]
 package-dir = {"" = "src"}
-packages = ["perturbopy"]
+
+[tool.setuptools.packages.find]
+where = ["src"]
 
 [project.scripts]
-input_generation = "perturbopy.generate_input:input_generation"
+input_generation = "perturbopy.generate_input:input_generation"
+auto_pipeline = "perturbopy.auto_pipeline.auto_pipeline:run_epr_calculation"

setup.py

@@ -29,6 +29,7 @@
     entry_points={
         'console_scripts': [
             'input_generation=perturbopy.generate_input:input_generation',
+            'auto_pipeline = perturbopy.auto_pipeline.auto_pipeline:run_epr_calculation',
         ],
     },
 )

src/perturbopy/auto_pipeline/__init__.py

@@ -0,0 +1 @@
+"The automated pipeline for the preliminary QE steps and qe2pert.x calculations."

src/perturbopy/auto_pipeline/ap_utils.py

@@ -0,0 +1,90 @@
+import os
+import shutil
+
+
+def make_computational_folder(perturbo_inputs_dir_path, config_machine, rm_preexist_dir=True):
+    """
+    Check if the COMP_FOLD variable is written in the config_machine file.
+    If not - use default location "/COMP_FOLD".
+    After it, this programm make the copy from folder with input files from inputs folder.
+
+
+    Parameters
+    ----------
+    source_folder : str
+        path of source directory
+
+    perturbo_inputs_dir_path : str
+        folder with all input files for the test
+
+    config_machine : dict
+        dictionary with computational information, which we'll use in this set of computations.
+
+    rm_preexist_dir : bool
+        whether to remove dir if it preexists
+
+    Returns
+    -------
+    comp_folder : str
+        string containing the path to generate directory in which
+        outputs for computation will be generated
+
+    """
+    # Read the perturbo_run variable from the environment
+    perturbo_folder_dir_prefix   = "COMP_FOLD"
+    try:
+        perturbo_folder_dir_prefix = config_machine['COMP_FOLD']
+    except KeyError:
+        print(f'COMP_FOLD not set in the config_machine. using default location -  {perturbo_folder_dir_prefix}')
+
+    # if folder already exist and we don't want to change it - simply pass rest of the function
+    perturbo_folder_dir = perturbo_folder_dir_prefix
+    if not rm_preexist_dir:
+        return perturbo_folder_dir
+
+    # else - copy over input files to scratch dir
+    src = perturbo_inputs_dir_path
+    dst = perturbo_folder_dir
+    if os.path.isdir(dst):
+        print(f'\n directory {dst} exists. Removing this directory ...\n')
+        shutil.rmtree(dst)
+        copy_folder_with_softlinks(src, dst)
+    else:
+        copy_folder_with_softlinks(src, dst)
+
+    return os.path.abspath(perturbo_folder_dir)
+
+
+def copy_folder_with_softlinks(src, dst):
+    """
+    Copy the src directory to the dst directory.
+
+    Parameters
+    ----------
+    src : str
+        folder with computational info, which is copied
+
+    dst : str
+        folder, where the files will be copied
+
+    Returns
+    -------
+    None
+    """
+    os.makedirs(dst)
+
+    # Get the list of files in the source folder
+    for root, dirs, files in os.walk(src):
+
+        # Determine the relative path of the current directory
+        relative_path = os.path.relpath(root, src)
+
+        # Create the corresponding subdirectory in the destination folder
+        dst_subfolder = os.path.join(dst, relative_path)
+        if not os.path.exists(dst_subfolder):
+            os.makedirs(dst_subfolder)
+
+        for file_name in files:
+            src_file_path = os.path.join(root, file_name)
+            dst_file_path = os.path.join(dst_subfolder, file_name)
+            shutil.copy2(src_file_path, dst_file_path)

src/perturbopy/test_utils/legacy/legacy.py → src/perturbopy/auto_pipeline/auto_pipeline.py

@@ -1,17 +1,14 @@
 # Legacy code, it may be possible to use it again in the future, so don't delete it yet.
 # from test_driver.py
 
-import numpy as np
 import os
 import sys
-import shlex
-import shutil
 import subprocess
+import argparse
 from perturbopy.io_utils.io import open_yaml
 from perturbopy.test_utils.run_test.env_utils import run_from_config_machine
-from perturbopy.test_utils.run_test.env_utils import perturbo_scratch_dir_config, move_qe2pert_files
-from perturbopy.test_utils.run_test.run_utils import print_test_info, setup_default_tol
 from perturbopy.test_utils.run_test.run_utils import ph_collection, define_nq_num
+from perturbopy.auto_pipeline.ap_utils import make_computational_folder
 
 
 def preliminary_commands(config_machine, step):
@@ -266,18 +263,14 @@ def run_qe2pert(source_folder, work_path, config_machine, prefix, input_name='qe
     os.symlink(softlink, f'tmp/{prefix}.save')
 
     # link rest files
-    softlink = f'../pw-ph-wann/wann/{prefix}_u.mat'
     print(f'\n = Link rest files = :\n {softlink};')
     sys.stdout.flush()
-    os.symlink(softlink, f'{prefix}_u.mat')
-    softlink = f'../pw-ph-wann/wann/{prefix}_u_dis.mat'
-    print(f'\n {softlink};')
-    sys.stdout.flush()
-    os.symlink(softlink, f'{prefix}_u_dis.mat')
-    softlink = f'../pw-ph-wann/wann/{prefix}_centres.xyz'
-    print(f'\n {softlink};')
-    sys.stdout.flush()
-    os.symlink(softlink, f'{prefix}_centres.xyz')
+    for file in os.listdir(f'{work_path}/pw-ph-wann/wann/'):
+        if file.startswith(prefix):
+            softlink = f'../pw-ph-wann/wann/{file}'
+            print(f'\n {softlink};')
+            sys.stdout.flush()
+            os.symlink(softlink, file)
 
     # run qe2pert
     print(f' = Running qe2pert = :\n {run}')
@@ -288,7 +281,7 @@ def run_qe2pert(source_folder, work_path, config_machine, prefix, input_name='qe
     os.chdir(source_folder)
 
 
-def run_epr_calculation(epr_name, config_machine, source_folder):
+def run_epr_calculation():
     """
     Run one test:
         #. Run scf calculation
@@ -299,8 +292,6 @@ def run_epr_calculation(epr_name, config_machine, source_folder):
 
     Parameters
     ----------
-    epr_name : str
-        name of computed epr_name file
     config_machine : str
         name of file with computational information, which we'll use in this set of computations.
         Should be in folder {source_folder}/config_machine.
@@ -311,23 +302,21 @@ def run_epr_calculation(epr_name, config_machine, source_folder):
     -----
     None
     """
+    parser  = argparse.ArgumentParser()
+    parser.add_argument('--config_machine', type=str, help='config_machine file', default='config_machine.yml', required=False)
+    parser.add_argument('--source_folder', type=str, help='source folder', required=True)
+    args = parser.parse_args()
+    config_machine = args.config_machine
+    source_folder = args.source_folder
+    source_folder = os.path.abspath(source_folder)
     # suffixes of paths needed to find driver/utils/references
-    inputs_path_suffix = f'epr_computation/{epr_name}'
     config_machine = open_yaml(os.path.join(source_folder, f'config_machine/{config_machine}'))
 
     # determine needed paths
-    inputs_dir_path = os.path.join(source_folder, inputs_path_suffix)
-    work_path = perturbo_scratch_dir_config(source_folder, inputs_dir_path, epr_name, config_machine, test_case='epr_calculation')
+    inputs_dir_path = source_folder
+    work_path = make_computational_folder(inputs_dir_path, config_machine)
 
-    # open input yaml-files with supplementary info
-    # and computational commands
-    input_yaml = open_yaml(os.path.join(source_folder, 'test_listing.yml'))
-
-    # print the test information before the run
-    print_test_info(epr_name, input_yaml, test_type='qe2pert')
-
-    # define the prefix - we'll need to have it in the later computations
-    prefix = input_yaml[epr_name]['prefix']
+    prefix = config_machine['prefix']
 
     # run scf
     run_scf(source_folder, work_path, config_machine)
@@ -343,5 +332,3 @@ def run_epr_calculation(epr_name, config_machine, source_folder):
 
     # run qe2pert
     run_qe2pert(source_folder, work_path, config_machine, prefix)
-
-    return