From 52445f78856623274e094ed37b4c347cad1c000b Mon Sep 17 00:00:00 2001 From: Henri Casanova Date: Wed, 24 Sep 2025 12:12:18 -1000 Subject: [PATCH 1/3] Added debug info to wfchef test --- tests/wfchef/test_wfchef.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/tests/wfchef/test_wfchef.py b/tests/wfchef/test_wfchef.py index c3311836..51a6b957 100644 --- a/tests/wfchef/test_wfchef.py +++ b/tests/wfchef/test_wfchef.py @@ -60,17 +60,23 @@ def test_create_recipe(self) -> None: assert((dirpath / "recipe_recipes" / "somename" / "recipe.py").exists()) assert((dirpath / "recipe_recipes" / "somename" / "microstructures").exists()) + sys.stderr.write("1. CALLING LS_RECIPE()\n") ls_recipe() # Install the recipe sys.stderr.write("Installing the recipe...\n") - subprocess.check_call([sys.executable, "-m", "pip", "install", "/tmp/recipe"]) + subprocess.check_call([sys.executable, "-m", "pip", "install", "/tmp/recipe/recipe_recipe"]) + + sys.stderr.write("2. CALLING LS_RECIPE()\n") + ls_recipe() # Uninstall the recipe # TODO: This does not uninstall the recipe (to fix) - # sys.stderr.write("Uninstalling the recipe...\n") - # uninstall_recipe("/tmp/recipe") - # ls_recipe() + sys.stderr.write("Uninstalling the recipe...\n") + uninstall_recipe("recipe_recipes.somename", dirpath) + sys.stderr.write("3. CALLING LS_RECIPE()\n") + + ls_recipe() # TODO: Do more extensive tests From fd0641a3e2f823c9c136416667945fc4a0512248 Mon Sep 17 00:00:00 2001 From: Henri Casanova Date: Wed, 24 Dec 2025 11:51:37 -1000 Subject: [PATCH 2/3] temporary commit --- pyproject.toml | 1 + tests/wfchef/test_wfchef.py | 6 +- wfcommons/wfchef/chef.py | 331 +++++++++++++++++++++++++++++------- 3 files changed, 271 insertions(+), 67 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index cb668e2a..3462be1a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -37,6 +37,7 @@ dependencies = [ "stringcase", "filelock", "pathos", + "tomli-w" ] dynamic = ["version", "entry-points", "scripts"] diff --git a/tests/wfchef/test_wfchef.py b/tests/wfchef/test_wfchef.py index 51a6b957..8ae8e529 100644 --- a/tests/wfchef/test_wfchef.py +++ b/tests/wfchef/test_wfchef.py @@ -52,10 +52,8 @@ def test_create_recipe(self) -> None: create_recipe(args["path"], args["out"], args["name"], cutoff=args["cutoff"], verbose=True) # Check that some of the expected files are there - assert((dirpath / "setup.py").exists()) + assert((dirpath / "pyproject.toml").exists()) assert((dirpath / "recipe_recipes" / "__init__.py").exists()) - assert((dirpath / "recipe_recipes" / "__init__.py").exists()) - assert((dirpath / "recipe_recipes" / "somename" / "__init__.py").exists()) assert((dirpath / "recipe_recipes" / "somename" / "__init__.py").exists()) assert((dirpath / "recipe_recipes" / "somename" / "recipe.py").exists()) assert((dirpath / "recipe_recipes" / "somename" / "microstructures").exists()) @@ -65,7 +63,7 @@ def test_create_recipe(self) -> None: # Install the recipe sys.stderr.write("Installing the recipe...\n") - subprocess.check_call([sys.executable, "-m", "pip", "install", "/tmp/recipe/recipe_recipe"]) + subprocess.check_call([sys.executable, "-m", "pip", "install", "/tmp/recipe/"]) sys.stderr.write("2. CALLING LS_RECIPE()\n") ls_recipe() diff --git a/wfcommons/wfchef/chef.py b/wfcommons/wfchef/chef.py index 31b3e6a5..1d775be4 100644 --- a/wfcommons/wfchef/chef.py +++ b/wfcommons/wfchef/chef.py @@ -17,7 +17,8 @@ import pandas as pd import pathlib import pickle -import pkg_resources +from importlib.metadata import entry_points +import tomli_w import subprocess import traceback import sys @@ -165,49 +166,149 @@ def analyzer_summary(path_to_instances: pathlib.Path) -> Dict: return stats_dict -def get_recipe(recipe: str) -> "Module": - for entry_point in pkg_resources.iter_entry_points('workflow_recipes'): - att = entry_point.attrs[0] - if att == recipe: +def get_recipe(recipe: str) -> Optional[type]: + """ + Load a recipe by name from installed entry points. + + :param recipe: Name of the recipe to load + :return: Recipe class or None if not found + """ + # For Python 3.10+, entry_points() returns a more convenient interface + # For Python 3.9, you may need to use entry_points().get('workflow_recipes', []) + try: + eps = entry_points(group='workflow_recipes') + except TypeError: + # Python 3.9 compatibility + eps = entry_points().get('workflow_recipes', []) + + for entry_point in eps: + # In importlib.metadata, entry points have 'name' instead of 'attrs' + if entry_point.name == recipe: return entry_point.load() + return None + def get_recipes() -> pd.DataFrame: + """ + Get a DataFrame of all available workflow recipes. + + :return: DataFrame with columns: name, module, import command + """ rows = [] - for entry_point in pkg_resources.iter_entry_points('workflow_recipes'): + + try: + eps = entry_points(group='workflow_recipes') + except TypeError: + # Python 3.9 compatibility + eps = entry_points().get('workflow_recipes', []) + + for entry_point in eps: try: Recipe = entry_point.load() - rows.append( - [Recipe.__name__, entry_point.module_name, f"from {entry_point.module_name} import {Recipe.__name__}"]) + # Extract module name from the entry point value + module_name = entry_point.value.split(':')[0] + class_name = Recipe.__name__ + rows.append([ + entry_point.name, # Use entry point name instead of class name + module_name, + f"from {module_name} import {class_name}" + ]) except Exception as e: + print(f"Could not load {entry_point.name}: {e}") traceback.print_exc() - print(f"Could not load {entry_point.module_name}") + return pd.DataFrame(rows, columns=["name", "module", "import command"]) def ls_recipe(): """ - Inspired by UNIX `ls` command, it lists the recipes already installed into the system and - how to import it to use. + Inspired by UNIX `ls` command, it lists the recipes already installed + into the system and how to import it to use. """ print(get_recipes()) -def uninstall_recipe(module_name:str, - savedir: pathlib.Path = this_dir.joinpath("recipes")): +def install_recipe(recipe_path: Union[str, pathlib.Path], + editable: bool = False, + verbose: bool = False): """ - Uninstalls a recipe installed in the system. + Installs a recipe from a local directory into the system. + + :param recipe_path: Path to the recipe directory (containing setup.py or pyproject.toml) + :param editable: If True, install in editable/development mode (-e flag) + :param verbose: If True, show detailed pip output """ + recipe_path = pathlib.Path(recipe_path).resolve() + + if not recipe_path.exists(): + print(f"Error: Recipe path does not exist: {recipe_path}") + return False + + # Check for setup.py or pyproject.toml + has_setup = recipe_path.joinpath("setup.py").exists() + has_pyproject = recipe_path.joinpath("pyproject.toml").exists() + + if not (has_setup or has_pyproject): + print(f"Error: No setup.py or pyproject.toml found in {recipe_path}") + return False - dst = f"wfcommons.wfchef.recipe.{savedir.stem}" try: - print([sys.executable, "-m", "pip", "uninstall", "-y", dst]) - subprocess.run([sys.executable, "-m", "pip", "uninstall", "-y", dst]) + cmd = [sys.executable, "-m", "pip", "install"] + + if editable: + cmd.append("-e") + + if verbose: + cmd.append("-v") + + cmd.append(str(recipe_path)) + + print(f"Installing recipe from: {recipe_path}") + print(f"Command: {' '.join(cmd)}") + + result = subprocess.run(cmd, capture_output=True, text=True) + + if result.returncode != 0: + print(f"Installation failed: {result.stderr}") + return False + else: + print(f"Successfully installed recipe from {recipe_path}") + if verbose: + print(result.stdout) + return True + + except Exception as e: + print(f"Could not install recipe from {recipe_path}: {e}") traceback.print_exc() + return False + + +def uninstall_recipe(module_name: str, + savedir: pathlib.Path = pathlib.Path(__file__).parent.joinpath("recipes")): + """ + Uninstalls a recipe installed in the system. + + :param module_name: Name of the module to uninstall + :param savedir: Directory where recipes are saved + """ + dst = f"wfcommons.wfchef.recipe.{savedir.stem}" + try: + cmd = [sys.executable, "-m", "pip", "uninstall", "-y", dst] + print(cmd) + result = subprocess.run(cmd, capture_output=True, text=True) + + if result.returncode != 0: + print(f"Uninstall failed: {result.stderr}") + return False + else: + print(f"Successfully uninstalled {dst}") + return True except Exception as e: + print(f"Could not uninstall recipe for {module_name}: {e}") traceback.print_exc() - print(f"Could not uninstall recipe for {module_name}") + return False def create_recipe(path_to_instances: Union[str, pathlib.Path], @@ -215,85 +316,189 @@ def create_recipe(path_to_instances: Union[str, pathlib.Path], wf_name: str, cutoff: int = 4000, verbose: bool = False, - runs: int = 1): + runs: int = 1, + author: str = "Workflow Chef", + author_email: str = "workflow@example.com", + package_version: str = "0.1.0"): """ - Creates a recipe for a workflow application by automatically replacing custom information - from the recipe skeleton. + Creates a recipe for a workflow application by automatically replacing + custom information from the recipe skeleton. - :param path_to_instances: name (for samples available in WfCommons) or path to the real workflow instances. - :type path_to_instances: str or pathlib.Path + :param path_to_instances: name (for samples available in WfCommons) or + path to the real workflow instances. :param savedir: path to save the recipe. - :type savedir: pathlib.Path :param wf_name: name of the workflow application. - :type wf_name: str :param cutoff: when set, only consider instances of smaller or equal sizes. - :type cutoff: int :param verbose: when set, prints status messages. - :type cutoff: bool - :param verbose: number of times to repeat the err calculation process (due to randomization). - :type runs: bool + :param runs: number of times to repeat the err calculation process + (due to randomization). + :param author: package author name. + :param author_email: package author email. + :param package_version: initial package version. """ + try: + import tomli_w + except ImportError: + raise ImportError( + "tomli_w is required for pyproject.toml generation. " + "Install it with: pip install tomli-w" + ) + + # Import these from your actual modules + from stringcase import capitalcase + + # Note: You'll need to define these paths in your actual code + # skeleton_path = pathlib.Path(__file__).parent.joinpath("skeleton") + camelname = capitalcase(wf_name) + package_name = f"{savedir.stem}_recipes" savedir.mkdir(exist_ok=True, parents=True) - dst = pathlib.Path(savedir, f"{savedir.stem}_recipes", wf_name).resolve() + dst = pathlib.Path(savedir, package_name, wf_name).resolve() dst.mkdir(exist_ok=True, parents=True) if verbose: print(f"Finding microstructures") + microstructures_path = dst.joinpath("microstructures") - save_microstructures(path_to_instances, microstructures_path, img_type=None, cutoff=cutoff) + # save_microstructures(path_to_instances, microstructures_path, + # img_type=None, cutoff=cutoff) if verbose: print(f"Generating Error Table") + err_savepath = microstructures_path.joinpath("metric", "err.csv") err_savepath.parent.mkdir(exist_ok=True, parents=True) - df = find_err(microstructures_path, runs=runs) - err_savepath.write_text(df.to_csv()) + # df = find_err(microstructures_path, runs=runs) + # err_savepath.write_text(df.to_csv()) - # Recipe - with skeleton_path.joinpath("recipe.py").open() as fp: - skeleton_str = fp.read() + # Recipe + # with skeleton_path.joinpath("recipe.py").open() as fp: + # skeleton_str = fp.read() if verbose: print(f"Generating Recipe Code") - skeleton_str = skeleton_str.replace("Skeleton", camelname) - skeleton_str = skeleton_str.replace("skeleton", wf_name) - with this_dir.joinpath(dst.joinpath("recipe.py")).open("w+") as fp: - fp.write(skeleton_str) + + # skeleton_str = skeleton_str.replace("Skeleton", camelname) + # skeleton_str = skeleton_str.replace("skeleton", wf_name) + # with dst.joinpath("recipe.py").open("w+") as fp: + # fp.write(skeleton_str) # recipe __init__.py - dst.joinpath("__init__.py").write_text(f"from .recipe import {camelname}Recipe") + dst.joinpath("__init__.py").write_text( + f"from .recipe import {camelname}Recipe\n" + ) - # setup.py - with skeleton_path.joinpath("setup.py").open() as fp: - skeleton_str = fp.read() + # __init__.py for recipes parent + dst.parent.joinpath("__init__.py").touch(exist_ok=True) - skeleton_str = skeleton_str.replace("PACKAGE_NAME", savedir.stem) + # Read existing imports or create new + parent_init = dst.parent.joinpath("__init__.py") + existing_imports = set() + if parent_init.exists() and parent_init.stat().st_size > 0: + existing_imports = set(parent_init.read_text().strip().split('\n')) - - print(f"The setup.py file being edit is located at {dst.parent.parent.joinpath('setup.py')}") - with this_dir.joinpath(dst.parent.parent.joinpath("setup.py")).open("w+") as fp: - fp.write(skeleton_str) + new_import = f"from .{wf_name} import {camelname}Recipe" + existing_imports.add(new_import) - # __init__.py - dst.parent.joinpath("__init__.py").touch(exist_ok=True) - with dst.parent.joinpath("__init__.py").open("a") as fp: - fp.write(f"from .{wf_name} import {camelname}Recipe\n") + parent_init.write_text('\n'.join(sorted(existing_imports)) + '\n') - # MANIFEST - with dst.parent.parent.joinpath("MANIFEST.in").open("a+") as fp: - fp.write(f"graft {savedir.stem}_recipes/{wf_name}/microstructures/**\n") - fp.write(f"graft {savedir.stem}_recipes/{wf_name}/microstructures\n") - fp.write(f"graft {savedir.stem}_recipes/{wf_name}\n") + # Create or update pyproject.toml + pyproject_path = dst.parent.parent.joinpath("pyproject.toml") - # workflow_recipes - with this_dir.joinpath(dst.parent.parent.joinpath("workflow_recipes.txt")).open("a+") as fp: - fp.write(f"{wf_name}_recipe = {savedir.stem}_recipes.{wf_name}:{camelname}Recipe\n") + if verbose: + print(f"Generating pyproject.toml at {pyproject_path}") + + # Load existing pyproject.toml or create new + if pyproject_path.exists(): + try: + import tomli + with pyproject_path.open("rb") as f: + config = tomli.load(f) + except ImportError: + print("Warning: tomli not installed. Creating new pyproject.toml") + config = {} + else: + config = {} + + # Ensure all required sections exist + if "build-system" not in config: + config["build-system"] = { + "requires": ["setuptools>=61.0", "wheel"], + "build-backend": "setuptools.build_meta" + } + + if "project" not in config: + config["project"] = { + "name": package_name, + "version": package_version, + "description": f"Workflow recipes package for {package_name}", + "authors": [{"name": author, "email": author_email}], + "requires-python": ">=3.8", + "dependencies": [ + "wfcommons>=1.0.0", + "pandas", + "numpy", + ], + } + + # Add README if it exists + readme_path = dst.parent.parent.joinpath("README.md") + if readme_path.exists(): + config["project"]["readme"] = "README.md" + + # Update or create entry points + if "entry-points" not in config["project"]: + config["project"]["entry-points"] = {} + + if "workflow_recipes" not in config["project"]["entry-points"]: + config["project"]["entry-points"]["workflow_recipes"] = {} + + # Add this recipe's entry point + entry_point_name = f"{wf_name}_recipe" + entry_point_value = f"{package_name}.{wf_name}:{camelname}Recipe" + config["project"]["entry-points"]["workflow_recipes"][entry_point_name] = entry_point_value + + # Configure setuptools to include package data + if "tool" not in config: + config["tool"] = {} + + if "setuptools" not in config["tool"]: + config["tool"]["setuptools"] = {} + + # Specify packages to include + if "packages" not in config["tool"]["setuptools"]: + config["tool"]["setuptools"]["packages"] = [package_name] + + # Include all package data (replaces MANIFEST.in) + if "package-data" not in config["tool"]["setuptools"]: + config["tool"]["setuptools"]["package-data"] = { + "*": ["**/*"] + } + + # Write pyproject.toml + with pyproject_path.open("wb") as f: + tomli_w.dump(config, f) + + if verbose: + print(f"Updated pyproject.toml with entry point: {entry_point_name}") if verbose: print(f"Analyzing Workflow Statistics") - stats = analyzer_summary(path_to_instances) - dst.joinpath("task_type_stats.json").write_text(json.dumps(stats)) + + # stats = analyzer_summary(path_to_instances) + # dst.joinpath("task_type_stats.json").write_text(json.dumps(stats)) + + if verbose: + print(f"\n{'=' * 60}") + print(f"Recipe created successfully!") + print(f"{'=' * 60}") + print(f"Recipe location: {dst}") + print(f"Package config: {pyproject_path}") + print(f"\nTo install this recipe, run:") + print(f" pip install -e {dst.parent.parent}") + print(f"\nOr use the install_recipe function:") + print(f" install_recipe('{dst.parent.parent}', editable=True)") + print(f"{'=' * 60}\n") def get_parser() -> argparse.ArgumentParser: From 81858090c47ca29c9835459bc69fb0143599b60a Mon Sep 17 00:00:00 2001 From: Henri Casanova Date: Wed, 24 Dec 2025 18:51:49 -1000 Subject: [PATCH 3/3] Refactored the recipe installing/uninstalling/ls feature --- tests/wfchef/test_wfchef.py | 83 ++++++-- wfcommons/wfchef/chef.py | 375 ++++++++++++++++++++++++++---------- 2 files changed, 337 insertions(+), 121 deletions(-) diff --git a/tests/wfchef/test_wfchef.py b/tests/wfchef/test_wfchef.py index 8ae8e529..cd5974a6 100644 --- a/tests/wfchef/test_wfchef.py +++ b/tests/wfchef/test_wfchef.py @@ -17,6 +17,7 @@ from tests.test_helpers import _create_fresh_local_dir from wfcommons.wfchef.chef import create_recipe +from wfcommons.wfchef.chef import install_recipe from wfcommons.wfchef.chef import uninstall_recipe from wfcommons.wfchef.chef import ls_recipe @@ -32,10 +33,11 @@ def test_create_recipe(self) -> None: dirpath = _create_fresh_local_dir("/tmp/recipe/") # Put a few JSON workflows in /tmp - urls = ["https://raw.githubusercontent.com/wfcommons/WfInstances/refs/heads/main/makeflow/blast/blast-chameleon-small-001.json", - "https://raw.githubusercontent.com/wfcommons/WfInstances/refs/heads/main/makeflow/blast/blast-chameleon-small-002.json", - "https://raw.githubusercontent.com/wfcommons/WfInstances/refs/heads/main/makeflow/blast/blast-chameleon-small-003.json", - ] + urls = [ + "https://raw.githubusercontent.com/wfcommons/WfInstances/refs/heads/main/makeflow/blast/blast-chameleon-small-001.json", + "https://raw.githubusercontent.com/wfcommons/WfInstances/refs/heads/main/makeflow/blast/blast-chameleon-small-002.json", + "https://raw.githubusercontent.com/wfcommons/WfInstances/refs/heads/main/makeflow/blast/blast-chameleon-small-003.json", + ] for url in urls: response = requests.get(url) local_file_name = url.split("/")[-1] @@ -49,33 +51,78 @@ def test_create_recipe(self) -> None: "name": "somename", "cutoff": 4000 } - create_recipe(args["path"], args["out"], args["name"], cutoff=args["cutoff"], verbose=True) - # Check that some of the expected files are there - assert((dirpath / "pyproject.toml").exists()) - assert((dirpath / "recipe_recipes" / "__init__.py").exists()) - assert((dirpath / "recipe_recipes" / "somename" / "__init__.py").exists()) - assert((dirpath / "recipe_recipes" / "somename" / "recipe.py").exists()) - assert((dirpath / "recipe_recipes" / "somename" / "microstructures").exists()) - - sys.stderr.write("1. CALLING LS_RECIPE()\n") + sys.stderr.write("\n" + "=" * 60 + "\n") + sys.stderr.write("Creating recipe...\n") + sys.stderr.write("=" * 60 + "\n") + + create_recipe( + args["path"], + args["out"], + args["name"], + cutoff=args["cutoff"], + verbose=True + ) + + # Check that expected files are there + sys.stderr.write("\nVerifying created files...\n") + assert (dirpath / "pyproject.toml").exists(), "pyproject.toml not found" + assert (dirpath / "wfchef_recipe_somename" / "__init__.py").exists(), "package __init__.py not found" + assert (dirpath / "wfchef_recipe_somename" / "recipe.py").exists(), "recipe.py not found" + assert (dirpath / "wfchef_recipe_somename" / "microstructures").exists(), "microstructures not found" + sys.stderr.write("✓ All expected files created\n") + + sys.stderr.write("\n" + "=" * 60 + "\n") + sys.stderr.write("Calling ls_recipe before the install:\n") + sys.stderr.write("=" * 60 + "\n") ls_recipe() # Install the recipe + sys.stderr.write("\n" + "=" * 60 + "\n") sys.stderr.write("Installing the recipe...\n") - subprocess.check_call([sys.executable, "-m", "pip", "install", "/tmp/recipe/"]) + sys.stderr.write("=" * 60 + "\n") + + success = install_recipe(dirpath, verbose=True) + assert success, "Recipe installation failed" + sys.stderr.write("✓ Recipe installed successfully\n") - sys.stderr.write("2. CALLING LS_RECIPE()\n") + sys.stderr.write("\n" + "=" * 60 + "\n") + sys.stderr.write("Calling ls_recipe after the install:\n") + sys.stderr.write("=" * 60 + "\n") ls_recipe() + # Verify the recipe can be loaded + sys.stderr.write("\n" + "=" * 60 + "\n") + sys.stderr.write("Testing the recipe import...\n") + sys.stderr.write("=" * 60 + "\n") + + try: + from wfchef_recipe_somename import SomenameRecipe + sys.stderr.write("✓ Successfully imported SomenameRecipe\n") + sys.stderr.write(f" Recipe class: {SomenameRecipe}\n") + sys.stderr.write(f" Module: {SomenameRecipe.__module__}\n") + except ImportError as e: + sys.stderr.write(f"✗ Failed to import recipe: {e}\n") + raise + # Uninstall the recipe - # TODO: This does not uninstall the recipe (to fix) + sys.stderr.write("\n" + "=" * 60 + "\n") sys.stderr.write("Uninstalling the recipe...\n") - uninstall_recipe("recipe_recipes.somename", dirpath) - sys.stderr.write("3. CALLING LS_RECIPE()\n") + sys.stderr.write("=" * 60 + "\n") + + success = uninstall_recipe("somename") + assert success, "Recipe uninstallation failed" + sys.stderr.write("✓ Recipe uninstalled successfully\n") + sys.stderr.write("\n" + "=" * 60 + "\n") + sys.stderr.write("Calling ls_recipe after the uninstall:\n") + sys.stderr.write("=" * 60 + "\n") ls_recipe() + sys.stderr.write("\n" + "=" * 60 + "\n") + sys.stderr.write("TEST COMPLETED SUCCESSFULLY\n") + sys.stderr.write("=" * 60 + "\n") + # TODO: Do more extensive tests # - Install/Uninstall the recipe diff --git a/wfcommons/wfchef/chef.py b/wfcommons/wfchef/chef.py index 1d775be4..9da4c81a 100644 --- a/wfcommons/wfchef/chef.py +++ b/wfcommons/wfchef/chef.py @@ -204,6 +204,7 @@ def get_recipes() -> pd.DataFrame: eps = entry_points().get('workflow_recipes', []) for entry_point in eps: + print("===> " + entry_point.name) try: Recipe = entry_point.load() # Extract module name from the entry point value @@ -229,6 +230,156 @@ def ls_recipe(): print(get_recipes()) +def install_recipe(recipe_path: Union[str, pathlib.Path], + verbose: bool = False): + """ + Installs a recipe from a local directory into the system. The recipe will be + + :param recipe_path: Path to the recipe directory (containing setup.py or pyproject.toml) + :param verbose: If True, show detailed pip output + """ + recipe_path = pathlib.Path(recipe_path).resolve() + + if not recipe_path.exists(): + print(f"Error: Recipe path does not exist: {recipe_path}") + return False + + # Check for setup.py or pyproject.toml + has_setup = recipe_path.joinpath("setup.py").exists() + has_pyproject = recipe_path.joinpath("pyproject.toml").exists() + + if not (has_setup or has_pyproject): + print(f"Error: No setup.py or pyproject.toml found in {recipe_path}") + return False + + try: + cmd = [sys.executable, "-m", "pip", "install"] + + # Add verbose flag before -e if needed + if verbose: + cmd.append("-v") + + cmd.append(str(recipe_path)) + + print(f"Installing recipe from: {recipe_path}") + print(f"Command: {' '.join(cmd)}") + + result = subprocess.run(cmd, capture_output=True, text=True) + + if result.returncode != 0: + print(f"Installation failed: {result.stderr}") + return False + else: + print(f"Successfully installed recipe from {recipe_path}") + if verbose: + print(result.stdout) + return True + + except Exception as e: + print(f"Could not install recipe from {recipe_path}: {e}") + traceback.print_exc() + return False + + +def install_recipe(recipe_path: Union[str, pathlib.Path], + editable: bool = False, + verbose: bool = False): + """ + Installs a recipe from a local directory into the system. + + :param recipe_path: Path to the recipe directory (containing setup.py or pyproject.toml) + :param editable: If True, install in editable/development mode (-e flag). + Default is False for production installs. + :param verbose: If True, show detailed pip output + """ + recipeimport + sys + + +import subprocess +import pathlib +import json +import traceback +from typing import Union, Optional +from importlib.metadata import entry_points +import pandas as pd + + +def get_recipe(recipe: str) -> Optional[type]: + """ + Load a recipe by name from installed entry points. + + :param recipe: Name of the recipe to load + :return: Recipe class or None if not found + """ + # For Python 3.10+, entry_points() returns a more convenient interface + # For Python 3.9, you may need to use entry_points().get('workflow_recipes', []) + try: + eps = entry_points(group='workflow_recipes') + except TypeError: + # Python 3.9 compatibility + eps = entry_points().get('workflow_recipes', []) + + for entry_point in eps: + # In importlib.metadata, entry points have 'name' instead of 'attrs' + if entry_point.name == recipe: + return entry_point.load() + + return None + + +def get_recipes() -> pd.DataFrame: + """ + Get a DataFrame of all available workflow recipes. + + :return: DataFrame with columns: name, module, import command + """ + rows = [] + + try: + eps = entry_points(group='workflow_recipes') + except TypeError: + # Python 3.9 compatibility + eps = entry_points().get('workflow_recipes', []) + + for entry_point in eps: + try: + Recipe = entry_point.load() + # Extract module name from the entry point value + module_name = entry_point.value.split(':')[0] + class_name = Recipe.__name__ + rows.append([ + entry_point.name, # Use entry point name instead of class name + module_name, + f"from {module_name} import {class_name}" + ]) + except Exception as e: + # Try to get module name even if load fails + try: + module_name = entry_point.value.split(':')[0] + class_name = entry_point.value.split(':')[1] if ':' in entry_point.value else "Unknown" + rows.append([ + entry_point.name, + module_name, + f"from {module_name} import {class_name} (NOT LOADED: {e})" + ]) + except: + pass + print(f"Could not load {entry_point.name}: {e}") + if "--verbose" in sys.argv or os.environ.get("WFCHEF_DEBUG"): + traceback.print_exc() + + return pd.DataFrame(rows, columns=["name", "module", "import command"]) + + +def ls_recipe(): + """ + Inspired by UNIX `ls` command, it lists the recipes already installed + into the system and how to import it to use. + """ + print(get_recipes()) + + def install_recipe(recipe_path: Union[str, pathlib.Path], editable: bool = False, verbose: bool = False): @@ -256,13 +407,15 @@ def install_recipe(recipe_path: Union[str, pathlib.Path], try: cmd = [sys.executable, "-m", "pip", "install"] - if editable: - cmd.append("-e") - + # Add verbose flag before -e if needed if verbose: cmd.append("-v") - cmd.append(str(recipe_path)) + # Add editable flag and path together + if editable: + cmd.extend(["-e", str(recipe_path)]) + else: + cmd.append(str(recipe_path)) print(f"Installing recipe from: {recipe_path}") print(f"Command: {' '.join(cmd)}") @@ -284,29 +437,33 @@ def install_recipe(recipe_path: Union[str, pathlib.Path], return False -def uninstall_recipe(module_name: str, - savedir: pathlib.Path = pathlib.Path(__file__).parent.joinpath("recipes")): +def uninstall_recipe(recipe_name: str): """ Uninstalls a recipe installed in the system. - :param module_name: Name of the module to uninstall - :param savedir: Directory where recipes are saved + :param recipe_name: Name of the recipe to uninstall (e.g., 'somename' or 'somename_recipe') """ - dst = f"wfcommons.wfchef.recipe.{savedir.stem}" + # Remove '_recipe' suffix if present + if recipe_name.endswith('_recipe'): + recipe_name = recipe_name[:-7] + + package_name = f"wfchef-recipe-{recipe_name}" + try: - cmd = [sys.executable, "-m", "pip", "uninstall", "-y", dst] - print(cmd) + cmd = [sys.executable, "-m", "pip", "uninstall", "-y", package_name] + print(f"Uninstalling: {package_name}") + print(f"Command: {' '.join(cmd)}") result = subprocess.run(cmd, capture_output=True, text=True) if result.returncode != 0: print(f"Uninstall failed: {result.stderr}") return False else: - print(f"Successfully uninstalled {dst}") + print(f"Successfully uninstalled {package_name}") return True except Exception as e: - print(f"Could not uninstall recipe for {module_name}: {e}") + print(f"Could not uninstall recipe for {recipe_name}: {e}") traceback.print_exc() return False @@ -321,15 +478,14 @@ def create_recipe(path_to_instances: Union[str, pathlib.Path], author_email: str = "workflow@example.com", package_version: str = "0.1.0"): """ - Creates a recipe for a workflow application by automatically replacing - custom information from the recipe skeleton. + Creates a standalone recipe package for a workflow application. :param path_to_instances: name (for samples available in WfCommons) or path to the real workflow instances. :param savedir: path to save the recipe. :param wf_name: name of the workflow application. :param cutoff: when set, only consider instances of smaller or equal sizes. - :param verbose: when set, prints status messages. + :param verbose: when set, prints status messages (and helpful how-to instructions!) :param runs: number of times to repeat the err calculation process (due to randomization). :param author: package author name. @@ -351,87 +507,69 @@ def create_recipe(path_to_instances: Union[str, pathlib.Path], # skeleton_path = pathlib.Path(__file__).parent.joinpath("skeleton") camelname = capitalcase(wf_name) - package_name = f"{savedir.stem}_recipes" + + # Create a standalone package name + package_name = f"wfchef-recipe-{wf_name}" + module_name = f"wfchef_recipe_{wf_name}" + + # Create simple directory structure: {module_name}/ savedir.mkdir(exist_ok=True, parents=True) - dst = pathlib.Path(savedir, package_name, wf_name).resolve() - dst.mkdir(exist_ok=True, parents=True) + + # Create the package directory + recipe_dir = savedir.joinpath(module_name) + recipe_dir.mkdir(exist_ok=True, parents=True) if verbose: print(f"Finding microstructures") - microstructures_path = dst.joinpath("microstructures") - # save_microstructures(path_to_instances, microstructures_path, - # img_type=None, cutoff=cutoff) + microstructures_path = recipe_dir.joinpath("microstructures") + save_microstructures(path_to_instances, microstructures_path, + img_type=None, cutoff=cutoff) if verbose: print(f"Generating Error Table") err_savepath = microstructures_path.joinpath("metric", "err.csv") err_savepath.parent.mkdir(exist_ok=True, parents=True) - # df = find_err(microstructures_path, runs=runs) - # err_savepath.write_text(df.to_csv()) + df = find_err(microstructures_path, runs=runs) + err_savepath.write_text(df.to_csv()) # Recipe - # with skeleton_path.joinpath("recipe.py").open() as fp: - # skeleton_str = fp.read() + with skeleton_path.joinpath("recipe.py").open() as fp: + skeleton_str = fp.read() if verbose: print(f"Generating Recipe Code") - # skeleton_str = skeleton_str.replace("Skeleton", camelname) - # skeleton_str = skeleton_str.replace("skeleton", wf_name) - # with dst.joinpath("recipe.py").open("w+") as fp: - # fp.write(skeleton_str) - - # recipe __init__.py - dst.joinpath("__init__.py").write_text( - f"from .recipe import {camelname}Recipe\n" + skeleton_str = skeleton_str.replace("Skeleton", camelname) + skeleton_str = skeleton_str.replace("skeleton", wf_name) + with recipe_dir.joinpath("recipe.py").open("w+") as fp: + fp.write(skeleton_str) + + # Package __init__.py - exports the recipe class + recipe_dir.joinpath("__init__.py").write_text( + f"\"\"\"WfChef recipe for {wf_name} workflow.\"\"\"\n\n" + f"from .recipe import {camelname}Recipe\n\n" + f"__version__ = '{package_version}'\n" + f"__all__ = ['{camelname}Recipe']\n" ) - # __init__.py for recipes parent - dst.parent.joinpath("__init__.py").touch(exist_ok=True) - - # Read existing imports or create new - parent_init = dst.parent.joinpath("__init__.py") - existing_imports = set() - if parent_init.exists() and parent_init.stat().st_size > 0: - existing_imports = set(parent_init.read_text().strip().split('\n')) - - new_import = f"from .{wf_name} import {camelname}Recipe" - existing_imports.add(new_import) - - parent_init.write_text('\n'.join(sorted(existing_imports)) + '\n') - - # Create or update pyproject.toml - pyproject_path = dst.parent.parent.joinpath("pyproject.toml") + # Create pyproject.toml at the package root + pyproject_path = savedir.joinpath("pyproject.toml") if verbose: print(f"Generating pyproject.toml at {pyproject_path}") - # Load existing pyproject.toml or create new - if pyproject_path.exists(): - try: - import tomli - with pyproject_path.open("rb") as f: - config = tomli.load(f) - except ImportError: - print("Warning: tomli not installed. Creating new pyproject.toml") - config = {} - else: - config = {} - - # Ensure all required sections exist - if "build-system" not in config: - config["build-system"] = { + # Create config for this standalone recipe package + config = { + "build-system": { "requires": ["setuptools>=61.0", "wheel"], "build-backend": "setuptools.build_meta" - } - - if "project" not in config: - config["project"] = { + }, + "project": { "name": package_name, "version": package_version, - "description": f"Workflow recipes package for {package_name}", + "description": f"WfChef recipe for {wf_name} workflow", "authors": [{"name": author, "email": author_email}], "requires-python": ">=3.8", "dependencies": [ @@ -439,65 +577,96 @@ def create_recipe(path_to_instances: Union[str, pathlib.Path], "pandas", "numpy", ], + }, + "tool": { + "setuptools": { + "packages": [module_name], + "package-data": { + module_name: ["**/*"] + } + } } + } - # Add README if it exists - readme_path = dst.parent.parent.joinpath("README.md") - if readme_path.exists(): - config["project"]["readme"] = "README.md" + # Add entry point for this recipe + config["project"]["entry-points"] = { + "workflow_recipes": { + f"{wf_name}_recipe": f"{module_name}:{camelname}Recipe" + } + } - # Update or create entry points - if "entry-points" not in config["project"]: - config["project"]["entry-points"] = {} + # Add README if it exists + readme_path = savedir.joinpath("README.md") + if readme_path.exists(): + config["project"]["readme"] = "README.md" + elif verbose: + # Create a basic README + readme_content = f"""# {package_name} - if "workflow_recipes" not in config["project"]["entry-points"]: - config["project"]["entry-points"]["workflow_recipes"] = {} +WfChef recipe for {wf_name} workflow. - # Add this recipe's entry point - entry_point_name = f"{wf_name}_recipe" - entry_point_value = f"{package_name}.{wf_name}:{camelname}Recipe" - config["project"]["entry-points"]["workflow_recipes"][entry_point_name] = entry_point_value +## Installation - # Configure setuptools to include package data - if "tool" not in config: - config["tool"] = {} +```bash +pip install -e . +``` - if "setuptools" not in config["tool"]: - config["tool"]["setuptools"] = {} +## Usage - # Specify packages to include - if "packages" not in config["tool"]["setuptools"]: - config["tool"]["setuptools"]["packages"] = [package_name] +```python +from {module_name} import {camelname}Recipe - # Include all package data (replaces MANIFEST.in) - if "package-data" not in config["tool"]["setuptools"]: - config["tool"]["setuptools"]["package-data"] = { - "*": ["**/*"] - } +recipe = {camelname}Recipe() +# Use the recipe... +``` + +## Entry Point + +This package registers the following workflow recipe entry point: +- `{wf_name}_recipe` -> `{module_name}:{camelname}Recipe` + +You can load it using: +```python +from wfcommons.wfchef import get_recipe + +Recipe = get_recipe("{wf_name}_recipe") +recipe = Recipe() +``` +""" + readme_path.write_text(readme_content) + config["project"]["readme"] = "README.md" # Write pyproject.toml with pyproject_path.open("wb") as f: tomli_w.dump(config, f) if verbose: - print(f"Updated pyproject.toml with entry point: {entry_point_name}") + print(f"Created pyproject.toml with entry point: {wf_name}_recipe") if verbose: print(f"Analyzing Workflow Statistics") - # stats = analyzer_summary(path_to_instances) - # dst.joinpath("task_type_stats.json").write_text(json.dumps(stats)) + stats = analyzer_summary(path_to_instances) + recipe_dir.joinpath("task_type_stats.json").write_text(json.dumps(stats)) if verbose: print(f"\n{'=' * 60}") print(f"Recipe created successfully!") print(f"{'=' * 60}") - print(f"Recipe location: {dst}") - print(f"Package config: {pyproject_path}") + print(f"Recipe location: {recipe_dir}") + print(f"Package root: {savedir}") + print(f"Package name: {package_name}") + print(f"Module name: {module_name}") + print(f"Entry point: {wf_name}_recipe -> {module_name}:{camelname}Recipe") print(f"\nTo install this recipe, run:") - print(f" pip install -e {dst.parent.parent}") + print(f" pip install -e {savedir}") print(f"\nOr use the install_recipe function:") - print(f" install_recipe('{dst.parent.parent}', editable=True)") + print(f" install_recipe('{savedir}', editable=True)") + print(f"\nAfter installation, import it as:") + print(f" from {module_name} import {camelname}Recipe") + print(f"\nOr load via entry point:") + print(f" from wfcommons.wfchef import get_recipe") + print(f" Recipe = get_recipe('{wf_name}_recipe')") print(f"{'=' * 60}\n")