diff --git a/AGENTS.md b/AGENTS.md index 35444084..2f8abea9 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -24,7 +24,7 @@ This repository contains DFODE-kit, a Python toolkit for sampling combustion sta - If adding a new invariant, encode it in tests or CI. ## Structure -- `dfode_kit/cli_tools/`: CLI entrypoints and subcommands +- `dfode_kit/cli/`: CLI entrypoints and subcommands - `dfode_kit/data_operations/`: dataset I/O, labeling, augmentation, integration utilities - `dfode_kit/dfode_core/`: models, training, preprocessing - `dfode_kit/df_interface/`: DeepFlame/OpenFOAM-facing helpers @@ -45,3 +45,4 @@ This repository contains DFODE-kit, a Python toolkit for sampling combustion sta - `docs/agents/verification.md` - `docs/agents/worktrees.md` - `docs/agents/roadmap.md` +- `docs/agents/package-topology-spec.md` diff --git a/README.md b/README.md index 50a5c614..a23d8e12 100644 --- a/README.md +++ b/README.md @@ -93,7 +93,7 @@ source /path/to/conda/etc/profile.d/conda.sh conda activate deepflame source /path/to/deepflame-dev/bashrc -python -m dfode_kit.cli_tools.main init oneD-flame \ +python -m dfode_kit.cli.main init oneD-flame \ --mech /path/to/mechanisms/CH4/gri30.yaml \ --fuel CH4:1 \ --oxidizer air \ @@ -105,7 +105,7 @@ python -m dfode_kit.cli_tools.main init oneD-flame \ ### 3. Run the case ```bash -python -m dfode_kit.cli_tools.main run-case \ +python -m dfode_kit.cli.main run-case \ --case /path/to/run/oneD_flame_CH4_phi1 \ --apply --json ``` @@ -113,7 +113,7 @@ python -m dfode_kit.cli_tools.main run-case \ ### 4. Sample the finished case into HDF5 ```bash -python -m dfode_kit.cli_tools.main sample \ +python -m dfode_kit.cli.main sample \ --mech /path/to/mechanisms/CH4/gri30.yaml \ --case /path/to/run/oneD_flame_CH4_phi1 \ --save /path/to/run/oneD_flame_CH4_phi1/ch4_phi1_sample.h5 \ @@ -133,10 +133,15 @@ If you are working on the repository itself, see: ## Repository layout -- `dfode_kit/cli_tools/` — CLI entrypoints and subcommands -- `dfode_kit/df_interface/` — DeepFlame/OpenFOAM-facing helpers and case setup -- `dfode_kit/data_operations/` — dataset I/O, sampling, augmentation, labeling -- `dfode_kit/dfode_core/` — model and training code +- `dfode_kit/cli/` — canonical CLI entrypoints and subcommands +- `dfode_kit/cli_tools/` — legacy compatibility shims for older CLI import paths +- `dfode_kit/cases/` — canonical case init/preset/sampling boundaries for DeepFlame/OpenFOAM workflows +- `dfode_kit/df_interface/` — legacy compatibility shims for case-facing helpers during the cases migration +- `dfode_kit/data/` — emerging canonical package for data contracts and HDF5 I/O helpers +- `dfode_kit/data_operations/` — legacy and transitional dataset I/O, augmentation, labeling, and integration helpers +- `dfode_kit/models/` — canonical model package +- `dfode_kit/training/` — canonical training package +- `dfode_kit/dfode_core/` — legacy compatibility surface for model/training code during migration - `canonical_cases/` — canonical flame case templates - `tutorials/` — tutorial notebooks and workflow examples - `docs/` — published project documentation diff --git a/dfode_kit/__init__.py b/dfode_kit/__init__.py index 321bdf90..761fafe4 100644 --- a/dfode_kit/__init__.py +++ b/dfode_kit/__init__.py @@ -30,10 +30,10 @@ "inverse_BCT": ("dfode_kit.utils", "inverse_BCT"), "BCT_torch": ("dfode_kit.utils", "BCT_torch"), "inverse_BCT_torch": ("dfode_kit.utils", "inverse_BCT_torch"), - "gather_species_arrays": ("dfode_kit.df_interface.sample_case", "gather_species_arrays"), - "df_to_h5": ("dfode_kit.df_interface.sample_case", "df_to_h5"), - "touch_h5": ("dfode_kit.data_operations.h5_kit", "touch_h5"), - "get_TPY_from_h5": ("dfode_kit.data_operations.h5_kit", "get_TPY_from_h5"), + "gather_species_arrays": ("dfode_kit.cases.sampling", "gather_species_arrays"), + "df_to_h5": ("dfode_kit.cases.sampling", "df_to_h5"), + "touch_h5": ("dfode_kit.data.io_hdf5", "touch_h5"), + "get_TPY_from_h5": ("dfode_kit.data.io_hdf5", "get_TPY_from_h5"), "advance_reactor": ("dfode_kit.data_operations.h5_kit", "advance_reactor"), "load_model": ("dfode_kit.data_operations.h5_kit", "load_model"), "predict_Y": ("dfode_kit.data_operations.h5_kit", "predict_Y"), diff --git a/dfode_kit/cases/__init__.py b/dfode_kit/cases/__init__.py new file mode 100644 index 00000000..32af3cae --- /dev/null +++ b/dfode_kit/cases/__init__.py @@ -0,0 +1,56 @@ +from importlib import import_module + + +__all__ = [ + 'AIR_OXIDIZER', + 'DEFAULT_ONE_D_FLAME_PRESET', + 'DEFAULT_ONE_D_FLAME_TEMPLATE', + 'ONE_D_FLAME_PRESETS', + 'OneDFlameInitInputs', + 'OneDFlamePreset', + 'OneDFreelyPropagatingFlameConfig', + 'df_to_h5', + 'dump_plan_json', + 'gather_species_arrays', + 'get_one_d_flame_preset', + 'load_plan_json', + 'one_d_flame_inputs_from_plan', + 'one_d_flame_overrides_from_plan', + 'one_d_flame_plan_dict', + 'resolve_oxidizer', + 'setup_one_d_flame_case', +] + +_ATTRIBUTE_MODULES = { + 'AIR_OXIDIZER': ('dfode_kit.cases.init', 'AIR_OXIDIZER'), + 'DEFAULT_ONE_D_FLAME_PRESET': ('dfode_kit.cases.init', 'DEFAULT_ONE_D_FLAME_PRESET'), + 'DEFAULT_ONE_D_FLAME_TEMPLATE': ('dfode_kit.cases.init', 'DEFAULT_ONE_D_FLAME_TEMPLATE'), + 'ONE_D_FLAME_PRESETS': ('dfode_kit.cases.init', 'ONE_D_FLAME_PRESETS'), + 'OneDFlameInitInputs': ('dfode_kit.cases.init', 'OneDFlameInitInputs'), + 'OneDFlamePreset': ('dfode_kit.cases.init', 'OneDFlamePreset'), + 'OneDFreelyPropagatingFlameConfig': ( + 'dfode_kit.cases.presets', + 'OneDFreelyPropagatingFlameConfig', + ), + 'df_to_h5': ('dfode_kit.cases.sampling', 'df_to_h5'), + 'dump_plan_json': ('dfode_kit.cases.init', 'dump_plan_json'), + 'gather_species_arrays': ('dfode_kit.cases.sampling', 'gather_species_arrays'), + 'get_one_d_flame_preset': ('dfode_kit.cases.init', 'get_one_d_flame_preset'), + 'load_plan_json': ('dfode_kit.cases.init', 'load_plan_json'), + 'one_d_flame_inputs_from_plan': ('dfode_kit.cases.init', 'one_d_flame_inputs_from_plan'), + 'one_d_flame_overrides_from_plan': ('dfode_kit.cases.init', 'one_d_flame_overrides_from_plan'), + 'one_d_flame_plan_dict': ('dfode_kit.cases.init', 'one_d_flame_plan_dict'), + 'resolve_oxidizer': ('dfode_kit.cases.init', 'resolve_oxidizer'), + 'setup_one_d_flame_case': ('dfode_kit.cases.deepflame', 'setup_one_d_flame_case'), +} + + +def __getattr__(name): + if name not in _ATTRIBUTE_MODULES: + raise AttributeError(f"module 'dfode_kit.cases' has no attribute '{name}'") + + module_name, attribute_name = _ATTRIBUTE_MODULES[name] + module = import_module(module_name) + value = getattr(module, attribute_name) + globals()[name] = value + return value diff --git a/dfode_kit/cases/deepflame.py b/dfode_kit/cases/deepflame.py new file mode 100644 index 00000000..e8c4d60c --- /dev/null +++ b/dfode_kit/cases/deepflame.py @@ -0,0 +1,105 @@ +import shutil +from pathlib import Path + +from dfode_kit.cases.presets import OneDFreelyPropagatingFlameConfig + + +def update_one_d_sample_config(cfg: OneDFreelyPropagatingFlameConfig, case_path): + case_path = Path(case_path).resolve() + orig_file_path = case_path / 'system/sampleConfigDict.orig' + new_file_path = case_path / 'system/sampleConfigDict' + shutil.copy(orig_file_path, new_file_path) + + replacements = { + 'CanteraMechanismFile_': f'"{Path(cfg.mech_path).resolve()}"', + 'inertSpecie_': f'"{cfg.inert_specie}"', + 'domainWidth': cfg.domain_width, + 'domainLength': cfg.domain_length, + 'ignitionRegion': cfg.ignition_region, + 'simTimeStep': cfg.sim_time_step, + 'simTime': cfg.sim_time, + 'simWriteInterval': cfg.sim_write_interval, + 'UInlet': cfg.inlet_speed, + 'pInternal': cfg.p0, + } + + with open(new_file_path, 'r') as file: + lines = file.readlines() + + for i, line in enumerate(lines): + for key, value in replacements.items(): + if key in line: + lines[i] = line.replace('placeHolder', str(value)) + + if 'unburntStates' in line: + state_strings = [f'{"TUnburnt":<20}{cfg.initial_gas.T:>16.10f};'] + state_strings += [ + f'{species}Unburnt'.ljust(20) + f'{cfg.initial_gas.Y[idx]:>16.10f};' + for idx, species in enumerate(cfg.species_names) + ] + lines[i] = '\n'.join(state_strings) + '\n\n' + + if 'equilibriumStates' in line: + state_strings = [f'{"TBurnt":<20}{cfg.burnt_gas.T:>16.10f};'] + state_strings += [ + f'{species}Burnt'.ljust(20) + f'{cfg.burnt_gas.Y[idx]:>16.10f};' + for idx, species in enumerate(cfg.species_names) + ] + lines[i] = '\n'.join(state_strings) + '\n\n' + + with open(new_file_path, 'w') as file: + file.writelines(lines) + + +def create_0_species_files(cfg: OneDFreelyPropagatingFlameConfig, case_path): + case_path = Path(case_path).resolve() + orig_0_file_path = case_path / '0/Ydefault.orig' + + for idx, species in enumerate(cfg.species_names): + new_0_file_path = case_path / '0' / f'{species}.orig' + shutil.copy(orig_0_file_path, new_0_file_path) + + with open(new_0_file_path, 'r') as file: + lines = file.readlines() + + for i, line in enumerate(lines): + if 'Ydefault' in line: + lines[i] = line.replace('Ydefault', f'{species}') + if 'uniform 0' in line: + lines[i] = line.replace('0', f'{cfg.initial_gas.Y[idx]}') + + with open(new_0_file_path, 'w') as file: + file.writelines(lines) + + +def update_set_fields_dict(cfg: OneDFreelyPropagatingFlameConfig, case_path): + case_path = Path(case_path).resolve() + orig_setFieldsDict_path = case_path / 'system/setFieldsDict.orig' + new_setFieldsDict_path = case_path / 'system/setFieldsDict' + shutil.copy(orig_setFieldsDict_path, new_setFieldsDict_path) + + with open(new_setFieldsDict_path, 'r') as file: + lines = file.readlines() + + for i, line in enumerate(lines): + if 'unburntStatesPlaceHolder' in line: + state_strings = [f'\tvolScalarFieldValue {"T":<10} $TUnburnt'] + for _, species in enumerate(cfg.species_names): + state_strings.append(f'volScalarFieldValue {species:<10} ${species}Unburnt') + lines[i] = '\n\t'.join(state_strings) + '\n' + if 'equilibriumStatesPlaceHolder' in line: + state_strings = [f'\t\t\tvolScalarFieldValue {"T":<10} $TBurnt'] + for _, species in enumerate(cfg.species_names): + state_strings.append(f'volScalarFieldValue {species:<10} ${species}Burnt') + lines[i] = '\n\t\t\t'.join(state_strings) + '\n' + + with open(new_setFieldsDict_path, 'w') as file: + file.writelines(lines) + + +def setup_one_d_flame_case(cfg: OneDFreelyPropagatingFlameConfig, case_path): + case_path = Path(case_path).resolve() + update_one_d_sample_config(cfg, case_path) + create_0_species_files(cfg, case_path) + update_set_fields_dict(cfg, case_path) + print(f'One-dimensional flame case setup completed at: {case_path}') diff --git a/dfode_kit/cases/init.py b/dfode_kit/cases/init.py new file mode 100644 index 00000000..91a7fc2e --- /dev/null +++ b/dfode_kit/cases/init.py @@ -0,0 +1,148 @@ +from __future__ import annotations + +import json +from dataclasses import asdict, dataclass +from pathlib import Path +from typing import Any + +from dfode_kit import DFODE_ROOT + + +DEFAULT_ONE_D_FLAME_TEMPLATE = ( + DFODE_ROOT / 'canonical_cases' / 'oneD_freely_propagating_flame' +) +DEFAULT_ONE_D_FLAME_PRESET = 'premixed-defaults-v1' +AIR_OXIDIZER = 'O2:1, N2:3.76' + + +@dataclass(frozen=True) +class OneDFlamePreset: + name: str + summary: str + assumptions: dict[str, str] + notes: list[str] + + +ONE_D_FLAME_PRESETS: dict[str, OneDFlamePreset] = { + DEFAULT_ONE_D_FLAME_PRESET: OneDFlamePreset( + name=DEFAULT_ONE_D_FLAME_PRESET, + summary=( + 'Current DFODE-kit empirical defaults for one-dimensional freely ' + 'propagating premixed flames.' + ), + assumptions={ + 'domain_length': 'flame_thickness / 10 * 500', + 'domain_width': 'domain_length / 10', + 'ignition_region': 'domain_length / 2', + 'sim_time_step': '1e-6', + 'num_output_steps': '100', + 'sim_write_interval': '(flame_thickness / flame_speed) * 10 / num_output_steps', + 'sim_time': 'sim_write_interval * (num_output_steps + 1)', + 'inlet_speed': 'flame_speed', + 'inert_specie': '"N2"', + }, + notes=[ + 'These values preserve the current hardcoded logic in OneDFreelyPropagatingFlameConfig.update_config().', + 'They are recommended starter defaults, not universal best practices.', + 'Override any resolved field explicitly when domain knowledge requires it.', + ], + ) +} + + +@dataclass(frozen=True) +class OneDFlameInitInputs: + mechanism: str + fuel: str + oxidizer: str + eq_ratio: float + T0: float + p0: float + preset: str = DEFAULT_ONE_D_FLAME_PRESET + template: str = str(DEFAULT_ONE_D_FLAME_TEMPLATE) + inert_specie: str = 'N2' + + +def resolve_oxidizer(oxidizer: str) -> str: + if oxidizer.strip().lower() == 'air': + return AIR_OXIDIZER + return oxidizer + + +def get_one_d_flame_preset(name: str) -> OneDFlamePreset: + try: + return ONE_D_FLAME_PRESETS[name] + except KeyError as exc: + raise ValueError( + f"Unknown oneD-flame preset: {name}. Available presets: {', '.join(sorted(ONE_D_FLAME_PRESETS))}" + ) from exc + + +def one_d_flame_plan_dict( + *, + inputs: OneDFlameInitInputs, + resolved: dict[str, Any], + output_dir: str | None, + config_path: str | None = None, +) -> dict[str, Any]: + preset = get_one_d_flame_preset(inputs.preset) + return { + 'schema_version': 1, + 'case_type': 'oneD-flame', + 'preset': preset.name, + 'preset_summary': preset.summary, + 'template': str(Path(inputs.template).resolve()), + 'output_dir': str(Path(output_dir).resolve()) if output_dir else None, + 'config_path': str(Path(config_path).resolve()) if config_path else None, + 'inputs': { + **asdict(inputs), + 'oxidizer': resolve_oxidizer(inputs.oxidizer), + 'template': str(Path(inputs.template).resolve()), + }, + 'assumptions': preset.assumptions, + 'notes': preset.notes, + 'resolved': resolved, + } + + +def dump_plan_json(plan: dict[str, Any], path: str | Path) -> Path: + output_path = Path(path).resolve() + output_path.parent.mkdir(parents=True, exist_ok=True) + output_path.write_text(json.dumps(plan, indent=2, sort_keys=True) + '\n', encoding='utf-8') + return output_path + + +def load_plan_json(path: str | Path) -> dict[str, Any]: + input_path = Path(path).resolve() + return json.loads(input_path.read_text(encoding='utf-8')) + + +def one_d_flame_inputs_from_plan(plan: dict[str, Any]) -> OneDFlameInitInputs: + if plan.get('case_type') != 'oneD-flame': + raise ValueError(f"Unsupported case_type in config: {plan.get('case_type')}") + + inputs = plan['inputs'] + return OneDFlameInitInputs( + mechanism=inputs['mechanism'], + fuel=inputs['fuel'], + oxidizer=inputs['oxidizer'], + eq_ratio=float(inputs['eq_ratio']), + T0=float(inputs['T0']), + p0=float(inputs['p0']), + preset=inputs.get('preset', plan.get('preset', DEFAULT_ONE_D_FLAME_PRESET)), + template=inputs.get('template', str(DEFAULT_ONE_D_FLAME_TEMPLATE)), + inert_specie=inputs.get('inert_specie', 'N2'), + ) + + +def one_d_flame_overrides_from_plan(plan: dict[str, Any]) -> dict[str, Any]: + overrides = dict(plan.get('resolved', {})) + overrides.pop('mechanism', None) + overrides.pop('fuel', None) + overrides.pop('oxidizer', None) + overrides.pop('eq_ratio', None) + overrides.pop('T0', None) + overrides.pop('p0', None) + overrides.pop('preset', None) + overrides.pop('template', None) + return overrides diff --git a/dfode_kit/cases/presets.py b/dfode_kit/cases/presets.py new file mode 100644 index 00000000..75fb90ea --- /dev/null +++ b/dfode_kit/cases/presets.py @@ -0,0 +1,117 @@ +from pathlib import Path +from dataclasses import dataclass, field + +import cantera as ct + + +@dataclass +class OneDFreelyPropagatingFlameConfig: + mechanism: str + T0: float + p0: float + fuel: str + oxidizer: str + eq_ratio: float + + initial_gas: ct.Solution = field(init=False) + burnt_gas: ct.Solution = field(init=False) + n_species: int = field(init=False) + species_names: list = field(init=False) + n_dims: int = field(init=False) + dim_names: list = field(init=False) + mech_path: Path = field(init=False) + + flame_speed: float = field(init=False, default=None) + flame_thickness: float = field(init=False, default=None) + flame: ct.SolutionArray = field(init=False, default=None) + + domain_length: float = field(init=False, default=None) + domain_width: float = field(init=False, default=None) + ignition_region: float = field(init=False, default=None) + sim_time_step: float = field(init=False, default=None) + sim_time: float = field(init=False, default=None) + sim_write_interval: float = field(init=False, default=None) + num_output_steps: int = field(init=False, default=None) + inlet_speed: float = field(init=False, default=None) + inert_specie: str = field(init=False, default='N2') + + def __post_init__(self): + """Post-initialization to set up initial and burnt gas states.""" + self.initial_gas = ct.Solution(self.mechanism) + self.initial_gas.TP = self.T0, self.p0 + self.initial_gas.set_equivalence_ratio(self.eq_ratio, self.fuel, self.oxidizer) + + self.burnt_gas = ct.Solution(self.mechanism) + self.burnt_gas.TP = self.T0, self.p0 + self.burnt_gas.set_equivalence_ratio(self.eq_ratio, self.fuel, self.oxidizer) + self.burnt_gas.equilibrate('HP') + + self.n_species = self.initial_gas.n_species + self.species_names = self.initial_gas.species_names + self.n_dims = 2 + self.n_species + self.dim_names = ['T', 'p'] + self.species_names + + self.mech_path = Path(self.mechanism).resolve() + + def calculate_laminar_flame_properties(self): + """Calculate laminar flame speed and thickness.""" + flame_speed_gas = ct.Solution(self.mechanism) + flame_speed_gas.TP = self.T0, self.p0 + flame_speed_gas.set_equivalence_ratio(self.eq_ratio, self.fuel, self.oxidizer) + + width = 0.1 + flame = ct.FreeFlame(flame_speed_gas, width=width) + flame.set_refine_criteria(ratio=3, slope=0.05, curve=0.1, prune=0.0) + + print('Solving premixed flame...') + flame.solve(loglevel=0, auto=True) + + laminar_flame_speed = flame.velocity[0] + print(f'{"Laminar Flame Speed":<25}:{laminar_flame_speed:>15.10f} m/s') + + z, T = flame.grid, flame.T + grad = (T[1:] - T[:-1]) / (z[1:] - z[:-1]) + laminar_flame_thickness = (max(T) - min(T)) / max(grad) + print(f'{"Laminar Flame Thickness":<25}:{laminar_flame_thickness:>15.10f} m') + + final_flame = flame.to_solution_array() + + self.flame_speed = laminar_flame_speed + self.flame_thickness = laminar_flame_thickness + self.flame = final_flame + + def update_config(self, params: dict): + """Update the configuration with new parameters.""" + for key, value in params.items(): + if hasattr(self, key): + setattr(self, key, value) + else: + raise AttributeError(f"'{key}' is not a valid attribute of OneDFreelyPropagatingFlameConfig") + + if self.flame_speed is None or self.flame_thickness is None: + self.calculate_laminar_flame_properties() + + if self.domain_length is None: + self.domain_length = self.flame_thickness / 10 * 500 + + if self.domain_width is None: + self.domain_width = self.domain_length / 10 + + if self.ignition_region is None: + self.ignition_region = self.domain_length / 2 + + if self.sim_time_step is None: + self.sim_time_step = 1e-6 + + if self.num_output_steps is None: + self.num_output_steps = 100 + + if self.sim_write_interval is None: + chem_time_scale = self.flame_thickness / self.flame_speed + self.sim_write_interval = chem_time_scale * 10 / self.num_output_steps + + if self.sim_time is None: + self.sim_time = self.sim_write_interval * (self.num_output_steps + 1) + + if self.inlet_speed is None: + self.inlet_speed = self.flame_speed diff --git a/dfode_kit/cases/sampling.py b/dfode_kit/cases/sampling.py new file mode 100644 index 00000000..3ca0dbab --- /dev/null +++ b/dfode_kit/cases/sampling.py @@ -0,0 +1,104 @@ +from pathlib import Path + +import h5py +import numpy as np +import cantera as ct + +from dfode_kit.utils import is_number, read_openfoam_scalar + + +def gather_species_arrays(species_names, directory_path) -> np.ndarray: + """ + Concatenate scalar arrays from OpenFOAM files for each species in the specified directory. + """ + all_arrays = [] + num_cell = None + directory_path = Path(directory_path) + + if not Path(directory_path).is_dir(): + raise ValueError(f'The directory does not exist: {directory_path}') + + for species in species_names: + file_path = directory_path / species + + if file_path.is_file(): + try: + species_array = read_openfoam_scalar(file_path) + + if isinstance(species_array, np.ndarray): + if num_cell is None: + num_cell = species_array.shape[0] + elif species_array.shape[0] != num_cell: + raise ValueError( + f'Shape mismatch for {species}: expected {num_cell}, got {species_array.shape[0]}.' + ) + + all_arrays.append(species_array) + except ValueError as e: + print(f'Error reading {file_path}: {e}') + else: + print(f'File not found: {file_path}') + + for i in range(len(all_arrays)): + if not isinstance(all_arrays[i], np.ndarray): + if isinstance(all_arrays[i], float): + all_arrays[i] = np.full((num_cell, 1), all_arrays[i]) + else: + print(f'Warning: {all_arrays[i]} is not a numpy array or float.') + + if all_arrays: + return np.concatenate(all_arrays, axis=1) + raise ValueError('No valid species arrays found to concatenate.') + + +def df_to_h5(root_dir, mechanism, hdf5_file_path, include_mesh=True): + """ + Iterate through directories in root_dir, concatenate arrays, and save to an HDF5 file. + """ + root_path = Path(root_dir).resolve() + mechanism = Path(mechanism).resolve() + hdf5_file_path = Path(hdf5_file_path) + gas = ct.Solution(mechanism) + species_names = ['T', 'p'] + gas.species_names + print(f'Species names: {species_names}') + + with h5py.File(hdf5_file_path, 'w') as hdf5_file: + hdf5_file.attrs['root_directory'] = str(root_path) + hdf5_file.attrs['mechanism'] = str(mechanism) + hdf5_file.attrs['species_names'] = species_names + + scalar_group = hdf5_file.create_group('scalar_fields') + + numeric_dirs = [ + dir_path for dir_path in root_path.iterdir() + if dir_path.is_dir() and is_number(dir_path.name) and dir_path.name != '0' + ] + numeric_dirs.sort(key=lambda x: float(x.name)) + + for dir_path in numeric_dirs: + try: + concatenated_array = gather_species_arrays(species_names, dir_path) + scalar_group.create_dataset(str(dir_path.name), data=concatenated_array) + except ValueError as e: + print(f'Error processing directory {dir_path}: {e}') + + if include_mesh: + mesh_group = hdf5_file.create_group('mesh') + mesh_files = [ + root_path / 'temp/0/Cx', + root_path / 'temp/0/Cy', + root_path / 'temp/0/Cz', + root_path / 'temp/0/V', + ] + + for mesh_file in mesh_files: + if mesh_file.is_file(): + try: + mesh_data = read_openfoam_scalar(mesh_file) + mesh_group.create_dataset(str(mesh_file.name), data=mesh_data) + except ValueError as e: + print(f'Error reading mesh file {mesh_file}: {e}') + else: + print(f'Mesh file not found: {mesh_file}') + + print(f'Saved concatenated arrays to {hdf5_file_path}') diff --git a/dfode_kit/dfode_core/test/__init__.py b/dfode_kit/cli/__init__.py similarity index 100% rename from dfode_kit/dfode_core/test/__init__.py rename to dfode_kit/cli/__init__.py diff --git a/dfode_kit/cli/command_loader.py b/dfode_kit/cli/command_loader.py new file mode 100644 index 00000000..3e3321c8 --- /dev/null +++ b/dfode_kit/cli/command_loader.py @@ -0,0 +1,51 @@ +import importlib +from collections import OrderedDict + + +_COMMAND_SPECS = { + 'augment': { + 'module': 'dfode_kit.cli.commands.augment', + 'help': 'Perform data augmentation.', + }, + 'h52npy': { + 'module': 'dfode_kit.cli.commands.h52npy', + 'help': 'Convert HDF5 scalar fields to NumPy array.', + }, + 'init': { + 'module': 'dfode_kit.cli.commands.init', + 'help': 'Initialize canonical cases from explicit presets.', + }, + 'config': { + 'module': 'dfode_kit.cli.commands.config', + 'help': 'Manage persistent runtime configuration.', + }, + 'label': { + 'module': 'dfode_kit.cli.commands.label', + 'help': 'Label data.', + }, + 'run-case': { + 'module': 'dfode_kit.cli.commands.run_case', + 'help': 'Run a DeepFlame/OpenFOAM case using stored configuration.', + }, + 'sample': { + 'module': 'dfode_kit.cli.commands.sample', + 'help': 'Perform sampling.', + }, + 'train': { + 'module': 'dfode_kit.cli.commands.train', + 'help': 'Train the model.', + }, +} + + +def load_command_specs(): + return OrderedDict(sorted(_COMMAND_SPECS.items(), key=lambda item: item[0])) + + +def load_command(command_name, command_specs=None): + command_specs = command_specs or load_command_specs() + if command_name not in command_specs: + raise KeyError(command_name) + + module_name = command_specs[command_name]['module'] + return importlib.import_module(module_name) diff --git a/dfode_kit/cli/commands/__init__.py b/dfode_kit/cli/commands/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/dfode_kit/cli/commands/augment.py b/dfode_kit/cli/commands/augment.py new file mode 100644 index 00000000..f2c7561b --- /dev/null +++ b/dfode_kit/cli/commands/augment.py @@ -0,0 +1,63 @@ +def add_command_parser(subparsers): + augment_parser = subparsers.add_parser('augment', help='Perform data augmentation.') + + augment_parser.add_argument( + '--mech', + required=True, + type=str, + help='Path to the YAML mechanism file.' + ) + augment_parser.add_argument( + '--h5_file', + required=True, + type=str, + help='Path to the h5 file to augment.' + ) + augment_parser.add_argument( + '--output_file', + required=True, + type=str, + help='Path to the output NUMPY file.' + ) + augment_parser.add_argument( + '--heat_limit', + type=bool, + default=False, + help='contraint perturbed data with heat release.' + ) + augment_parser.add_argument( + '--element_limit', + type=bool, + default=True, + help='contraint perturbed data with element ratio.' + ) + augment_parser.add_argument( + '--dataset_num', + required=True, + type=int, + help='num of dataset.' + ) + augment_parser.add_argument( + '--perturb_factor', + type=float, + default=0.1, + help='Factor to perturb the data by.' + ) + + +def handle_command(args): + import numpy as np + + from dfode_kit.data_operations.augment_data import random_perturb + from dfode_kit.data.io_hdf5 import get_TPY_from_h5 + + print('Handling augment command') + print(f'Loading data from h5 file: {args.h5_file}') + data = get_TPY_from_h5(args.h5_file) + print('Data shape:', data.shape) + + all_data = random_perturb(data, args.mech, args.dataset_num, args.heat_limit, args.element_limit) + + np.save(args.output_file, all_data) + print('Saved augmented data shape:', all_data.shape) + print(f'Saved augmented data to {args.output_file}') diff --git a/dfode_kit/cli/commands/config.py b/dfode_kit/cli/commands/config.py new file mode 100644 index 00000000..7855545c --- /dev/null +++ b/dfode_kit/cli/commands/config.py @@ -0,0 +1,86 @@ +from __future__ import annotations + +import argparse +import json + + +def add_command_parser(subparsers): + config_parser = subparsers.add_parser( + 'config', + help='Manage persistent DFODE-kit runtime configuration.', + ) + config_subparsers = config_parser.add_subparsers(dest='config_command') + + path_parser = config_subparsers.add_parser('path', help='Print the runtime config file path.') + path_parser.add_argument('--json', action='store_true', help='Print structured JSON output.') + + show_parser = config_subparsers.add_parser('show', help='Show the current runtime configuration.') + show_parser.add_argument('--json', action='store_true', help='Print structured JSON output.') + + schema_parser = config_subparsers.add_parser('schema', help='Show supported config keys and defaults.') + schema_parser.add_argument('--json', action='store_true', help='Print structured JSON output.') + + set_parser = config_subparsers.add_parser('set', help='Persist a config key/value pair.') + set_parser.add_argument('key', type=str) + set_parser.add_argument('value', type=str) + set_parser.add_argument('--json', action='store_true', help='Print structured JSON output.') + + unset_parser = config_subparsers.add_parser('unset', help='Reset a config key to its default.') + unset_parser.add_argument('key', type=str) + unset_parser.add_argument('--json', action='store_true', help='Print structured JSON output.') + + +def handle_command(args): + from dfode_kit.runtime.config import ( + describe_config_schema, + get_config_path, + load_runtime_config, + set_config_value, + unset_config_value, + ) + + if args.config_command == 'path': + path = str(get_config_path()) + if args.json: + print(json.dumps({'config_path': path}, indent=2, sort_keys=True)) + else: + print(path) + return + + if args.config_command == 'show': + config = load_runtime_config() + if args.json: + print(json.dumps(config, indent=2, sort_keys=True)) + else: + for key, value in config.items(): + print(f'{key}: {value}') + return + + if args.config_command == 'schema': + schema = describe_config_schema() + if args.json: + print(json.dumps(schema, indent=2, sort_keys=True)) + else: + for key, meta in schema.items(): + print(f'{key}:') + print(f' description: {meta["description"]}') + print(f' default: {meta["default"]}') + return + + if args.config_command == 'set': + config, path = set_config_value(args.key, args.value) + if args.json: + print(json.dumps({'event': 'config_set', 'key': args.key, 'path': str(path), 'config': config}, indent=2, sort_keys=True)) + else: + print(f'Set {args.key} in {path}') + return + + if args.config_command == 'unset': + config, path = unset_config_value(args.key) + if args.json: + print(json.dumps({'event': 'config_unset', 'key': args.key, 'path': str(path), 'config': config}, indent=2, sort_keys=True)) + else: + print(f'Unset {args.key} in {path}') + return + + raise ValueError('The config command requires a subcommand: path, show, schema, set, or unset.') diff --git a/dfode_kit/cli/commands/h52npy.py b/dfode_kit/cli/commands/h52npy.py new file mode 100644 index 00000000..e3c57253 --- /dev/null +++ b/dfode_kit/cli/commands/h52npy.py @@ -0,0 +1,28 @@ +import argparse + +def add_command_parser(subparsers): + h52npy_parser = subparsers.add_parser('h52npy', help='Convert HDF5 scalar fields to NumPy array.') + h52npy_parser.add_argument('--source', + required=True, + type=str, + help='Path to the HDF5 file.') + h52npy_parser.add_argument('--save_to', + required=True, + type=str, + help='Path for the output NumPy file.') + +def handle_command(args): + print("Handling h52npy command") + # Load the HDF5 file and concatenate datasets + concatenate_datasets_to_npy(args.source, args.save_to) + +def concatenate_datasets_to_npy(hdf5_file_path, output_npy_file): + """Concatenate all datasets under the ``scalar_fields`` group and save to NPY.""" + import numpy as np + + from dfode_kit.data.contracts import stack_scalar_field_datasets + + concatenated_array = stack_scalar_field_datasets(hdf5_file_path) + print(f"Shape of the final concatenated array: {concatenated_array.shape}") + np.save(output_npy_file, concatenated_array) + print(f"Saved concatenated array to {output_npy_file}") diff --git a/dfode_kit/cli/commands/init.py b/dfode_kit/cli/commands/init.py new file mode 100644 index 00000000..afc15de7 --- /dev/null +++ b/dfode_kit/cli/commands/init.py @@ -0,0 +1,152 @@ +from __future__ import annotations + +import argparse +import json +import shutil +from pathlib import Path + + +def add_command_parser(subparsers): + init_parser = subparsers.add_parser( + 'init', + help='Initialize canonical case templates from explicit presets.', + ) + init_subparsers = init_parser.add_subparsers(dest='init_command') + + one_d_flame = init_subparsers.add_parser( + 'oneD-flame', + help='Initialize a one-dimensional freely propagating premixed flame case.', + ) + one_d_flame.add_argument('--mech', type=str, help='Path to the mechanism file.') + one_d_flame.add_argument('--fuel', type=str, help='Fuel composition string, e.g. CH4:1.') + one_d_flame.add_argument( + '--oxidizer', + type=str, + help='Oxidizer composition string, or the alias "air".', + ) + one_d_flame.add_argument('--phi', type=float, help='Equivalence ratio.') + one_d_flame.add_argument('--T0', type=float, default=300.0, help='Initial temperature [K].') + one_d_flame.add_argument('--p0', type=float, default=101325.0, help='Initial pressure [Pa].') + one_d_flame.add_argument( + '--preset', + type=str, + default='premixed-defaults-v1', + help='Named initialization preset. Preserves current empirical DFODE-kit defaults.', + ) + one_d_flame.add_argument( + '--template', + type=str, + help='Path to the case template directory. Defaults to DFODE-kit canonical 1D flame template.', + ) + one_d_flame.add_argument('--out', type=str, help='Output case directory for generated files.') + one_d_flame.add_argument( + '--from-config', + type=str, + help='Load an init plan/config JSON produced by --write-config.', + ) + one_d_flame.add_argument( + '--write-config', + type=str, + help='Write the fully resolved init plan/config to JSON.', + ) + one_d_flame.add_argument( + '--preview', + action='store_true', + help='Preview the resolved plan without copying the case template.', + ) + one_d_flame.add_argument( + '--apply', + action='store_true', + help='Generate the case directory from the resolved plan.', + ) + one_d_flame.add_argument( + '--json', + action='store_true', + help='Print resolved plan/output as JSON for agent-readable consumption.', + ) + one_d_flame.add_argument( + '--force', + action='store_true', + help='Overwrite the output directory if it already exists.', + ) + one_d_flame.add_argument( + '--inert-specie', + type=str, + default='N2', + help='Inert species name to write into CanteraTorchProperties.', + ) + + one_d_flame.add_argument('--domain-length', type=float, help='Override resolved domain length [m].') + one_d_flame.add_argument('--domain-width', type=float, help='Override resolved domain width [m].') + one_d_flame.add_argument('--ignition-region', type=float, help='Override resolved ignition region [m].') + one_d_flame.add_argument('--sim-time-step', type=float, help='Override resolved time step [s].') + one_d_flame.add_argument('--sim-time', type=float, help='Override resolved end time [s].') + one_d_flame.add_argument( + '--sim-write-interval', + type=float, + help='Override resolved write interval [s].', + ) + one_d_flame.add_argument('--num-output-steps', type=int, help='Override resolved number of output steps.') + one_d_flame.add_argument('--inlet-speed', type=float, help='Override resolved inlet speed [m/s].') + + +PREVIEW_ONLY_KEYS = ('preview', 'apply', 'write_config', 'json', 'force', 'out', 'from_config', 'init_command', 'command') + + +def handle_command(args): + if args.init_command != 'oneD-flame': + raise ValueError('The init command currently supports only the oneD-flame subcommand.') + _handle_one_d_flame(args) + + +def _handle_one_d_flame(args): + from dfode_kit.cli.commands.init_helpers import resolve_one_d_flame_plan, apply_one_d_flame_plan + + if not args.preview and not args.apply and not args.write_config: + raise ValueError('Specify at least one action: --preview, --apply, or --write-config.') + + plan = resolve_one_d_flame_plan(args) + json_result = {'case_type': 'oneD-flame'} if args.json else None + + if args.write_config: + from dfode_kit.df_interface.case_init import dump_plan_json + + config_path = dump_plan_json(plan, args.write_config) + if args.json: + json_result['config_written'] = {'path': str(config_path)} + else: + print(f'Wrote init config: {config_path}') + + if args.preview: + if args.json: + json_result['plan'] = plan + else: + _print_human_plan(plan) + + if args.apply: + result = apply_one_d_flame_plan(plan, force=args.force, quiet=args.json) + if args.json: + json_result['apply'] = result + else: + print(f"Initialized case at: {result['case_dir']}") + print(f"Metadata: {result['metadata_path']}") + + if args.json: + print(json.dumps(json_result, indent=2, sort_keys=True)) + + +def _print_human_plan(plan: dict): + resolved = plan['resolved'] + print('Resolved oneD-flame init plan') + print(f"preset: {plan['preset']}") + print(f"template: {plan['template']}") + print(f"output_dir: {plan['output_dir']}") + print('inputs:') + for key, value in plan['inputs'].items(): + print(f' {key}: {value}') + print('resolved:') + for key in sorted(resolved): + print(f' {key}: {resolved[key]}') + print('assumptions:') + for key, value in plan['assumptions'].items(): + print(f' {key}: {value}') diff --git a/dfode_kit/cli/commands/init_helpers.py b/dfode_kit/cli/commands/init_helpers.py new file mode 100644 index 00000000..9867224e --- /dev/null +++ b/dfode_kit/cli/commands/init_helpers.py @@ -0,0 +1,174 @@ +from __future__ import annotations + +import io +import shutil +from contextlib import redirect_stdout +from pathlib import Path +from typing import Any + +from dfode_kit.df_interface.case_init import ( + DEFAULT_ONE_D_FLAME_TEMPLATE, + OneDFlameInitInputs, + dump_plan_json, + load_plan_json, + one_d_flame_inputs_from_plan, + one_d_flame_overrides_from_plan, + one_d_flame_plan_dict, + resolve_oxidizer, +) + + +OVERRIDE_FIELDS = { + 'domain_length': 'domain_length', + 'domain_width': 'domain_width', + 'ignition_region': 'ignition_region', + 'sim_time_step': 'sim_time_step', + 'sim_time': 'sim_time', + 'sim_write_interval': 'sim_write_interval', + 'num_output_steps': 'num_output_steps', + 'inlet_speed': 'inlet_speed', +} + + +def resolve_one_d_flame_plan(args) -> dict[str, Any]: + if args.from_config: + plan = load_plan_json(args.from_config) + inputs = one_d_flame_inputs_from_plan(plan) + overrides = one_d_flame_overrides_from_plan(plan) + template = Path(inputs.template) + out_dir = args.out or plan.get('output_dir') + else: + template = Path(args.template).resolve() if args.template else DEFAULT_ONE_D_FLAME_TEMPLATE.resolve() + _validate_required_args(args, ('mech', 'fuel', 'oxidizer', 'phi')) + inputs = OneDFlameInitInputs( + mechanism=args.mech, + fuel=args.fuel, + oxidizer=resolve_oxidizer(args.oxidizer), + eq_ratio=float(args.phi), + T0=float(args.T0), + p0=float(args.p0), + preset=args.preset, + template=str(template), + inert_specie=args.inert_specie, + ) + overrides = _extract_override_args(args) + out_dir = args.out + + if args.out: + out_dir = args.out + + if args.apply and not out_dir: + raise ValueError('The --out path is required when using --apply.') + + template = Path(inputs.template).resolve() + if not template.is_dir(): + raise ValueError(f'Template directory does not exist: {template}') + + cfg = _build_one_d_flame_config(inputs, overrides, quiet=getattr(args, 'json', False)) + + resolved = { + 'flame_speed': cfg.flame_speed, + 'flame_thickness': cfg.flame_thickness, + 'domain_length': cfg.domain_length, + 'domain_width': cfg.domain_width, + 'ignition_region': cfg.ignition_region, + 'sim_time_step': cfg.sim_time_step, + 'num_output_steps': cfg.num_output_steps, + 'sim_write_interval': cfg.sim_write_interval, + 'sim_time': cfg.sim_time, + 'inlet_speed': cfg.inlet_speed, + 'inert_specie': cfg.inert_specie, + } + + return one_d_flame_plan_dict( + inputs=OneDFlameInitInputs( + mechanism=inputs.mechanism, + fuel=inputs.fuel, + oxidizer=inputs.oxidizer, + eq_ratio=inputs.eq_ratio, + T0=inputs.T0, + p0=inputs.p0, + preset=inputs.preset, + template=str(template), + inert_specie=cfg.inert_specie, + ), + resolved=resolved, + output_dir=out_dir, + config_path=args.from_config, + ) + + +def apply_one_d_flame_plan( + plan: dict[str, Any], + force: bool = False, + quiet: bool = False, +) -> dict[str, Any]: + case_dir = Path(plan['output_dir']).resolve() + template_dir = Path(plan['template']).resolve() + + if case_dir.exists(): + if not force: + raise ValueError(f'Output directory already exists: {case_dir}. Use --force to replace it.') + shutil.rmtree(case_dir) + + shutil.copytree(template_dir, case_dir) + + inputs = one_d_flame_inputs_from_plan(plan) + overrides = one_d_flame_overrides_from_plan(plan) + cfg = _build_one_d_flame_config(inputs, overrides, quiet=quiet) + + from dfode_kit.df_interface.oneDflame_setup import setup_one_d_flame_case + + if quiet: + with redirect_stdout(io.StringIO()): + setup_one_d_flame_case(cfg, case_dir) + else: + setup_one_d_flame_case(cfg, case_dir) + + metadata_path = dump_plan_json(plan, case_dir / 'dfode-init-plan.json') + return { + 'event': 'case_initialized', + 'case_dir': str(case_dir), + 'metadata_path': str(metadata_path), + 'preset': plan['preset'], + } + + +def _build_one_d_flame_config( + inputs: OneDFlameInitInputs, + overrides: dict[str, Any], + quiet: bool = False, +): + from dfode_kit.df_interface.flame_configurations import OneDFreelyPropagatingFlameConfig + + cfg = OneDFreelyPropagatingFlameConfig( + mechanism=inputs.mechanism, + T0=inputs.T0, + p0=inputs.p0, + fuel=inputs.fuel, + oxidizer=inputs.oxidizer, + eq_ratio=inputs.eq_ratio, + ) + update_params = dict(overrides) + update_params['inert_specie'] = inputs.inert_specie + if quiet: + with redirect_stdout(io.StringIO()): + cfg.update_config(update_params) + else: + cfg.update_config(update_params) + return cfg + + +def _extract_override_args(args) -> dict[str, Any]: + overrides = {} + for cli_name, field_name in OVERRIDE_FIELDS.items(): + value = getattr(args, cli_name) + if value is not None: + overrides[field_name] = value + return overrides + + +def _validate_required_args(args, names: tuple[str, ...]): + missing = [f'--{name.replace("_", "-")}' for name in names if getattr(args, name) is None] + if missing: + raise ValueError(f'Missing required arguments: {", ".join(missing)}') diff --git a/dfode_kit/cli/commands/label.py b/dfode_kit/cli/commands/label.py new file mode 100644 index 00000000..10102c05 --- /dev/null +++ b/dfode_kit/cli/commands/label.py @@ -0,0 +1,51 @@ +import argparse + + +def add_command_parser(subparsers): + label_parser = subparsers.add_parser('label', help='Label data.') + + label_parser.add_argument( + '--mech', + required=True, + type=str, + help='Path to the YAML mechanism file.' + ) + label_parser.add_argument( + '--time', + required=True, + type=float, + help='Time step for reactor advancement' + ) + label_parser.add_argument( + '--source', + required=True, + type=str, + help='Path to the original dataset.' + ) + label_parser.add_argument( + '--save', + required=True, + type=str, + help='Path to save the labeled dataset.' + ) + label_parser.set_defaults(func=handle_command) + + +def handle_command(args): + import numpy as np + + from dfode_kit.data_operations import label_npy as label_main + + try: + labeled_data = label_main( + mech_path=args.mech, + time_step=float(args.time), + source_path=args.source, + ) + np.save(args.save, labeled_data) + print(f'Labeled data saved to: {args.save}') + + except (FileNotFoundError, ValueError) as e: + print(f'Error: {e}') + except Exception as e: + print(f'An unexpected error occurred: {e}') diff --git a/dfode_kit/cli/commands/run_case.py b/dfode_kit/cli/commands/run_case.py new file mode 100644 index 00000000..bf765576 --- /dev/null +++ b/dfode_kit/cli/commands/run_case.py @@ -0,0 +1,99 @@ +from __future__ import annotations + +import argparse +import json +from pathlib import Path + + +def add_command_parser(subparsers): + run_case_parser = subparsers.add_parser( + 'run-case', + help='Run a DeepFlame/OpenFOAM case using stored runtime configuration.', + ) + run_case_parser.add_argument('--case', required=True, type=str, help='Path to the case directory.') + run_case_parser.add_argument( + '--runner', + default='Allrun', + type=str, + help='Case runner script to execute inside the case directory. Defaults to Allrun.', + ) + run_case_parser.add_argument( + '--np', + type=int, + help='MPI rank count hint recorded in metadata. Defaults to config.default_np.', + ) + run_case_parser.add_argument('--preview', action='store_true', help='Preview the resolved runtime command without executing it.') + run_case_parser.add_argument('--apply', action='store_true', help='Execute the case runner.') + run_case_parser.add_argument('--json', action='store_true', help='Print structured JSON output.') + run_case_parser.add_argument( + '--openfoam-bashrc', + type=str, + help='Override config.openfoam_bashrc for this invocation.', + ) + run_case_parser.add_argument( + '--conda-sh', + type=str, + help='Override config.conda_sh for this invocation.', + ) + run_case_parser.add_argument( + '--conda-env', + type=str, + help='Override config.conda_env_name for this invocation.', + ) + run_case_parser.add_argument( + '--deepflame-bashrc', + type=str, + help='Override config.deepflame_bashrc for this invocation.', + ) + run_case_parser.add_argument( + '--python-executable', + type=str, + help='Override config.python_executable for this invocation.', + ) + run_case_parser.add_argument( + '--mpirun-command', + type=str, + help='Override config.mpirun_command for this invocation.', + ) + + +def handle_command(args): + from dfode_kit.runtime.run_case import execute_run_case, resolve_run_case_plan + + if not args.preview and not args.apply: + raise ValueError('Specify at least one action: --preview or --apply.') + + plan = resolve_run_case_plan(args) + json_result = {'case_type': 'deepflame-run-case'} if args.json else None + + if args.preview: + if args.json: + json_result['plan'] = plan + else: + _print_human_plan(plan) + + if args.apply: + result = execute_run_case(plan, quiet=args.json) + if args.json: + json_result['apply'] = result + else: + print(f"Completed run-case in: {result['case_dir']}") + print(f"exit_code: {result['exit_code']}") + if result.get('stdout_log'): + print(f"stdout_log: {result['stdout_log']}") + print(f"stderr_log: {result['stderr_log']}") + + if args.json: + print(json.dumps(json_result, indent=2, sort_keys=True)) + + +def _print_human_plan(plan: dict): + print('Resolved run-case plan') + print(f"case_dir: {plan['case_dir']}") + print(f"runner: {plan['runner']}") + print('runtime_config:') + for key, value in plan['runtime_config'].items(): + print(f' {key}: {value}') + print('shell_lines:') + for line in plan['shell_lines']: + print(f' {line}') diff --git a/dfode_kit/cli/commands/run_case_helpers.py b/dfode_kit/cli/commands/run_case_helpers.py new file mode 100644 index 00000000..2b5b87c0 --- /dev/null +++ b/dfode_kit/cli/commands/run_case_helpers.py @@ -0,0 +1,3 @@ +"""Compatibility shim for :mod:`dfode_kit.runtime.run_case`.""" + +from dfode_kit.runtime.run_case import * # noqa: F401,F403 diff --git a/dfode_kit/cli/commands/sample.py b/dfode_kit/cli/commands/sample.py new file mode 100644 index 00000000..6e0cfe04 --- /dev/null +++ b/dfode_kit/cli/commands/sample.py @@ -0,0 +1,39 @@ +import argparse + + +def add_command_parser(subparsers): + sample_parser = subparsers.add_parser('sample', help='Perform sampling.') + + sample_parser.add_argument( + '--mech', + required=True, + type=str, + help='Path to the mechanism file.' + ) + sample_parser.add_argument( + '--case', + required=True, + type=str, + help='Root directory containing data.' + ) + sample_parser.add_argument( + '--save', + required=True, + type=str, + help='Path where the HDF5 file will be saved.' + ) + sample_parser.add_argument( + '--include_mesh', + action='store_true', + help='Include mesh data in the HDF5 file.' + ) + + +def handle_command(args): + from dfode_kit.data.io_hdf5 import touch_h5 + from dfode_kit.df_interface.sample_case import df_to_h5 + + print('Handling sample command') + df_to_h5(args.case, args.mech, args.save, include_mesh=args.include_mesh) + print() + touch_h5(args.save) diff --git a/dfode_kit/cli/commands/train.py b/dfode_kit/cli/commands/train.py new file mode 100644 index 00000000..2a4b62e9 --- /dev/null +++ b/dfode_kit/cli/commands/train.py @@ -0,0 +1,28 @@ +def add_command_parser(subparsers): + train_parser = subparsers.add_parser('train', help='Train the model.') + train_parser.add_argument( + '--mech', + required=True, + type=str, + help='Path to the YAML mechanism file.' + ) + train_parser.add_argument( + '--source_file', + required=True, + type=str, + help='Path to the source NUMPY file. (With source data and labeled data)' + ) + train_parser.add_argument( + '--output_path', + required=True, + type=str, + help='Path to the output model.' + ) + + +def handle_command(args): + from dfode_kit.training.train import train + + print('Handling train command') + train(args.mech, args.source_file, args.output_path) + print(f'Saved Model to {args.output_path}') diff --git a/dfode_kit/cli/main.py b/dfode_kit/cli/main.py new file mode 100644 index 00000000..ff0e4581 --- /dev/null +++ b/dfode_kit/cli/main.py @@ -0,0 +1,81 @@ +import argparse +import sys + +from dfode_kit.cli.command_loader import load_command, load_command_specs + + +DESCRIPTION = ( + 'dfode-kit provides a command-line interface for performing various tasks ' + 'related to deep learning and reacting flow simulations. This toolkit allows ' + 'users to efficiently augment data, label datasets, sample from low-dimensional ' + 'flame simulations, and train deep learning models. It is designed to support ' + 'physics-informed methodologies for accurate and reliable simulations.' +) + + +def build_parser(command_specs, selected_command=None): + parser = argparse.ArgumentParser(prog='dfode-kit', description=DESCRIPTION) + parser.add_argument( + '--list-commands', + action='store_true', + help='List available commands in deterministic order and exit.', + ) + subparsers = parser.add_subparsers(dest='command') + + for command_name, command_spec in command_specs.items(): + if command_name == selected_command: + command_module = load_command(command_name, command_specs) + command_module.add_command_parser(subparsers) + else: + subparsers.add_parser( + command_name, + help=command_spec['help'], + add_help=False, + ) + + return parser + + +def main(argv=None): + argv = sys.argv[1:] if argv is None else argv + command_specs = load_command_specs() + + lightweight_parser = build_parser(command_specs) + known_args, _ = lightweight_parser.parse_known_args(argv) + + if known_args.list_commands: + for command_name in command_specs: + print(command_name) + return 0 + + if known_args.command is None: + lightweight_parser.print_usage(sys.stderr) + return 2 + + try: + command_module = load_command(known_args.command, command_specs) + except KeyError: + print(f"Unknown command: {known_args.command}", file=sys.stderr) + return 2 + except Exception as exc: + print(f"Command '{known_args.command}' is unavailable: {exc}", file=sys.stderr) + return 1 + + parser = build_parser(command_specs, selected_command=known_args.command) + args = parser.parse_args(argv) + + if not hasattr(command_module, 'handle_command'): + print(f"Unknown command: {args.command}", file=sys.stderr) + return 2 + + try: + command_module.handle_command(args) + except Exception as exc: + print(f"Command '{args.command}' failed: {exc}", file=sys.stderr) + return 1 + + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/dfode_kit/cli_tools/__init__.py b/dfode_kit/cli_tools/__init__.py index e69de29b..6104fb9a 100644 --- a/dfode_kit/cli_tools/__init__.py +++ b/dfode_kit/cli_tools/__init__.py @@ -0,0 +1,3 @@ +"""Compatibility shims for the renamed ``dfode_kit.cli_tools`` package.""" + +from dfode_kit.cli import * # noqa: F401,F403 diff --git a/dfode_kit/cli_tools/command_loader.py b/dfode_kit/cli_tools/command_loader.py index e28df9f7..36eba4b5 100644 --- a/dfode_kit/cli_tools/command_loader.py +++ b/dfode_kit/cli_tools/command_loader.py @@ -1,51 +1,3 @@ -import importlib -from collections import OrderedDict +"""Compatibility shim for :mod:`dfode_kit.cli.command_loader`.""" - -_COMMAND_SPECS = { - 'augment': { - 'module': 'dfode_kit.cli_tools.commands.augment', - 'help': 'Perform data augmentation.', - }, - 'h52npy': { - 'module': 'dfode_kit.cli_tools.commands.h52npy', - 'help': 'Convert HDF5 scalar fields to NumPy array.', - }, - 'init': { - 'module': 'dfode_kit.cli_tools.commands.init', - 'help': 'Initialize canonical cases from explicit presets.', - }, - 'config': { - 'module': 'dfode_kit.cli_tools.commands.config', - 'help': 'Manage persistent runtime configuration.', - }, - 'label': { - 'module': 'dfode_kit.cli_tools.commands.label', - 'help': 'Label data.', - }, - 'run-case': { - 'module': 'dfode_kit.cli_tools.commands.run_case', - 'help': 'Run a DeepFlame/OpenFOAM case using stored configuration.', - }, - 'sample': { - 'module': 'dfode_kit.cli_tools.commands.sample', - 'help': 'Perform sampling.', - }, - 'train': { - 'module': 'dfode_kit.cli_tools.commands.train', - 'help': 'Train the model.', - }, -} - - -def load_command_specs(): - return OrderedDict(sorted(_COMMAND_SPECS.items(), key=lambda item: item[0])) - - -def load_command(command_name, command_specs=None): - command_specs = command_specs or load_command_specs() - if command_name not in command_specs: - raise KeyError(command_name) - - module_name = command_specs[command_name]['module'] - return importlib.import_module(module_name) +from dfode_kit.cli.command_loader import * # noqa: F401,F403 diff --git a/dfode_kit/cli_tools/commands/__init__.py b/dfode_kit/cli_tools/commands/__init__.py index e69de29b..21eb97d0 100644 --- a/dfode_kit/cli_tools/commands/__init__.py +++ b/dfode_kit/cli_tools/commands/__init__.py @@ -0,0 +1 @@ +"""Compatibility shims for :mod:`dfode_kit.cli.commands`.""" diff --git a/dfode_kit/cli_tools/commands/augment.py b/dfode_kit/cli_tools/commands/augment.py index 7e3dad6b..5f4c9047 100644 --- a/dfode_kit/cli_tools/commands/augment.py +++ b/dfode_kit/cli_tools/commands/augment.py @@ -1,63 +1,3 @@ -def add_command_parser(subparsers): - augment_parser = subparsers.add_parser('augment', help='Perform data augmentation.') +"""Compatibility shim for :mod:`dfode_kit.cli.commands.augment`.""" - augment_parser.add_argument( - '--mech', - required=True, - type=str, - help='Path to the YAML mechanism file.' - ) - augment_parser.add_argument( - '--h5_file', - required=True, - type=str, - help='Path to the h5 file to augment.' - ) - augment_parser.add_argument( - '--output_file', - required=True, - type=str, - help='Path to the output NUMPY file.' - ) - augment_parser.add_argument( - '--heat_limit', - type=bool, - default=False, - help='contraint perturbed data with heat release.' - ) - augment_parser.add_argument( - '--element_limit', - type=bool, - default=True, - help='contraint perturbed data with element ratio.' - ) - augment_parser.add_argument( - '--dataset_num', - required=True, - type=int, - help='num of dataset.' - ) - augment_parser.add_argument( - '--perturb_factor', - type=float, - default=0.1, - help='Factor to perturb the data by.' - ) - - -def handle_command(args): - import numpy as np - - from dfode_kit.data_operations.augment_data import random_perturb - from dfode_kit.data_operations.h5_kit import get_TPY_from_h5 - - print('Handling augment command') - print(f'Loading data from h5 file: {args.h5_file}') - data = get_TPY_from_h5(args.h5_file) - print('Data shape:', data.shape) - - all_data = random_perturb(data, args.mech, args.dataset_num, args.heat_limit, args.element_limit) - - np.save(args.output_file, all_data) - print('Saved augmented data shape:', all_data.shape) - print(f'Saved augmented data to {args.output_file}') +from dfode_kit.cli.commands.augment import * # noqa: F401,F403 diff --git a/dfode_kit/cli_tools/commands/config.py b/dfode_kit/cli_tools/commands/config.py index 72d5a5f6..a17f3b72 100644 --- a/dfode_kit/cli_tools/commands/config.py +++ b/dfode_kit/cli_tools/commands/config.py @@ -1,86 +1,3 @@ -from __future__ import annotations +"""Compatibility shim for :mod:`dfode_kit.cli.commands.config`.""" -import argparse -import json - - -def add_command_parser(subparsers): - config_parser = subparsers.add_parser( - 'config', - help='Manage persistent DFODE-kit runtime configuration.', - ) - config_subparsers = config_parser.add_subparsers(dest='config_command') - - path_parser = config_subparsers.add_parser('path', help='Print the runtime config file path.') - path_parser.add_argument('--json', action='store_true', help='Print structured JSON output.') - - show_parser = config_subparsers.add_parser('show', help='Show the current runtime configuration.') - show_parser.add_argument('--json', action='store_true', help='Print structured JSON output.') - - schema_parser = config_subparsers.add_parser('schema', help='Show supported config keys and defaults.') - schema_parser.add_argument('--json', action='store_true', help='Print structured JSON output.') - - set_parser = config_subparsers.add_parser('set', help='Persist a config key/value pair.') - set_parser.add_argument('key', type=str) - set_parser.add_argument('value', type=str) - set_parser.add_argument('--json', action='store_true', help='Print structured JSON output.') - - unset_parser = config_subparsers.add_parser('unset', help='Reset a config key to its default.') - unset_parser.add_argument('key', type=str) - unset_parser.add_argument('--json', action='store_true', help='Print structured JSON output.') - - -def handle_command(args): - from dfode_kit.runtime_config import ( - describe_config_schema, - get_config_path, - load_runtime_config, - set_config_value, - unset_config_value, - ) - - if args.config_command == 'path': - path = str(get_config_path()) - if args.json: - print(json.dumps({'config_path': path}, indent=2, sort_keys=True)) - else: - print(path) - return - - if args.config_command == 'show': - config = load_runtime_config() - if args.json: - print(json.dumps(config, indent=2, sort_keys=True)) - else: - for key, value in config.items(): - print(f'{key}: {value}') - return - - if args.config_command == 'schema': - schema = describe_config_schema() - if args.json: - print(json.dumps(schema, indent=2, sort_keys=True)) - else: - for key, meta in schema.items(): - print(f'{key}:') - print(f' description: {meta["description"]}') - print(f' default: {meta["default"]}') - return - - if args.config_command == 'set': - config, path = set_config_value(args.key, args.value) - if args.json: - print(json.dumps({'event': 'config_set', 'key': args.key, 'path': str(path), 'config': config}, indent=2, sort_keys=True)) - else: - print(f'Set {args.key} in {path}') - return - - if args.config_command == 'unset': - config, path = unset_config_value(args.key) - if args.json: - print(json.dumps({'event': 'config_unset', 'key': args.key, 'path': str(path), 'config': config}, indent=2, sort_keys=True)) - else: - print(f'Unset {args.key} in {path}') - return - - raise ValueError('The config command requires a subcommand: path, show, schema, set, or unset.') +from dfode_kit.cli.commands.config import * # noqa: F401,F403 diff --git a/dfode_kit/cli_tools/commands/h52npy.py b/dfode_kit/cli_tools/commands/h52npy.py index 48fc639d..1e808f60 100644 --- a/dfode_kit/cli_tools/commands/h52npy.py +++ b/dfode_kit/cli_tools/commands/h52npy.py @@ -1,28 +1,3 @@ -import argparse +"""Compatibility shim for :mod:`dfode_kit.cli.commands.h52npy`.""" -def add_command_parser(subparsers): - h52npy_parser = subparsers.add_parser('h52npy', help='Convert HDF5 scalar fields to NumPy array.') - h52npy_parser.add_argument('--source', - required=True, - type=str, - help='Path to the HDF5 file.') - h52npy_parser.add_argument('--save_to', - required=True, - type=str, - help='Path for the output NumPy file.') - -def handle_command(args): - print("Handling h52npy command") - # Load the HDF5 file and concatenate datasets - concatenate_datasets_to_npy(args.source, args.save_to) - -def concatenate_datasets_to_npy(hdf5_file_path, output_npy_file): - """Concatenate all datasets under the ``scalar_fields`` group and save to NPY.""" - import numpy as np - - from dfode_kit.data_operations.contracts import stack_scalar_field_datasets - - concatenated_array = stack_scalar_field_datasets(hdf5_file_path) - print(f"Shape of the final concatenated array: {concatenated_array.shape}") - np.save(output_npy_file, concatenated_array) - print(f"Saved concatenated array to {output_npy_file}") +from dfode_kit.cli.commands.h52npy import * # noqa: F401,F403 diff --git a/dfode_kit/cli_tools/commands/init.py b/dfode_kit/cli_tools/commands/init.py index a4141cc3..a8a7c249 100644 --- a/dfode_kit/cli_tools/commands/init.py +++ b/dfode_kit/cli_tools/commands/init.py @@ -1,152 +1,3 @@ -from __future__ import annotations +"""Compatibility shim for :mod:`dfode_kit.cli.commands.init`.""" -import argparse -import json -import shutil -from pathlib import Path - - -def add_command_parser(subparsers): - init_parser = subparsers.add_parser( - 'init', - help='Initialize canonical case templates from explicit presets.', - ) - init_subparsers = init_parser.add_subparsers(dest='init_command') - - one_d_flame = init_subparsers.add_parser( - 'oneD-flame', - help='Initialize a one-dimensional freely propagating premixed flame case.', - ) - one_d_flame.add_argument('--mech', type=str, help='Path to the mechanism file.') - one_d_flame.add_argument('--fuel', type=str, help='Fuel composition string, e.g. CH4:1.') - one_d_flame.add_argument( - '--oxidizer', - type=str, - help='Oxidizer composition string, or the alias "air".', - ) - one_d_flame.add_argument('--phi', type=float, help='Equivalence ratio.') - one_d_flame.add_argument('--T0', type=float, default=300.0, help='Initial temperature [K].') - one_d_flame.add_argument('--p0', type=float, default=101325.0, help='Initial pressure [Pa].') - one_d_flame.add_argument( - '--preset', - type=str, - default='premixed-defaults-v1', - help='Named initialization preset. Preserves current empirical DFODE-kit defaults.', - ) - one_d_flame.add_argument( - '--template', - type=str, - help='Path to the case template directory. Defaults to DFODE-kit canonical 1D flame template.', - ) - one_d_flame.add_argument('--out', type=str, help='Output case directory for generated files.') - one_d_flame.add_argument( - '--from-config', - type=str, - help='Load an init plan/config JSON produced by --write-config.', - ) - one_d_flame.add_argument( - '--write-config', - type=str, - help='Write the fully resolved init plan/config to JSON.', - ) - one_d_flame.add_argument( - '--preview', - action='store_true', - help='Preview the resolved plan without copying the case template.', - ) - one_d_flame.add_argument( - '--apply', - action='store_true', - help='Generate the case directory from the resolved plan.', - ) - one_d_flame.add_argument( - '--json', - action='store_true', - help='Print resolved plan/output as JSON for agent-readable consumption.', - ) - one_d_flame.add_argument( - '--force', - action='store_true', - help='Overwrite the output directory if it already exists.', - ) - one_d_flame.add_argument( - '--inert-specie', - type=str, - default='N2', - help='Inert species name to write into CanteraTorchProperties.', - ) - - one_d_flame.add_argument('--domain-length', type=float, help='Override resolved domain length [m].') - one_d_flame.add_argument('--domain-width', type=float, help='Override resolved domain width [m].') - one_d_flame.add_argument('--ignition-region', type=float, help='Override resolved ignition region [m].') - one_d_flame.add_argument('--sim-time-step', type=float, help='Override resolved time step [s].') - one_d_flame.add_argument('--sim-time', type=float, help='Override resolved end time [s].') - one_d_flame.add_argument( - '--sim-write-interval', - type=float, - help='Override resolved write interval [s].', - ) - one_d_flame.add_argument('--num-output-steps', type=int, help='Override resolved number of output steps.') - one_d_flame.add_argument('--inlet-speed', type=float, help='Override resolved inlet speed [m/s].') - - -PREVIEW_ONLY_KEYS = ('preview', 'apply', 'write_config', 'json', 'force', 'out', 'from_config', 'init_command', 'command') - - -def handle_command(args): - if args.init_command != 'oneD-flame': - raise ValueError('The init command currently supports only the oneD-flame subcommand.') - _handle_one_d_flame(args) - - -def _handle_one_d_flame(args): - from dfode_kit.cli_tools.commands.init_helpers import resolve_one_d_flame_plan, apply_one_d_flame_plan - - if not args.preview and not args.apply and not args.write_config: - raise ValueError('Specify at least one action: --preview, --apply, or --write-config.') - - plan = resolve_one_d_flame_plan(args) - json_result = {'case_type': 'oneD-flame'} if args.json else None - - if args.write_config: - from dfode_kit.df_interface.case_init import dump_plan_json - - config_path = dump_plan_json(plan, args.write_config) - if args.json: - json_result['config_written'] = {'path': str(config_path)} - else: - print(f'Wrote init config: {config_path}') - - if args.preview: - if args.json: - json_result['plan'] = plan - else: - _print_human_plan(plan) - - if args.apply: - result = apply_one_d_flame_plan(plan, force=args.force, quiet=args.json) - if args.json: - json_result['apply'] = result - else: - print(f"Initialized case at: {result['case_dir']}") - print(f"Metadata: {result['metadata_path']}") - - if args.json: - print(json.dumps(json_result, indent=2, sort_keys=True)) - - -def _print_human_plan(plan: dict): - resolved = plan['resolved'] - print('Resolved oneD-flame init plan') - print(f"preset: {plan['preset']}") - print(f"template: {plan['template']}") - print(f"output_dir: {plan['output_dir']}") - print('inputs:') - for key, value in plan['inputs'].items(): - print(f' {key}: {value}') - print('resolved:') - for key in sorted(resolved): - print(f' {key}: {resolved[key]}') - print('assumptions:') - for key, value in plan['assumptions'].items(): - print(f' {key}: {value}') +from dfode_kit.cli.commands.init import * # noqa: F401,F403 diff --git a/dfode_kit/cli_tools/commands/init_helpers.py b/dfode_kit/cli_tools/commands/init_helpers.py index 9867224e..2db40d4a 100644 --- a/dfode_kit/cli_tools/commands/init_helpers.py +++ b/dfode_kit/cli_tools/commands/init_helpers.py @@ -1,174 +1,3 @@ -from __future__ import annotations +"""Compatibility shim for :mod:`dfode_kit.cli.commands.init_helpers`.""" -import io -import shutil -from contextlib import redirect_stdout -from pathlib import Path -from typing import Any - -from dfode_kit.df_interface.case_init import ( - DEFAULT_ONE_D_FLAME_TEMPLATE, - OneDFlameInitInputs, - dump_plan_json, - load_plan_json, - one_d_flame_inputs_from_plan, - one_d_flame_overrides_from_plan, - one_d_flame_plan_dict, - resolve_oxidizer, -) - - -OVERRIDE_FIELDS = { - 'domain_length': 'domain_length', - 'domain_width': 'domain_width', - 'ignition_region': 'ignition_region', - 'sim_time_step': 'sim_time_step', - 'sim_time': 'sim_time', - 'sim_write_interval': 'sim_write_interval', - 'num_output_steps': 'num_output_steps', - 'inlet_speed': 'inlet_speed', -} - - -def resolve_one_d_flame_plan(args) -> dict[str, Any]: - if args.from_config: - plan = load_plan_json(args.from_config) - inputs = one_d_flame_inputs_from_plan(plan) - overrides = one_d_flame_overrides_from_plan(plan) - template = Path(inputs.template) - out_dir = args.out or plan.get('output_dir') - else: - template = Path(args.template).resolve() if args.template else DEFAULT_ONE_D_FLAME_TEMPLATE.resolve() - _validate_required_args(args, ('mech', 'fuel', 'oxidizer', 'phi')) - inputs = OneDFlameInitInputs( - mechanism=args.mech, - fuel=args.fuel, - oxidizer=resolve_oxidizer(args.oxidizer), - eq_ratio=float(args.phi), - T0=float(args.T0), - p0=float(args.p0), - preset=args.preset, - template=str(template), - inert_specie=args.inert_specie, - ) - overrides = _extract_override_args(args) - out_dir = args.out - - if args.out: - out_dir = args.out - - if args.apply and not out_dir: - raise ValueError('The --out path is required when using --apply.') - - template = Path(inputs.template).resolve() - if not template.is_dir(): - raise ValueError(f'Template directory does not exist: {template}') - - cfg = _build_one_d_flame_config(inputs, overrides, quiet=getattr(args, 'json', False)) - - resolved = { - 'flame_speed': cfg.flame_speed, - 'flame_thickness': cfg.flame_thickness, - 'domain_length': cfg.domain_length, - 'domain_width': cfg.domain_width, - 'ignition_region': cfg.ignition_region, - 'sim_time_step': cfg.sim_time_step, - 'num_output_steps': cfg.num_output_steps, - 'sim_write_interval': cfg.sim_write_interval, - 'sim_time': cfg.sim_time, - 'inlet_speed': cfg.inlet_speed, - 'inert_specie': cfg.inert_specie, - } - - return one_d_flame_plan_dict( - inputs=OneDFlameInitInputs( - mechanism=inputs.mechanism, - fuel=inputs.fuel, - oxidizer=inputs.oxidizer, - eq_ratio=inputs.eq_ratio, - T0=inputs.T0, - p0=inputs.p0, - preset=inputs.preset, - template=str(template), - inert_specie=cfg.inert_specie, - ), - resolved=resolved, - output_dir=out_dir, - config_path=args.from_config, - ) - - -def apply_one_d_flame_plan( - plan: dict[str, Any], - force: bool = False, - quiet: bool = False, -) -> dict[str, Any]: - case_dir = Path(plan['output_dir']).resolve() - template_dir = Path(plan['template']).resolve() - - if case_dir.exists(): - if not force: - raise ValueError(f'Output directory already exists: {case_dir}. Use --force to replace it.') - shutil.rmtree(case_dir) - - shutil.copytree(template_dir, case_dir) - - inputs = one_d_flame_inputs_from_plan(plan) - overrides = one_d_flame_overrides_from_plan(plan) - cfg = _build_one_d_flame_config(inputs, overrides, quiet=quiet) - - from dfode_kit.df_interface.oneDflame_setup import setup_one_d_flame_case - - if quiet: - with redirect_stdout(io.StringIO()): - setup_one_d_flame_case(cfg, case_dir) - else: - setup_one_d_flame_case(cfg, case_dir) - - metadata_path = dump_plan_json(plan, case_dir / 'dfode-init-plan.json') - return { - 'event': 'case_initialized', - 'case_dir': str(case_dir), - 'metadata_path': str(metadata_path), - 'preset': plan['preset'], - } - - -def _build_one_d_flame_config( - inputs: OneDFlameInitInputs, - overrides: dict[str, Any], - quiet: bool = False, -): - from dfode_kit.df_interface.flame_configurations import OneDFreelyPropagatingFlameConfig - - cfg = OneDFreelyPropagatingFlameConfig( - mechanism=inputs.mechanism, - T0=inputs.T0, - p0=inputs.p0, - fuel=inputs.fuel, - oxidizer=inputs.oxidizer, - eq_ratio=inputs.eq_ratio, - ) - update_params = dict(overrides) - update_params['inert_specie'] = inputs.inert_specie - if quiet: - with redirect_stdout(io.StringIO()): - cfg.update_config(update_params) - else: - cfg.update_config(update_params) - return cfg - - -def _extract_override_args(args) -> dict[str, Any]: - overrides = {} - for cli_name, field_name in OVERRIDE_FIELDS.items(): - value = getattr(args, cli_name) - if value is not None: - overrides[field_name] = value - return overrides - - -def _validate_required_args(args, names: tuple[str, ...]): - missing = [f'--{name.replace("_", "-")}' for name in names if getattr(args, name) is None] - if missing: - raise ValueError(f'Missing required arguments: {", ".join(missing)}') +from dfode_kit.cli.commands.init_helpers import * # noqa: F401,F403 diff --git a/dfode_kit/cli_tools/commands/label.py b/dfode_kit/cli_tools/commands/label.py index 10102c05..bb1936d2 100644 --- a/dfode_kit/cli_tools/commands/label.py +++ b/dfode_kit/cli_tools/commands/label.py @@ -1,51 +1,3 @@ -import argparse +"""Compatibility shim for :mod:`dfode_kit.cli.commands.label`.""" - -def add_command_parser(subparsers): - label_parser = subparsers.add_parser('label', help='Label data.') - - label_parser.add_argument( - '--mech', - required=True, - type=str, - help='Path to the YAML mechanism file.' - ) - label_parser.add_argument( - '--time', - required=True, - type=float, - help='Time step for reactor advancement' - ) - label_parser.add_argument( - '--source', - required=True, - type=str, - help='Path to the original dataset.' - ) - label_parser.add_argument( - '--save', - required=True, - type=str, - help='Path to save the labeled dataset.' - ) - label_parser.set_defaults(func=handle_command) - - -def handle_command(args): - import numpy as np - - from dfode_kit.data_operations import label_npy as label_main - - try: - labeled_data = label_main( - mech_path=args.mech, - time_step=float(args.time), - source_path=args.source, - ) - np.save(args.save, labeled_data) - print(f'Labeled data saved to: {args.save}') - - except (FileNotFoundError, ValueError) as e: - print(f'Error: {e}') - except Exception as e: - print(f'An unexpected error occurred: {e}') +from dfode_kit.cli.commands.label import * # noqa: F401,F403 diff --git a/dfode_kit/cli_tools/commands/run_case.py b/dfode_kit/cli_tools/commands/run_case.py index c3ae8ea9..494a264e 100644 --- a/dfode_kit/cli_tools/commands/run_case.py +++ b/dfode_kit/cli_tools/commands/run_case.py @@ -1,99 +1,3 @@ -from __future__ import annotations +"""Compatibility shim for :mod:`dfode_kit.cli.commands.run_case`.""" -import argparse -import json -from pathlib import Path - - -def add_command_parser(subparsers): - run_case_parser = subparsers.add_parser( - 'run-case', - help='Run a DeepFlame/OpenFOAM case using stored runtime configuration.', - ) - run_case_parser.add_argument('--case', required=True, type=str, help='Path to the case directory.') - run_case_parser.add_argument( - '--runner', - default='Allrun', - type=str, - help='Case runner script to execute inside the case directory. Defaults to Allrun.', - ) - run_case_parser.add_argument( - '--np', - type=int, - help='MPI rank count hint recorded in metadata. Defaults to config.default_np.', - ) - run_case_parser.add_argument('--preview', action='store_true', help='Preview the resolved runtime command without executing it.') - run_case_parser.add_argument('--apply', action='store_true', help='Execute the case runner.') - run_case_parser.add_argument('--json', action='store_true', help='Print structured JSON output.') - run_case_parser.add_argument( - '--openfoam-bashrc', - type=str, - help='Override config.openfoam_bashrc for this invocation.', - ) - run_case_parser.add_argument( - '--conda-sh', - type=str, - help='Override config.conda_sh for this invocation.', - ) - run_case_parser.add_argument( - '--conda-env', - type=str, - help='Override config.conda_env_name for this invocation.', - ) - run_case_parser.add_argument( - '--deepflame-bashrc', - type=str, - help='Override config.deepflame_bashrc for this invocation.', - ) - run_case_parser.add_argument( - '--python-executable', - type=str, - help='Override config.python_executable for this invocation.', - ) - run_case_parser.add_argument( - '--mpirun-command', - type=str, - help='Override config.mpirun_command for this invocation.', - ) - - -def handle_command(args): - from dfode_kit.cli_tools.commands.run_case_helpers import execute_run_case, resolve_run_case_plan - - if not args.preview and not args.apply: - raise ValueError('Specify at least one action: --preview or --apply.') - - plan = resolve_run_case_plan(args) - json_result = {'case_type': 'deepflame-run-case'} if args.json else None - - if args.preview: - if args.json: - json_result['plan'] = plan - else: - _print_human_plan(plan) - - if args.apply: - result = execute_run_case(plan, quiet=args.json) - if args.json: - json_result['apply'] = result - else: - print(f"Completed run-case in: {result['case_dir']}") - print(f"exit_code: {result['exit_code']}") - if result.get('stdout_log'): - print(f"stdout_log: {result['stdout_log']}") - print(f"stderr_log: {result['stderr_log']}") - - if args.json: - print(json.dumps(json_result, indent=2, sort_keys=True)) - - -def _print_human_plan(plan: dict): - print('Resolved run-case plan') - print(f"case_dir: {plan['case_dir']}") - print(f"runner: {plan['runner']}") - print('runtime_config:') - for key, value in plan['runtime_config'].items(): - print(f' {key}: {value}') - print('shell_lines:') - for line in plan['shell_lines']: - print(f' {line}') +from dfode_kit.cli.commands.run_case import * # noqa: F401,F403 diff --git a/dfode_kit/cli_tools/commands/run_case_helpers.py b/dfode_kit/cli_tools/commands/run_case_helpers.py index ba482cc1..2b5b87c0 100644 --- a/dfode_kit/cli_tools/commands/run_case_helpers.py +++ b/dfode_kit/cli_tools/commands/run_case_helpers.py @@ -1,97 +1,3 @@ -from __future__ import annotations +"""Compatibility shim for :mod:`dfode_kit.runtime.run_case`.""" -import shlex -import subprocess -from pathlib import Path -from typing import Any - -from dfode_kit.runtime_config import resolve_runtime_config - - -def resolve_run_case_plan(args) -> dict[str, Any]: - case_dir = Path(args.case).resolve() - if not case_dir.is_dir(): - raise ValueError(f'Case directory does not exist: {case_dir}') - - runner_path = case_dir / args.runner - if not runner_path.is_file(): - raise ValueError(f'Case runner does not exist: {runner_path}') - - runtime_config = resolve_runtime_config( - { - 'openfoam_bashrc': args.openfoam_bashrc, - 'conda_sh': args.conda_sh, - 'conda_env_name': args.conda_env, - 'deepflame_bashrc': args.deepflame_bashrc, - 'python_executable': args.python_executable, - 'mpirun_command': args.mpirun_command, - 'default_np': args.np, - } - ) - _validate_runtime_config(runtime_config) - - shell_lines = [ - f'source {shlex.quote(runtime_config["openfoam_bashrc"])}', - f'source {shlex.quote(runtime_config["conda_sh"])}', - f'conda activate {shlex.quote(runtime_config["conda_env_name"])}', - f'source {shlex.quote(runtime_config["deepflame_bashrc"])}', - 'set -eo pipefail', - 'which dfLowMachFoam', - f'cd {shlex.quote(str(case_dir))}', - f'chmod +x {shlex.quote(args.runner)}', - f'./{shlex.quote(args.runner)}', - ] - - return { - 'schema_version': 1, - 'case_type': 'deepflame-run-case', - 'case_dir': str(case_dir), - 'runner': args.runner, - 'np': runtime_config['default_np'], - 'runtime_config': runtime_config, - 'shell_lines': shell_lines, - 'shell_script': '\n'.join(shell_lines), - } - - -def execute_run_case(plan: dict[str, Any], quiet: bool = False) -> dict[str, Any]: - case_dir = Path(plan['case_dir']).resolve() - command = ['bash', '-lc', plan['shell_script']] - - if quiet: - stdout_log = case_dir / 'log.dfode-run-case.stdout' - stderr_log = case_dir / 'log.dfode-run-case.stderr' - with stdout_log.open('w', encoding='utf-8') as stdout_handle, stderr_log.open('w', encoding='utf-8') as stderr_handle: - completed = subprocess.run(command, stdout=stdout_handle, stderr=stderr_handle, text=True) - result = { - 'event': 'run_case_completed', - 'case_dir': str(case_dir), - 'runner': plan['runner'], - 'exit_code': completed.returncode, - 'stdout_log': str(stdout_log), - 'stderr_log': str(stderr_log), - } - else: - completed = subprocess.run(command) - result = { - 'event': 'run_case_completed', - 'case_dir': str(case_dir), - 'runner': plan['runner'], - 'exit_code': completed.returncode, - } - - if completed.returncode != 0: - raise ValueError(f"Case runner failed with exit code {completed.returncode}: {case_dir / plan['runner']}") - - return result - - -def _validate_runtime_config(config: dict[str, Any]): - required = ['openfoam_bashrc', 'conda_sh', 'conda_env_name', 'deepflame_bashrc'] - missing = [key for key in required if not config.get(key)] - if missing: - raise ValueError( - 'Missing runtime config values: ' - + ', '.join(missing) - + '. Use `dfode-kit config set ...` or per-command overrides.' - ) +from dfode_kit.runtime.run_case import * # noqa: F401,F403 diff --git a/dfode_kit/cli_tools/commands/sample.py b/dfode_kit/cli_tools/commands/sample.py index 88cb4152..d2dcd1bb 100644 --- a/dfode_kit/cli_tools/commands/sample.py +++ b/dfode_kit/cli_tools/commands/sample.py @@ -1,39 +1,3 @@ -import argparse +"""Compatibility shim for :mod:`dfode_kit.cli.commands.sample`.""" - -def add_command_parser(subparsers): - sample_parser = subparsers.add_parser('sample', help='Perform sampling.') - - sample_parser.add_argument( - '--mech', - required=True, - type=str, - help='Path to the mechanism file.' - ) - sample_parser.add_argument( - '--case', - required=True, - type=str, - help='Root directory containing data.' - ) - sample_parser.add_argument( - '--save', - required=True, - type=str, - help='Path where the HDF5 file will be saved.' - ) - sample_parser.add_argument( - '--include_mesh', - action='store_true', - help='Include mesh data in the HDF5 file.' - ) - - -def handle_command(args): - from dfode_kit.data_operations.h5_kit import touch_h5 - from dfode_kit.df_interface.sample_case import df_to_h5 - - print('Handling sample command') - df_to_h5(args.case, args.mech, args.save, include_mesh=args.include_mesh) - print() - touch_h5(args.save) +from dfode_kit.cli.commands.sample import * # noqa: F401,F403 diff --git a/dfode_kit/cli_tools/commands/train.py b/dfode_kit/cli_tools/commands/train.py index 6baa5d56..9f99574f 100644 --- a/dfode_kit/cli_tools/commands/train.py +++ b/dfode_kit/cli_tools/commands/train.py @@ -1,28 +1,3 @@ -def add_command_parser(subparsers): - train_parser = subparsers.add_parser('train', help='Train the model.') - train_parser.add_argument( - '--mech', - required=True, - type=str, - help='Path to the YAML mechanism file.' - ) - train_parser.add_argument( - '--source_file', - required=True, - type=str, - help='Path to the source NUMPY file. (With source data and labeled data)' - ) - train_parser.add_argument( - '--output_path', - required=True, - type=str, - help='Path to the output model.' - ) +"""Compatibility shim for :mod:`dfode_kit.cli.commands.train`.""" - -def handle_command(args): - from dfode_kit.dfode_core.train.train import train - - print('Handling train command') - train(args.mech, args.source_file, args.output_path) - print(f'Saved Model to {args.output_path}') +from dfode_kit.cli.commands.train import * # noqa: F401,F403 diff --git a/dfode_kit/cli_tools/main.py b/dfode_kit/cli_tools/main.py index ad32a79f..5732201b 100644 --- a/dfode_kit/cli_tools/main.py +++ b/dfode_kit/cli_tools/main.py @@ -1,80 +1,6 @@ -import argparse -import sys +"""Compatibility shim for :mod:`dfode_kit.cli.main`.""" -from dfode_kit.cli_tools.command_loader import load_command, load_command_specs - - -DESCRIPTION = ( - 'dfode-kit provides a command-line interface for performing various tasks ' - 'related to deep learning and reacting flow simulations. This toolkit allows ' - 'users to efficiently augment data, label datasets, sample from low-dimensional ' - 'flame simulations, and train deep learning models. It is designed to support ' - 'physics-informed methodologies for accurate and reliable simulations.' -) - - -def build_parser(command_specs, selected_command=None): - parser = argparse.ArgumentParser(prog='dfode-kit', description=DESCRIPTION) - parser.add_argument( - '--list-commands', - action='store_true', - help='List available commands in deterministic order and exit.', - ) - subparsers = parser.add_subparsers(dest='command') - - for command_name, command_spec in command_specs.items(): - if command_name == selected_command: - command_module = load_command(command_name, command_specs) - command_module.add_command_parser(subparsers) - else: - subparsers.add_parser( - command_name, - help=command_spec['help'], - add_help=False, - ) - - return parser - - -def main(argv=None): - argv = sys.argv[1:] if argv is None else argv - command_specs = load_command_specs() - - lightweight_parser = build_parser(command_specs) - known_args, _ = lightweight_parser.parse_known_args(argv) - - if known_args.list_commands: - for command_name in command_specs: - print(command_name) - return 0 - - if known_args.command is None: - lightweight_parser.print_usage(sys.stderr) - return 2 - - try: - command_module = load_command(known_args.command, command_specs) - except KeyError: - print(f"Unknown command: {known_args.command}", file=sys.stderr) - return 2 - except Exception as exc: - print(f"Command '{known_args.command}' is unavailable: {exc}", file=sys.stderr) - return 1 - - parser = build_parser(command_specs, selected_command=known_args.command) - args = parser.parse_args(argv) - - if not hasattr(command_module, 'handle_command'): - print(f"Unknown command: {args.command}", file=sys.stderr) - return 2 - - try: - command_module.handle_command(args) - except Exception as exc: - print(f"Command '{args.command}' failed: {exc}", file=sys.stderr) - return 1 - - return 0 +from dfode_kit.cli.main import * # noqa: F401,F403 if __name__ == "__main__": diff --git a/dfode_kit/data/__init__.py b/dfode_kit/data/__init__.py new file mode 100644 index 00000000..32587c5f --- /dev/null +++ b/dfode_kit/data/__init__.py @@ -0,0 +1,35 @@ +from importlib import import_module + + +__all__ = [ + "SCALAR_FIELDS_GROUP", + "MECHANISM_ATTR", + "read_scalar_field_datasets", + "stack_scalar_field_datasets", + "require_h5_attr", + "require_h5_group", + "touch_h5", + "get_TPY_from_h5", +] + +_ATTRIBUTE_MODULES = { + "SCALAR_FIELDS_GROUP": ("dfode_kit.data.contracts", "SCALAR_FIELDS_GROUP"), + "MECHANISM_ATTR": ("dfode_kit.data.contracts", "MECHANISM_ATTR"), + "read_scalar_field_datasets": ("dfode_kit.data.contracts", "read_scalar_field_datasets"), + "stack_scalar_field_datasets": ("dfode_kit.data.contracts", "stack_scalar_field_datasets"), + "require_h5_attr": ("dfode_kit.data.contracts", "require_h5_attr"), + "require_h5_group": ("dfode_kit.data.contracts", "require_h5_group"), + "touch_h5": ("dfode_kit.data.io_hdf5", "touch_h5"), + "get_TPY_from_h5": ("dfode_kit.data.io_hdf5", "get_TPY_from_h5"), +} + + +def __getattr__(name): + if name not in _ATTRIBUTE_MODULES: + raise AttributeError(f"module 'dfode_kit.data' has no attribute '{name}'") + + module_name, attribute_name = _ATTRIBUTE_MODULES[name] + module = import_module(module_name) + value = getattr(module, attribute_name) + globals()[name] = value + return value diff --git a/dfode_kit/data/contracts.py b/dfode_kit/data/contracts.py new file mode 100644 index 00000000..1e5ade0e --- /dev/null +++ b/dfode_kit/data/contracts.py @@ -0,0 +1,69 @@ +from __future__ import annotations + +from typing import Dict + +import h5py +import numpy as np + +SCALAR_FIELDS_GROUP = "scalar_fields" +MECHANISM_ATTR = "mechanism" + + +def require_h5_group(hdf5_file: h5py.File, group_name: str) -> h5py.Group: + if group_name not in hdf5_file: + raise ValueError(f"'{group_name}' group not found in HDF5 file") + return hdf5_file[group_name] + + +def require_h5_attr(hdf5_file: h5py.File, attr_name: str): + if attr_name not in hdf5_file.attrs: + raise ValueError(f"Required HDF5 root attribute '{attr_name}' is missing") + return hdf5_file.attrs[attr_name] + + +def ordered_group_dataset_names(group: h5py.Group) -> list[str]: + names = list(group.keys()) + return sorted(names, key=_dataset_name_sort_key) + + +def read_scalar_field_datasets(hdf5_file_path: str) -> Dict[str, np.ndarray]: + with h5py.File(hdf5_file_path, "r") as hdf5_file: + scalar_group = require_h5_group(hdf5_file, SCALAR_FIELDS_GROUP) + dataset_names = ordered_group_dataset_names(scalar_group) + + data = {name: scalar_group[name][:] for name in dataset_names} + + _validate_scalar_field_datasets(data, source=hdf5_file_path) + return data + + +def stack_scalar_field_datasets(hdf5_file_path: str) -> np.ndarray: + datasets = read_scalar_field_datasets(hdf5_file_path) + return np.concatenate(list(datasets.values()), axis=0) + + +def _validate_scalar_field_datasets(datasets: Dict[str, np.ndarray], source: str) -> None: + if not datasets: + raise ValueError(f"No datasets found in '{SCALAR_FIELDS_GROUP}' for {source}") + + expected_columns = None + for dataset_name, dataset in datasets.items(): + if dataset.ndim != 2: + raise ValueError( + f"Dataset '{dataset_name}' in '{SCALAR_FIELDS_GROUP}' must be 2D; got shape {dataset.shape}" + ) + + if expected_columns is None: + expected_columns = dataset.shape[1] + elif dataset.shape[1] != expected_columns: + raise ValueError( + "All datasets in 'scalar_fields' must share the same column count; " + f"expected {expected_columns}, got {dataset.shape[1]} for dataset '{dataset_name}'" + ) + + +def _dataset_name_sort_key(name: str): + try: + return (0, float(name)) + except ValueError: + return (1, name) diff --git a/dfode_kit/data/io_hdf5.py b/dfode_kit/data/io_hdf5.py new file mode 100644 index 00000000..a77282e8 --- /dev/null +++ b/dfode_kit/data/io_hdf5.py @@ -0,0 +1,48 @@ +import h5py +import numpy as np + +from dfode_kit.data.contracts import SCALAR_FIELDS_GROUP, read_scalar_field_datasets + + +def touch_h5(hdf5_file_path): + """ + Load an HDF5 file and print its contents and metadata. + + Parameters + ---------- + hdf5_file_path : str + The path to the HDF5 file to be opened. + + Returns + ------- + None + This function does not return any value. It prints the metadata, groups, + and datasets contained in the HDF5 file. + + Raises + ------ + FileNotFoundError + If the specified HDF5 file does not exist. + OSError + If the file cannot be opened as an HDF5 file. + """ + print(f"Inspecting HDF5 file: {hdf5_file_path}\n") + + with h5py.File(hdf5_file_path, "r") as hdf5_file: + print("Metadata in the HDF5 file:") + for attr in hdf5_file.attrs: + print(f"{attr}: {hdf5_file.attrs[attr]}") + + print("\nGroups and datasets in the HDF5 file:") + for group_name, group in hdf5_file.items(): + print(f"Group: {group_name}") + for dataset_name in group.keys(): + dataset = group[dataset_name] + print(f" Dataset: {dataset_name}, Shape: {dataset.shape}") + + +def get_TPY_from_h5(file_path): + """Read and stack all datasets from the ``scalar_fields`` HDF5 group.""" + datasets = read_scalar_field_datasets(file_path) + print(f"Number of datasets in {SCALAR_FIELDS_GROUP} group: {len(datasets)}") + return np.concatenate(list(datasets.values()), axis=0) diff --git a/dfode_kit/data_operations/__init__.py b/dfode_kit/data_operations/__init__.py index ad090283..336d74d6 100644 --- a/dfode_kit/data_operations/__init__.py +++ b/dfode_kit/data_operations/__init__.py @@ -20,8 +20,8 @@ ] _ATTRIBUTE_MODULES = { - "touch_h5": ("dfode_kit.data_operations.h5_kit", "touch_h5"), - "get_TPY_from_h5": ("dfode_kit.data_operations.h5_kit", "get_TPY_from_h5"), + "touch_h5": ("dfode_kit.data.io_hdf5", "touch_h5"), + "get_TPY_from_h5": ("dfode_kit.data.io_hdf5", "get_TPY_from_h5"), "integrate_h5": ("dfode_kit.data_operations.h5_kit", "integrate_h5"), "load_model": ("dfode_kit.data_operations.h5_kit", "load_model"), "nn_integrate": ("dfode_kit.data_operations.h5_kit", "nn_integrate"), @@ -29,12 +29,12 @@ "calculate_error": ("dfode_kit.data_operations.h5_kit", "calculate_error"), "random_perturb": ("dfode_kit.data_operations.augment_data", "random_perturb"), "label_npy": ("dfode_kit.data_operations.label_data", "label_npy"), - "SCALAR_FIELDS_GROUP": ("dfode_kit.data_operations.contracts", "SCALAR_FIELDS_GROUP"), - "MECHANISM_ATTR": ("dfode_kit.data_operations.contracts", "MECHANISM_ATTR"), - "read_scalar_field_datasets": ("dfode_kit.data_operations.contracts", "read_scalar_field_datasets"), - "stack_scalar_field_datasets": ("dfode_kit.data_operations.contracts", "stack_scalar_field_datasets"), - "require_h5_attr": ("dfode_kit.data_operations.contracts", "require_h5_attr"), - "require_h5_group": ("dfode_kit.data_operations.contracts", "require_h5_group"), + "SCALAR_FIELDS_GROUP": ("dfode_kit.data.contracts", "SCALAR_FIELDS_GROUP"), + "MECHANISM_ATTR": ("dfode_kit.data.contracts", "MECHANISM_ATTR"), + "read_scalar_field_datasets": ("dfode_kit.data.contracts", "read_scalar_field_datasets"), + "stack_scalar_field_datasets": ("dfode_kit.data.contracts", "stack_scalar_field_datasets"), + "require_h5_attr": ("dfode_kit.data.contracts", "require_h5_attr"), + "require_h5_group": ("dfode_kit.data.contracts", "require_h5_group"), } diff --git a/dfode_kit/data_operations/augment_data.py b/dfode_kit/data_operations/augment_data.py index c04475fe..d9892bed 100644 --- a/dfode_kit/data_operations/augment_data.py +++ b/dfode_kit/data_operations/augment_data.py @@ -2,7 +2,7 @@ import cantera as ct import time from dfode_kit.data_operations.h5_kit import advance_reactor -from dfode_kit.dfode_core.train.formation import formation_calculate +from dfode_kit.training.formation import formation_calculate def single_step(npstate, chem, time_step=1e-6): gas = ct.Solution(chem) diff --git a/dfode_kit/data_operations/contracts.py b/dfode_kit/data_operations/contracts.py index 1e5ade0e..5a860e3e 100644 --- a/dfode_kit/data_operations/contracts.py +++ b/dfode_kit/data_operations/contracts.py @@ -1,69 +1 @@ -from __future__ import annotations - -from typing import Dict - -import h5py -import numpy as np - -SCALAR_FIELDS_GROUP = "scalar_fields" -MECHANISM_ATTR = "mechanism" - - -def require_h5_group(hdf5_file: h5py.File, group_name: str) -> h5py.Group: - if group_name not in hdf5_file: - raise ValueError(f"'{group_name}' group not found in HDF5 file") - return hdf5_file[group_name] - - -def require_h5_attr(hdf5_file: h5py.File, attr_name: str): - if attr_name not in hdf5_file.attrs: - raise ValueError(f"Required HDF5 root attribute '{attr_name}' is missing") - return hdf5_file.attrs[attr_name] - - -def ordered_group_dataset_names(group: h5py.Group) -> list[str]: - names = list(group.keys()) - return sorted(names, key=_dataset_name_sort_key) - - -def read_scalar_field_datasets(hdf5_file_path: str) -> Dict[str, np.ndarray]: - with h5py.File(hdf5_file_path, "r") as hdf5_file: - scalar_group = require_h5_group(hdf5_file, SCALAR_FIELDS_GROUP) - dataset_names = ordered_group_dataset_names(scalar_group) - - data = {name: scalar_group[name][:] for name in dataset_names} - - _validate_scalar_field_datasets(data, source=hdf5_file_path) - return data - - -def stack_scalar_field_datasets(hdf5_file_path: str) -> np.ndarray: - datasets = read_scalar_field_datasets(hdf5_file_path) - return np.concatenate(list(datasets.values()), axis=0) - - -def _validate_scalar_field_datasets(datasets: Dict[str, np.ndarray], source: str) -> None: - if not datasets: - raise ValueError(f"No datasets found in '{SCALAR_FIELDS_GROUP}' for {source}") - - expected_columns = None - for dataset_name, dataset in datasets.items(): - if dataset.ndim != 2: - raise ValueError( - f"Dataset '{dataset_name}' in '{SCALAR_FIELDS_GROUP}' must be 2D; got shape {dataset.shape}" - ) - - if expected_columns is None: - expected_columns = dataset.shape[1] - elif dataset.shape[1] != expected_columns: - raise ValueError( - "All datasets in 'scalar_fields' must share the same column count; " - f"expected {expected_columns}, got {dataset.shape[1]} for dataset '{dataset_name}'" - ) - - -def _dataset_name_sort_key(name: str): - try: - return (0, float(name)) - except ValueError: - return (1, name) +from dfode_kit.data.contracts import * # noqa: F401,F403 diff --git a/dfode_kit/data_operations/h5_kit.py b/dfode_kit/data_operations/h5_kit.py index e9202124..64a62b27 100644 --- a/dfode_kit/data_operations/h5_kit.py +++ b/dfode_kit/data_operations/h5_kit.py @@ -3,78 +3,10 @@ import numpy as np import cantera as ct -from dfode_kit.data_operations.contracts import ( - MECHANISM_ATTR, - SCALAR_FIELDS_GROUP, - read_scalar_field_datasets, - require_h5_attr, -) +from dfode_kit.data.contracts import MECHANISM_ATTR, require_h5_attr +from dfode_kit.data.io_hdf5 import get_TPY_from_h5, touch_h5 from dfode_kit.utils import BCT, inverse_BCT -def touch_h5(hdf5_file_path): - """ - Load an HDF5 file and print its contents and metadata. - - Parameters - ---------- - hdf5_file_path : str - The path to the HDF5 file to be opened. - - Returns - ------- - None - This function does not return any value. It prints the metadata, groups, - and datasets contained in the HDF5 file. - - Raises - ------ - FileNotFoundError - If the specified HDF5 file does not exist. - OSError - If the file cannot be opened as an HDF5 file. - - Notes - ----- - This function provides a simple way to inspect the structure and metadata of - an HDF5 file, making it useful for debugging and understanding data organization. - - Examples - -------- - >>> touch_h5('/path/to/file.h5') - Metadata in the HDF5 file: - root_directory: /path/to/root - mechanism: /path/to/mechanism.yaml - species_names: ['T', 'p', 'species1', ...] - - Groups and datasets in the HDF5 file: - Group: scalar_fields - Dataset: 0, Shape: (100, 5) - Dataset: 1, Shape: (100, 5) - Group: mesh - Dataset: Cx, Shape: (100, 1) - """ - print(f"Inspecting HDF5 file: {hdf5_file_path}\n") - - with h5py.File(hdf5_file_path, 'r') as hdf5_file: - # Print the metadata - print("Metadata in the HDF5 file:") - for attr in hdf5_file.attrs: - print(f"{attr}: {hdf5_file.attrs[attr]}") - - # Print the names of the groups and datasets in the file - print("\nGroups and datasets in the HDF5 file:") - for group_name, group in hdf5_file.items(): - print(f"Group: {group_name}") - for dataset_name in group.keys(): - dataset = group[dataset_name] - print(f" Dataset: {dataset_name}, Shape: {dataset.shape}") - -def get_TPY_from_h5(file_path): - """Read and stack all datasets from the ``scalar_fields`` HDF5 group.""" - datasets = read_scalar_field_datasets(file_path) - print(f"Number of datasets in {SCALAR_FIELDS_GROUP} group: {len(datasets)}") - return np.concatenate(list(datasets.values()), axis=0) - def advance_reactor(gas, state, reactor, reactor_net, time_step): """Advance the reactor simulation for a given state.""" state = state.flatten() diff --git a/dfode_kit/df_interface/__init__.py b/dfode_kit/df_interface/__init__.py index 7cab800f..9ae45942 100644 --- a/dfode_kit/df_interface/__init__.py +++ b/dfode_kit/df_interface/__init__.py @@ -8,13 +8,13 @@ ] _ATTRIBUTE_MODULES = { - 'df_to_h5': ('dfode_kit.df_interface.sample_case', 'df_to_h5'), + 'df_to_h5': ('dfode_kit.cases.sampling', 'df_to_h5'), 'OneDFreelyPropagatingFlameConfig': ( - 'dfode_kit.df_interface.flame_configurations', + 'dfode_kit.cases.presets', 'OneDFreelyPropagatingFlameConfig', ), 'setup_one_d_flame_case': ( - 'dfode_kit.df_interface.oneDflame_setup', + 'dfode_kit.cases.deepflame', 'setup_one_d_flame_case', ), } diff --git a/dfode_kit/df_interface/case_init.py b/dfode_kit/df_interface/case_init.py index 87c4d831..17f93ca4 100644 --- a/dfode_kit/df_interface/case_init.py +++ b/dfode_kit/df_interface/case_init.py @@ -1,148 +1,3 @@ -from __future__ import annotations +"""Compatibility shim for the future dfode_kit.cases.init module.""" -import json -from dataclasses import asdict, dataclass -from pathlib import Path -from typing import Any - -from dfode_kit import DFODE_ROOT - - -DEFAULT_ONE_D_FLAME_TEMPLATE = ( - DFODE_ROOT / "canonical_cases" / "oneD_freely_propagating_flame" -) -DEFAULT_ONE_D_FLAME_PRESET = "premixed-defaults-v1" -AIR_OXIDIZER = "O2:1, N2:3.76" - - -@dataclass(frozen=True) -class OneDFlamePreset: - name: str - summary: str - assumptions: dict[str, str] - notes: list[str] - - -ONE_D_FLAME_PRESETS: dict[str, OneDFlamePreset] = { - DEFAULT_ONE_D_FLAME_PRESET: OneDFlamePreset( - name=DEFAULT_ONE_D_FLAME_PRESET, - summary=( - "Current DFODE-kit empirical defaults for one-dimensional freely " - "propagating premixed flames." - ), - assumptions={ - "domain_length": "flame_thickness / 10 * 500", - "domain_width": "domain_length / 10", - "ignition_region": "domain_length / 2", - "sim_time_step": "1e-6", - "num_output_steps": "100", - "sim_write_interval": "(flame_thickness / flame_speed) * 10 / num_output_steps", - "sim_time": "sim_write_interval * (num_output_steps + 1)", - "inlet_speed": "flame_speed", - "inert_specie": '"N2"', - }, - notes=[ - "These values preserve the current hardcoded logic in OneDFreelyPropagatingFlameConfig.update_config().", - "They are recommended starter defaults, not universal best practices.", - "Override any resolved field explicitly when domain knowledge requires it.", - ], - ) -} - - -@dataclass(frozen=True) -class OneDFlameInitInputs: - mechanism: str - fuel: str - oxidizer: str - eq_ratio: float - T0: float - p0: float - preset: str = DEFAULT_ONE_D_FLAME_PRESET - template: str = str(DEFAULT_ONE_D_FLAME_TEMPLATE) - inert_specie: str = "N2" - - -def resolve_oxidizer(oxidizer: str) -> str: - if oxidizer.strip().lower() == "air": - return AIR_OXIDIZER - return oxidizer - - -def get_one_d_flame_preset(name: str) -> OneDFlamePreset: - try: - return ONE_D_FLAME_PRESETS[name] - except KeyError as exc: - raise ValueError( - f"Unknown oneD-flame preset: {name}. Available presets: {', '.join(sorted(ONE_D_FLAME_PRESETS))}" - ) from exc - - -def one_d_flame_plan_dict( - *, - inputs: OneDFlameInitInputs, - resolved: dict[str, Any], - output_dir: str | None, - config_path: str | None = None, -) -> dict[str, Any]: - preset = get_one_d_flame_preset(inputs.preset) - return { - "schema_version": 1, - "case_type": "oneD-flame", - "preset": preset.name, - "preset_summary": preset.summary, - "template": str(Path(inputs.template).resolve()), - "output_dir": str(Path(output_dir).resolve()) if output_dir else None, - "config_path": str(Path(config_path).resolve()) if config_path else None, - "inputs": { - **asdict(inputs), - "oxidizer": resolve_oxidizer(inputs.oxidizer), - "template": str(Path(inputs.template).resolve()), - }, - "assumptions": preset.assumptions, - "notes": preset.notes, - "resolved": resolved, - } - - -def dump_plan_json(plan: dict[str, Any], path: str | Path) -> Path: - output_path = Path(path).resolve() - output_path.parent.mkdir(parents=True, exist_ok=True) - output_path.write_text(json.dumps(plan, indent=2, sort_keys=True) + "\n", encoding="utf-8") - return output_path - - -def load_plan_json(path: str | Path) -> dict[str, Any]: - input_path = Path(path).resolve() - return json.loads(input_path.read_text(encoding="utf-8")) - - -def one_d_flame_inputs_from_plan(plan: dict[str, Any]) -> OneDFlameInitInputs: - if plan.get("case_type") != "oneD-flame": - raise ValueError(f"Unsupported case_type in config: {plan.get('case_type')}") - - inputs = plan["inputs"] - return OneDFlameInitInputs( - mechanism=inputs["mechanism"], - fuel=inputs["fuel"], - oxidizer=inputs["oxidizer"], - eq_ratio=float(inputs["eq_ratio"]), - T0=float(inputs["T0"]), - p0=float(inputs["p0"]), - preset=inputs.get("preset", plan.get("preset", DEFAULT_ONE_D_FLAME_PRESET)), - template=inputs.get("template", str(DEFAULT_ONE_D_FLAME_TEMPLATE)), - inert_specie=inputs.get("inert_specie", "N2"), - ) - - -def one_d_flame_overrides_from_plan(plan: dict[str, Any]) -> dict[str, Any]: - overrides = dict(plan.get("resolved", {})) - overrides.pop("mechanism", None) - overrides.pop("fuel", None) - overrides.pop("oxidizer", None) - overrides.pop("eq_ratio", None) - overrides.pop("T0", None) - overrides.pop("p0", None) - overrides.pop("preset", None) - overrides.pop("template", None) - return overrides +from dfode_kit.cases.init import * # noqa: F401,F403 diff --git a/dfode_kit/df_interface/flame_configurations.py b/dfode_kit/df_interface/flame_configurations.py index 5194865f..dee0353c 100644 --- a/dfode_kit/df_interface/flame_configurations.py +++ b/dfode_kit/df_interface/flame_configurations.py @@ -1,126 +1,4 @@ -from pathlib import Path -from dataclasses import dataclass, field +"""Compatibility shim for the future dfode_kit.cases.presets module.""" -import cantera as ct - -@dataclass -class OneDFreelyPropagatingFlameConfig: - mechanism: str - T0: float - p0: float - fuel: str - oxidizer: str - eq_ratio: float - - initial_gas: ct.Solution = field(init=False) - burnt_gas: ct.Solution = field(init=False) - n_species: int = field(init=False) - species_names: list = field(init=False) - n_dims: int = field(init=False) - dim_names: list = field(init=False) - mech_path: Path = field(init=False) - - flame_speed: float = field(init=False, default=None) - flame_thickness: float = field(init=False, default=None) - flame: ct.SolutionArray = field(init=False, default=None) - - domain_length: float = field(init=False, default=None) - domain_width: float = field(init=False, default=None) - ignition_region: float = field(init=False, default=None) - sim_time_step: float = field(init=False, default=None) - sim_time: float = field(init=False, default=None) - sim_write_interval: float = field(init=False, default=None) - num_output_steps: int = field(init=False, default=None) - inlet_speed: float = field(init=False, default=None) - inert_specie: str = field(init=False, default='N2') - - - def __post_init__(self): - """Post-initialization to set up initial and burnt gas states.""" - self.initial_gas = ct.Solution(self.mechanism) - self.initial_gas.TP = self.T0, self.p0 - self.initial_gas.set_equivalence_ratio(self.eq_ratio, self.fuel, self.oxidizer) - - self.burnt_gas = ct.Solution(self.mechanism) - self.burnt_gas.TP = self.T0, self.p0 - self.burnt_gas.set_equivalence_ratio(self.eq_ratio, self.fuel, self.oxidizer) - self.burnt_gas.equilibrate('HP') - - self.n_species = self.initial_gas.n_species - self.species_names = self.initial_gas.species_names - self.n_dims = 2 + self.n_species - self.dim_names = ['T', 'p'] + self.species_names - - self.mech_path = Path(self.mechanism).resolve() - - def calculate_laminar_flame_properties(self): - """Calculate laminar flame speed and thickness.""" - - # Initialize the gas object - flame_speed_gas = ct.Solution(self.mechanism) - flame_speed_gas.TP = self.T0, self.p0 - - # Set the equivalence ratio - flame_speed_gas.set_equivalence_ratio(self.eq_ratio, self.fuel, self.oxidizer) - - # Create and solve the flame - width = 0.1 - flame = ct.FreeFlame(flame_speed_gas, width=width) - flame.set_refine_criteria(ratio=3, slope=0.05, curve=0.1, prune=0.0) - - print("Solving premixed flame...") - flame.solve(loglevel=0, auto=True) - - # Access laminar flame speed - laminar_flame_speed = flame.velocity[0] - print(f'{"Laminar Flame Speed":<25}:{laminar_flame_speed:>15.10f} m/s') - - # Calculate laminar flame thickness - z, T = flame.grid, flame.T - grad = (T[1:] - T[:-1]) / (z[1:] - z[:-1]) - laminar_flame_thickness = (max(T) - min(T)) / max(grad) - print(f'{"Laminar Flame Thickness":<25}:{laminar_flame_thickness:>15.10f} m') - - final_flame = flame.to_solution_array() - - self.flame_speed = laminar_flame_speed - self.flame_thickness = laminar_flame_thickness - self.flame = final_flame - - def update_config(self, params: dict): - """Update the configuration with new parameters.""" - for key, value in params.items(): - if hasattr(self, key): - setattr(self, key, value) - else: - raise AttributeError(f"'{key}' is not a valid attribute of OneDFreelyPropagatingFlameConfig") - - if self.flame_speed is None or self.flame_thickness is None: - self.calculate_laminar_flame_properties() - - if self.domain_length is None: - self.domain_length = self.flame_thickness / 10 * 500 - - if self.domain_width is None: - self.domain_width = self.domain_length / 10 - - if self.ignition_region is None: - self.ignition_region = self.domain_length / 2 - - if self.sim_time_step is None: - self.sim_time_step = 1e-6 - - if self.num_output_steps is None: - self.num_output_steps = 100 - - if self.sim_write_interval is None: - chem_time_scale = self.flame_thickness / self.flame_speed - self.sim_write_interval = chem_time_scale * 10 / self.num_output_steps - - if self.sim_time is None: - self.sim_time = self.sim_write_interval * (self.num_output_steps + 1) - - if self.inlet_speed is None: - self.inlet_speed = self.flame_speed - +from dfode_kit.cases.presets import * # noqa: F401,F403 \ No newline at end of file diff --git a/dfode_kit/df_interface/oneDflame_setup.py b/dfode_kit/df_interface/oneDflame_setup.py index d7ade640..d469142e 100755 --- a/dfode_kit/df_interface/oneDflame_setup.py +++ b/dfode_kit/df_interface/oneDflame_setup.py @@ -1,117 +1,3 @@ -import cantera as ct -import numpy as np -import shutil +"""Compatibility shim for the future dfode_kit.cases.deepflame module.""" -from pathlib import Path - -from dfode_kit.df_interface.flame_configurations import OneDFreelyPropagatingFlameConfig - -def update_one_d_sample_config(cfg: OneDFreelyPropagatingFlameConfig, case_path): - case_path = Path(case_path).resolve() - orig_file_path = case_path / 'system/sampleConfigDict.orig' - new_file_path = case_path / 'system/sampleConfigDict' - shutil.copy(orig_file_path, new_file_path) - - replacements = { - "CanteraMechanismFile_": f'"{Path(cfg.mech_path).resolve()}"', - "inertSpecie_": f'"{cfg.inert_specie}"', - - "domainWidth": cfg.domain_width, - "domainLength": cfg.domain_length, - "ignitionRegion": cfg.ignition_region, - - "simTimeStep": cfg.sim_time_step, - "simTime": cfg.sim_time, - "simWriteInterval": cfg.sim_write_interval, - - "UInlet": cfg.inlet_speed, - "pInternal": cfg.p0, - } - - with open(new_file_path, 'r') as file: - lines = file.readlines() - - for i, line in enumerate(lines): - for key, value in replacements.items(): - if key in line: - lines[i] = line.replace("placeHolder", str(value)) - - # Update unburnt states - if "unburntStates" in line: - state_strings = [f'{"TUnburnt":<20}{cfg.initial_gas.T:>16.10f};'] - state_strings += [ - f'{species}Unburnt'.ljust(20) + f'{cfg.initial_gas.Y[idx]:>16.10f};' - for idx, species in enumerate(cfg.species_names) - ] - lines[i] = '\n'.join(state_strings) + '\n\n' - - # Update equilibrium states - if "equilibriumStates" in line: - state_strings = [f'{"TBurnt":<20}{cfg.burnt_gas.T:>16.10f};'] - state_strings += [ - f'{species}Burnt'.ljust(20) + f'{cfg.burnt_gas.Y[idx]:>16.10f};' - for idx, species in enumerate(cfg.species_names) - ] - lines[i] = '\n'.join(state_strings) + '\n\n' - - with open(new_file_path, 'w') as file: - file.writelines(lines) - -def create_0_species_files(cfg: OneDFreelyPropagatingFlameConfig, case_path): - case_path = Path(case_path).resolve() - orig_0_file_path = case_path / '0/Ydefault.orig' - - for idx, species in enumerate(cfg.species_names): - new_0_file_path = case_path / '0' / f'{species}.orig' - shutil.copy(orig_0_file_path, new_0_file_path) - - with open(new_0_file_path, 'r') as file: - lines = file.readlines() - - for i, line in enumerate(lines): - if "Ydefault" in line: - lines[i] = line.replace("Ydefault", f'{species}') - if "uniform 0" in line: - lines[i] = line.replace("0", f'{cfg.initial_gas.Y[idx]}') - - with open(new_0_file_path, 'w') as file: - file.writelines(lines) - -def update_set_fields_dict(cfg: OneDFreelyPropagatingFlameConfig, case_path): - case_path = Path(case_path).resolve() - orig_setFieldsDict_path = case_path / 'system/setFieldsDict.orig' - new_setFieldsDict_path = case_path / 'system/setFieldsDict' - shutil.copy(orig_setFieldsDict_path, new_setFieldsDict_path) - - with open(new_setFieldsDict_path, 'r') as file: - lines = file.readlines() - - for i, line in enumerate(lines): - if "unburntStatesPlaceHolder" in line: - state_strings = [f'\tvolScalarFieldValue {"T":<10} $TUnburnt'] - for _, species in enumerate(cfg.species_names): - state_strings.append(f'volScalarFieldValue {species:<10} ${species}Unburnt') - lines[i] = '\n\t'.join(state_strings) + '\n' - if "equilibriumStatesPlaceHolder" in line: - state_strings = [f'\t\t\tvolScalarFieldValue {"T":<10} $TBurnt'] - for _, species in enumerate(cfg.species_names): - state_strings.append(f'volScalarFieldValue {species:<10} ${species}Burnt') - lines[i] = '\n\t\t\t'.join(state_strings) + '\n' - - - with open(new_setFieldsDict_path, 'w') as file: - file.writelines(lines) - -def setup_one_d_flame_case(cfg: OneDFreelyPropagatingFlameConfig, case_path): - case_path = Path(case_path).resolve() - - # Update sampleConfigDict - update_one_d_sample_config(cfg, case_path) - - # Create 0/ species files - create_0_species_files(cfg, case_path) - - # Update setFieldsDict - update_set_fields_dict(cfg, case_path) - - print(f"One-dimensional flame case setup completed at: {case_path}") \ No newline at end of file +from dfode_kit.cases.deepflame import * # noqa: F401,F403 diff --git a/dfode_kit/df_interface/sample_case.py b/dfode_kit/df_interface/sample_case.py index 2e237654..57f3b438 100644 --- a/dfode_kit/df_interface/sample_case.py +++ b/dfode_kit/df_interface/sample_case.py @@ -1,182 +1,4 @@ -from pathlib import Path +"""Compatibility shim for the future dfode_kit.cases.sampling module.""" -import h5py -import numpy as np -import cantera as ct +from dfode_kit.cases.sampling import * # noqa: F401,F403 -from dfode_kit.utils import is_number, read_openfoam_scalar - -def gather_species_arrays(species_names, directory_path) -> np.ndarray: - """ - Concatenate scalar arrays from OpenFOAM files for each species in the specified directory. - - Parameters - ---------- - species_names : list of str - A list of species names corresponding to the files in the directory. - directory_path : str - The path to the directory containing the OpenFOAM files. - - Returns - ------- - numpy.ndarray - A 2D numpy array containing the concatenated scalar arrays for each species. - - Raises - ------ - ValueError - If the provided directory does not exist, if there is a shape mismatch - between arrays, or if no valid species arrays are found to concatenate. - - Notes - ----- - This function reads scalar values from files named after the species and - ensures that all arrays have the same number of cells. If a scalar value - is a float, it is converted into a uniform numpy array. - - Examples - -------- - >>> gather_species_arrays(['species1', 'species2'], '/path/to/directory') - array([[...], [...], ...]) - """ - all_arrays = [] - num_cell = None - directory_path = Path(directory_path) - - # Ensure the provided directory exists - if not Path(directory_path).is_dir(): - raise ValueError(f"The directory does not exist: {directory_path}") - - for species in species_names: - # Construct the file name based on species name - file_path = directory_path / species - - # Check if the file exists - if file_path.is_file(): - try: - # Read the scalar values using the existing function - species_array = read_openfoam_scalar(file_path) - - # Check if species_array is a numpy array - if isinstance(species_array, np.ndarray): - # Assign num_cell on the first valid species_array - if num_cell is None: - num_cell = species_array.shape[0] - else: - # Ensure the shape matches num_cell - if species_array.shape[0] != num_cell: - raise ValueError(f"Shape mismatch for {species}: expected {num_cell}, got {species_array.shape[0]}.") - - all_arrays.append(species_array) - except ValueError as e: - print(f"Error reading {file_path}: {e}") - else: - print(f"File not found: {file_path}") - - # Replace non-numpy arrays (that are floats) with numpy arrays of uniform values - for i in range(len(all_arrays)): - if not isinstance(all_arrays[i], np.ndarray): - if isinstance(all_arrays[i], float): - all_arrays[i] = np.full((num_cell, 1), all_arrays[i]) # Create a uniform array - else: - print(f"Warning: {all_arrays[i]} is not a numpy array or float.") - - # Concatenate all arrays into one big array if there are valid arrays - if all_arrays: - concatenated_array = np.concatenate(all_arrays, axis=1) - return concatenated_array - else: - raise ValueError("No valid species arrays found to concatenate.") - -def df_to_h5(root_dir, mechanism, hdf5_file_path, include_mesh=True): - """ - Iterate through directories in root_dir, concatenate arrays, and save to an HDF5 file. - - Parameters - ---------- - root_dir : str - The path to the root directory containing subdirectories with data. - mechanism : str - The path to the mechanism file to be used by the Cantera solution. - hdf5_file_path : str - The path where the HDF5 file will be saved. - include_mesh : bool, optional - Whether to include mesh data in the HDF5 file (default is True). - - Returns - ------- - None - This function does not return any value. It saves the concatenated data - directly to an HDF5 file. - - Raises - ------ - ValueError - If there are issues with reading directories or files, or if the data - cannot be processed. - - Notes - ----- - This function processes directories containing numerical data, concatenates - scalar arrays for each species, and saves the results in an HDF5 file. - It also optionally includes mesh data from predefined mesh files. - - Examples - -------- - >>> df_to_h5('/path/to/root', '/path/to/mechanism.yaml', '/path/to/output.h5') - Saved concatenated arrays to /path/to/output.h5 - """ - root_path = Path(root_dir).resolve() - mechanism = Path(mechanism).resolve() - hdf5_file_path = Path(hdf5_file_path) - gas = ct.Solution(mechanism) - species_names = ['T', 'p'] + gas.species_names - print(f"Species names: {species_names}") - - - with h5py.File(hdf5_file_path, 'w') as hdf5_file: - hdf5_file.attrs['root_directory'] = str(root_path) - hdf5_file.attrs['mechanism'] = str(mechanism) - hdf5_file.attrs['species_names'] = species_names - - scalar_group = hdf5_file.create_group('scalar_fields') - - numeric_dirs = [ - dir_path for dir_path in root_path.iterdir() - if dir_path.is_dir() and is_number(dir_path.name) and dir_path.name != '0' - ] - - # Sort the directories based on their numeric values - numeric_dirs.sort(key=lambda x: float(x.name)) - - for dir_path in numeric_dirs: - # Concatenate arrays for the current directory - try: - concatenated_array = gather_species_arrays(species_names, dir_path) - - # Create a dataset in HDF5 with the directory path as the key - scalar_group.create_dataset(str(dir_path.name), data=concatenated_array) - except ValueError as e: - print(f"Error processing directory {dir_path}: {e}") - - if include_mesh: - mesh_group = hdf5_file.create_group('mesh') - mesh_files = [ - root_path / 'temp/0/Cx', - root_path / 'temp/0/Cy', - root_path / 'temp/0/Cz', - root_path / 'temp/0/V', - ] - - for mesh_file in mesh_files: - if mesh_file.is_file(): - try: - mesh_data = read_openfoam_scalar(mesh_file) - mesh_group.create_dataset(str(mesh_file.name), data=mesh_data) - except ValueError as e: - print(f"Error reading mesh file {mesh_file}: {e}") - else: - print(f"Mesh file not found: {mesh_file}") - - print(f"Saved concatenated arrays to {hdf5_file_path}") - diff --git a/dfode_kit/dfode_core/model/__init__.py b/dfode_kit/dfode_core/model/__init__.py index 258e7d6d..8dd28eb9 100644 --- a/dfode_kit/dfode_core/model/__init__.py +++ b/dfode_kit/dfode_core/model/__init__.py @@ -1 +1,33 @@ -"""Model package for DFODE training.""" +"""Compatibility shims for :mod:`dfode_kit.models`.""" + +from importlib import import_module + + +__all__ = [ + "MLP", + "build_mlp", + "create_model", + "get_model_factory", + "register_model", + "registered_models", +] + +_ATTRIBUTE_MODULES = { + "MLP": ("dfode_kit.models.mlp", "MLP"), + "build_mlp": ("dfode_kit.models.mlp", "build_mlp"), + "create_model": ("dfode_kit.models.registry", "create_model"), + "get_model_factory": ("dfode_kit.models.registry", "get_model_factory"), + "register_model": ("dfode_kit.models.registry", "register_model"), + "registered_models": ("dfode_kit.models.registry", "registered_models"), +} + + +def __getattr__(name): + if name not in _ATTRIBUTE_MODULES: + raise AttributeError(f"module 'dfode_kit.dfode_core.model' has no attribute '{name}'") + + module_name, attribute_name = _ATTRIBUTE_MODULES[name] + module = import_module(module_name) + value = getattr(module, attribute_name) + globals()[name] = value + return value diff --git a/dfode_kit/dfode_core/model/mlp.py b/dfode_kit/dfode_core/model/mlp.py index f6fe3cb3..03ce8988 100644 --- a/dfode_kit/dfode_core/model/mlp.py +++ b/dfode_kit/dfode_core/model/mlp.py @@ -1,26 +1,5 @@ -from __future__ import annotations +"""Compatibility shim for :mod:`dfode_kit.models.mlp`.""" -import torch +from dfode_kit.models.mlp import MLP, build_mlp -from dfode_kit.dfode_core.train.config import ModelConfig - - -class MLP(torch.nn.Module): - def __init__(self, layer_info): - super().__init__() - - self.net = torch.nn.Sequential() - n = len(layer_info) - 1 - for i in range(n - 1): - self.net.add_module(f"linear_layer_{i}", torch.nn.Linear(layer_info[i], layer_info[i + 1])) - self.net.add_module(f"gelu_layer_{i}", torch.nn.GELU()) - self.net.add_module(f"linear_layer_{n - 1}", torch.nn.Linear(layer_info[n - 1], layer_info[n])) - - def forward(self, x): - return self.net(x) - - -def build_mlp(*, model_config: ModelConfig, n_species: int, device): - hidden_layers = list(model_config.params.get("hidden_layers", [400, 400, 400, 400])) - layer_info = [2 + n_species, *hidden_layers, n_species - 1] - return MLP(layer_info).to(device) +__all__ = ["MLP", "build_mlp"] diff --git a/dfode_kit/dfode_core/model/registry.py b/dfode_kit/dfode_core/model/registry.py index 88c33e01..f4fbe082 100644 --- a/dfode_kit/dfode_core/model/registry.py +++ b/dfode_kit/dfode_core/model/registry.py @@ -1,29 +1,10 @@ -from __future__ import annotations +"""Compatibility shim for :mod:`dfode_kit.models.registry`.""" -from typing import Callable, Dict +from dfode_kit.models.registry import create_model, get_model_factory, register_model, registered_models - -ModelFactory = Callable[..., object] -_MODEL_REGISTRY: Dict[str, ModelFactory] = {} - - -def register_model(name: str, factory: ModelFactory) -> None: - if not name: - raise ValueError("Model name must be non-empty.") - _MODEL_REGISTRY[name] = factory - - -def get_model_factory(name: str) -> ModelFactory: - try: - return _MODEL_REGISTRY[name] - except KeyError as exc: - available = ", ".join(sorted(_MODEL_REGISTRY)) or "" - raise KeyError(f"Unknown model '{name}'. Available models: {available}") from exc - - -def create_model(name: str, **kwargs): - return get_model_factory(name)(**kwargs) - - -def registered_models(): - return tuple(sorted(_MODEL_REGISTRY)) +__all__ = [ + "create_model", + "get_model_factory", + "register_model", + "registered_models", +] diff --git a/dfode_kit/dfode_core/test/test.py b/dfode_kit/dfode_core/test/test.py deleted file mode 100644 index 70e36da4..00000000 --- a/dfode_kit/dfode_core/test/test.py +++ /dev/null @@ -1,17 +0,0 @@ -import torch -import numpy as np -import os -import cantera as ct - -from dfode_kit.data_operations.h5_kit import advance_reactor - -DFODE_ROOT = os.environ['DFODE_ROOT'] -def test_npy( - mech_path: str, - source_file: str, - output_path: str, - time_step: float = 1e-6, -) -> np.ndarray: - - test_data = np.load(source_file) - \ No newline at end of file diff --git a/dfode_kit/dfode_core/train/__init__.py b/dfode_kit/dfode_core/train/__init__.py index 0052703b..d7a55251 100644 --- a/dfode_kit/dfode_core/train/__init__.py +++ b/dfode_kit/dfode_core/train/__init__.py @@ -1 +1,43 @@ -"""Training package for DFODE core.""" +"""Compatibility shims for :mod:`dfode_kit.training`.""" + +from importlib import import_module + + +__all__ = [ + "ModelConfig", + "OptimizerConfig", + "TrainerConfig", + "TrainingConfig", + "create_trainer", + "default_training_config", + "get_trainer_factory", + "register_trainer", + "registered_trainers", + "train", + "with_overrides", +] + +_ATTRIBUTE_MODULES = { + "ModelConfig": ("dfode_kit.training.config", "ModelConfig"), + "OptimizerConfig": ("dfode_kit.training.config", "OptimizerConfig"), + "TrainerConfig": ("dfode_kit.training.config", "TrainerConfig"), + "TrainingConfig": ("dfode_kit.training.config", "TrainingConfig"), + "create_trainer": ("dfode_kit.training.registry", "create_trainer"), + "default_training_config": ("dfode_kit.training.config", "default_training_config"), + "get_trainer_factory": ("dfode_kit.training.registry", "get_trainer_factory"), + "register_trainer": ("dfode_kit.training.registry", "register_trainer"), + "registered_trainers": ("dfode_kit.training.registry", "registered_trainers"), + "train": ("dfode_kit.training.train", "train"), + "with_overrides": ("dfode_kit.training.config", "with_overrides"), +} + + +def __getattr__(name): + if name not in _ATTRIBUTE_MODULES: + raise AttributeError(f"module 'dfode_kit.dfode_core.train' has no attribute '{name}'") + + module_name, attribute_name = _ATTRIBUTE_MODULES[name] + module = import_module(module_name) + value = getattr(module, attribute_name) + globals()[name] = value + return value diff --git a/dfode_kit/dfode_core/train/config.py b/dfode_kit/dfode_core/train/config.py index c5ec4947..66b7fce3 100644 --- a/dfode_kit/dfode_core/train/config.py +++ b/dfode_kit/dfode_core/train/config.py @@ -1,57 +1,19 @@ -from __future__ import annotations - -from dataclasses import dataclass, field, replace -from typing import Any, Dict, Optional - - -@dataclass(frozen=True) -class ModelConfig: - name: str = "mlp" - params: Dict[str, Any] = field(default_factory=lambda: { - "hidden_layers": [400, 400, 400, 400], - }) - - -@dataclass(frozen=True) -class OptimizerConfig: - name: str = "adam" - lr: float = 1e-3 - - -@dataclass(frozen=True) -class TrainerConfig: - name: str = "supervised_physics" - max_epochs: int = 1500 - lr_decay_epoch: int = 500 - lr_decay_factor: float = 0.1 - batch_size: int = 20000 - - -@dataclass(frozen=True) -class TrainingConfig: - model: ModelConfig = field(default_factory=ModelConfig) - optimizer: OptimizerConfig = field(default_factory=OptimizerConfig) - trainer: TrainerConfig = field(default_factory=TrainerConfig) - time_step: float = 1e-6 - - -def default_training_config() -> TrainingConfig: - return TrainingConfig() - - -def with_overrides( - config: Optional[TrainingConfig] = None, - *, - model: Optional[ModelConfig] = None, - optimizer: Optional[OptimizerConfig] = None, - trainer: Optional[TrainerConfig] = None, - time_step: Optional[float] = None, -) -> TrainingConfig: - base = config or default_training_config() - return replace( - base, - model=model or base.model, - optimizer=optimizer or base.optimizer, - trainer=trainer or base.trainer, - time_step=base.time_step if time_step is None else time_step, - ) +"""Compatibility shim for :mod:`dfode_kit.training.config`.""" + +from dfode_kit.training.config import ( + ModelConfig, + OptimizerConfig, + TrainerConfig, + TrainingConfig, + default_training_config, + with_overrides, +) + +__all__ = [ + "ModelConfig", + "OptimizerConfig", + "TrainerConfig", + "TrainingConfig", + "default_training_config", + "with_overrides", +] diff --git a/dfode_kit/dfode_core/train/formation.py b/dfode_kit/dfode_core/train/formation.py index 4f036c69..20b18ee1 100644 --- a/dfode_kit/dfode_core/train/formation.py +++ b/dfode_kit/dfode_core/train/formation.py @@ -1,9 +1,5 @@ -import cantera as ct -import numpy as np +"""Compatibility shim for :mod:`dfode_kit.training.formation`.""" -def formation_calculate(mechanism): - gas = ct.Solution(mechanism) - gas.TPY = 298.15, ct.one_atm,'O2:1' - partial_molar_enthalpy = gas.partial_molar_enthalpies/gas.molecular_weights - print(partial_molar_enthalpy) - return partial_molar_enthalpy \ No newline at end of file +from dfode_kit.training.formation import formation_calculate + +__all__ = ["formation_calculate"] diff --git a/dfode_kit/dfode_core/train/registry.py b/dfode_kit/dfode_core/train/registry.py index 5658e168..26011aa7 100644 --- a/dfode_kit/dfode_core/train/registry.py +++ b/dfode_kit/dfode_core/train/registry.py @@ -1,29 +1,10 @@ -from __future__ import annotations +"""Compatibility shim for :mod:`dfode_kit.training.registry`.""" -from typing import Callable, Dict +from dfode_kit.training.registry import create_trainer, get_trainer_factory, register_trainer, registered_trainers - -TrainerFactory = Callable[..., object] -_TRAINER_REGISTRY: Dict[str, TrainerFactory] = {} - - -def register_trainer(name: str, factory: TrainerFactory) -> None: - if not name: - raise ValueError("Trainer name must be non-empty.") - _TRAINER_REGISTRY[name] = factory - - -def get_trainer_factory(name: str) -> TrainerFactory: - try: - return _TRAINER_REGISTRY[name] - except KeyError as exc: - available = ", ".join(sorted(_TRAINER_REGISTRY)) or "" - raise KeyError(f"Unknown trainer '{name}'. Available trainers: {available}") from exc - - -def create_trainer(name: str, **kwargs): - return get_trainer_factory(name)(**kwargs) - - -def registered_trainers(): - return tuple(sorted(_TRAINER_REGISTRY)) +__all__ = [ + "create_trainer", + "get_trainer_factory", + "register_trainer", + "registered_trainers", +] diff --git a/dfode_kit/dfode_core/train/supervised_physics.py b/dfode_kit/dfode_core/train/supervised_physics.py index 336b1884..447392d8 100644 --- a/dfode_kit/dfode_core/train/supervised_physics.py +++ b/dfode_kit/dfode_core/train/supervised_physics.py @@ -1,102 +1,8 @@ -from __future__ import annotations +"""Compatibility shim for :mod:`dfode_kit.training.supervised_physics`.""" -import torch +from dfode_kit.training.supervised_physics import ( + SupervisedPhysicsTrainer, + build_supervised_physics_trainer, +) -from dfode_kit.dfode_core.train.config import OptimizerConfig, TrainerConfig - - -class SupervisedPhysicsTrainer: - def __init__( - self, - trainer_config: TrainerConfig, - optimizer_config: OptimizerConfig, - ) -> None: - self.trainer_config = trainer_config - self.optimizer_config = optimizer_config - - def _build_optimizer(self, model): - if self.optimizer_config.name != "adam": - raise ValueError( - f"Unsupported optimizer '{self.optimizer_config.name}'. Only 'adam' is implemented in this slice." - ) - return torch.optim.Adam(model.parameters(), lr=self.optimizer_config.lr) - - def fit( - self, - *, - model, - features, - labels, - features_mean, - features_std, - labels_mean, - labels_std, - formation_enthalpies, - time_step: float, - ) -> None: - loss_fn = torch.nn.L1Loss() - optimizer = self._build_optimizer(model) - model.train() - - for epoch in range(self.trainer_config.max_epochs): - if epoch > 0 and epoch % self.trainer_config.lr_decay_epoch == 0: - for param_group in optimizer.param_groups: - param_group["lr"] *= self.trainer_config.lr_decay_factor - - total_loss1 = 0.0 - total_loss2 = 0.0 - total_loss3 = 0.0 - total_loss = 0.0 - - for i in range(0, len(features), self.trainer_config.batch_size): - batch_features = features[i:i + self.trainer_config.batch_size] - batch_labels = labels[i:i + self.trainer_config.batch_size] - - optimizer.zero_grad() - - preds = model(batch_features) - loss1 = loss_fn(preds, batch_labels) - - base_y = batch_features[:, 2:-1] * features_std[2:-1] + features_mean[2:-1] - Y_in = (base_y * 0.1 + 1) ** 10 - Y_out = (((preds * labels_std + labels_mean) + base_y) * 0.1 + 1) ** 10 - Y_target = (((batch_labels * labels_std + labels_mean) + base_y) * 0.1 + 1) ** 10 - - loss2 = loss_fn(Y_out.sum(axis=1), Y_in.sum(axis=1)) - - Y_out_total = torch.cat((Y_out, (1 - Y_out.sum(axis=1)).reshape(Y_out.shape[0], 1)), axis=1) - Y_target_total = torch.cat((Y_target, (1 - Y_target.sum(axis=1)).reshape(Y_target.shape[0], 1)), axis=1) - - loss3 = loss_fn( - (formation_enthalpies * Y_out_total).sum(axis=1), - (formation_enthalpies * Y_target_total).sum(axis=1), - ) / time_step - loss = loss1 + loss2 + loss3 / 1e13 - - total_loss1 += loss1.item() - total_loss2 += loss2.item() - total_loss3 += loss3.item() - total_loss += loss.item() - - loss.backward() - optimizer.step() - - batches = len(features) / self.trainer_config.batch_size - total_loss1 /= batches - total_loss2 /= batches - total_loss3 /= batches - total_loss /= batches - - print( - "Epoch: {}, Loss1: {:4e}, Loss2: {:4e}, Loss3: {:4e}, Loss: {:4e}".format( - epoch + 1, - total_loss1, - total_loss2, - total_loss3, - total_loss, - ) - ) - - -def build_supervised_physics_trainer(*, trainer_config: TrainerConfig, optimizer_config: OptimizerConfig): - return SupervisedPhysicsTrainer(trainer_config=trainer_config, optimizer_config=optimizer_config) +__all__ = ["SupervisedPhysicsTrainer", "build_supervised_physics_trainer"] diff --git a/dfode_kit/dfode_core/train/train.py b/dfode_kit/dfode_core/train/train.py index fa0b1ce8..7fbcb327 100644 --- a/dfode_kit/dfode_core/train/train.py +++ b/dfode_kit/dfode_core/train/train.py @@ -1,127 +1,5 @@ -from __future__ import annotations +"""Compatibility shim for :mod:`dfode_kit.training.train`.""" -import numpy as np -import cantera as ct -import torch +from dfode_kit.training.train import train -from dfode_kit.dfode_core.model.mlp import build_mlp -from dfode_kit.dfode_core.model.registry import create_model, register_model -from dfode_kit.dfode_core.train.config import TrainingConfig, default_training_config, with_overrides -from dfode_kit.dfode_core.train.formation import formation_calculate -from dfode_kit.dfode_core.train.registry import create_trainer, register_trainer -from dfode_kit.dfode_core.train.supervised_physics import build_supervised_physics_trainer -from dfode_kit.utils import BCT - - -def _prepare_training_tensors(labeled_data: np.ndarray, n_species: int, device): - thermochem_states1 = labeled_data[:, 0 : 2 + n_species].copy() - thermochem_states2 = labeled_data[:, 2 + n_species :].copy() - - print(thermochem_states1.shape, thermochem_states2.shape) - thermochem_states1[:, 2:] = np.clip(thermochem_states1[:, 2:], 0, 1) - thermochem_states2[:, 2:] = np.clip(thermochem_states2[:, 2:], 0, 1) - - features = torch.tensor( - np.hstack((thermochem_states1[:, :2], BCT(thermochem_states1[:, 2:]))), - dtype=torch.float32, - ).to(device) - labels = torch.tensor( - BCT(thermochem_states2[:, 2:-1]) - BCT(thermochem_states1[:, 2:-1]), - dtype=torch.float32, - ).to(device) - - features_mean = torch.mean(features, dim=0) - features_std = torch.std(features, dim=0) - labels_mean = torch.mean(labels, dim=0) - labels_std = torch.std(labels, dim=0) - - normalized_features = (features - features_mean) / features_std - normalized_labels = (labels - labels_mean) / labels_std - - return { - "features": normalized_features, - "labels": normalized_labels, - "features_mean": features_mean, - "features_std": features_std, - "labels_mean": labels_mean, - "labels_std": labels_std, - } - - -def _register_defaults() -> None: - register_model("mlp", build_mlp) - register_trainer("supervised_physics", build_supervised_physics_trainer) - - - -def train( - mech_path: str, - source_file: str, - output_path: str, - time_step: float = 1e-6, - config: TrainingConfig | None = None, -) -> np.ndarray: - """Train a model using registry-selected components. - - The default config preserves the previous hard-coded MLP + Adam + - supervised-physics training behavior while making model/trainer selection - replaceable without editing this entrypoint. - """ - - _register_defaults() - effective_config = with_overrides(config or default_training_config(), time_step=time_step) - labeled_data = np.load(source_file) - - gas = ct.Solution(mech_path) - n_species = gas.n_species - formation_enthalpies = formation_calculate(mech_path) - - device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") - model = create_model( - effective_config.model.name, - model_config=effective_config.model, - n_species=n_species, - device=device, - ) - - training_tensors = _prepare_training_tensors(labeled_data, n_species, device) - training_tensors["formation_enthalpies"] = torch.tensor( - formation_enthalpies, - dtype=torch.float32, - ).to(device) - - trainer = create_trainer( - effective_config.trainer.name, - trainer_config=effective_config.trainer, - optimizer_config=effective_config.optimizer, - ) - trainer.fit(model=model, time_step=effective_config.time_step, **training_tensors) - - torch.save( - { - "net": model.state_dict(), - "data_in_mean": training_tensors["features_mean"].cpu().numpy(), - "data_in_std": training_tensors["features_std"].cpu().numpy(), - "data_target_mean": training_tensors["labels_mean"].cpu().numpy(), - "data_target_std": training_tensors["labels_std"].cpu().numpy(), - "training_config": { - "model": { - "name": effective_config.model.name, - "params": dict(effective_config.model.params), - }, - "optimizer": { - "name": effective_config.optimizer.name, - "lr": effective_config.optimizer.lr, - }, - "trainer": { - "name": effective_config.trainer.name, - "max_epochs": effective_config.trainer.max_epochs, - "lr_decay_epoch": effective_config.trainer.lr_decay_epoch, - "lr_decay_factor": effective_config.trainer.lr_decay_factor, - "batch_size": effective_config.trainer.batch_size, - }, - "time_step": effective_config.time_step, - }, - }, - output_path, - ) +__all__ = ["train"] diff --git a/dfode_kit/models/__init__.py b/dfode_kit/models/__init__.py new file mode 100644 index 00000000..c571f0f0 --- /dev/null +++ b/dfode_kit/models/__init__.py @@ -0,0 +1,33 @@ +"""Model architectures and registry helpers.""" + +from importlib import import_module + + +__all__ = [ + "MLP", + "build_mlp", + "create_model", + "get_model_factory", + "register_model", + "registered_models", +] + +_ATTRIBUTE_MODULES = { + "MLP": ("dfode_kit.models.mlp", "MLP"), + "build_mlp": ("dfode_kit.models.mlp", "build_mlp"), + "create_model": ("dfode_kit.models.registry", "create_model"), + "get_model_factory": ("dfode_kit.models.registry", "get_model_factory"), + "register_model": ("dfode_kit.models.registry", "register_model"), + "registered_models": ("dfode_kit.models.registry", "registered_models"), +} + + +def __getattr__(name): + if name not in _ATTRIBUTE_MODULES: + raise AttributeError(f"module 'dfode_kit.models' has no attribute '{name}'") + + module_name, attribute_name = _ATTRIBUTE_MODULES[name] + module = import_module(module_name) + value = getattr(module, attribute_name) + globals()[name] = value + return value diff --git a/dfode_kit/models/mlp.py b/dfode_kit/models/mlp.py new file mode 100644 index 00000000..d7d6aca0 --- /dev/null +++ b/dfode_kit/models/mlp.py @@ -0,0 +1,26 @@ +from __future__ import annotations + +import torch + +from dfode_kit.training.config import ModelConfig + + +class MLP(torch.nn.Module): + def __init__(self, layer_info): + super().__init__() + + self.net = torch.nn.Sequential() + n = len(layer_info) - 1 + for i in range(n - 1): + self.net.add_module(f"linear_layer_{i}", torch.nn.Linear(layer_info[i], layer_info[i + 1])) + self.net.add_module(f"gelu_layer_{i}", torch.nn.GELU()) + self.net.add_module(f"linear_layer_{n - 1}", torch.nn.Linear(layer_info[n - 1], layer_info[n])) + + def forward(self, x): + return self.net(x) + + +def build_mlp(*, model_config: ModelConfig, n_species: int, device): + hidden_layers = list(model_config.params.get("hidden_layers", [400, 400, 400, 400])) + layer_info = [2 + n_species, *hidden_layers, n_species - 1] + return MLP(layer_info).to(device) diff --git a/dfode_kit/models/registry.py b/dfode_kit/models/registry.py new file mode 100644 index 00000000..88c33e01 --- /dev/null +++ b/dfode_kit/models/registry.py @@ -0,0 +1,29 @@ +from __future__ import annotations + +from typing import Callable, Dict + + +ModelFactory = Callable[..., object] +_MODEL_REGISTRY: Dict[str, ModelFactory] = {} + + +def register_model(name: str, factory: ModelFactory) -> None: + if not name: + raise ValueError("Model name must be non-empty.") + _MODEL_REGISTRY[name] = factory + + +def get_model_factory(name: str) -> ModelFactory: + try: + return _MODEL_REGISTRY[name] + except KeyError as exc: + available = ", ".join(sorted(_MODEL_REGISTRY)) or "" + raise KeyError(f"Unknown model '{name}'. Available models: {available}") from exc + + +def create_model(name: str, **kwargs): + return get_model_factory(name)(**kwargs) + + +def registered_models(): + return tuple(sorted(_MODEL_REGISTRY)) diff --git a/dfode_kit/runtime/__init__.py b/dfode_kit/runtime/__init__.py new file mode 100644 index 00000000..1868a255 --- /dev/null +++ b/dfode_kit/runtime/__init__.py @@ -0,0 +1,36 @@ +from dfode_kit.runtime.config import ( + APP_NAME, + CONFIG_FILENAME, + CONFIG_KEYS, + DEFAULT_CONFIG, + coerce_config_value, + describe_config_schema, + get_config_dir, + get_config_path, + load_runtime_config, + resolve_runtime_config, + save_runtime_config, + set_config_value, + unset_config_value, + validate_config_key, +) +from dfode_kit.runtime.run_case import execute_run_case, resolve_run_case_plan + +__all__ = [ + 'APP_NAME', + 'CONFIG_FILENAME', + 'CONFIG_KEYS', + 'DEFAULT_CONFIG', + 'coerce_config_value', + 'describe_config_schema', + 'execute_run_case', + 'get_config_dir', + 'get_config_path', + 'load_runtime_config', + 'resolve_run_case_plan', + 'resolve_runtime_config', + 'save_runtime_config', + 'set_config_value', + 'unset_config_value', + 'validate_config_key', +] diff --git a/dfode_kit/runtime/config.py b/dfode_kit/runtime/config.py new file mode 100644 index 00000000..7117a808 --- /dev/null +++ b/dfode_kit/runtime/config.py @@ -0,0 +1,104 @@ +from __future__ import annotations + +import json +import os +from pathlib import Path +from typing import Any + + +APP_NAME = "dfode-kit" +CONFIG_FILENAME = "config.json" + +CONFIG_KEYS: dict[str, str] = { + "openfoam_bashrc": "Path to the OpenFOAM bashrc that should be sourced first.", + "conda_sh": "Path to conda.sh used to activate a Conda environment before sourcing DeepFlame.", + "conda_env_name": "Conda environment name used for Cantera/DeepFlame-compatible Python packages.", + "deepflame_bashrc": "Path to the DeepFlame bashrc sourced after OpenFOAM and Conda activation.", + "python_executable": "Optional Python executable path for future workflow commands and diagnostics.", + "default_np": "Default MPI rank count for case execution commands.", + "mpirun_command": "MPI launcher command used by case execution workflows.", +} + +DEFAULT_CONFIG: dict[str, Any] = { + "openfoam_bashrc": None, + "conda_sh": None, + "conda_env_name": None, + "deepflame_bashrc": None, + "python_executable": None, + "default_np": 4, + "mpirun_command": "mpirun", +} + + +def get_config_dir() -> Path: + xdg = os.environ.get("XDG_CONFIG_HOME") + if xdg: + return Path(xdg).expanduser().resolve() / APP_NAME + return Path.home().resolve() / ".config" / APP_NAME + + +def get_config_path() -> Path: + return get_config_dir() / CONFIG_FILENAME + + +def load_runtime_config() -> dict[str, Any]: + path = get_config_path() + config = dict(DEFAULT_CONFIG) + if path.is_file(): + loaded = json.loads(path.read_text(encoding="utf-8")) + config.update(loaded) + return config + + +def save_runtime_config(config: dict[str, Any]) -> Path: + path = get_config_path() + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(config, indent=2, sort_keys=True) + "\n", encoding="utf-8") + return path + + +def validate_config_key(key: str) -> str: + if key not in CONFIG_KEYS: + raise ValueError( + f"Unknown config key: {key}. Available keys: {', '.join(sorted(CONFIG_KEYS))}" + ) + return key + + +def coerce_config_value(key: str, value: str) -> Any: + if key == "default_np": + return int(value) + return value + + +def set_config_value(key: str, value: str) -> tuple[dict[str, Any], Path]: + validate_config_key(key) + config = load_runtime_config() + config[key] = coerce_config_value(key, value) + return config, save_runtime_config(config) + + +def unset_config_value(key: str) -> tuple[dict[str, Any], Path]: + validate_config_key(key) + config = load_runtime_config() + config[key] = DEFAULT_CONFIG[key] + return config, save_runtime_config(config) + + +def describe_config_schema() -> dict[str, dict[str, Any]]: + return { + key: { + "description": CONFIG_KEYS[key], + "default": DEFAULT_CONFIG[key], + } + for key in sorted(CONFIG_KEYS) + } + + +def resolve_runtime_config(overrides: dict[str, Any] | None = None) -> dict[str, Any]: + config = load_runtime_config() + for key, value in (overrides or {}).items(): + if value is not None: + validate_config_key(key) + config[key] = value + return config diff --git a/dfode_kit/runtime/run_case.py b/dfode_kit/runtime/run_case.py new file mode 100644 index 00000000..deca95ad --- /dev/null +++ b/dfode_kit/runtime/run_case.py @@ -0,0 +1,97 @@ +from __future__ import annotations + +import shlex +import subprocess +from pathlib import Path +from typing import Any + +from dfode_kit.runtime.config import resolve_runtime_config + + +def resolve_run_case_plan(args) -> dict[str, Any]: + case_dir = Path(args.case).resolve() + if not case_dir.is_dir(): + raise ValueError(f'Case directory does not exist: {case_dir}') + + runner_path = case_dir / args.runner + if not runner_path.is_file(): + raise ValueError(f'Case runner does not exist: {runner_path}') + + runtime_config = resolve_runtime_config( + { + 'openfoam_bashrc': args.openfoam_bashrc, + 'conda_sh': args.conda_sh, + 'conda_env_name': args.conda_env, + 'deepflame_bashrc': args.deepflame_bashrc, + 'python_executable': args.python_executable, + 'mpirun_command': args.mpirun_command, + 'default_np': args.np, + } + ) + validate_run_case_runtime_config(runtime_config) + + shell_lines = [ + f'source {shlex.quote(runtime_config["openfoam_bashrc"])}', + f'source {shlex.quote(runtime_config["conda_sh"])}', + f'conda activate {shlex.quote(runtime_config["conda_env_name"])}', + f'source {shlex.quote(runtime_config["deepflame_bashrc"])}', + 'set -eo pipefail', + 'which dfLowMachFoam', + f'cd {shlex.quote(str(case_dir))}', + f'chmod +x {shlex.quote(args.runner)}', + f'./{shlex.quote(args.runner)}', + ] + + return { + 'schema_version': 1, + 'case_type': 'deepflame-run-case', + 'case_dir': str(case_dir), + 'runner': args.runner, + 'np': runtime_config['default_np'], + 'runtime_config': runtime_config, + 'shell_lines': shell_lines, + 'shell_script': '\n'.join(shell_lines), + } + + +def execute_run_case(plan: dict[str, Any], quiet: bool = False) -> dict[str, Any]: + case_dir = Path(plan['case_dir']).resolve() + command = ['bash', '-lc', plan['shell_script']] + + if quiet: + stdout_log = case_dir / 'log.dfode-run-case.stdout' + stderr_log = case_dir / 'log.dfode-run-case.stderr' + with stdout_log.open('w', encoding='utf-8') as stdout_handle, stderr_log.open('w', encoding='utf-8') as stderr_handle: + completed = subprocess.run(command, stdout=stdout_handle, stderr=stderr_handle, text=True) + result = { + 'event': 'run_case_completed', + 'case_dir': str(case_dir), + 'runner': plan['runner'], + 'exit_code': completed.returncode, + 'stdout_log': str(stdout_log), + 'stderr_log': str(stderr_log), + } + else: + completed = subprocess.run(command) + result = { + 'event': 'run_case_completed', + 'case_dir': str(case_dir), + 'runner': plan['runner'], + 'exit_code': completed.returncode, + } + + if completed.returncode != 0: + raise ValueError(f"Case runner failed with exit code {completed.returncode}: {case_dir / plan['runner']}") + + return result + + +def validate_run_case_runtime_config(config: dict[str, Any]): + required = ['openfoam_bashrc', 'conda_sh', 'conda_env_name', 'deepflame_bashrc'] + missing = [key for key in required if not config.get(key)] + if missing: + raise ValueError( + 'Missing runtime config values: ' + + ', '.join(missing) + + '. Use `dfode-kit config set ...` or per-command overrides.' + ) diff --git a/dfode_kit/runtime_config.py b/dfode_kit/runtime_config.py index 7117a808..aad82335 100644 --- a/dfode_kit/runtime_config.py +++ b/dfode_kit/runtime_config.py @@ -1,104 +1 @@ -from __future__ import annotations - -import json -import os -from pathlib import Path -from typing import Any - - -APP_NAME = "dfode-kit" -CONFIG_FILENAME = "config.json" - -CONFIG_KEYS: dict[str, str] = { - "openfoam_bashrc": "Path to the OpenFOAM bashrc that should be sourced first.", - "conda_sh": "Path to conda.sh used to activate a Conda environment before sourcing DeepFlame.", - "conda_env_name": "Conda environment name used for Cantera/DeepFlame-compatible Python packages.", - "deepflame_bashrc": "Path to the DeepFlame bashrc sourced after OpenFOAM and Conda activation.", - "python_executable": "Optional Python executable path for future workflow commands and diagnostics.", - "default_np": "Default MPI rank count for case execution commands.", - "mpirun_command": "MPI launcher command used by case execution workflows.", -} - -DEFAULT_CONFIG: dict[str, Any] = { - "openfoam_bashrc": None, - "conda_sh": None, - "conda_env_name": None, - "deepflame_bashrc": None, - "python_executable": None, - "default_np": 4, - "mpirun_command": "mpirun", -} - - -def get_config_dir() -> Path: - xdg = os.environ.get("XDG_CONFIG_HOME") - if xdg: - return Path(xdg).expanduser().resolve() / APP_NAME - return Path.home().resolve() / ".config" / APP_NAME - - -def get_config_path() -> Path: - return get_config_dir() / CONFIG_FILENAME - - -def load_runtime_config() -> dict[str, Any]: - path = get_config_path() - config = dict(DEFAULT_CONFIG) - if path.is_file(): - loaded = json.loads(path.read_text(encoding="utf-8")) - config.update(loaded) - return config - - -def save_runtime_config(config: dict[str, Any]) -> Path: - path = get_config_path() - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(json.dumps(config, indent=2, sort_keys=True) + "\n", encoding="utf-8") - return path - - -def validate_config_key(key: str) -> str: - if key not in CONFIG_KEYS: - raise ValueError( - f"Unknown config key: {key}. Available keys: {', '.join(sorted(CONFIG_KEYS))}" - ) - return key - - -def coerce_config_value(key: str, value: str) -> Any: - if key == "default_np": - return int(value) - return value - - -def set_config_value(key: str, value: str) -> tuple[dict[str, Any], Path]: - validate_config_key(key) - config = load_runtime_config() - config[key] = coerce_config_value(key, value) - return config, save_runtime_config(config) - - -def unset_config_value(key: str) -> tuple[dict[str, Any], Path]: - validate_config_key(key) - config = load_runtime_config() - config[key] = DEFAULT_CONFIG[key] - return config, save_runtime_config(config) - - -def describe_config_schema() -> dict[str, dict[str, Any]]: - return { - key: { - "description": CONFIG_KEYS[key], - "default": DEFAULT_CONFIG[key], - } - for key in sorted(CONFIG_KEYS) - } - - -def resolve_runtime_config(overrides: dict[str, Any] | None = None) -> dict[str, Any]: - config = load_runtime_config() - for key, value in (overrides or {}).items(): - if value is not None: - validate_config_key(key) - config[key] = value - return config +from dfode_kit.runtime.config import * # noqa: F401,F403 diff --git a/dfode_kit/training/__init__.py b/dfode_kit/training/__init__.py new file mode 100644 index 00000000..8293bf0c --- /dev/null +++ b/dfode_kit/training/__init__.py @@ -0,0 +1,43 @@ +"""Training configuration, registries, and execution helpers.""" + +from importlib import import_module + + +__all__ = [ + "ModelConfig", + "OptimizerConfig", + "TrainerConfig", + "TrainingConfig", + "default_training_config", + "with_overrides", + "create_trainer", + "get_trainer_factory", + "register_trainer", + "registered_trainers", + "train", +] + +_ATTRIBUTE_MODULES = { + "ModelConfig": ("dfode_kit.training.config", "ModelConfig"), + "OptimizerConfig": ("dfode_kit.training.config", "OptimizerConfig"), + "TrainerConfig": ("dfode_kit.training.config", "TrainerConfig"), + "TrainingConfig": ("dfode_kit.training.config", "TrainingConfig"), + "default_training_config": ("dfode_kit.training.config", "default_training_config"), + "with_overrides": ("dfode_kit.training.config", "with_overrides"), + "create_trainer": ("dfode_kit.training.registry", "create_trainer"), + "get_trainer_factory": ("dfode_kit.training.registry", "get_trainer_factory"), + "register_trainer": ("dfode_kit.training.registry", "register_trainer"), + "registered_trainers": ("dfode_kit.training.registry", "registered_trainers"), + "train": ("dfode_kit.training.train", "train"), +} + + +def __getattr__(name): + if name not in _ATTRIBUTE_MODULES: + raise AttributeError(f"module 'dfode_kit.training' has no attribute '{name}'") + + module_name, attribute_name = _ATTRIBUTE_MODULES[name] + module = import_module(module_name) + value = getattr(module, attribute_name) + globals()[name] = value + return value diff --git a/dfode_kit/training/config.py b/dfode_kit/training/config.py new file mode 100644 index 00000000..c5ec4947 --- /dev/null +++ b/dfode_kit/training/config.py @@ -0,0 +1,57 @@ +from __future__ import annotations + +from dataclasses import dataclass, field, replace +from typing import Any, Dict, Optional + + +@dataclass(frozen=True) +class ModelConfig: + name: str = "mlp" + params: Dict[str, Any] = field(default_factory=lambda: { + "hidden_layers": [400, 400, 400, 400], + }) + + +@dataclass(frozen=True) +class OptimizerConfig: + name: str = "adam" + lr: float = 1e-3 + + +@dataclass(frozen=True) +class TrainerConfig: + name: str = "supervised_physics" + max_epochs: int = 1500 + lr_decay_epoch: int = 500 + lr_decay_factor: float = 0.1 + batch_size: int = 20000 + + +@dataclass(frozen=True) +class TrainingConfig: + model: ModelConfig = field(default_factory=ModelConfig) + optimizer: OptimizerConfig = field(default_factory=OptimizerConfig) + trainer: TrainerConfig = field(default_factory=TrainerConfig) + time_step: float = 1e-6 + + +def default_training_config() -> TrainingConfig: + return TrainingConfig() + + +def with_overrides( + config: Optional[TrainingConfig] = None, + *, + model: Optional[ModelConfig] = None, + optimizer: Optional[OptimizerConfig] = None, + trainer: Optional[TrainerConfig] = None, + time_step: Optional[float] = None, +) -> TrainingConfig: + base = config or default_training_config() + return replace( + base, + model=model or base.model, + optimizer=optimizer or base.optimizer, + trainer=trainer or base.trainer, + time_step=base.time_step if time_step is None else time_step, + ) diff --git a/dfode_kit/training/formation.py b/dfode_kit/training/formation.py new file mode 100644 index 00000000..5cdede04 --- /dev/null +++ b/dfode_kit/training/formation.py @@ -0,0 +1,10 @@ +import cantera as ct +import numpy as np + + +def formation_calculate(mechanism): + gas = ct.Solution(mechanism) + gas.TPY = 298.15, ct.one_atm, 'O2:1' + partial_molar_enthalpy = gas.partial_molar_enthalpies / gas.molecular_weights + print(partial_molar_enthalpy) + return partial_molar_enthalpy diff --git a/dfode_kit/training/registry.py b/dfode_kit/training/registry.py new file mode 100644 index 00000000..5658e168 --- /dev/null +++ b/dfode_kit/training/registry.py @@ -0,0 +1,29 @@ +from __future__ import annotations + +from typing import Callable, Dict + + +TrainerFactory = Callable[..., object] +_TRAINER_REGISTRY: Dict[str, TrainerFactory] = {} + + +def register_trainer(name: str, factory: TrainerFactory) -> None: + if not name: + raise ValueError("Trainer name must be non-empty.") + _TRAINER_REGISTRY[name] = factory + + +def get_trainer_factory(name: str) -> TrainerFactory: + try: + return _TRAINER_REGISTRY[name] + except KeyError as exc: + available = ", ".join(sorted(_TRAINER_REGISTRY)) or "" + raise KeyError(f"Unknown trainer '{name}'. Available trainers: {available}") from exc + + +def create_trainer(name: str, **kwargs): + return get_trainer_factory(name)(**kwargs) + + +def registered_trainers(): + return tuple(sorted(_TRAINER_REGISTRY)) diff --git a/dfode_kit/training/supervised_physics.py b/dfode_kit/training/supervised_physics.py new file mode 100644 index 00000000..59f97576 --- /dev/null +++ b/dfode_kit/training/supervised_physics.py @@ -0,0 +1,102 @@ +from __future__ import annotations + +import torch + +from dfode_kit.training.config import OptimizerConfig, TrainerConfig + + +class SupervisedPhysicsTrainer: + def __init__( + self, + trainer_config: TrainerConfig, + optimizer_config: OptimizerConfig, + ) -> None: + self.trainer_config = trainer_config + self.optimizer_config = optimizer_config + + def _build_optimizer(self, model): + if self.optimizer_config.name != "adam": + raise ValueError( + f"Unsupported optimizer '{self.optimizer_config.name}'. Only 'adam' is implemented in this slice." + ) + return torch.optim.Adam(model.parameters(), lr=self.optimizer_config.lr) + + def fit( + self, + *, + model, + features, + labels, + features_mean, + features_std, + labels_mean, + labels_std, + formation_enthalpies, + time_step: float, + ) -> None: + loss_fn = torch.nn.L1Loss() + optimizer = self._build_optimizer(model) + model.train() + + for epoch in range(self.trainer_config.max_epochs): + if epoch > 0 and epoch % self.trainer_config.lr_decay_epoch == 0: + for param_group in optimizer.param_groups: + param_group["lr"] *= self.trainer_config.lr_decay_factor + + total_loss1 = 0.0 + total_loss2 = 0.0 + total_loss3 = 0.0 + total_loss = 0.0 + + for i in range(0, len(features), self.trainer_config.batch_size): + batch_features = features[i:i + self.trainer_config.batch_size] + batch_labels = labels[i:i + self.trainer_config.batch_size] + + optimizer.zero_grad() + + preds = model(batch_features) + loss1 = loss_fn(preds, batch_labels) + + base_y = batch_features[:, 2:-1] * features_std[2:-1] + features_mean[2:-1] + Y_in = (base_y * 0.1 + 1) ** 10 + Y_out = (((preds * labels_std + labels_mean) + base_y) * 0.1 + 1) ** 10 + Y_target = (((batch_labels * labels_std + labels_mean) + base_y) * 0.1 + 1) ** 10 + + loss2 = loss_fn(Y_out.sum(axis=1), Y_in.sum(axis=1)) + + Y_out_total = torch.cat((Y_out, (1 - Y_out.sum(axis=1)).reshape(Y_out.shape[0], 1)), axis=1) + Y_target_total = torch.cat((Y_target, (1 - Y_target.sum(axis=1)).reshape(Y_target.shape[0], 1)), axis=1) + + loss3 = loss_fn( + (formation_enthalpies * Y_out_total).sum(axis=1), + (formation_enthalpies * Y_target_total).sum(axis=1), + ) / time_step + loss = loss1 + loss2 + loss3 / 1e13 + + total_loss1 += loss1.item() + total_loss2 += loss2.item() + total_loss3 += loss3.item() + total_loss += loss.item() + + loss.backward() + optimizer.step() + + batches = len(features) / self.trainer_config.batch_size + total_loss1 /= batches + total_loss2 /= batches + total_loss3 /= batches + total_loss /= batches + + print( + "Epoch: {}, Loss1: {:4e}, Loss2: {:4e}, Loss3: {:4e}, Loss: {:4e}".format( + epoch + 1, + total_loss1, + total_loss2, + total_loss3, + total_loss, + ) + ) + + +def build_supervised_physics_trainer(*, trainer_config: TrainerConfig, optimizer_config: OptimizerConfig): + return SupervisedPhysicsTrainer(trainer_config=trainer_config, optimizer_config=optimizer_config) diff --git a/dfode_kit/training/train.py b/dfode_kit/training/train.py new file mode 100644 index 00000000..c6b11a9d --- /dev/null +++ b/dfode_kit/training/train.py @@ -0,0 +1,127 @@ +from __future__ import annotations + +import numpy as np +import cantera as ct +import torch + +from dfode_kit.models.mlp import build_mlp +from dfode_kit.models.registry import create_model, register_model +from dfode_kit.training.config import TrainingConfig, default_training_config, with_overrides +from dfode_kit.training.formation import formation_calculate +from dfode_kit.training.registry import create_trainer, register_trainer +from dfode_kit.training.supervised_physics import build_supervised_physics_trainer +from dfode_kit.utils import BCT + + +def _prepare_training_tensors(labeled_data: np.ndarray, n_species: int, device): + thermochem_states1 = labeled_data[:, 0 : 2 + n_species].copy() + thermochem_states2 = labeled_data[:, 2 + n_species :].copy() + + print(thermochem_states1.shape, thermochem_states2.shape) + thermochem_states1[:, 2:] = np.clip(thermochem_states1[:, 2:], 0, 1) + thermochem_states2[:, 2:] = np.clip(thermochem_states2[:, 2:], 0, 1) + + features = torch.tensor( + np.hstack((thermochem_states1[:, :2], BCT(thermochem_states1[:, 2:]))), + dtype=torch.float32, + ).to(device) + labels = torch.tensor( + BCT(thermochem_states2[:, 2:-1]) - BCT(thermochem_states1[:, 2:-1]), + dtype=torch.float32, + ).to(device) + + features_mean = torch.mean(features, dim=0) + features_std = torch.std(features, dim=0) + labels_mean = torch.mean(labels, dim=0) + labels_std = torch.std(labels, dim=0) + + normalized_features = (features - features_mean) / features_std + normalized_labels = (labels - labels_mean) / labels_std + + return { + "features": normalized_features, + "labels": normalized_labels, + "features_mean": features_mean, + "features_std": features_std, + "labels_mean": labels_mean, + "labels_std": labels_std, + } + + +def _register_defaults() -> None: + register_model("mlp", build_mlp) + register_trainer("supervised_physics", build_supervised_physics_trainer) + + + +def train( + mech_path: str, + source_file: str, + output_path: str, + time_step: float = 1e-6, + config: TrainingConfig | None = None, +) -> np.ndarray: + """Train a model using registry-selected components. + + The default config preserves the previous hard-coded MLP + Adam + + supervised-physics training behavior while making model/trainer selection + replaceable without editing this entrypoint. + """ + + _register_defaults() + effective_config = with_overrides(config or default_training_config(), time_step=time_step) + labeled_data = np.load(source_file) + + gas = ct.Solution(mech_path) + n_species = gas.n_species + formation_enthalpies = formation_calculate(mech_path) + + device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") + model = create_model( + effective_config.model.name, + model_config=effective_config.model, + n_species=n_species, + device=device, + ) + + training_tensors = _prepare_training_tensors(labeled_data, n_species, device) + training_tensors["formation_enthalpies"] = torch.tensor( + formation_enthalpies, + dtype=torch.float32, + ).to(device) + + trainer = create_trainer( + effective_config.trainer.name, + trainer_config=effective_config.trainer, + optimizer_config=effective_config.optimizer, + ) + trainer.fit(model=model, time_step=effective_config.time_step, **training_tensors) + + torch.save( + { + "net": model.state_dict(), + "data_in_mean": training_tensors["features_mean"].cpu().numpy(), + "data_in_std": training_tensors["features_std"].cpu().numpy(), + "data_target_mean": training_tensors["labels_mean"].cpu().numpy(), + "data_target_std": training_tensors["labels_std"].cpu().numpy(), + "training_config": { + "model": { + "name": effective_config.model.name, + "params": dict(effective_config.model.params), + }, + "optimizer": { + "name": effective_config.optimizer.name, + "lr": effective_config.optimizer.lr, + }, + "trainer": { + "name": effective_config.trainer.name, + "max_epochs": effective_config.trainer.max_epochs, + "lr_decay_epoch": effective_config.trainer.lr_decay_epoch, + "lr_decay_factor": effective_config.trainer.lr_decay_factor, + "batch_size": effective_config.trainer.batch_size, + }, + "time_step": effective_config.time_step, + }, + }, + output_path, + ) diff --git a/docs/agents/README.md b/docs/agents/README.md index 79a3686f..518bee56 100644 --- a/docs/agents/README.md +++ b/docs/agents/README.md @@ -14,6 +14,8 @@ Use this docs tree for information that is too detailed or too volatile for `AGE - `cli-usability-plan.md`: CLI agent-usability improvement plan - `data-io-contract-plan.md`: data I/O contract and refactor plan - `train-config-plan.md`: training/config refactor plan for experiment throughput +- `package-topology-spec.md`: target package organization and module-boundary spec +- `package-topology-migration-plan.md`: staged migration plan toward the target package topology ## Philosophy - `AGENTS.md` is the entrypoint for repository workflow. diff --git a/docs/agents/cli-usability-plan.md b/docs/agents/cli-usability-plan.md index 01da66f8..747a5952 100644 --- a/docs/agents/cli-usability-plan.md +++ b/docs/agents/cli-usability-plan.md @@ -4,7 +4,7 @@ Improve only the DFODE-kit CLI surface for coding-agent use: deterministic behavior, explicit command dispatch, and cleaner automation semantics. ## Audit summary -Current CLI issues observed in `dfode_kit/cli_tools/`: +Current CLI issues observed in `dfode_kit/cli/`: - `main.py` does not return exit codes; command success/failure is not standardized for automation. - command discovery in `command_loader.py` depends on dynamic module walk order, which is not guaranteed to be stable. - command dispatch is implicit and only prints a fallback message for missing commands instead of failing predictably. diff --git a/docs/agents/data-io-contract-plan.md b/docs/agents/data-io-contract-plan.md index fb640154..27ecb166 100644 --- a/docs/agents/data-io-contract-plan.md +++ b/docs/agents/data-io-contract-plan.md @@ -6,7 +6,7 @@ Keep this branch focused on dataset layout and HDF5/NPY boundaries in `dfode_kit ## Audit summary - `dfode_kit/data_operations/h5_kit.py` currently mixes three concerns: HDF5 inspection/loading, reactor/model integration, and error reporting. - HDF5 layout assumptions are implicit: code assumes a root `mechanism` attribute and a `scalar_fields` group of stack-compatible datasets. -- `dfode_kit/cli_tools/commands/h52npy.py` reimplements part of the HDF5 read path instead of sharing one validated loader. +- `dfode_kit/cli/commands/h52npy.py` reimplements part of the HDF5 read path instead of sharing one validated loader. - Some runtime contracts still rely on loose assumptions or non-deterministic dataset iteration. ## First slice diff --git a/docs/agents/package-topology-migration-plan.md b/docs/agents/package-topology-migration-plan.md new file mode 100644 index 00000000..2fcbd8c3 --- /dev/null +++ b/docs/agents/package-topology-migration-plan.md @@ -0,0 +1,174 @@ +# Package topology migration plan + +## Status +Draft staged plan for moving toward the target package topology in `package-topology-spec.md`. + +## Goal +Reach the target topology through small, behavior-preserving slices that are easy to review and verify. + +## Non-goals +- no big-bang rename of the whole package +- no simultaneous redesign of APIs and directory layout +- no broad feature work mixed into topology-only branches + +## Refactor strategy +Prefer slices that follow this pattern: +1. extract or split logic behind tests +2. add compatibility imports/shims if needed +3. switch call sites +4. remove old path only after verification + +This minimizes breakage and keeps diffs understandable. + +## Phase 0: invariants before moves +Before major moves, preserve or add tests around: +- CLI command listing and help paths +- HDF5 contract helpers +- init plan behavior +- runtime config behavior +- run-case planning behavior +- training config/registry behavior + +If a move lacks a small invariant test, add one first. + +## Phase 1: rename only the most stable package boundary +### Candidate +`cli_tools/` -> `cli/` + +### Why first +- package boundary is already conceptually clean +- command behavior is already tested +- low domain ambiguity + +### Slice contents +- create `dfode_kit/cli/` +- move `main.py`, `command_loader.py`, `commands/` +- keep import compatibility from old paths temporarily if needed +- update tests/docs/import sites + +### Success check +- `dfode-kit --list-commands` unchanged +- per-command `--help` behavior unchanged +- lazy command loading preserved + +## Phase 2: split `h5_kit.py` by concern before broader package renames +### Why now +`h5_kit.py` is one of the clearest mixed-responsibility hotspots. + +### Target outcome +- keep schema checks in `data/contracts.py` +- move pure HDF5 I/O helpers to `data/io_hdf5.py` +- move model/integration helpers to more appropriate modules later + +### Suggested slices +1. extract read/write/stack helpers +2. switch callers +3. only then rename package path if useful + +### Success check +- `h52npy` and sampling-related HDF5 paths behave identically +- no model/training logic remains hidden in an I/O-named module unless clearly temporary + +## Phase 3: `data_operations/` -> `data/` +### Why after Phase 2 +It is easier once the most confusing mixed module is split first. + +### Target mapping +- `contracts.py` -> `data/contracts.py` +- `augment_data.py` -> `data/augment.py` +- `label_data.py` -> `data/label.py` +- extracted HDF5 helpers -> `data/io_hdf5.py` + +### Success check +- user-facing CLI behavior unchanged +- internal imports simpler and more domain-named + +## Phase 4: `df_interface/` -> `cases/` +### Why this is a later slice +There is some ambiguity in how to divide: +- presets +- case planning +- DeepFlame-specific setup +- case sampling + +### Recommended sequence +1. stabilize public responsibilities first +2. rename package second +3. split files only where the split improves clarity immediately + +### Target mapping +- `case_init.py` -> `cases/init.py` +- `flame_configurations.py` -> `cases/presets.py` +- `sample_case.py` -> `cases/sampling.py` +- `oneDflame_setup.py` -> `cases/deepflame.py` or split further + +### Success check +- `init` and `sample` commands still work the same way +- preset behavior remains explicit and documented + +## Phase 5: `dfode_core/` -> `models/` + `training/` +### Why this is larger +This is really two reorganizations: +- model code +- trainer/config/train loop code + +### Recommended sequence +1. move model registry and model implementations to `models/` +2. move trainer config/registry/train loop to `training/` +3. relocate or split `preprocess.py` only after its actual ownership is clear +4. remove in-package `dfode_core/test/` + +### Success check +- train command behavior preserved +- model/trainer registries still work +- package navigation becomes clearer for experimentation work + +## Phase 6: `runtime_config.py` -> `runtime/config.py` +### Why this is a clean later slice +This is small and conceptually stable, but not urgent. + +### Possible follow-up +- move reusable run-case planning/application helpers from CLI helper modules into `runtime/run_case.py` + +### Success check +- runtime config file path/schema unchanged +- `config` and `run-case` CLI behavior unchanged + +## Compatibility rules during migration +- prefer re-export shims for old import paths during transition +- mark old paths as deprecated in docs/comments before removing them +- remove shims only after downstream call sites are updated and verified + +## Branching guidance +Keep branches focused by package boundary, for example: +- `refactor/cli-package-rename` +- `refactor/h5-io-split` +- `refactor/data-package-rename` +- `refactor/cases-package-rename` +- `refactor/models-training-split` +- `refactor/runtime-package` + +## Verification checklist per slice +- update or add targeted unit tests +- run `make verify` +- build docs if docs or import paths changed +- smoke-check relevant CLI commands +- check for accidental eager heavy imports on help/list paths + +## Initial recommended order +1. `cli_tools/` -> `cli/` +2. split `h5_kit.py` +3. `data_operations/` -> `data/` +4. `df_interface/` -> `cases/` +5. `dfode_core/model` -> `models/` +6. `dfode_core/train` -> `training/` +7. `runtime_config.py` -> `runtime/config.py` +8. evaluate remaining `preprocess.py` / utility cleanup + +## Definition of done for the overall direction +The topology migration is meaningfully complete when: +- major package names match stable responsibilities +- mixed-responsibility files have been split at the obvious boundaries +- old compatibility paths are removed or minimized +- docs and tests refer to the new package structure +- a new contributor can infer where to add code without reading large amounts of historical context diff --git a/docs/agents/package-topology-spec.md b/docs/agents/package-topology-spec.md new file mode 100644 index 00000000..3deb037a --- /dev/null +++ b/docs/agents/package-topology-spec.md @@ -0,0 +1,226 @@ +# Package topology target spec + +## Status +Draft target architecture. This is a direction-setting spec, not a big-bang rewrite plan. + +## Goal +Evolve `dfode_kit/` toward clearer subsystem boundaries that match the user-visible workflow and reduce mixed-responsibility modules. + +Target shape: + +```text +dfode_kit/ + cli/ + main.py + command_loader.py + commands/ + cases/ + init.py + presets.py + sampling.py + deepflame.py + data/ + contracts.py + io_hdf5.py + augment.py + label.py + models/ + mlp.py + registry.py + training/ + config.py + registry.py + supervised_physics.py + train.py + runtime/ + config.py + run_case.py + utils/ +``` + +## Why this direction +The current package layout has the right broad pieces, but some names are historical and some modules mix concerns: + +- `cli_tools/` is clear, but the `*_tools` naming is heavier than needed. +- `data_operations/` currently mixes contracts, HDF5 I/O, augmentation, labeling, and some model/integration helpers. +- `df_interface/` includes case initialization, case setup, and sampling concerns under a vague name. +- `dfode_core/` mixes models, training, preprocessing, and an in-package test directory. +- some modules carry more than one abstraction level, especially `h5_kit.py`. + +The target layout moves toward domain-first naming and narrower module responsibility. + +## Design principles + +### 1. Name packages after stable responsibilities +Prefer names that tell a new contributor where code belongs: + +- `cli`: command-line entrypoints and dispatch only +- `cases`: canonical case setup, case-local sampling, DeepFlame-facing case operations +- `data`: data contracts, storage I/O, augmentation, labeling +- `models`: neural-network architectures and model registry +- `training`: trainer config, trainer registry, training loops +- `runtime`: machine-local configuration and case execution helpers +- `utils`: small generic helpers only + +### 2. Keep orchestration separate from primitives +Examples: +- `cases.init` should assemble a case plan and apply it +- `data.io_hdf5` should do HDF5 read/write/validation work +- `training.train` should orchestrate a training run +- `models.mlp` should define one architecture, not run the whole workflow + +### 3. Avoid mixed abstraction levels in one module +A file named like an I/O helper should not also be the home for model inference or numerical integration orchestration. + +### 4. Keep CLI modules thin +CLI command modules should: +- define arguments +- validate command-level options +- call importable library functions +- handle output formatting and exit behavior + +They should not be the only place where business logic exists. + +### 5. Preserve agent-friendly behavior +Refactors in this direction must preserve: +- lazy imports for heavy dependencies where possible +- deterministic CLI command ordering +- clean stdout/stderr behavior +- small, verifiable change slices + +## Target package responsibilities + +## `dfode_kit/cli/` +Owns command parsing, command registration, and command dispatch. + +### Intended contents +- `main.py`: top-level argument parsing and exit-code contract +- `command_loader.py`: deterministic command specification/loading +- `commands/`: one module per user-facing subcommand + +### Non-goals +- no domain logic that cannot be reused outside the CLI +- no heavy imports at top level when avoidable + +## `dfode_kit/cases/` +Owns canonical case planning and DeepFlame/OpenFOAM-facing case operations. + +### Intended contents +- `init.py`: case plan assembly and application helpers +- `presets.py`: named case-setup presets and preset metadata +- `sampling.py`: sample extraction from completed cases +- `deepflame.py`: DeepFlame/OpenFOAM case-specific helpers + +### Notes +This package replaces the vaguer `df_interface/` name with a user-visible workflow concept: cases. + +## `dfode_kit/data/` +Owns dataset contracts, file I/O, and state transformation steps. + +### Intended contents +- `contracts.py`: HDF5/NPY schema checks and dataset ordering rules +- `io_hdf5.py`: HDF5 reading/writing and stack/reshape helpers +- `augment.py`: augmentation primitives and workflow helpers +- `label.py`: labeling primitives and workflow helpers + +### Notes +This package should not become a second training or inference package. If a function is primarily model inference or time integration orchestration, it likely belongs elsewhere. + +## `dfode_kit/models/` +Owns model architectures and model registration. + +### Intended contents +- `mlp.py`: current MLP implementation +- `registry.py`: model factory registration and lookup + +### Notes +Keep model creation separate from training-loop concerns. + +## `dfode_kit/training/` +Owns trainer configuration, trainer registration, and training execution. + +### Intended contents +- `config.py`: typed training configuration +- `registry.py`: trainer registry +- `supervised_physics.py`: current trainer implementation +- `train.py`: orchestration entrypoint for training runs + +### Notes +Preprocessing that is training-specific may move here later if it is not reusable elsewhere. + +## `dfode_kit/runtime/` +Owns machine-local runtime config and execution planning. + +### Intended contents +- `config.py`: runtime config persistence and schema +- `run_case.py`: case execution planning/application helpers + +### Notes +This separates workstation/runtime concerns from case-definition concerns. + +## `dfode_kit/utils/` +Owns small generic helpers. + +### Rule +If a helper is domain-specific enough to obviously belong to `cases`, `data`, `models`, `training`, or `runtime`, do not place it in `utils`. + +## Migration mapping from current layout + +### Current -> target +- `cli_tools/` -> `cli/` +- `df_interface/` -> `cases/` +- `data_operations/` -> `data/` +- `dfode_core/model/` -> `models/` +- `dfode_core/train/` -> `training/` +- `runtime_config.py` -> `runtime/config.py` + +### Likely file moves +- `cli_tools/main.py` -> `cli/main.py` +- `cli_tools/command_loader.py` -> `cli/command_loader.py` +- `cli_tools/commands/*.py` -> `cli/commands/*.py` +- `df_interface/case_init.py` -> `cases/init.py` +- `df_interface/flame_configurations.py` -> `cases/presets.py` +- `df_interface/sample_case.py` -> `cases/sampling.py` +- `df_interface/oneDflame_setup.py` -> `cases/deepflame.py` or split across `init.py` + `deepflame.py` +- `data_operations/contracts.py` -> `data/contracts.py` +- `data_operations/h5_kit.py` -> split; pure HDF5 pieces to `data/io_hdf5.py` +- `data_operations/augment_data.py` -> `data/augment.py` +- `data_operations/label_data.py` -> `data/label.py` +- `dfode_core/model/mlp.py` -> `models/mlp.py` +- `dfode_core/model/registry.py` -> `models/registry.py` +- `dfode_core/train/config.py` -> `training/config.py` +- `dfode_core/train/registry.py` -> `training/registry.py` +- `dfode_core/train/supervised_physics.py` -> `training/supervised_physics.py` +- `dfode_core/train/train.py` -> `training/train.py` +- `runtime_config.py` -> `runtime/config.py` +- `cli_tools/commands/run_case_helpers.py` -> `runtime/run_case.py` or split between CLI and runtime library + +## Compatibility expectations +This refactor direction should preserve current user-facing behavior during migration: + +- keep the `dfode-kit` CLI command name unchanged +- keep subcommand names stable unless there is a strong reason to rename them +- preserve current import paths temporarily via compatibility shims where needed +- prefer deprecation windows over abrupt breakage + +## What should not happen +- do not rewrite all packages in one PR +- do not move files without adding or updating tests for the moved contract +- do not change user-facing CLI behavior incidentally while only intending package moves +- do not put more domain-specific logic into `utils.py` +- do not mix rename-only moves with unrelated feature work + +## Success criteria +A refactor slice is aligned with this spec if it: +- makes package names more domain-specific +- reduces mixed-responsibility modules +- keeps behavior stable or makes changes explicitly documented +- preserves agent-friendly CLI/test behavior +- leaves the repository easier to navigate for a new contributor + +## Deferred questions +These can be answered incrementally rather than upfront: +- should `preprocess.py` become `training/preprocess.py`, `data/preprocess.py`, or be split? +- should case sampling stay under `cases/` or move partly into `data/` after extraction? +- should `runtime/` eventually hold more execution-plan/provenance helpers? +- should a future `workflows/` package exist for higher-level chained operations? diff --git a/docs/agents/roadmap.md b/docs/agents/roadmap.md index 268702e7..b92f086c 100644 --- a/docs/agents/roadmap.md +++ b/docs/agents/roadmap.md @@ -15,6 +15,8 @@ - Split mixed-responsibility modules (`h5_kit.py` first) - Introduce model/training registries or config-driven training - Add architecture-specific worktrees and smoke tests +- Move toward the target package topology in `docs/agents/package-topology-spec.md` +- Prefer small package-boundary refactors over one-shot tree rewrites ## Phase 4: Experiment velocity - Support multiple model architectures without editing core train loop diff --git a/docs/agents/train-config-plan.md b/docs/agents/train-config-plan.md index a26f46db..883a1448 100644 --- a/docs/agents/train-config-plan.md +++ b/docs/agents/train-config-plan.md @@ -1,10 +1,10 @@ # Training/config refactor plan ## Scope -Keep changes tightly limited to `dfode_kit/dfode_core/{model,train}` and the train CLI surface needed to select registered components. Do not redesign data loading, labeling, or DeepFlame integration in this slice. +Keep changes tightly limited to the model/training package boundary (`dfode_kit/models`, `dfode_kit/training`, plus compatibility shims under `dfode_kit/dfode_core/{model,train}`) and the train CLI surface needed to select registered components. Do not redesign data loading, labeling, or DeepFlame integration in this slice. ## Current audit -- `dfode_kit/dfode_core/train/train.py` hard-codes: +- `dfode_kit/training/train.py` hard-codes: - model architecture (`MLP([2+n_species, 400, 400, 400, 400, n_species-1])`) - optimizer (`Adam`) - loop hyperparameters (`max_epochs`, LR schedule, batch size) diff --git a/docs/architecture.md b/docs/architecture.md index fa7394c3..986964ab 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -2,10 +2,11 @@ ## Current repository structure -- `dfode_kit/cli_tools/`: CLI entrypoints and subcommands +- `dfode_kit/cli/`: CLI entrypoints and subcommands - `dfode_kit/data_operations/`: dataset I/O, contracts, labeling, augmentation, integration utilities - `dfode_kit/dfode_core/`: models, training, registries, preprocessing -- `dfode_kit/df_interface/`: DeepFlame/OpenFOAM-facing helpers +- `dfode_kit/cases/`: explicit case init, preset, sampling, and DeepFlame-facing helpers +- `dfode_kit/df_interface/`: compatibility layer for the legacy case-facing import paths - `docs/agents/`: agent-facing operational and planning docs - `tests/`: lightweight repository and harness tests diff --git a/docs/init.md b/docs/init.md index 45272a7e..81ed8e92 100644 --- a/docs/init.md +++ b/docs/init.md @@ -36,9 +36,11 @@ Create or preview a parameterized copy of the canonical one-dimensional freely p This preset preserves the current hardcoded empirical logic from: -- `dfode_kit/df_interface/flame_configurations.py` +- `dfode_kit/cases/presets.py` - method: `OneDFreelyPropagatingFlameConfig.update_config()` +Legacy `dfode_kit/df_interface/*` imports remain as compatibility shims during the package-topology migration. + ## Stable intent The command is a **preset instantiator**, not a claim of universal best practice. diff --git a/docs/run-case.md b/docs/run-case.md index d5be16f9..b40fc8af 100644 --- a/docs/run-case.md +++ b/docs/run-case.md @@ -7,6 +7,8 @@ DFODE-kit provides two CLI entrypoints for running DeepFlame/OpenFOAM cases repr This document is the shared reference for both humans and AI agents. +Implementation note: runtime config helpers now live under `dfode_kit.runtime.config`, and reusable run planning/execution helpers live under `dfode_kit.runtime.run_case`. The legacy import paths remain as compatibility shims during migration. + ## Why a persistent runtime config exists Running DeepFlame cases usually requires machine-local paths and environment activation steps that do not belong in case templates: @@ -155,7 +157,7 @@ Returns a JSON object containing: - `case_dir` - `runner` - `np` -- `runtime_config` +- `dfode_kit.runtime.config` - `shell_lines` - `shell_script` @@ -206,7 +208,7 @@ source /path/to/conda/etc/profile.d/conda.sh conda activate deepflame source /path/to/deepflame-dev/bashrc -python -m dfode_kit.cli_tools.main init oneD-flame \ +python -m dfode_kit.cli.main init oneD-flame \ --mech /path/to/mechanisms/CH4/gri30.yaml \ --fuel CH4:1 \ --oxidizer air \ @@ -228,7 +230,7 @@ dfode-kit run-case \ ```bash source /path/to/conda/etc/profile.d/conda.sh conda activate deepflame -python -m dfode_kit.cli_tools.main sample \ +python -m dfode_kit.cli.main sample \ --mech /path/to/mechanisms/CH4/gri30.yaml \ --case /path/to/run/oneD_flame_CH4_phi1 \ --save /path/to/run/oneD_flame_CH4_phi1/ch4_phi1_sample.h5 \ diff --git a/mkdocs.yml b/mkdocs.yml index bc7219b8..6807fc78 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -43,3 +43,5 @@ nav: - CLI Usability Plan: agents/cli-usability-plan.md - Data I/O Contract Plan: agents/data-io-contract-plan.md - Train Config Plan: agents/train-config-plan.md + - Package Topology Spec: agents/package-topology-spec.md + - Package Topology Migration Plan: agents/package-topology-migration-plan.md diff --git a/pyproject.toml b/pyproject.toml index 1e47a51d..04bf4b31 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,7 +19,7 @@ dev = [ ] [project.scripts] -dfode-kit = "dfode_kit.cli_tools.main:main" +dfode-kit = "dfode_kit.cli.main:main" [tool.setuptools.packages.find] -include = ["dfode_kit"] \ No newline at end of file +include = ["dfode_kit*"] diff --git a/tests/test_case_init.py b/tests/test_case_init.py index 7395122b..456a1db2 100644 --- a/tests/test_case_init.py +++ b/tests/test_case_init.py @@ -1,6 +1,6 @@ from pathlib import Path -from dfode_kit.df_interface.case_init import ( +from dfode_kit.cases.init import ( AIR_OXIDIZER, DEFAULT_ONE_D_FLAME_PRESET, OneDFlameInitInputs, diff --git a/tests/test_cases_shims.py b/tests/test_cases_shims.py new file mode 100644 index 00000000..42263b3c --- /dev/null +++ b/tests/test_cases_shims.py @@ -0,0 +1,7 @@ +from dfode_kit.cases import init as cases_init +from dfode_kit.df_interface import case_init as legacy_case_init + + +def test_df_interface_case_init_reexports_cases_symbols(): + assert legacy_case_init.resolve_oxidizer is cases_init.resolve_oxidizer + assert legacy_case_init.DEFAULT_ONE_D_FLAME_PRESET == cases_init.DEFAULT_ONE_D_FLAME_PRESET diff --git a/tests/test_cli_main.py b/tests/test_cli_main.py index 30dcc86e..2caa36e8 100644 --- a/tests/test_cli_main.py +++ b/tests/test_cli_main.py @@ -1,4 +1,5 @@ from collections import OrderedDict +from importlib import import_module from importlib.util import module_from_spec, spec_from_file_location from pathlib import Path from types import SimpleNamespace @@ -8,8 +9,8 @@ ROOT = Path(__file__).resolve().parents[1] PACKAGE_INIT_PATH = ROOT / "dfode_kit" / "__init__.py" -COMMAND_LOADER_PATH = ROOT / "dfode_kit" / "cli_tools" / "command_loader.py" -MAIN_PATH = ROOT / "dfode_kit" / "cli_tools" / "main.py" +COMMAND_LOADER_PATH = ROOT / "dfode_kit" / "cli" / "command_loader.py" +MAIN_PATH = ROOT / "dfode_kit" / "cli" / "main.py" package_spec = spec_from_file_location("dfode_kit", PACKAGE_INIT_PATH) @@ -18,19 +19,19 @@ sys.modules["dfode_kit"] = dfode_pkg package_spec.loader.exec_module(dfode_pkg) -cli_pkg = sys.modules.setdefault("dfode_kit.cli_tools", types.ModuleType("dfode_kit.cli_tools")) -cli_pkg.__path__ = [str(ROOT / "dfode_kit" / "cli_tools")] +cli_pkg = sys.modules.setdefault("dfode_kit.cli", types.ModuleType("dfode_kit.cli")) +cli_pkg.__path__ = [str(ROOT / "dfode_kit" / "cli")] -command_loader_spec = spec_from_file_location("dfode_kit.cli_tools.command_loader", COMMAND_LOADER_PATH) +command_loader_spec = spec_from_file_location("dfode_kit.cli.command_loader", COMMAND_LOADER_PATH) command_loader = module_from_spec(command_loader_spec) assert command_loader_spec.loader is not None -sys.modules["dfode_kit.cli_tools.command_loader"] = command_loader +sys.modules["dfode_kit.cli.command_loader"] = command_loader command_loader_spec.loader.exec_module(command_loader) -main_spec = spec_from_file_location("dfode_kit.cli_tools.main", MAIN_PATH) +main_spec = spec_from_file_location("dfode_kit.cli.main", MAIN_PATH) main = module_from_spec(main_spec) assert main_spec.loader is not None -sys.modules["dfode_kit.cli_tools.main"] = main +sys.modules["dfode_kit.cli.main"] = main main_spec.loader.exec_module(main) @@ -137,3 +138,11 @@ def test_main_returns_two_for_missing_handler(monkeypatch, capsys): captured = capsys.readouterr() assert exit_code == 2 assert "Unknown command: dummy" in captured.err + + +def test_cli_tools_command_loader_shim_re_exports_cli_symbols(): + shim = import_module("dfode_kit.cli_tools.command_loader") + direct = import_module("dfode_kit.cli.command_loader") + + assert shim.load_command_specs is direct.load_command_specs + assert shim.load_command is direct.load_command diff --git a/tests/test_data_contracts.py b/tests/test_data_contracts.py index 0b3044d3..28d3e4df 100644 --- a/tests/test_data_contracts.py +++ b/tests/test_data_contracts.py @@ -7,7 +7,7 @@ import pytest -CONTRACTS_PATH = Path(__file__).resolve().parents[1] / "dfode_kit" / "data_operations" / "contracts.py" +CONTRACTS_PATH = Path(__file__).resolve().parents[1] / "dfode_kit" / "data" / "contracts.py" SPEC = spec_from_file_location("dfode_contracts_module", CONTRACTS_PATH) contracts = module_from_spec(SPEC) assert SPEC.loader is not None @@ -23,10 +23,18 @@ def write_scalar_fields_h5(path: Path, datasets: dict[str, np.ndarray], mechanis def test_importing_data_contracts_does_not_require_cantera_or_torch(): - contracts_module = importlib.import_module("dfode_kit.data_operations.contracts") + contracts_module = importlib.import_module("dfode_kit.data.contracts") assert contracts_module.SCALAR_FIELDS_GROUP == "scalar_fields" +def test_legacy_data_operations_contracts_path_re_exports_new_contracts_module(): + legacy_contracts = importlib.import_module("dfode_kit.data_operations.contracts") + new_contracts = importlib.import_module("dfode_kit.data.contracts") + + assert legacy_contracts.SCALAR_FIELDS_GROUP == new_contracts.SCALAR_FIELDS_GROUP + assert legacy_contracts.stack_scalar_field_datasets is new_contracts.stack_scalar_field_datasets + + def test_stack_scalar_field_datasets_uses_deterministic_numeric_order(tmp_path): path = tmp_path / "states.h5" write_scalar_fields_h5( diff --git a/tests/test_data_io_hdf5.py b/tests/test_data_io_hdf5.py new file mode 100644 index 00000000..dfdd8339 --- /dev/null +++ b/tests/test_data_io_hdf5.py @@ -0,0 +1,52 @@ +import importlib +from pathlib import Path + +import h5py +import numpy as np + +from dfode_kit.data.contracts import MECHANISM_ATTR, SCALAR_FIELDS_GROUP + + +def write_scalar_fields_h5(path: Path, datasets: dict[str, np.ndarray], mechanism: str = "mech.yaml"): + with h5py.File(path, "w") as h5_file: + h5_file.attrs[MECHANISM_ATTR] = mechanism + group = h5_file.create_group(SCALAR_FIELDS_GROUP) + for name, data in datasets.items(): + group.create_dataset(name, data=data) + + +def test_importing_data_io_hdf5_does_not_require_cantera_or_torch(): + io_module = importlib.import_module("dfode_kit.data.io_hdf5") + assert io_module.get_TPY_from_h5 is not None + + +def test_get_tpy_from_h5_uses_contract_ordering_and_stacks_datasets(tmp_path, capsys): + path = tmp_path / "states.h5" + write_scalar_fields_h5( + path, + { + "10": np.array([[10.0, 10.1]]), + "2": np.array([[2.0, 2.1]]), + "1": np.array([[1.0, 1.1]]), + }, + ) + + io_module = importlib.import_module("dfode_kit.data.io_hdf5") + stacked = io_module.get_TPY_from_h5(path) + + assert np.allclose( + stacked, + np.array([[1.0, 1.1], [2.0, 2.1], [10.0, 10.1]]), + ) + assert f"Number of datasets in {SCALAR_FIELDS_GROUP} group: 3" in capsys.readouterr().out + + +def test_legacy_package_exports_point_to_extracted_io_helpers(): + legacy_data_operations = importlib.import_module("dfode_kit.data_operations") + root_package = importlib.import_module("dfode_kit") + io_module = importlib.import_module("dfode_kit.data.io_hdf5") + + assert legacy_data_operations.touch_h5 is io_module.touch_h5 + assert legacy_data_operations.get_TPY_from_h5 is io_module.get_TPY_from_h5 + assert root_package.touch_h5 is io_module.touch_h5 + assert root_package.get_TPY_from_h5 is io_module.get_TPY_from_h5 diff --git a/tests/test_init_cli.py b/tests/test_init_cli.py index e86b8721..779194d2 100644 --- a/tests/test_init_cli.py +++ b/tests/test_init_cli.py @@ -2,7 +2,7 @@ from types import ModuleType, SimpleNamespace import sys -from dfode_kit.cli_tools.commands import init_helpers +from dfode_kit.cli.commands import init_helpers class FakeCfg: @@ -100,14 +100,14 @@ def test_apply_one_d_flame_plan_copies_template_and_writes_metadata(tmp_path, mo monkeypatch.setattr(init_helpers, '_build_one_d_flame_config', lambda inputs, overrides, quiet=False: FakeCfg()) - fake_module = ModuleType('dfode_kit.df_interface.oneDflame_setup') + fake_module = ModuleType('dfode_kit.cases.deepflame') calls = {} def fake_setup(cfg, case_path): calls['case_path'] = str(case_path) fake_module.setup_one_d_flame_case = fake_setup - monkeypatch.setitem(sys.modules, 'dfode_kit.df_interface.oneDflame_setup', fake_module) + monkeypatch.setitem(sys.modules, 'dfode_kit.cases.deepflame', fake_module) result = init_helpers.apply_one_d_flame_plan(plan, force=False) diff --git a/tests/test_run_case_cli.py b/tests/test_run_case_cli.py index 0ad5a829..bb693297 100644 --- a/tests/test_run_case_cli.py +++ b/tests/test_run_case_cli.py @@ -1,7 +1,7 @@ from pathlib import Path from types import SimpleNamespace -from dfode_kit.cli_tools.commands import run_case_helpers +from dfode_kit.runtime import run_case as run_case_helpers class DummyArgs(SimpleNamespace): @@ -84,3 +84,11 @@ def fake_run(command, stdout=None, stderr=None, text=None): assert Path(result['stdout_log']).is_file() assert Path(result['stderr_log']).is_file() assert calls['command'] == ['bash', '-lc', 'echo hello'] + + + +def test_legacy_run_case_helpers_shim_matches_new_module(): + from dfode_kit.cli_tools.commands import run_case_helpers as legacy_run_case_helpers + + assert legacy_run_case_helpers.resolve_run_case_plan is run_case_helpers.resolve_run_case_plan + assert legacy_run_case_helpers.execute_run_case is run_case_helpers.execute_run_case diff --git a/tests/test_runtime_config.py b/tests/test_runtime_config.py index 14c4d1ed..b005b5f2 100644 --- a/tests/test_runtime_config.py +++ b/tests/test_runtime_config.py @@ -1,7 +1,4 @@ -import json -from pathlib import Path - -from dfode_kit import runtime_config +from dfode_kit.runtime import config as runtime_config def test_runtime_config_round_trip(tmp_path, monkeypatch): @@ -28,3 +25,10 @@ def test_validate_config_key_rejects_unknown(): assert 'Unknown config key' in str(exc) else: raise AssertionError('expected ValueError') + + +def test_legacy_runtime_config_shim_matches_new_module(): + from dfode_kit import runtime_config as legacy_runtime_config + + assert legacy_runtime_config.get_config_path is runtime_config.get_config_path + assert legacy_runtime_config.resolve_runtime_config is runtime_config.resolve_runtime_config diff --git a/tests/test_train_config_architecture.py b/tests/test_train_config_architecture.py index 0c761c52..f7502ae8 100644 --- a/tests/test_train_config_architecture.py +++ b/tests/test_train_config_architecture.py @@ -1,3 +1,4 @@ +from importlib import import_module from importlib.util import module_from_spec, spec_from_file_location from pathlib import Path import sys @@ -6,9 +7,9 @@ ROOT = Path(__file__).resolve().parents[1] -CONFIG_PATH = ROOT / "dfode_kit" / "dfode_core" / "train" / "config.py" -MODEL_REGISTRY_PATH = ROOT / "dfode_kit" / "dfode_core" / "model" / "registry.py" -TRAINER_REGISTRY_PATH = ROOT / "dfode_kit" / "dfode_core" / "train" / "registry.py" +CONFIG_PATH = ROOT / "dfode_kit" / "training" / "config.py" +MODEL_REGISTRY_PATH = ROOT / "dfode_kit" / "models" / "registry.py" +TRAINER_REGISTRY_PATH = ROOT / "dfode_kit" / "training" / "registry.py" PLAN_PATH = ROOT / "docs" / "agents" / "train-config-plan.md" @@ -80,8 +81,23 @@ def test_registry_errors_include_available_names(): assert "Unknown model 'does_not_exist'" in str(excinfo.value) +def test_legacy_dfode_core_imports_remain_available_as_shims(): + canonical_config = import_module("dfode_kit.training.config") + canonical_model_registry = import_module("dfode_kit.models.registry") + canonical_trainer_registry = import_module("dfode_kit.training.registry") + legacy_config = import_module("dfode_kit.dfode_core.train.config") + legacy_model_registry = import_module("dfode_kit.dfode_core.model.registry") + legacy_trainer_registry = import_module("dfode_kit.dfode_core.train.registry") + + assert legacy_config.TrainingConfig is canonical_config.TrainingConfig + assert legacy_model_registry.register_model is canonical_model_registry.register_model + assert legacy_trainer_registry.register_trainer is canonical_trainer_registry.register_trainer + + def test_training_plan_doc_exists_and_mentions_registry_design(): content = PLAN_PATH.read_text() assert "Model registry" in content assert "Typed training config" in content assert "First implementation slice" in content + assert "dfode_kit/models" in content + assert "dfode_kit/training" in content diff --git a/tutorials/oneD_freely_propagating_flame/2_model_test/priori/test.py b/tutorials/oneD_freely_propagating_flame/2_model_test/priori/test.py index 6f7ccd04..af787954 100644 --- a/tutorials/oneD_freely_propagating_flame/2_model_test/priori/test.py +++ b/tutorials/oneD_freely_propagating_flame/2_model_test/priori/test.py @@ -5,8 +5,7 @@ from dfode_kit import DFODE_ROOT from dfode_kit.data_operations import integrate_h5, touch_h5, calculate_error -# from dfode_kit.dfode_core.test.test import test_npy -from dfode_kit.dfode_core.model.mlp import MLP +from dfode_kit.models.mlp import MLP mech_path = f'{DFODE_ROOT}/mechanisms/Burke2012_s9r23.yaml' gas = ct.Solution(mech_path) diff --git a/tutorials/twoD_HIT_flame/2_model_test/priori/test.py b/tutorials/twoD_HIT_flame/2_model_test/priori/test.py index cbafb126..c0514f04 100644 --- a/tutorials/twoD_HIT_flame/2_model_test/priori/test.py +++ b/tutorials/twoD_HIT_flame/2_model_test/priori/test.py @@ -5,8 +5,7 @@ from dfode_kit import DFODE_ROOT from dfode_kit.data_operations import integrate_h5, touch_h5, calculate_error -# from dfode_kit.dfode_core.test.test import test_npy -from dfode_kit.dfode_core.model.mlp import MLP +from dfode_kit.models.mlp import MLP mech_path = f'{DFODE_ROOT}/mechanisms/Burke2012_s9r23.yaml' gas = ct.Solution(mech_path)