From 5e2e33208f8b39de9a55ad446d9bf56e94498ec7 Mon Sep 17 00:00:00 2001 From: rettigl Date: Tue, 14 Jan 2025 23:56:23 +0100 Subject: [PATCH 01/44] add logging, elabftw metadata retrieval, and angle and offset sliders for crop tool --- .cspell/custom-dictionary.txt | 10 ++ src/specsanalyzer/config.py | 93 +++++++++++- src/specsanalyzer/core.py | 69 ++++++--- src/specsanalyzer/logging.py | 122 +++++++++++++++ src/specsscan/core.py | 88 +++++++++-- src/specsscan/helpers.py | 94 ++---------- src/specsscan/metadata.py | 276 ++++++++++++++++++++++++++++++++++ 7 files changed, 639 insertions(+), 113 deletions(-) create mode 100644 src/specsanalyzer/logging.py create mode 100644 src/specsscan/metadata.py diff --git a/.cspell/custom-dictionary.txt b/.cspell/custom-dictionary.txt index bf46422..522f42e 100644 --- a/.cspell/custom-dictionary.txt +++ b/.cspell/custom-dictionary.txt @@ -2,6 +2,8 @@ allclose ALLUSERSPROFILE amperemeter +appauthor +appname arange archiver argwhere @@ -38,10 +40,14 @@ dxde dyda dyde Ekin +elab +elabapi +elabid electronanalyser elems endstation energydispersion +entityid eshift faddr Faradayweg @@ -72,6 +78,7 @@ kwds labview Laurenz lensmodes +levelname lineh linev listf @@ -102,6 +109,7 @@ Nxpix Nxpixels Nypixels OPCPA +orcid pcolormesh Phoibos polyfit @@ -120,6 +128,7 @@ rrvec rtol rtype scanvector +sharelink specsanalyzer Specslab specsscan @@ -134,6 +143,7 @@ tqdm typehints TZCYXS undoc +userid venv viewcode vline diff --git a/src/specsanalyzer/config.py b/src/specsanalyzer/config.py index f946744..c90fdf4 100755 --- a/src/specsanalyzer/config.py +++ b/src/specsanalyzer/config.py @@ -8,9 +8,17 @@ from pathlib import Path import yaml +from platformdirs import user_config_path + +from specsanalyzer.logging import setup_logging package_dir = os.path.dirname(find_spec("specsanalyzer").origin) +USER_CONFIG_PATH = user_config_path(appname="specsscan", appauthor="OpenCOMPES", ensure_exists=True) + +# Configure logging +logger = setup_logging("config") + def parse_config( config: dict | str = None, @@ -61,7 +69,7 @@ def parse_config( else: config_dict = load_config(config) if verbose: - print(f"Configuration loaded from: [{str(Path(config).resolve())}]") + logger.info(f"Configuration loaded from: [{str(Path(config).resolve())}]") folder_dict: dict = None if isinstance(folder_config, dict): @@ -72,7 +80,7 @@ def parse_config( if Path(folder_config).exists(): folder_dict = load_config(folder_config) if verbose: - print(f"Folder config loaded from: [{str(Path(folder_config).resolve())}]") + logger.info(f"Folder config loaded from: [{str(Path(folder_config).resolve())}]") user_dict: dict = None if isinstance(user_config, dict): @@ -85,7 +93,7 @@ def parse_config( if Path(user_config).exists(): user_dict = load_config(user_config) if verbose: - print(f"User config loaded from: [{str(Path(user_config).resolve())}]") + logger.info(f"User config loaded from: [{str(Path(user_config).resolve())}]") system_dict: dict = None if isinstance(system_config, dict): @@ -105,14 +113,14 @@ def parse_config( if Path(system_config).exists(): system_dict = load_config(system_config) if verbose: - print(f"System config loaded from: [{str(Path(system_config).resolve())}]") + logger.info(f"System config loaded from: [{str(Path(system_config).resolve())}]") if isinstance(default_config, dict): default_dict = default_config else: default_dict = load_config(default_config) if verbose: - print(f"Default config loaded from: [{str(Path(default_config).resolve())}]") + logger.info(f"Default config loaded from: [{str(Path(default_config).resolve())}]") if folder_dict is not None: config_dict = complete_dictionary( @@ -226,3 +234,78 @@ def complete_dictionary(dictionary: dict, base_dictionary: dict) -> dict: dictionary[k] = v return dictionary + + +def _parse_env_file(file_path: Path) -> dict: + """Helper function to parse a .env file into a dictionary. + + Args: + file_path (Path): Path to the .env file + + Returns: + dict: Dictionary of environment variables from the file + """ + env_content = {} + if file_path.exists(): + with open(file_path) as f: + for line in f: + line = line.strip() + if line and "=" in line: + key, val = line.split("=", 1) + env_content[key.strip()] = val.strip() + return env_content + + +def read_env_var(var_name: str) -> str | None: + """Read an environment variable from multiple locations in order: + 1. OS environment variables + 2. .env file in current directory + 3. .env file in user config directory + + Args: + var_name (str): Name of the environment variable to read + + Returns: + str | None: Value of the environment variable or None if not found + """ + # First check OS environment variables + value = os.getenv(var_name) + if value is not None: + logger.debug(f"Found {var_name} in OS environment variables") + return value + + # Then check .env in current directory + local_vars = _parse_env_file(Path(".env")) + if var_name in local_vars: + logger.debug(f"Found {var_name} in ./.env file") + return local_vars[var_name] + + # Finally check .env in user config directory + user_vars = _parse_env_file(USER_CONFIG_PATH / ".env") + if var_name in user_vars: + logger.debug(f"Found {var_name} in user config .env file") + return user_vars[var_name] + + logger.debug(f"Environment variable {var_name} not found in any location") + return None + + +def save_env_var(var_name: str, value: str) -> None: + """Save an environment variable to the .env file in the user config directory. + If the file exists, preserves other variables. If not, creates a new file. + + Args: + var_name (str): Name of the environment variable to save + value (str): Value to save for the environment variable + """ + env_path = USER_CONFIG_PATH / ".env" + env_content = _parse_env_file(env_path) + + # Update or add new variable + env_content[var_name] = value + + # Write all variables back to file + with open(env_path, "w") as f: + for key, val in env_content.items(): + f.write(f"{key}={val}\n") + logger.debug(f"Environment variable {var_name} saved to .env file") diff --git a/src/specsanalyzer/core.py b/src/specsanalyzer/core.py index c461bcf..883c1d0 100755 --- a/src/specsanalyzer/core.py +++ b/src/specsanalyzer/core.py @@ -3,7 +3,6 @@ import os from typing import Any -from typing import Generator import imutils import ipywidgets as ipw @@ -382,6 +381,8 @@ def crop_tool( - ek_range_max - ang_range_min - ang_range_max + - angle_offset_px + - rotation_angle Other parameters are passed to ``convert_image()``. """ @@ -410,18 +411,10 @@ def crop_tool( linev2 = ax.axvline(x=data_array.Ekin[-1]) try: - ang_range_min = ( - kwds["ang_range_min"] if "ang_range_min" in kwds else self._config["ang_range_min"] - ) - ang_range_max = ( - kwds["ang_range_max"] if "ang_range_max" in kwds else self._config["ang_range_max"] - ) - ek_range_min = ( - kwds["ek_range_min"] if "ek_range_min" in kwds else self._config["ek_range_min"] - ) - ek_range_max = ( - kwds["ek_range_max"] if "ek_range_max" in kwds else self._config["ek_range_max"] - ) + ang_range_min = kwds.get("ang_range_min", self._config["ang_range_min"]) + ang_range_max = kwds.get("ang_range_max", self._config["ang_range_max"]) + ek_range_min = kwds.get("ek_range_min", self._config["ek_range_min"]) + ek_range_max = kwds.get("ek_range_max", self._config["ek_range_max"]) ang_min = ( ang_range_min * ( @@ -473,6 +466,15 @@ def crop_tool( vline_range = [ek_min, ek_max] hline_range = [ang_min, ang_max] + angle_offset_px = kwds.get("angle_offset_px", self._config.get("angle_offset_px", 0)) + rotation_angle = kwds.get("rotation_angle", self._config.get("rotation_angle", 0)) + + clim_slider = ipw.FloatRangeSlider( + description="colorbar limits", + value=[data_array.data.min(), data_array.data.max()], + min=data_array.data.min(), + max=data_array.data.max(), + ) vline_slider = ipw.FloatRangeSlider( description="Ekin", value=vline_range, @@ -487,14 +489,33 @@ def crop_tool( max=data_array.Angle[-1], step=0.1, ) - clim_slider = ipw.FloatRangeSlider( - description="colorbar limits", - value=[data_array.data.min(), data_array.data.max()], - min=data_array.data.min(), - max=data_array.data.max(), + ang_offset_slider = ipw.FloatSlider( + description="Angle offset", + value=angle_offset_px, + min=-20, + max=20, + step=1, + ) + rotation_slider = ipw.FloatSlider( + description="Rotation angle", + value=rotation_angle, + min=-5, + max=5, + step=0.1, ) - def update(hline, vline, v_vals): + def update(hline, vline, v_vals, angle_offset_px, rotation_angle): + data_array = self.convert_image( + raw_img=raw_img, + lens_mode=lens_mode, + kinetic_energy=kinetic_energy, + pass_energy=pass_energy, + work_function=work_function, + crop=False, + angle_offset_px=angle_offset_px, + rotation_angle=rotation_angle, + ) + mesh_obj.update({"array": data_array.data}) lineh1.set_ydata([hline[0]]) lineh2.set_ydata([hline[1]]) linev1.set_xdata([vline[0]]) @@ -507,9 +528,11 @@ def update(hline, vline, v_vals): hline=hline_slider, vline=vline_slider, v_vals=clim_slider, + angle_offset_px=ang_offset_slider, + rotation_angle=rotation_slider, ) - def cropit(val): # pylint: disable=unused-argument + def cropit(val): # noqa: ARG001 ang_min = min(hline_slider.value) ang_max = max(hline_slider.value) ek_min = min(vline_slider.value) @@ -552,6 +575,8 @@ def cropit(val): # pylint: disable=unused-argument ) ).item() self._config["crop"] = True + self._config["angle_offset_px"] = ang_offset_slider.value + self._config["rotation_angle"] = rotation_slider.value ax.cla() self._data_array.plot(ax=ax, add_colorbar=False) @@ -561,6 +586,8 @@ def cropit(val): # pylint: disable=unused-argument hline_slider.close() clim_slider.close() apply_button.close() + ang_offset_slider.close() + rotation_slider.close() apply_button = ipw.Button(description="Crop") display(apply_button) @@ -727,7 +754,7 @@ def update(v_vals, pos_x, pos_y, sigma_x, sigma_y, amplitude): v_vals=clim_slider, ) - def apply_fft(apply: bool): # pylint: disable=unused-argument + def apply_fft(apply: bool): # noqa: ARG001 amplitude = amplitude_slider.value pos_x = pos_x_slider.value pos_y = pos_y_slider.value diff --git a/src/specsanalyzer/logging.py b/src/specsanalyzer/logging.py new file mode 100644 index 0000000..cd51ac0 --- /dev/null +++ b/src/specsanalyzer/logging.py @@ -0,0 +1,122 @@ +""" +This module provides a function to set up logging for the application. It configures +both console and file logging handlers, allowing different log levels for each. The +log files are stored in a user-specific log directory. + +""" +from __future__ import annotations + +import logging +import os +import sys +from datetime import datetime +from functools import wraps +from typing import Callable + +# Default log directory +DEFAULT_LOG_DIR = os.path.join(os.getcwd(), "logs") +CONSOLE_VERBOSITY = logging.INFO +FILE_VERBOSITY = logging.DEBUG + + +def setup_logging( + name: str, + set_base_handler: bool = False, + user_log_path: str | None = None, +) -> logging.Logger: + """ + Configures and returns a logger with specified log levels for console and file handlers. + + Args: + name (str): The name of the logger. + set_base_handler (bool, optional): Option to re-initialize the base handler logging to the + logfile. Defaults to False. + user_log_path (str, optional): Path to the user-specific log directory. + Defaults to DEFAULT_LOG_DIR. + + Returns: + logging.Logger: The configured logger instance. + + The logger will always write DEBUG level messages to a file located in the user's log + directory, while the console log level can be adjusted based on the 'verbosity' parameter. + """ + # Create base logger + base_logger = logging.getLogger("specsanalyzer") + base_logger.setLevel(logging.DEBUG) # Set the minimum log level for the logger + if set_base_handler or not base_logger.hasHandlers(): + if base_logger.hasHandlers(): + base_logger.handlers.clear() + + # Determine log file path + if user_log_path is None: + user_log_path = DEFAULT_LOG_DIR + os.makedirs(user_log_path, exist_ok=True) + log_file = os.path.join(user_log_path, f"sed_{datetime.now().strftime('%Y-%m-%d')}.log") + + # Create file handler and set level to debug + file_handler = logging.FileHandler(log_file) + file_handler.setLevel(FILE_VERBOSITY) + + # Create formatter for file + file_formatter = logging.Formatter( + "%(asctime)s - %(name)s - %(levelname)s - %(message)s in %(filename)s:%(lineno)d", + ) + file_handler.setFormatter(file_formatter) + + # Add file handler to logger + base_logger.addHandler(file_handler) + + # create named logger + logger = base_logger.getChild(name) + + if logger.hasHandlers(): + logger.handlers.clear() + + # Create console handler and set level + console_handler = logging.StreamHandler(sys.stdout) + console_handler.setLevel(CONSOLE_VERBOSITY) + + # Create formatter for console + console_formatter = logging.Formatter("%(levelname)s - %(message)s") + console_handler.setFormatter(console_formatter) + + # Add console handler to logger + logger.addHandler(console_handler) + + # Capture warnings with the logging system + logging.captureWarnings(True) + + return logger + + +def set_verbosity(logger: logging.Logger, verbose: bool) -> None: + """Sets log level for the given logger's default handler. + + Args: + logger (logging.Logger): The logger on which to set the log level. + verbose (bool): Sets loglevel to INFO if True, to WARNING otherwise. + """ + handler = logger.handlers[0] + if verbose: + handler.setLevel(logging.INFO) + else: + handler.setLevel(logging.WARNING) + + +def call_logger(logger: logging.Logger): + def log_call(func: Callable): + @wraps(func) + def new_func(*args, **kwargs): + saved_args = locals() + args_str = "" + for arg in saved_args["args"][1:]: + args_str += f"{arg}, " + for name, arg in saved_args["kwargs"].items(): + args_str += f"{name}={arg}, " + args_str = args_str.rstrip(", ") + logger.debug(f"Call {func.__name__}({args_str})") + return func(*args, **kwargs) + + return new_func + + return log_call diff --git a/src/specsscan/core.py b/src/specsscan/core.py index 81929c7..14cebd7 100755 --- a/src/specsscan/core.py +++ b/src/specsscan/core.py @@ -17,9 +17,11 @@ from specsanalyzer import SpecsAnalyzer from specsanalyzer.config import parse_config +from specsanalyzer.config import save_config from specsanalyzer.io import to_h5 from specsanalyzer.io import to_nexus from specsanalyzer.io import to_tiff +from specsanalyzer.logging import setup_logging from specsscan.helpers import get_coords from specsscan.helpers import get_scan_path from specsscan.helpers import handle_meta @@ -30,6 +32,9 @@ package_dir = os.path.dirname(find_spec("specsscan").origin) +# Configure logging +logger = setup_logging("specsscan") + class SpecsScan: """SpecsAnalyzer class for loading scans and data from SPECS Phoibos electron analyzers, @@ -144,6 +149,7 @@ def load_scan( xr.DataArray: xarray DataArray object with kinetic energy, angle/position and optionally a third scanned axis (for ex., delay, temperature) as coordinates. """ + token = kwds.pop("token", None) scan_path = get_scan_path(path, scan, self._config["data_path"]) df_lut = parse_lut_to_df(scan_path) @@ -269,14 +275,16 @@ def load_scan( self.metadata.update( **handle_meta( - df_lut, - self._scan_info, - self.config, + df_lut=df_lut, + scan_info=self._scan_info, + config=self.config["metadata"], + scan=scan, fast_axes=list(fast_axes), # type: ignore slow_axes=list(slow_axes), projection=projection, metadata=copy.deepcopy(metadata), collect_metadata=collect_metadata, + token=token, ), **{"loader": loader_dict}, **{"conversion_parameters": conversion_metadata}, @@ -323,6 +331,38 @@ def crop_tool(self, scan: int = None, path: Path | str = "", **kwds): **kwds, ) + def save_crop_params( + self, + filename: str = None, + overwrite: bool = False, + ): + """Save the generated crop parameters to the folder config file. + + Args: + filename (str, optional): Filename of the config dictionary to save to. + Defaults to "specs_config.yaml" in the current folder. + overwrite (bool, optional): Option to overwrite the present dictionary. + Defaults to False. + """ + if filename is None: + filename = "specs_config.yaml" + if "ek_range_min" not in self.spa.config: + raise ValueError("No crop parameters to save!") + + config = { + "spa_params": { + "crop": self.spa.config["crop"], + "ek_range_min": self.spa.config["ek_range_min"], + "ek_range_max": self.spa.config["ek_range_max"], + "ang_range_min": self.spa.config["ang_range_min"], + "ang_range_max": self.spa.config["ang_range_max"], + "angle_offset_px": self.spa.config["angle_offset_px"], + "rotation_angle": self.spa.config["rotation_angle"], + }, + } + save_config(config, filename, overwrite) + logger.info(f'Saved crop parameters to "{filename}".') + def fft_tool(self, scan: int = None, path: Path | str = "", **kwds): """FFT tool to play around with the peak parameters in the Fourier plane. Built to filter out the meshgrid appearing in the raw data images. The optimized parameters are stored in @@ -360,6 +400,33 @@ def fft_tool(self, scan: int = None, path: Path | str = "", **kwds): **kwds, ) + def save_fft_params( + self, + filename: str = None, + overwrite: bool = False, + ): + """Save the generated fft filter parameters to the folder config file. + + Args: + filename (str, optional): Filename of the config dictionary to save to. + Defaults to "specs_config.yaml" in the current folder. + overwrite (bool, optional): Option to overwrite the present dictionary. + Defaults to False. + """ + if filename is None: + filename = "specs_config.yaml" + if len(self.spa.config["fft_filter_peaks"]) == 0: + raise ValueError("No fft parameters to save!") + + config = { + "spa_params": { + "fft_filter_peaks": self.spa.config["fft_filter_peaks"], + "apply_fft_filter": self.spa.config["apply_fft_filter"], + }, + } + save_config(config, filename, overwrite) + logger.info(f'Saved fft parameters to "{filename}".') + def check_scan( self, scan: int, @@ -390,6 +457,7 @@ def check_scan( Returns: xr.DataArray: 3-D xarray of dimensions (Ekin, Angle, Iterations) """ + token = kwds.pop("token", None) scan_path = get_scan_path(path, scan, self._config["data_path"]) df_lut = parse_lut_to_df(scan_path) @@ -437,7 +505,7 @@ def check_scan( conversion_metadata = xr_list[0].attrs["conversion_parameters"] - dims = get_coords( + dims = get_coords( # noqa: F841 scan_path=scan_path, scan_type=scan_type, scan_info=self._scan_info, @@ -465,14 +533,16 @@ def check_scan( self.metadata.update( **handle_meta( - df_lut, - self._scan_info, - self.config, + df_lut=df_lut, + scan_info=self._scan_info, + config=self.config["metadata"], + scan=scan, fast_axes=list(fast_axes), # type: ignore slow_axes=list(slow_axes), projection=projection, - metadata=metadata, + metadata=copy.deepcopy(metadata), collect_metadata=collect_metadata, + token=token, ), **{"loader": loader_dict}, **{"conversion_parameters": conversion_metadata}, @@ -616,7 +686,7 @@ def process_sweep_scan( ) or not self.spa.config["crop"] ): - warn("No valid cropping parameters found, consider using crop_tool() to set.") + logger.warning("No valid cropping parameters found, consider using crop_tool() to set.") e_step = converted.Ekin[1] - converted.Ekin[0] e0 = converted.Ekin[-1] - ekin_step diff --git a/src/specsscan/helpers.py b/src/specsscan/helpers.py index 02e396c..b66ff96 100644 --- a/src/specsscan/helpers.py +++ b/src/specsscan/helpers.py @@ -2,19 +2,16 @@ from __future__ import annotations import datetime as dt -import json from pathlib import Path from typing import Any from typing import Sequence -from urllib.error import HTTPError -from urllib.error import URLError -from urllib.request import urlopen import numpy as np import pandas as pd from tqdm.auto import tqdm from specsanalyzer.config import complete_dictionary +from specsscan.metadata import MetadataRetriever def get_scan_path(path: Path | str, scan: int, basepath: Path | str) -> Path: @@ -348,11 +345,13 @@ def handle_meta( df_lut: pd.DataFrame, scan_info: dict, config: dict, + scan: int, fast_axes: list[str], slow_axes: list[str], projection: str, metadata: dict = None, collect_metadata: bool = False, + token: str = None, ) -> dict: """Helper function for the handling metadata from different files @@ -361,16 +360,18 @@ def handle_meta( from ``parse_lut_to_df()`` scan_info (dict): scan_info class dict containing containing the contents of info.txt file config (dict): config dictionary containing the contents of config.yaml file + scan (int): Scan number fast_axes (list[str]): The fast-axis dimensions of the scan slow_axes (list[str]): The slow-axis dimensions of the scan metadata (dict, optional): Metadata dictionary with additional metadata for the scan. Defaults to empty dictionary. collect_metadata (bool, optional): Option to collect further metadata e.g. from EPICS archiver needed for NeXus conversion. Defaults to False. + token (str, optional):: The elabFTW api token to use for fetching metadata Returns: dict: metadata dictionary containing additional metadata from the EPICS - archive. + archive and elabFTW. """ if metadata is None: @@ -421,53 +422,18 @@ def handle_meta( } if collect_metadata: - # Get metadata from Epics archive if not present already - start = dt.datetime.utcfromtimestamp(ts_from).isoformat() + metadata_retriever = MetadataRetriever(config, token) - # replace metadata names by epics channels - try: - replace_dict = config["epics_channels"] - for key in list(metadata["scan_info"]): - if key.lower() in replace_dict: - metadata["scan_info"][replace_dict[key.lower()]] = metadata["scan_info"][key] - metadata["scan_info"].pop(key) - epics_channels = replace_dict.values() - except KeyError: - epics_channels = [] - - channels_missing = set(epics_channels) - set(metadata["scan_info"].keys()) - if channels_missing: - print("Collecting data from the EPICS archive...") - for channel in channels_missing: - try: - _, vals = get_archiver_data( - archiver_url=config.get("archiver_url"), - archiver_channel=channel, - ts_from=ts_from, - ts_to=ts_to, - ) - metadata["scan_info"][f"{channel}"] = np.mean(vals) + metadata = metadata_retriever.fetch_epics_metadata( + ts_from=ts_from, + ts_to=ts_to, + metadata=metadata, + ) - except IndexError: - metadata["scan_info"][f"{channel}"] = np.nan - print( - f"Data for channel {channel} doesn't exist for time {start}", - ) - except HTTPError as exc: - print( - f"Incorrect URL for the archive channel {channel}. " - "Make sure that the channel name and file start and end times are " - "correct.", - ) - print("Error code: ", exc) - except URLError as exc: - print( - f"Cannot access the archive URL for channel {channel}. " - f"Make sure that you are within the FHI network." - f"Skipping over channels {channels_missing}.", - ) - print("Error code: ", exc) - break + metadata = metadata_retriever.fetch_elab_metadata( + scan=scan, + metadata=metadata, + ) metadata["scan_info"]["energy_scan_mode"] = energy_scan_mode @@ -484,34 +450,6 @@ def handle_meta( return metadata -def get_archiver_data( - archiver_url: str, - archiver_channel: str, - ts_from: float, - ts_to: float, -) -> tuple[np.ndarray, np.ndarray]: - """Extract time stamps and corresponding data from and EPICS archiver instance - - Args: - archiver_url (str): URL of the archiver data extraction interface - archiver_channel (str): EPICS channel to extract data for - ts_from (float): starting time stamp of the range of interest - ts_to (float): ending time stamp of the range of interest - - Returns: - tuple[List, List]: The extracted time stamps and corresponding data - """ - iso_from = dt.datetime.utcfromtimestamp(ts_from).isoformat() - iso_to = dt.datetime.utcfromtimestamp(ts_to).isoformat() - req_str = archiver_url + archiver_channel + "&from=" + iso_from + "Z&to=" + iso_to + "Z" - with urlopen(req_str) as req: - data = json.load(req) - secs = [x["secs"] + x["nanos"] * 1e-9 for x in data[0]["data"]] - vals = [x["val"] for x in data[0]["data"]] - - return (np.asarray(secs), np.asarray(vals)) - - def find_scan(path: Path, scan: int) -> list[Path]: """Search function to locate the scan folder diff --git a/src/specsscan/metadata.py b/src/specsscan/metadata.py new file mode 100644 index 0000000..e98c867 --- /dev/null +++ b/src/specsscan/metadata.py @@ -0,0 +1,276 @@ +""" +The module provides a MetadataRetriever class for retrieving metadata +from an EPICS archiver and an elabFTW instance. +""" +from __future__ import annotations + +import datetime +import json +from urllib.error import HTTPError +from urllib.error import URLError +from urllib.request import urlopen + +import elabapi_python +import numpy as np + +from specsanalyzer.config import read_env_var +from specsanalyzer.config import save_env_var +from specsanalyzer.logging import setup_logging + +logger = setup_logging("mpes_metadata_retriever") + + +class MetadataRetriever: + """ + A class for retrieving metadata from an EPICS archiver and an elabFTW instance. + """ + + def __init__(self, metadata_config: dict, token: str = None) -> None: + """ + Initializes the MetadataRetriever class. + + Args: + metadata_config (dict): Takes a dict containing at least url for the EPICS archiver and + elabFTW instance. + token (str, optional): The token to use for fetching metadata. If provided, + will be saved to .env file for future use. + """ + # Token handling + if token: + self.token = token + save_env_var("ELAB_TOKEN", self.token) + else: + # Try to load token from config or .env file + self.token = read_env_var("ELAB_TOKEN") + + self._config = metadata_config + + self.url = str(metadata_config.get("elab_url")) + if not self.url: + raise ValueError("No URL provided for fetching metadata from elabFTW.") + + # Config + self.configuration = elabapi_python.Configuration() + self.configuration.api_key["api_key"] = self.token + self.configuration.api_key_prefix["api_key"] = "Authorization" + self.configuration.host = self.url + self.configuration.debug = False + self.configuration.verify_ssl = False + + # create an instance of the API class + self.api_client = elabapi_python.ApiClient(self.configuration) + # fix issue with Authorization header not being properly set by the generated lib + self.api_client.set_default_header(header_name="Authorization", header_value=self.token) + + # create an instance of Items + self.itemsApi = elabapi_python.ItemsApi(self.api_client) + self.experimentsApi = elabapi_python.ExperimentsApi(self.api_client) + self.linksApi = elabapi_python.LinksToItemsApi(self.api_client) + self.experimentsLinksApi = elabapi_python.LinksToExperimentsApi(self.api_client) + self.usersApi = elabapi_python.UsersApi(self.api_client) + + def fetch_epics_metadata(self, ts_from: float, ts_to: float, metadata: dict) -> dict: + """Fetch metadata from an EPICS archiver instance for times between ts_from and ts_to. + Channels are defined in the config. + + Args: + ts_from (float): Start timestamp of the range to collect data from. + ts_to (float): End timestamp of the range to collect data from. + metadata (dict): Input metadata dictionary. Will be updated + + Returns: + dict: Updated metadata dictionary. + """ + start = datetime.datetime.utcfromtimestamp(ts_from) + + # replace metadata names by epics channels + try: + replace_dict = self._config["epics_channels"] + for key in list(metadata["scan_info"]): + if key.lower() in replace_dict: + metadata["scan_info"][replace_dict[key.lower()]] = metadata["scan_info"][key] + metadata["scan_info"].pop(key) + epics_channels = replace_dict.values() + except KeyError: + epics_channels = [] + + channels_missing = set(epics_channels) - set(metadata["scan_info"].keys()) + if channels_missing: + logger.info("Collecting data from the EPICS archive...") + for channel in channels_missing: + try: + _, vals = get_archiver_data( + archiver_url=str(self._config.get("archiver_url")), + archiver_channel=channel, + ts_from=ts_from, + ts_to=ts_to, + ) + metadata["scan_info"][f"{channel}"] = np.mean(vals) + + except IndexError: + metadata["scan_info"][f"{channel}"] = np.nan + logger.info( + f"Data for channel {channel} doesn't exist for time {start}", + ) + except HTTPError as exc: + logger.warning( + f"Incorrect URL for the archive channel {channel}. " + "Make sure that the channel name and file start and end times are " + "correct.", + ) + logger.warning(f"Error code: {exc}") + except URLError as exc: + logger.warning( + f"Cannot access the archive URL for channel {channel}. " + f"Make sure that you are within the FHI network." + f"Skipping over channels {channels_missing}.", + ) + logger.warning(f"Error code: {exc}") + break + + return metadata + + def fetch_elab_metadata(self, scan: int, metadata: dict) -> dict: + """Fetch metadata from an elabFTW instance + + Args: + scan (int): Scan number for which to fetch metadata + metadata (dict): Input metadata dictionary. Will be updated + + Returns: + dict: Updated metadata dictionary + """ + if not self.token: + logger.warning( + "No valid token found. Token is required for metadata collection. Either provide " + "a token parameter or set the ELAB_TOKEN environment variable.", + ) + return metadata + logger.info("Collecting data from the elabFTW instance...") + # Get the experiment + try: + experiment = self.experimentsApi.read_experiments(q=f"'Phoibos scan {scan}'")[0] + except IndexError: + logger.warning(f"No elabFTW entry found for run {scan}") + return metadata + + if "elabFTW" not in metadata: + metadata["elabFTW"] = {} + + exp_id = experiment.id + # Get user information + user = self.usersApi.read_user(experiment.userid) + metadata["elabFTW"]["user"] = {} + metadata["elabFTW"]["user"]["name"] = user.fullname + metadata["elabFTW"]["user"]["email"] = user.email + metadata["elabFTW"]["user"]["id"] = user.userid + if user.orcid: + metadata["elabFTW"]["user"]["orcid"] = user.orcid + # Get the links to items + links = self.linksApi.read_entity_items_links(entity_type="experiments", id=exp_id) + # Get the items + items = [self.itemsApi.get_item(link.entityid) for link in links] + items_dict = {item.category_title: item for item in items} + items_dict["scan"] = experiment + + # Sort the metadata + for category, item in items_dict.items(): + category = category.replace(":", "").replace(" ", "_").lower() + if category not in metadata["elabFTW"]: + metadata["elabFTW"][category] = {} + metadata["elabFTW"][category]["title"] = item.title + metadata["elabFTW"][category]["summary"] = item.body + metadata["elabFTW"][category]["id"] = item.id + metadata["elabFTW"][category]["elabid"] = item.elabid + if item.sharelink: + metadata["elabFTW"][category]["link"] = item.sharelink + if item.metadata is not None: + metadata_json = json.loads(item.metadata) + for key, val in metadata_json["extra_fields"].items(): + if val["value"] and val["value"] != ["None"]: + try: + metadata["elabFTW"][category][key] = float(val["value"]) + except ValueError: + metadata["elabFTW"][category][key] = val["value"] + + # group beam profiles: + if ( + "laser_status" in metadata["elabFTW"] + and "pump_profile_x" in metadata["elabFTW"]["laser_status"] + and "pump_profile_y" in metadata["elabFTW"]["laser_status"] + ): + metadata["elabFTW"]["laser_status"]["pump_profile"] = [ + float(metadata["elabFTW"]["laser_status"]["pump_profile_x"]), + float(metadata["elabFTW"]["laser_status"]["pump_profile_y"]), + ] + if ( + "laser_status" in metadata["elabFTW"] + and "probe_profile_x" in metadata["elabFTW"]["laser_status"] + and "probe_profile_y" in metadata["elabFTW"]["laser_status"] + ): + metadata["elabFTW"]["laser_status"]["probe_profile"] = [ + float(metadata["elabFTW"]["laser_status"]["probe_profile_x"]), + float(metadata["elabFTW"]["laser_status"]["probe_profile_y"]), + ] + + # fix preparation date + if "sample" in metadata["elabFTW"] and "preparation_date" in metadata["elabFTW"]["sample"]: + metadata["elabFTW"]["sample"]["preparation_date"] = ( + datetime.datetime.strptime( + metadata["elabFTW"]["sample"]["preparation_date"], + "%Y-%m-%d", + ) + .replace(tzinfo=datetime.timezone.utc) + .isoformat() + ) + + # fix polarizations + if ( + "scan" in metadata["elabFTW"] + and "pump_polarization" in metadata["elabFTW"]["scan"] + and isinstance(metadata["elabFTW"]["scan"]["pump_polarization"], str) + ): + if metadata["elabFTW"]["scan"]["pump_polarization"] == "s": + metadata["elabFTW"]["scan"]["pump_polarization"] = 90 + elif metadata["elabFTW"]["scan"]["pump_polarization"] == "p": + metadata["elabFTW"]["scan"]["pump_polarization"] = 0 + + if ( + "scan" in metadata["elabFTW"] + and "probe_polarization" in metadata["elabFTW"]["scan"] + and isinstance(metadata["elabFTW"]["scan"]["probe_polarization"], str) + ): + if metadata["elabFTW"]["scan"]["probe_polarization"] == "s": + metadata["elabFTW"]["scan"]["probe_polarization"] = 90 + elif metadata["elabFTW"]["scan"]["probe_polarization"] == "p": + metadata["elabFTW"]["scan"]["probe_polarization"] = 0 + + return metadata + + +def get_archiver_data( + archiver_url: str, + archiver_channel: str, + ts_from: float, + ts_to: float, +) -> tuple[np.ndarray, np.ndarray]: + """Extract time stamps and corresponding data from and EPICS archiver instance + + Args: + archiver_url (str): URL of the archiver data extraction interface + archiver_channel (str): EPICS channel to extract data for + ts_from (float): starting time stamp of the range of interest + ts_to (float): ending time stamp of the range of interest + + Returns: + tuple[np.ndarray, np.ndarray]: The extracted time stamps and corresponding data + """ + iso_from = datetime.datetime.utcfromtimestamp(ts_from).isoformat() + iso_to = datetime.datetime.utcfromtimestamp(ts_to).isoformat() + req_str = archiver_url + archiver_channel + "&from=" + iso_from + "Z&to=" + iso_to + "Z" + with urlopen(req_str) as req: + data = json.load(req) + secs = [x["secs"] + x["nanos"] * 1e-9 for x in data[0]["data"]] + vals = [x["val"] for x in data[0]["data"]] + + return (np.asarray(secs), np.asarray(vals)) From 1eb138f6d19037b54d8b2a8157e3851bee5b6d70 Mon Sep 17 00:00:00 2001 From: rettigl Date: Wed, 15 Jan 2025 00:37:51 +0100 Subject: [PATCH 02/44] keep 0-valued entries --- src/specsscan/metadata.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/specsscan/metadata.py b/src/specsscan/metadata.py index e98c867..226aa2e 100644 --- a/src/specsscan/metadata.py +++ b/src/specsscan/metadata.py @@ -187,7 +187,7 @@ def fetch_elab_metadata(self, scan: int, metadata: dict) -> dict: if item.metadata is not None: metadata_json = json.loads(item.metadata) for key, val in metadata_json["extra_fields"].items(): - if val["value"] and val["value"] != ["None"]: + if val["value"] is not None and val["value"] != "" and val["value"] != ["None"]: try: metadata["elabFTW"][category][key] = float(val["value"]) except ValueError: From 165ace41709b0ef2a765c09f71d8479b10e9beda Mon Sep 17 00:00:00 2001 From: rettigl Date: Wed, 15 Jan 2025 09:31:42 +0100 Subject: [PATCH 03/44] add requirements --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index 8f1ce02..6b9ba34 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -32,6 +32,7 @@ classifiers = [ "Operating System :: OS Independent", ] dependencies = [ + "elabapi-python>=5.0", "h5py>=3.6.0", "imutils>=0.5.4", "ipympl>=0.9.1", From 0f70b6657212c7d24dda04af05bac199405e9a79 Mon Sep 17 00:00:00 2001 From: rettigl Date: Wed, 15 Jan 2025 10:15:59 +0100 Subject: [PATCH 04/44] fix tests --- src/specsanalyzer/core.py | 16 ++++++++--- src/specsscan/config/example_config_FHI.yaml | 29 +++++++++++--------- src/specsscan/core.py | 4 +-- src/specsscan/metadata.py | 4 +++ tests/data | 2 +- 5 files changed, 35 insertions(+), 20 deletions(-) diff --git a/src/specsanalyzer/core.py b/src/specsanalyzer/core.py index 883c1d0..5ad97e5 100755 --- a/src/specsanalyzer/core.py +++ b/src/specsanalyzer/core.py @@ -411,10 +411,18 @@ def crop_tool( linev2 = ax.axvline(x=data_array.Ekin[-1]) try: - ang_range_min = kwds.get("ang_range_min", self._config["ang_range_min"]) - ang_range_max = kwds.get("ang_range_max", self._config["ang_range_max"]) - ek_range_min = kwds.get("ek_range_min", self._config["ek_range_min"]) - ek_range_max = kwds.get("ek_range_max", self._config["ek_range_max"]) + ang_range_min = ( + kwds["ang_range_min"] if "ang_range_min" in kwds else self._config["ang_range_min"] + ) + ang_range_max = ( + kwds["ang_range_max"] if "ang_range_max" in kwds else self._config["ang_range_max"] + ) + ek_range_min = ( + kwds["ek_range_min"] if "ek_range_min" in kwds else self._config["ek_range_min"] + ) + ek_range_max = ( + kwds["ek_range_max"] if "ek_range_max" in kwds else self._config["ek_range_max"] + ) ang_min = ( ang_range_min * ( diff --git a/src/specsscan/config/example_config_FHI.yaml b/src/specsscan/config/example_config_FHI.yaml index 5f5aec9..bf4976b 100644 --- a/src/specsscan/config/example_config_FHI.yaml +++ b/src/specsscan/config/example_config_FHI.yaml @@ -36,19 +36,22 @@ units: spatial0: "mm" voltage: "V" -# URL of the epics archiver request engine -archiver_url: "http://__epicsarchiver_host__:17668/retrieval/data/getData.json?pv=" -# dictionary containing axis names with Epics channels to request from the EPICS archiver -epics_channels: - tempa: "trARPES:Carving:TEMP_RBV" - x: "trARPES:Carving:TRX.RBV" - y: "trARPES:Carving:TRY.RBV" - z: "trARPES:Carving:TRZ.RBV" - polar: "trARPES:Carving:THT.RBV" - tilt: "trARPES:Carving:PHI.RBV" - azimuth: "trARPES:Carving:OMG.RBV" - drain_current: "trARPES:Sample:Measure" - pressure: "trARPES:XGS600:PressureAC:P_RD" +metadata: + # URL of the elabFTW instance API interface + elab_url: "https://__elabftw_host__/api/v2" + # URL of the epics archiver request engine + archiver_url: "http://__epicsarchiver_host__:17668/retrieval/data/getData.json?pv=" + # dictionary containing axis names with Epics channels to request from the EPICS archiver + epics_channels: + tempa: "trARPES:Carving:TEMP_RBV" + x: "trARPES:Carving:TRX.RBV" + y: "trARPES:Carving:TRY.RBV" + z: "trARPES:Carving:TRZ.RBV" + polar: "trARPES:Carving:THT.RBV" + tilt: "trARPES:Carving:PHI.RBV" + azimuth: "trARPES:Carving:OMG.RBV" + drain_current: "trARPES:Sample:Measure" + pressure: "trARPES:XGS600:PressureAC:P_RD" # parameters for NeXus conversion nexus: diff --git a/src/specsscan/core.py b/src/specsscan/core.py index 14cebd7..45bc5b4 100755 --- a/src/specsscan/core.py +++ b/src/specsscan/core.py @@ -277,7 +277,7 @@ def load_scan( **handle_meta( df_lut=df_lut, scan_info=self._scan_info, - config=self.config["metadata"], + config=self.config.get("metadata", {}), scan=scan, fast_axes=list(fast_axes), # type: ignore slow_axes=list(slow_axes), @@ -535,7 +535,7 @@ def check_scan( **handle_meta( df_lut=df_lut, scan_info=self._scan_info, - config=self.config["metadata"], + config=self.config.get("metadata", {}), scan=scan, fast_axes=list(fast_axes), # type: ignore slow_axes=list(slow_axes), diff --git a/src/specsscan/metadata.py b/src/specsscan/metadata.py index 226aa2e..41d0bc1 100644 --- a/src/specsscan/metadata.py +++ b/src/specsscan/metadata.py @@ -12,6 +12,7 @@ import elabapi_python import numpy as np +from urllib3.exceptions import MaxRetryError from specsanalyzer.config import read_env_var from specsanalyzer.config import save_env_var @@ -153,6 +154,9 @@ def fetch_elab_metadata(self, scan: int, metadata: dict) -> dict: except IndexError: logger.warning(f"No elabFTW entry found for run {scan}") return metadata + except MaxRetryError: + logger.warning("Connection to elabFTW could not be established. Check accessibility") + return metadata if "elabFTW" not in metadata: metadata["elabFTW"] = {} diff --git a/tests/data b/tests/data index e921fe9..bbede1b 160000 --- a/tests/data +++ b/tests/data @@ -1 +1 @@ -Subproject commit e921fe9c198d263c900d8568476e12e4dcef82b8 +Subproject commit bbede1bfb99020f3fb71d4b03570b83bfab07c83 From e5746fd91e915cdfd4e13d5df73936e54a8c95f5 Mon Sep 17 00:00:00 2001 From: rettigl Date: Thu, 16 Jan 2025 15:33:20 +0100 Subject: [PATCH 05/44] remove pump laser beam section if pump closed --- src/specsscan/metadata.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/specsscan/metadata.py b/src/specsscan/metadata.py index 41d0bc1..afc6c9d 100644 --- a/src/specsscan/metadata.py +++ b/src/specsscan/metadata.py @@ -249,6 +249,11 @@ def fetch_elab_metadata(self, scan: int, metadata: dict) -> dict: elif metadata["elabFTW"]["scan"]["probe_polarization"] == "p": metadata["elabFTW"]["scan"]["probe_polarization"] = 0 + # remove pump information if pump not applied: + if not metadata["elabFTW"]["scan"].get("pump_status", 0): + if "pump_photon_energy" in metadata["elabFTW"].get("laser_status", {}): + del metadata["elabFTW"]["laser_status"]["pump_photon_energy"] + return metadata From 0f3101ca67dda406d1439ae792c49cfdf05190c9 Mon Sep 17 00:00:00 2001 From: rettigl Date: Thu, 16 Jan 2025 16:25:56 +0100 Subject: [PATCH 06/44] use default user config path look for .env file also in system config folder --- docs/specsanalyzer/config.rst | 2 +- src/specsanalyzer/config.py | 42 ++++++++++++++++++++--------------- 2 files changed, 25 insertions(+), 19 deletions(-) diff --git a/docs/specsanalyzer/config.rst b/docs/specsanalyzer/config.rst index 4165f69..d098923 100644 --- a/docs/specsanalyzer/config.rst +++ b/docs/specsanalyzer/config.rst @@ -4,7 +4,7 @@ The config module contains a mechanics to collect configuration parameters from It will load an (optional) provided config file, or alternatively use a passed python dictionary as initial config dictionary, and subsequently look for the following additional config files to load: * ``folder_config``: A config file of name :file:`specs_config.yaml` in the current working directory. This is mostly intended to pass calibration parameters of the workflow between different notebook instances. -* ``user_config``: A config file provided by the user, stored as :file:`.specsanalyzer/config.yaml` in the current user's home directly. This is intended to give a user the option for individual configuration modifications of system settings. +* ``user_config``: A config file provided by the user, stored as :file:`.config/specsanalyzer/config.yaml` in the current user's home directly. This is intended to give a user the option for individual configuration modifications of system settings. * ``system_config``: A config file provided by the system administrator, stored as :file:`/etc/specsanalyzer/config.yaml` on Linux-based systems, and :file:`%ALLUSERSPROFILE%/specsanalyzer/config.yaml` on Windows. This should provide all necessary default parameters for using the specsanalyzer processor with a given setup. For an example for the setup at the Fritz Haber Institute setup, see :ref:`example_config` * ``default_config``: The default configuration shipped with the package. Typically, all parameters here should be overwritten by any of the other configuration files. diff --git a/src/specsanalyzer/config.py b/src/specsanalyzer/config.py index c90fdf4..626d096 100755 --- a/src/specsanalyzer/config.py +++ b/src/specsanalyzer/config.py @@ -14,7 +14,16 @@ package_dir = os.path.dirname(find_spec("specsanalyzer").origin) -USER_CONFIG_PATH = user_config_path(appname="specsscan", appauthor="OpenCOMPES", ensure_exists=True) +USER_CONFIG_PATH = user_config_path( + appname="specsanalyzer", + appauthor="OpenCOMPES", + ensure_exists=True, +) +SYSTEM_CONFIG_PATH = ( + Path(os.environ["ALLUSERSPROFILE"]).joinpath("specsanalyzer") + if platform.system() == "Windows" + else Path("/etc/").joinpath("specsanalyzer") +) # Configure logging logger = setup_logging("config") @@ -44,7 +53,8 @@ def parse_config( user_config (dict | str, optional): user-based config dictionary or file path. The loaded dictionary is completed with the user-based values, taking preference over system and default values. - Defaults to the file ".specsanalyzer/config.yaml" in the current user's home directory. + Defaults to the file ".config/specsanalyzer/config.yaml" in the current user's home + directory. system_config (dict | str, optional): system-wide config dictionary or file path. The loaded dictionary is completed with the system-wide values, taking preference over default values. @@ -87,9 +97,7 @@ def parse_config( user_dict = user_config else: if user_config is None: - user_config = str( - Path.home().joinpath(".specsanalyzer").joinpath("config.yaml"), - ) + user_config = str(USER_CONFIG_PATH.joinpath("config.yaml")) if Path(user_config).exists(): user_dict = load_config(user_config) if verbose: @@ -100,16 +108,7 @@ def parse_config( system_dict = system_config else: if system_config is None: - if platform.system() in ["Linux", "Darwin"]: - system_config = str( - Path("/etc/").joinpath("specsanalyzer").joinpath("config.yaml"), - ) - elif platform.system() == "Windows": - system_config = str( - Path(os.environ["ALLUSERSPROFILE"]) - .joinpath("specsanalyzer") - .joinpath("config.yaml"), - ) + system_config = str(SYSTEM_CONFIG_PATH.joinpath("config.yaml")) if Path(system_config).exists(): system_dict = load_config(system_config) if verbose: @@ -261,6 +260,7 @@ def read_env_var(var_name: str) -> str | None: 1. OS environment variables 2. .env file in current directory 3. .env file in user config directory + 4. .env file in system config directory Args: var_name (str): Name of the environment variable to read @@ -268,24 +268,30 @@ def read_env_var(var_name: str) -> str | None: Returns: str | None: Value of the environment variable or None if not found """ - # First check OS environment variables + # 1. check OS environment variables value = os.getenv(var_name) if value is not None: logger.debug(f"Found {var_name} in OS environment variables") return value - # Then check .env in current directory + # 2. check .env in current directory local_vars = _parse_env_file(Path(".env")) if var_name in local_vars: logger.debug(f"Found {var_name} in ./.env file") return local_vars[var_name] - # Finally check .env in user config directory + # 3. check .env in user config directory user_vars = _parse_env_file(USER_CONFIG_PATH / ".env") if var_name in user_vars: logger.debug(f"Found {var_name} in user config .env file") return user_vars[var_name] + # 4. check .env in system config directory + system_vars = _parse_env_file(SYSTEM_CONFIG_PATH / ".env") + if var_name in system_vars: + logger.debug(f"Found {var_name} in system config .env file") + return system_vars[var_name] + logger.debug(f"Environment variable {var_name} not found in any location") return None From ae9b44e2b0f9555a1ccb982cc543079f898a0885 Mon Sep 17 00:00:00 2001 From: rettigl Date: Thu, 16 Jan 2025 16:33:37 +0100 Subject: [PATCH 07/44] use config_v1.yaml --- docs/specsanalyzer/config.rst | 4 ++-- src/specsanalyzer/config.py | 10 +++++----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/specsanalyzer/config.rst b/docs/specsanalyzer/config.rst index d098923..34ce2f1 100644 --- a/docs/specsanalyzer/config.rst +++ b/docs/specsanalyzer/config.rst @@ -4,8 +4,8 @@ The config module contains a mechanics to collect configuration parameters from It will load an (optional) provided config file, or alternatively use a passed python dictionary as initial config dictionary, and subsequently look for the following additional config files to load: * ``folder_config``: A config file of name :file:`specs_config.yaml` in the current working directory. This is mostly intended to pass calibration parameters of the workflow between different notebook instances. -* ``user_config``: A config file provided by the user, stored as :file:`.config/specsanalyzer/config.yaml` in the current user's home directly. This is intended to give a user the option for individual configuration modifications of system settings. -* ``system_config``: A config file provided by the system administrator, stored as :file:`/etc/specsanalyzer/config.yaml` on Linux-based systems, and :file:`%ALLUSERSPROFILE%/specsanalyzer/config.yaml` on Windows. This should provide all necessary default parameters for using the specsanalyzer processor with a given setup. For an example for the setup at the Fritz Haber Institute setup, see :ref:`example_config` +* ``user_config``: A config file provided by the user, stored as :file:`.config/specsanalyzer/config_v1.yaml` in the current user's home directly. This is intended to give a user the option for individual configuration modifications of system settings. +* ``system_config``: A config file provided by the system administrator, stored as :file:`/etc/specsanalyzer/config_v1.yaml` on Linux-based systems, and :file:`%ALLUSERSPROFILE%/specsanalyzer/config_v1.yaml` on Windows. This should provide all necessary default parameters for using the specsanalyzer processor with a given setup. For an example for the setup at the Fritz Haber Institute setup, see :ref:`example_config` * ``default_config``: The default configuration shipped with the package. Typically, all parameters here should be overwritten by any of the other configuration files. The config mechanism returns the combined dictionary, and reports the loaded configuration files. In order to disable or overwrite any of the configuration files, they can be also given as optional parameters (path to a file, or python dictionary). diff --git a/src/specsanalyzer/config.py b/src/specsanalyzer/config.py index 626d096..f3806ef 100755 --- a/src/specsanalyzer/config.py +++ b/src/specsanalyzer/config.py @@ -53,13 +53,13 @@ def parse_config( user_config (dict | str, optional): user-based config dictionary or file path. The loaded dictionary is completed with the user-based values, taking preference over system and default values. - Defaults to the file ".config/specsanalyzer/config.yaml" in the current user's home + Defaults to the file ".config/specsanalyzer/config_v1.yaml" in the current user's home directory. system_config (dict | str, optional): system-wide config dictionary or file path. The loaded dictionary is completed with the system-wide values, taking preference over default values. - Defaults to the file "/etc/specsanalyzer/config.yaml" on linux, - and "%ALLUSERSPROFILE%/specsanalyzer/config.yaml" on windows. + Defaults to the file "/etc/specsanalyzer/config_v1.yaml" on linux, + and "%ALLUSERSPROFILE%/specsanalyzer/config_v1.yaml" on windows. default_config (dict | str, optional): default config dictionary or file path. The loaded dictionary is completed with the default values. Defaults to *package_dir*/config/default.yaml". @@ -97,7 +97,7 @@ def parse_config( user_dict = user_config else: if user_config is None: - user_config = str(USER_CONFIG_PATH.joinpath("config.yaml")) + user_config = str(USER_CONFIG_PATH.joinpath("config_v1.yaml")) if Path(user_config).exists(): user_dict = load_config(user_config) if verbose: @@ -108,7 +108,7 @@ def parse_config( system_dict = system_config else: if system_config is None: - system_config = str(SYSTEM_CONFIG_PATH.joinpath("config.yaml")) + system_config = str(SYSTEM_CONFIG_PATH.joinpath("config_v1.yaml")) if Path(system_config).exists(): system_dict = load_config(system_config) if verbose: From 0896ce71c60b5266cf130df7d774bffae7145dce Mon Sep 17 00:00:00 2001 From: rettigl Date: Fri, 17 Jan 2025 13:52:33 +0100 Subject: [PATCH 08/44] add logging and delay as metadata --- src/specsanalyzer/convert.py | 7 ++++++- src/specsanalyzer/core.py | 20 +++++++++++++----- src/specsscan/core.py | 11 +++++++++- src/specsscan/helpers.py | 39 +++++++++++++++++++++++------------- src/specsscan/metadata.py | 2 +- 5 files changed, 57 insertions(+), 22 deletions(-) diff --git a/src/specsanalyzer/convert.py b/src/specsanalyzer/convert.py index aaa015f..0fa8de4 100755 --- a/src/specsanalyzer/convert.py +++ b/src/specsanalyzer/convert.py @@ -1,9 +1,14 @@ """Specsanalyzer image conversion module""" from __future__ import annotations +import logging + import numpy as np from scipy.ndimage import map_coordinates +# Configure logging +logger = logging.getLogger("specsanalyzer.specsscan") + def get_damatrix_from_calib2d( lens_mode: str, @@ -82,7 +87,7 @@ def get_damatrix_from_calib2d( elif lens_mode in supported_space_modes: # use the mode defaults - print("This is a spatial mode, using default " + lens_mode + " config") + logger.info("This is a spatial mode, using default " + lens_mode + " config") rr_vec, da_matrix_full = get_rr_da(lens_mode, calib2d_dict) a_inner = da_matrix_full[0][0] da_matrix = da_matrix_full[1:][:] diff --git a/src/specsanalyzer/core.py b/src/specsanalyzer/core.py index 5ad97e5..b5a8922 100755 --- a/src/specsanalyzer/core.py +++ b/src/specsanalyzer/core.py @@ -20,9 +20,14 @@ from specsanalyzer.convert import physical_unit_data from specsanalyzer.img_tools import crop_xarray from specsanalyzer.img_tools import fourier_filter_2d +from specsanalyzer.logging import set_verbosity +from specsanalyzer.logging import setup_logging package_dir = os.path.dirname(__file__) +# Configure logging +logger = setup_logging("specsanalyzer") + class SpecsAnalyzer: """SpecsAnalyzer: A class to convert photoemission data from a SPECS Phoibos analyzer from @@ -38,6 +43,7 @@ def __init__( self, metadata: dict[Any, Any] = {}, config: dict[Any, Any] | str = {}, + verbose: bool = True, **kwds, ): """SpecsAnalyzer constructor. @@ -45,12 +51,14 @@ def __init__( Args: metadata (dict, optional): Metadata dictionary. Defaults to {}. config (dict | str, optional): Metadata dictionary or file path. Defaults to {}. + verbose (bool, optional): Disable info logs if set to False. **kwds: Keyword arguments passed to ``parse_config``. """ self._config = parse_config( config, **kwds, ) + set_verbosity(logger, verbose) self.metadata = metadata self._data_array = None self.print_msg = True @@ -286,7 +294,7 @@ def convert_image( ek_min = range_dict["ek_min"] ek_max = range_dict["ek_max"] if self.print_msg: - print("Using saved crop parameters...") + logger.info("Using saved crop parameters...") data_array = crop_xarray(data_array, ang_min, ang_max, ek_min, ek_max) except KeyError: try: @@ -343,11 +351,13 @@ def convert_image( + data_array.coords[data_array.dims[1]][0] ) if self.print_msg: - print("Cropping parameters not found, using cropping ranges from config...") + logger.info( + "Cropping parameters not found, using cropping ranges from config...", + ) data_array = crop_xarray(data_array, ang_min, ang_max, ek_min, ek_max) except KeyError: if self.print_msg: - print( + logger.warning( "Warning: Cropping parameters not found, " "use method crop_tool() after loading.", ) @@ -402,7 +412,7 @@ def crop_tool( try: mesh_obj = data_array.plot(ax=ax) except AttributeError: - print("Load the scan first!") + logger.info("Load the scan first!") raise lineh1 = ax.axhline(y=data_array.Angle[0]) @@ -650,7 +660,7 @@ def fft_tool( filtered = fourier_filter_2d(raw_image, peaks=fft_filter_peaks, ret="filtered") except IndexError: - print("Load the scan first!") + logger.warning("Load the scan first!") raise fig = plt.figure() diff --git a/src/specsscan/core.py b/src/specsscan/core.py index 45bc5b4..8a0dcaf 100755 --- a/src/specsscan/core.py +++ b/src/specsscan/core.py @@ -21,6 +21,7 @@ from specsanalyzer.io import to_h5 from specsanalyzer.io import to_nexus from specsanalyzer.io import to_tiff +from specsanalyzer.logging import set_verbosity from specsanalyzer.logging import setup_logging from specsscan.helpers import get_coords from specsscan.helpers import get_scan_path @@ -50,6 +51,7 @@ def __init__( self, metadata: dict = {}, config: dict | str = {}, + verbose: bool = True, **kwds, ): """SpecsScan constructor. @@ -57,6 +59,7 @@ def __init__( Args: metadata (dict, optional): Metadata dictionary. Defaults to {}. config (Union[dict, str], optional): Metadata dictionary or file path. Defaults to {}. + verbose (bool, optional): Disable info logs if set to False. **kwds: Keyword arguments passed to ``parse_config``. """ self._config = parse_config( @@ -65,6 +68,8 @@ def __init__( **kwds, ) + set_verbosity(logger, verbose) + self.metadata = metadata self._scan_info: dict[Any, Any] = {} @@ -75,12 +80,14 @@ def __init__( folder_config={}, user_config={}, system_config={}, + verbose=verbose, ) except KeyError: self.spa = SpecsAnalyzer( folder_config={}, user_config={}, system_config={}, + verbose=verbose, ) self._result: xr.DataArray = None @@ -242,10 +249,11 @@ def load_scan( k: coordinate_mapping[k] for k in coordinate_mapping.keys() if k in res_xarray.dims } depends_dict = { - rename_dict[k]: coordinate_depends[k] + rename_dict.get(k, k): coordinate_depends[k] for k in coordinate_depends.keys() if k in res_xarray.dims } + res_xarray = res_xarray.rename(rename_dict) for k, v in coordinate_mapping.items(): if k in fast_axes: @@ -260,6 +268,7 @@ def load_scan( "/entry/sample/transformations/sample_polar": "Polar", "/entry/sample/transformations/sample_tilt": "Tilt", "/entry/sample/transformations/sample_azimuth": "Azimuth", + "/entry/instrument/beam_pump/pulse_delay": "delay", } # store data for resolved axis coordinates diff --git a/src/specsscan/helpers.py b/src/specsscan/helpers.py index b66ff96..0ad89cd 100644 --- a/src/specsscan/helpers.py +++ b/src/specsscan/helpers.py @@ -2,17 +2,22 @@ from __future__ import annotations import datetime as dt +import logging from pathlib import Path from typing import Any from typing import Sequence import numpy as np import pandas as pd +import xarray as xr from tqdm.auto import tqdm from specsanalyzer.config import complete_dictionary from specsscan.metadata import MetadataRetriever +# Configure logging +logger = logging.getLogger("specsanalyzer.specsscan") + def get_scan_path(path: Path | str, scan: int, basepath: Path | str) -> Path: """Returns the path to the given scan. @@ -123,7 +128,7 @@ def load_images( "load_scan method.", ) from exc - print(f"Averaging over {avg_dim}...") + logger.info(f"Averaging over {avg_dim}...") for dim in tqdm(raw_2d_sliced): avg_list = [] for image in tqdm(dim, leave=False, disable=not tqdm_enable_nested): @@ -210,14 +215,14 @@ def parse_lut_to_df(scan_path: Path) -> pd.DataFrame: df_lut.reset_index(inplace=True) new_cols = df_lut.columns.to_list()[1:] - new_cols[new_cols.index("delaystage")] = "Delay" + new_cols[new_cols.index("delaystage")] = "DelayStage" new_cols.insert(3, "delay (fs)") # Create label to drop the column later df_lut = df_lut.set_axis(new_cols, axis="columns") df_lut.drop(columns="delay (fs)", inplace=True) except FileNotFoundError: - print( + logger.info( "LUT.txt not found. Storing metadata from info.txt", ) return None @@ -265,7 +270,7 @@ def get_coords( return (np.array([]), "") if df_lut is not None: - print("scanvector.txt not found. Obtaining coordinates from LUT") + logger.info("scanvector.txt not found. Obtaining coordinates from LUT") df_new: pd.DataFrame = df_lut.loc[:, df_lut.columns[2:]] @@ -276,13 +281,18 @@ def get_coords( raise FileNotFoundError("scanvector.txt file not found!") from exc if scan_type == "delay": - t_0 = scan_info["TimeZero"] - coords -= t_0 - coords *= 2 / 3e11 * 1e15 + t0 = scan_info["TimeZero"] + coords = mm_to_fs(coords, t0) return coords, dim +def mm_to_fs(delaystage: xr.DataArray | np.ndarray | float, t0: float) -> float: + delay = delaystage - t0 + delay *= 2 / 2.99792458e11 * 1e15 + return delay + + def compare_coords(axis_data: np.ndarray) -> tuple[np.ndarray, int]: """Identifies the most changing column in a given 2-D numpy array. @@ -338,6 +348,9 @@ def parse_info_to_dict(path: Path) -> dict: except FileNotFoundError as exc: raise FileNotFoundError("info.txt file not found.") from exc + if "DelayStage" in info_dict and "TimeZero" in info_dict: + info_dict["delay"] = mm_to_fs(info_dict["DelayStage"], info_dict["TimeZero"]) + return info_dict @@ -377,7 +390,7 @@ def handle_meta( if metadata is None: metadata = {} - print("Gathering metadata from different locations") + logger.info("Gathering metadata from different locations") # get metadata from LUT dataframe lut_meta = {} energy_scan_mode = "snapshot" @@ -395,10 +408,10 @@ def handle_meta( metadata["scan_info"] = complete_dictionary( metadata.get("scan_info", {}), - complete_dictionary(lut_meta, scan_info), + complete_dictionary(scan_info, lut_meta), ) # merging dictionaries - print("Collecting time stamps...") + logger.info("Collecting time stamps...") if "time" in metadata["scan_info"]: time_list = [metadata["scan_info"]["time"][0], metadata["scan_info"]["time"][-1]] elif "StartTime" in metadata["scan_info"]: @@ -445,8 +458,6 @@ def handle_meta( metadata["scan_info"]["slow_axes"] = slow_axes metadata["scan_info"]["fast_axes"] = fast_axes - print("Done!") - return metadata @@ -460,7 +471,7 @@ def find_scan(path: Path, scan: int) -> list[Path]: Returns: List[Path]: scan_path: Path object pointing to the scan folder """ - print("Scan path not provided, searching directories...") + logger.info("Scan path not provided, searching directories...") for file in path.iterdir(): if file.is_dir(): try: @@ -474,7 +485,7 @@ def find_scan(path: Path, scan: int) -> list[Path]: file.glob(f"*/*/Raw Data/{scan}"), ) if scan_path: - print("Scan found at path:", scan_path[0]) + logger.info(f"Scan found at path: {scan_path[0]}") break else: scan_path = [] diff --git a/src/specsscan/metadata.py b/src/specsscan/metadata.py index afc6c9d..8ec96e7 100644 --- a/src/specsscan/metadata.py +++ b/src/specsscan/metadata.py @@ -90,7 +90,7 @@ def fetch_epics_metadata(self, ts_from: float, ts_to: float, metadata: dict) -> for key in list(metadata["scan_info"]): if key.lower() in replace_dict: metadata["scan_info"][replace_dict[key.lower()]] = metadata["scan_info"][key] - metadata["scan_info"].pop(key) + del metadata["scan_info"][key] epics_channels = replace_dict.values() except KeyError: epics_channels = [] From a942aeed38c84435eb3447ae004e96a8185d079b Mon Sep 17 00:00:00 2001 From: Laurenz Rettig Date: Fri, 17 Jan 2025 22:24:55 +0000 Subject: [PATCH 09/44] remove typing --- src/specsscan/helpers.py | 3 +-- src/specsscan/metadata.py | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/src/specsscan/helpers.py b/src/specsscan/helpers.py index 0ad89cd..f67d580 100644 --- a/src/specsscan/helpers.py +++ b/src/specsscan/helpers.py @@ -9,7 +9,6 @@ import numpy as np import pandas as pd -import xarray as xr from tqdm.auto import tqdm from specsanalyzer.config import complete_dictionary @@ -287,7 +286,7 @@ def get_coords( return coords, dim -def mm_to_fs(delaystage: xr.DataArray | np.ndarray | float, t0: float) -> float: +def mm_to_fs(delaystage, t0): delay = delaystage - t0 delay *= 2 / 2.99792458e11 * 1e15 return delay diff --git a/src/specsscan/metadata.py b/src/specsscan/metadata.py index 8ec96e7..f7bd4e7 100644 --- a/src/specsscan/metadata.py +++ b/src/specsscan/metadata.py @@ -250,7 +250,7 @@ def fetch_elab_metadata(self, scan: int, metadata: dict) -> dict: metadata["elabFTW"]["scan"]["probe_polarization"] = 0 # remove pump information if pump not applied: - if not metadata["elabFTW"]["scan"].get("pump_status", 0): + if not metadata["elabFTW"]["scan"].get("pump_status", 1): if "pump_photon_energy" in metadata["elabFTW"].get("laser_status", {}): del metadata["elabFTW"]["laser_status"]["pump_photon_energy"] From b36c20b669147fc7e753bce4a7f9f6f4a4e1b9ab Mon Sep 17 00:00:00 2001 From: rettigl Date: Tue, 21 Jan 2025 00:25:44 +0100 Subject: [PATCH 10/44] add tests --- tests/test_specsscan_metadata.py | 112 +++++++++++++++++++++++++++++++ 1 file changed, 112 insertions(+) create mode 100644 tests/test_specsscan_metadata.py diff --git a/tests/test_specsscan_metadata.py b/tests/test_specsscan_metadata.py new file mode 100644 index 0000000..13960de --- /dev/null +++ b/tests/test_specsscan_metadata.py @@ -0,0 +1,112 @@ +"""Tests specific for metadata retrieval""" +from __future__ import annotations + +import datetime +import json +from unittest.mock import MagicMock +from unittest.mock import patch + +import numpy as np +import pytest + +from specsscan.metadata import get_archiver_data +from specsscan.metadata import MetadataRetriever + + +@pytest.fixture +def metadata_config(): + return { + "elab_url": "http://example.com", + "epics_channels": {"channel1": "channel1"}, + "archiver_url": "http://archiver.example.com", + "aperture_config": { + datetime.datetime.fromisoformat("2023-01-01T00:00:00"): { + "fa_size": {"1.0": [(0, 1), (0, 1)]}, + "ca_size": {"1.0": (0, 1)}, + }, + }, + "lens_mode_config": {"mode1": {"lens1": 1.0, "lens2": 2.0}}, + "fa_in_channel": "fa_in", + "fa_hor_channel": "fa_hor", + "ca_in_channel": "ca_in", + } + + +@pytest.fixture +def metadata_retriever(metadata_config): + return MetadataRetriever(metadata_config, "dummy_token") + + +def test_metadata_retriever_init(metadata_retriever): + assert metadata_retriever.token == "dummy_token" + assert metadata_retriever.url == "http://example.com" + + +@patch("specsscan.metadata.urlopen") +def test_get_archiver_data(mock_urlopen): + """Test get_archiver_data using a mock of urlopen.""" + mock_response = MagicMock() + mock_response.read.return_value = json.dumps( + [{"data": [{"secs": 1, "nanos": 500000000, "val": 10}]}], + ) + mock_urlopen.return_value.__enter__.return_value = mock_response + + ts_from = datetime.datetime(2023, 1, 1).timestamp() + ts_to = datetime.datetime(2023, 1, 2).timestamp() + archiver_url = "http://archiver.example.com" + archiver_channel = "channel1" + + secs, vals = get_archiver_data(archiver_url, archiver_channel, ts_from, ts_to) + + assert np.array_equal(secs, np.array([1.5])) + assert np.array_equal(vals, np.array([10])) + + +@patch("specsscan.metadata.get_archiver_data") +def test_fetch_epics_metadata(mock_get_archiver_data, metadata_retriever): + """Test fetch_epics_metadata using a mock of get_archiver_data.""" + mock_get_archiver_data.return_value = (np.array([1.5]), np.array([10])) + metadata = {"scan_info": {}} + ts_from = datetime.datetime(2023, 1, 1).timestamp() + ts_to = datetime.datetime(2023, 1, 2).timestamp() + + updated_metadata = metadata_retriever.fetch_epics_metadata(ts_from, ts_to, metadata) + + assert updated_metadata["scan_info"]["channel1"] == 10 + + +@patch("specsscan.metadata.elabapi_python") +def test_fetch_elab_metadata(mock_elabapi_python, metadata_config): + """Test fetch_elab_metadata using a mock of elabapi_python.""" + mock_experiment = MagicMock() + mock_experiment.id = 1 + mock_experiment.userid = 1 + mock_experiment.title = "Test Experiment" + mock_experiment.body = "Test Body" + mock_experiment.metadata = json.dumps({"extra_fields": {"key": {"value": "value"}}}) + mock_elabapi_python.ExperimentsApi.return_value.read_experiments.return_value = [ + mock_experiment, + ] + mock_user = MagicMock() + mock_user.fullname = "Test User" + mock_user.email = "test@example.com" + mock_user.userid = 1 + mock_user.orcid = "0000-0000-0000-0000" + mock_elabapi_python.UsersApi.return_value.read_user.return_value = mock_user + mock_elabapi_python.LinksToItemsApi.return_value.read_entity_items_links.return_value = [] + + metadata_retriever = MetadataRetriever(metadata_config, "dummy_token") + + metadata = {} + runs = ["run1"] + + updated_metadata = metadata_retriever.fetch_elab_metadata(runs, metadata) + + assert updated_metadata["elabFTW"]["user"]["name"] == "Test User" + assert updated_metadata["elabFTW"]["user"]["email"] == "test@example.com" + assert updated_metadata["elabFTW"]["user"]["id"] == 1 + assert updated_metadata["elabFTW"]["user"]["orcid"] == "0000-0000-0000-0000" + assert updated_metadata["elabFTW"]["scan"]["title"] == "Test Experiment" + assert updated_metadata["elabFTW"]["scan"]["summary"] == "Test Body" + assert updated_metadata["elabFTW"]["scan"]["id"] == 1 + assert updated_metadata["elabFTW"]["scan"]["key"] == "value" From 24e6bc4f3c8a8dff7607f97bf67f772613194fea Mon Sep 17 00:00:00 2001 From: rettigl Date: Tue, 21 Jan 2025 00:31:38 +0100 Subject: [PATCH 11/44] remove call logger --- src/specsanalyzer/logging.py | 21 --------------------- 1 file changed, 21 deletions(-) diff --git a/src/specsanalyzer/logging.py b/src/specsanalyzer/logging.py index cd51ac0..bcb1977 100644 --- a/src/specsanalyzer/logging.py +++ b/src/specsanalyzer/logging.py @@ -10,8 +10,6 @@ import os import sys from datetime import datetime -from functools import wraps -from typing import Callable # Default log directory DEFAULT_LOG_DIR = os.path.join(os.getcwd(), "logs") @@ -101,22 +99,3 @@ def set_verbosity(logger: logging.Logger, verbose: bool) -> None: handler.setLevel(logging.INFO) else: handler.setLevel(logging.WARNING) - - -def call_logger(logger: logging.Logger): - def log_call(func: Callable): - @wraps(func) - def new_func(*args, **kwargs): - saved_args = locals() - args_str = "" - for arg in saved_args["args"][1:]: - args_str += f"{arg}, " - for name, arg in saved_args["kwargs"].items(): - args_str += f"{name}={arg}, " - args_str = args_str.rstrip(", ") - logger.debug(f"Call {func.__name__}({args_str})") - return func(*args, **kwargs) - - return new_func - - return log_call From 0ebee0e7a67a2046d7972167c2084a4528e957a3 Mon Sep 17 00:00:00 2001 From: rettigl Date: Wed, 22 Jan 2025 16:02:20 +0100 Subject: [PATCH 12/44] small fixes --- src/specsscan/metadata.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/specsscan/metadata.py b/src/specsscan/metadata.py index f7bd4e7..de9425b 100644 --- a/src/specsscan/metadata.py +++ b/src/specsscan/metadata.py @@ -194,7 +194,7 @@ def fetch_elab_metadata(self, scan: int, metadata: dict) -> dict: if val["value"] is not None and val["value"] != "" and val["value"] != ["None"]: try: metadata["elabFTW"][category][key] = float(val["value"]) - except ValueError: + except (ValueError, TypeError): metadata["elabFTW"][category][key] = val["value"] # group beam profiles: From a0bf9d21bc52aaeda5d558ee3e52f96d60cbd588 Mon Sep 17 00:00:00 2001 From: rettigl Date: Wed, 22 Jan 2025 21:06:06 +0100 Subject: [PATCH 13/44] reset metadata and config --- src/specsanalyzer/config.py | 11 ++++++----- src/specsscan/core.py | 6 ++++++ 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/src/specsanalyzer/config.py b/src/specsanalyzer/config.py index f3806ef..519e9da 100755 --- a/src/specsanalyzer/config.py +++ b/src/specsanalyzer/config.py @@ -1,6 +1,7 @@ """This module contains a config library for loading yaml/json files into dicts""" from __future__ import annotations +import copy import json import os import platform @@ -75,7 +76,7 @@ def parse_config( config = {} if isinstance(config, dict): - config_dict = config + config_dict = copy.deepcopy(config) else: config_dict = load_config(config) if verbose: @@ -83,7 +84,7 @@ def parse_config( folder_dict: dict = None if isinstance(folder_config, dict): - folder_dict = folder_config + folder_dict = copy.deepcopy(folder_config) else: if folder_config is None: folder_config = "./specs_config.yaml" @@ -94,7 +95,7 @@ def parse_config( user_dict: dict = None if isinstance(user_config, dict): - user_dict = user_config + user_dict = copy.deepcopy(user_config) else: if user_config is None: user_config = str(USER_CONFIG_PATH.joinpath("config_v1.yaml")) @@ -105,7 +106,7 @@ def parse_config( system_dict: dict = None if isinstance(system_config, dict): - system_dict = system_config + system_dict = copy.deepcopy(system_config) else: if system_config is None: system_config = str(SYSTEM_CONFIG_PATH.joinpath("config_v1.yaml")) @@ -115,7 +116,7 @@ def parse_config( logger.info(f"System config loaded from: [{str(Path(system_config).resolve())}]") if isinstance(default_config, dict): - default_dict = default_config + default_dict = copy.deepcopy(default_config) else: default_dict = load_config(default_config) if verbose: diff --git a/src/specsscan/core.py b/src/specsscan/core.py index 8a0dcaf..5676e7d 100755 --- a/src/specsscan/core.py +++ b/src/specsscan/core.py @@ -282,6 +282,9 @@ def load_scan( except KeyError: pass + # reset metadata + self.metadata = {} + self.metadata.update( **handle_meta( df_lut=df_lut, @@ -540,6 +543,9 @@ def check_scan( fast_axes = set(res_xarray.dims) - slow_axes projection = "reciprocal" if "Angle" in fast_axes else "real" + # reset metadata + self.metadata = {} + self.metadata.update( **handle_meta( df_lut=df_lut, From b9478cfb622b3cb461f7e13ab415265704bdb7d7 Mon Sep 17 00:00:00 2001 From: rettigl Date: Wed, 22 Jan 2025 21:38:55 +0100 Subject: [PATCH 14/44] fix tests --- tests/test_specsscan.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_specsscan.py b/tests/test_specsscan.py index 3511b93..713ba3c 100755 --- a/tests/test_specsscan.py +++ b/tests/test_specsscan.py @@ -277,7 +277,6 @@ def test_fft_tool(): sigma_y=23, apply=True, ) - assert sps.config["spa_params"]["fft_filter_peaks"] == fft_filter_peaks assert sps.spa.config["fft_filter_peaks"] == fft_filter_peaks res_xarray = sps.load_scan(scan=3610, path=data_dir, apply_fft_filter=True) np.testing.assert_almost_equal(res_xarray.data.sum(), 62197237155.50347, decimal=3) From 38513cb2d4bf6fcb48ecf84220adee7cee238379 Mon Sep 17 00:00:00 2001 From: rettigl Date: Thu, 23 Jan 2025 14:23:10 +0100 Subject: [PATCH 15/44] add metadata support for pump2 --- src/specsscan/core.py | 16 ++++------------ src/specsscan/metadata.py | 25 ++++++++++++++++++++++++- 2 files changed, 28 insertions(+), 13 deletions(-) diff --git a/src/specsscan/core.py b/src/specsscan/core.py index 5676e7d..039d7b4 100755 --- a/src/specsscan/core.py +++ b/src/specsscan/core.py @@ -254,6 +254,10 @@ def load_scan( if k in res_xarray.dims } + # store data for resolved axis coordinates + for axis in res_xarray.dims: + self._scan_info[axis] = res_xarray.coords[axis].data + res_xarray = res_xarray.rename(rename_dict) for k, v in coordinate_mapping.items(): if k in fast_axes: @@ -264,18 +268,6 @@ def load_scan( slow_axes.add(v) self._scan_info["coordinate_depends"] = depends_dict - axis_dict = { - "/entry/sample/transformations/sample_polar": "Polar", - "/entry/sample/transformations/sample_tilt": "Tilt", - "/entry/sample/transformations/sample_azimuth": "Azimuth", - "/entry/instrument/beam_pump/pulse_delay": "delay", - } - - # store data for resolved axis coordinates - for k, v in depends_dict.items(): - if v in axis_dict: - self._scan_info[axis_dict[v]] = res_xarray.coords[k].data - for name in res_xarray.dims: try: res_xarray[name].attrs["unit"] = self._config["units"][name] diff --git a/src/specsscan/metadata.py b/src/specsscan/metadata.py index de9425b..643d2fc 100644 --- a/src/specsscan/metadata.py +++ b/src/specsscan/metadata.py @@ -216,6 +216,15 @@ def fetch_elab_metadata(self, scan: int, metadata: dict) -> dict: float(metadata["elabFTW"]["laser_status"]["probe_profile_x"]), float(metadata["elabFTW"]["laser_status"]["probe_profile_y"]), ] + if ( + "laser_status" in metadata["elabFTW"] + and "pump2_profile_x" in metadata["elabFTW"]["laser_status"] + and "pump2_profile_y" in metadata["elabFTW"]["laser_status"] + ): + metadata["elabFTW"]["laser_status"]["pump2_profile"] = [ + float(metadata["elabFTW"]["laser_status"]["pump2_profile_x"]), + float(metadata["elabFTW"]["laser_status"]["pump2_profile_y"]), + ] # fix preparation date if "sample" in metadata["elabFTW"] and "preparation_date" in metadata["elabFTW"]["sample"]: @@ -249,11 +258,25 @@ def fetch_elab_metadata(self, scan: int, metadata: dict) -> dict: elif metadata["elabFTW"]["scan"]["probe_polarization"] == "p": metadata["elabFTW"]["scan"]["probe_polarization"] = 0 + if ( + "scan" in metadata["elabFTW"] + and "pump2_polarization" in metadata["elabFTW"]["scan"] + and isinstance(metadata["elabFTW"]["scan"]["pump2_polarization"], str) + ): + if metadata["elabFTW"]["scan"]["pump2_polarization"] == "s": + metadata["elabFTW"]["scan"]["pump2_polarization"] = 90 + elif metadata["elabFTW"]["scan"]["pump2_polarization"] == "p": + metadata["elabFTW"]["scan"]["pump2_polarization"] = 0 + # remove pump information if pump not applied: - if not metadata["elabFTW"]["scan"].get("pump_status", 1): + if metadata["elabFTW"]["scan"].get("pump_status", "closed") == "closed": if "pump_photon_energy" in metadata["elabFTW"].get("laser_status", {}): del metadata["elabFTW"]["laser_status"]["pump_photon_energy"] + if metadata["elabFTW"]["scan"].get("pump2_status", "closed") == "closed": + if "pump2_photon_energy" in metadata["elabFTW"].get("laser_status", {}): + del metadata["elabFTW"]["laser_status"]["pump2_photon_energy"] + return metadata From 1050b1d9c3b70e49267ea8ff0ea55f82b34bb84b Mon Sep 17 00:00:00 2001 From: rettigl Date: Sun, 26 Jan 2025 21:40:50 +0100 Subject: [PATCH 16/44] return as float32 to save memory --- src/specsanalyzer/core.py | 2 +- tests/test_specsscan.py | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/specsanalyzer/core.py b/src/specsanalyzer/core.py index b5a8922..7e80ef1 100755 --- a/src/specsanalyzer/core.py +++ b/src/specsanalyzer/core.py @@ -280,7 +280,7 @@ def convert_image( }, dims=conversion_parameters["dims"], attrs={"conversion_parameters": conversion_parameters}, - ) + ).astype(np.float32) # Handle cropping based on parameters stored in correction dictionary crop = kwds.pop("crop", self._config.get("crop", False)) diff --git a/tests/test_specsscan.py b/tests/test_specsscan.py index 713ba3c..ed39670 100755 --- a/tests/test_specsscan.py +++ b/tests/test_specsscan.py @@ -68,7 +68,7 @@ def test_conversion_3d(): path=data_dir, iterations=np.s_[0:2], ) - np.testing.assert_allclose(res_xarray, res_xarray2) + np.testing.assert_allclose(res_xarray, res_xarray2, rtol=1e-6) with pytest.raises(IndexError): sps.load_scan( @@ -259,7 +259,7 @@ def test_fft_tool(): apply_fft_filter=False, ) - np.testing.assert_almost_equal(res_xarray.data.sum(), 62145561928.15108, decimal=3) + np.testing.assert_allclose(res_xarray.data.sum(), 62145556000.0) res_xarray = sps.load_scan( scan=3610, @@ -267,7 +267,7 @@ def test_fft_tool(): fft_filter_peaks=fft_filter_peaks, apply_fft_filter=True, ) - np.testing.assert_almost_equal(res_xarray.data.sum(), 62197237155.50347, decimal=3) + np.testing.assert_allclose(res_xarray.data.sum(), 62197240000.0) sps.fft_tool( amplitude=1, @@ -279,7 +279,7 @@ def test_fft_tool(): ) assert sps.spa.config["fft_filter_peaks"] == fft_filter_peaks res_xarray = sps.load_scan(scan=3610, path=data_dir, apply_fft_filter=True) - np.testing.assert_almost_equal(res_xarray.data.sum(), 62197237155.50347, decimal=3) + np.testing.assert_allclose(res_xarray.data.sum(), 62197240000.0) def test_conversion_and_save_to_nexus(): From 7d57d01a55e486e78364795af63e5ae905adaf0d Mon Sep 17 00:00:00 2001 From: rettigl Date: Tue, 28 Jan 2025 16:01:32 +0100 Subject: [PATCH 17/44] add further tests and fixes --- .cspell/custom-dictionary.txt | 1 + docs/specsscan/helpers.rst | 4 + src/specsanalyzer/config.py | 3 +- src/specsscan/metadata.py | 28 +++++-- tests/test_config.py | 139 +++++++++++++++++++++++++++++++ tests/test_specsscan_metadata.py | 42 +++++++++- 6 files changed, 209 insertions(+), 8 deletions(-) diff --git a/.cspell/custom-dictionary.txt b/.cspell/custom-dictionary.txt index 522f42e..9ce01cb 100644 --- a/.cspell/custom-dictionary.txt +++ b/.cspell/custom-dictionary.txt @@ -34,6 +34,7 @@ dapolymatrix dataconverter dataframe delaystage +delenv dtype dxda dxde diff --git a/docs/specsscan/helpers.rst b/docs/specsscan/helpers.rst index aeca3c1..dc4ce92 100644 --- a/docs/specsscan/helpers.rst +++ b/docs/specsscan/helpers.rst @@ -3,3 +3,7 @@ Helpers .. automodule:: specsscan.helpers :members: :undoc-members: + +.. automodule:: specsscan.metadata + :members: + :undoc-members: diff --git a/src/specsanalyzer/config.py b/src/specsanalyzer/config.py index 519e9da..edeb50f 100755 --- a/src/specsanalyzer/config.py +++ b/src/specsanalyzer/config.py @@ -25,6 +25,7 @@ if platform.system() == "Windows" else Path("/etc/").joinpath("specsanalyzer") ) +ENV_DIR = Path(".env") # Configure logging logger = setup_logging("config") @@ -276,7 +277,7 @@ def read_env_var(var_name: str) -> str | None: return value # 2. check .env in current directory - local_vars = _parse_env_file(Path(".env")) + local_vars = _parse_env_file(ENV_DIR) if var_name in local_vars: logger.debug(f"Found {var_name} in ./.env file") return local_vars[var_name] diff --git a/src/specsscan/metadata.py b/src/specsscan/metadata.py index 643d2fc..e2c1776 100644 --- a/src/specsscan/metadata.py +++ b/src/specsscan/metadata.py @@ -6,6 +6,7 @@ import datetime import json +from copy import deepcopy from urllib.error import HTTPError from urllib.error import URLError from urllib.request import urlopen @@ -36,6 +37,8 @@ def __init__(self, metadata_config: dict, token: str = None) -> None: token (str, optional): The token to use for fetching metadata. If provided, will be saved to .env file for future use. """ + self._config = deepcopy(metadata_config) + # Token handling if token: self.token = token @@ -44,17 +47,24 @@ def __init__(self, metadata_config: dict, token: str = None) -> None: # Try to load token from config or .env file self.token = read_env_var("ELAB_TOKEN") - self._config = metadata_config + if not self.token: + logger.warning( + "No valid token provided for elabFTW. Fetching elabFTW metadata will be skipped.", + ) + return - self.url = str(metadata_config.get("elab_url")) + self.url = metadata_config.get("elab_url") if not self.url: - raise ValueError("No URL provided for fetching metadata from elabFTW.") + logger.warning( + "No URL provided for elabFTW. Fetching elabFTW metadata will be skipped.", + ) + return # Config self.configuration = elabapi_python.Configuration() self.configuration.api_key["api_key"] = self.token self.configuration.api_key_prefix["api_key"] = "Authorization" - self.configuration.host = self.url + self.configuration.host = str(self.url) self.configuration.debug = False self.configuration.verify_ssl = False @@ -95,7 +105,7 @@ def fetch_epics_metadata(self, ts_from: float, ts_to: float, metadata: dict) -> except KeyError: epics_channels = [] - channels_missing = set(epics_channels) - set(metadata["scan_info"].keys()) + channels_missing = set(epics_channels) - set(metadata.get("scan_info", {}).keys()) if channels_missing: logger.info("Collecting data from the EPICS archive...") for channel in channels_missing: @@ -147,6 +157,14 @@ def fetch_elab_metadata(self, scan: int, metadata: dict) -> dict: "a token parameter or set the ELAB_TOKEN environment variable.", ) return metadata + + if not self.url: + logger.warning( + "No URL provided for fetching metadata from elabFTW. " + "Fetching elabFTW metadata will be skipped.", + ) + return metadata + logger.info("Collecting data from the elabFTW instance...") # Get the experiment try: diff --git a/tests/test_config.py b/tests/test_config.py index 20742d0..5b3b091 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -8,7 +8,9 @@ from specsanalyzer.config import complete_dictionary from specsanalyzer.config import load_config from specsanalyzer.config import parse_config +from specsanalyzer.config import read_env_var from specsanalyzer.config import save_config +from specsanalyzer.config import save_env_var test_dir = os.path.dirname(__file__) default_config_keys = [ @@ -88,3 +90,140 @@ def test_save_dict(): save_config(config_dict, filename, overwrite=True) config = load_config(filename) assert "test_entry" not in config.keys() + + +@pytest.fixture +def mock_env_file(tmp_path, monkeypatch): + """Mock the .env file for testing""" + monkeypatch.setattr("specsanalyzer.config.USER_CONFIG_PATH", tmp_path) + yield tmp_path + + +def test_env_var_read_write(mock_env_file): # noqa: ARG001 + """Test reading and writing environment variables.""" + # Test writing a new variable + save_env_var("TEST_VAR", "test_value") + assert read_env_var("TEST_VAR") == "test_value" + + # Test writing multiple variables + save_env_var("TEST_VAR2", "test_value2") + assert read_env_var("TEST_VAR") == "test_value" + assert read_env_var("TEST_VAR2") == "test_value2" + + # Test overwriting an existing variable + save_env_var("TEST_VAR", "new_value") + assert read_env_var("TEST_VAR") == "new_value" + assert read_env_var("TEST_VAR2") == "test_value2" # Other variables unchanged + + # Test reading non-existent variable + assert read_env_var("NON_EXISTENT_VAR") is None + + +def test_env_var_read_no_file(mock_env_file): # noqa: ARG001 + """Test reading environment variables when .env file doesn't exist.""" + # Test reading from non-existent file + assert read_env_var("TEST_VAR") is None + + +def test_env_var_special_characters(mock_env_file): # noqa: ARG001 + """Test reading and writing environment variables with special characters.""" + test_cases = { + "TEST_URL": "http://example.com/path?query=value", + "TEST_PATH": "/path/to/something/with/spaces and special=chars", + "TEST_QUOTES": "value with 'single' and \"double\" quotes", + } + + for var_name, value in test_cases.items(): + save_env_var(var_name, value) + assert read_env_var(var_name) == value + + +def test_env_var_precedence(mock_env_file, tmp_path, monkeypatch): # noqa: ARG001 + """Test that environment variables are read in correct order of precedence""" + # Create local .env directory if it doesn't exist + local_env_dir = tmp_path / "local" + local_env_dir.mkdir(exist_ok=True) + system_env_dir = tmp_path / "system" + system_env_dir.mkdir(exist_ok=True) + monkeypatch.setattr("specsanalyzer.config.ENV_DIR", local_env_dir / ".env") + monkeypatch.setattr("specsanalyzer.config.SYSTEM_CONFIG_PATH", system_env_dir) + + # Set up test values in different locations + os.environ["TEST_VAR"] = "os_value" + + # Save to system config first (4th precedence) + with open(system_env_dir / ".env", "w") as f: + f.write("TEST_VAR=system_value\n") + + # Save to user config first (3rd precedence) + save_env_var("TEST_VAR", "user_value") + + # Create local .env file (2nd precedence) + with open(local_env_dir / ".env", "w") as f: + f.write("TEST_VAR=local_value\n") + + assert read_env_var("TEST_VAR") == "os_value" + + # Remove from OS env to test other precedence levels + monkeypatch.delenv("TEST_VAR", raising=False) + assert read_env_var("TEST_VAR") == "local_value" + + # Remove local .env and should get user config value + (local_env_dir / ".env").unlink() + assert read_env_var("TEST_VAR") == "user_value" + + # Remove user config and should get system value + (mock_env_file / ".env").unlink() + assert read_env_var("TEST_VAR") == "system_value" + + # Remove system config and should get None + (system_env_dir / ".env").unlink() + assert read_env_var("TEST_VAR") is None + + +def test_env_var_save_and_load(mock_env_file, monkeypatch): # noqa: ARG001 + """Test saving and loading environment variables""" + # Clear any existing OS environment variables + monkeypatch.delenv("TEST_VAR", raising=False) + monkeypatch.delenv("OTHER_VAR", raising=False) + + # Save a variable + save_env_var("TEST_VAR", "test_value") + + # Should be able to read it back + assert read_env_var("TEST_VAR") == "test_value" + + # Save another variable - should preserve existing ones + save_env_var("OTHER_VAR", "other_value") + assert read_env_var("TEST_VAR") == "test_value" + assert read_env_var("OTHER_VAR") == "other_value" + + +def test_env_var_not_found(mock_env_file): # noqa: ARG001 + """Test behavior when environment variable is not found""" + assert read_env_var("NONEXISTENT_VAR") is None + + +def test_env_file_format(mock_env_file, monkeypatch): # noqa: ARG001 + """Test that .env file parsing handles different formats correctly""" + # Clear any existing OS environment variables + monkeypatch.delenv("TEST_VAR", raising=False) + monkeypatch.delenv("SPACES_VAR", raising=False) + monkeypatch.delenv("EMPTY_VAR", raising=False) + monkeypatch.delenv("COMMENT", raising=False) + + with open(mock_env_file / ".env", "w") as f: + f.write( + """ + TEST_VAR=value1 + SPACES_VAR = value2 + EMPTY_VAR= + #COMMENT=value3 + INVALID_LINE + """, + ) + + assert read_env_var("TEST_VAR") == "value1" + assert read_env_var("SPACES_VAR") == "value2" + assert read_env_var("EMPTY_VAR") == "" + assert read_env_var("COMMENT") is None diff --git a/tests/test_specsscan_metadata.py b/tests/test_specsscan_metadata.py index 13960de..68e44f3 100644 --- a/tests/test_specsscan_metadata.py +++ b/tests/test_specsscan_metadata.py @@ -11,6 +11,7 @@ from specsscan.metadata import get_archiver_data from specsscan.metadata import MetadataRetriever +from tests.test_config import mock_env_file # noqa: F401 @pytest.fixture @@ -33,7 +34,7 @@ def metadata_config(): @pytest.fixture -def metadata_retriever(metadata_config): +def metadata_retriever(metadata_config, mock_env_file): # noqa: ARG001 return MetadataRetriever(metadata_config, "dummy_token") @@ -42,6 +43,30 @@ def test_metadata_retriever_init(metadata_retriever): assert metadata_retriever.url == "http://example.com" +def test_metadata_retriever_no_token(metadata_config, tmp_path, monkeypatch): + monkeypatch.setattr("specsanalyzer.config.ENV_DIR", tmp_path / ".dummy_env") + monkeypatch.setattr("specsanalyzer.config.SYSTEM_CONFIG_PATH", tmp_path / "system") + monkeypatch.setattr("specsanalyzer.config.USER_CONFIG_PATH", tmp_path / "user") + retriever = MetadataRetriever(metadata_config) + assert retriever.token is None + + metadata = {} + runs = ["run1"] + updated_metadata = retriever.fetch_elab_metadata(runs, metadata) + assert updated_metadata == metadata + + +def test_metadata_retriever_no_url(metadata_config, mock_env_file): # noqa: ARG001 + metadata_config.pop("elab_url") + retriever = MetadataRetriever(metadata_config, "dummy_token") + assert retriever.url is None + + metadata = {} + runs = ["run1"] + updated_metadata = retriever.fetch_elab_metadata(runs, metadata) + assert updated_metadata == metadata + + @patch("specsscan.metadata.urlopen") def test_get_archiver_data(mock_urlopen): """Test get_archiver_data using a mock of urlopen.""" @@ -75,8 +100,21 @@ def test_fetch_epics_metadata(mock_get_archiver_data, metadata_retriever): assert updated_metadata["scan_info"]["channel1"] == 10 +@patch("sed.loader.mpes.metadata.get_archiver_data") +def test_fetch_epics_metadata_missing_channels(mock_get_archiver_data, metadata_retriever): + """Test fetch_epics_metadata with missing EPICS channels.""" + mock_get_archiver_data.return_value = (np.array([1.5]), np.array([10])) + metadata = {"file": {"channel1": 10}} + ts_from = datetime.datetime(2023, 1, 1).timestamp() + ts_to = datetime.datetime(2023, 1, 2).timestamp() + + updated_metadata = metadata_retriever.fetch_epics_metadata(ts_from, ts_to, metadata) + + assert "channel1" in updated_metadata["file"] + + @patch("specsscan.metadata.elabapi_python") -def test_fetch_elab_metadata(mock_elabapi_python, metadata_config): +def test_fetch_elab_metadata(mock_elabapi_python, metadata_config, mock_env_file): # noqa: ARG001 """Test fetch_elab_metadata using a mock of elabapi_python.""" mock_experiment = MagicMock() mock_experiment.id = 1 From d5858d559e9fd7336c32a28c05f0b26be13f676e Mon Sep 17 00:00:00 2001 From: rettigl Date: Tue, 28 Jan 2025 16:04:36 +0100 Subject: [PATCH 18/44] rename metadata tests --- tests/{test_specsscan_metadata.py => test_metadata.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename tests/{test_specsscan_metadata.py => test_metadata.py} (100%) diff --git a/tests/test_specsscan_metadata.py b/tests/test_metadata.py similarity index 100% rename from tests/test_specsscan_metadata.py rename to tests/test_metadata.py From 29ce768c419a3f57a162d867b5125fdfb56ade20 Mon Sep 17 00:00:00 2001 From: rettigl Date: Tue, 28 Jan 2025 16:10:01 +0100 Subject: [PATCH 19/44] fix typo --- tests/test_metadata.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_metadata.py b/tests/test_metadata.py index 68e44f3..064643f 100644 --- a/tests/test_metadata.py +++ b/tests/test_metadata.py @@ -100,7 +100,7 @@ def test_fetch_epics_metadata(mock_get_archiver_data, metadata_retriever): assert updated_metadata["scan_info"]["channel1"] == 10 -@patch("sed.loader.mpes.metadata.get_archiver_data") +@patch("specsscan.metadata.get_archiver_data") def test_fetch_epics_metadata_missing_channels(mock_get_archiver_data, metadata_retriever): """Test fetch_epics_metadata with missing EPICS channels.""" mock_get_archiver_data.return_value = (np.array([1.5]), np.array([10])) From c9a88558bcef61a0b4c2665d04d99440e214f216 Mon Sep 17 00:00:00 2001 From: rettigl Date: Tue, 28 Jan 2025 16:27:14 +0100 Subject: [PATCH 20/44] remove coverage restrictions --- pyproject.toml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 6b9ba34..b10deec 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -84,12 +84,6 @@ all = [ "specsanalyzer[dev,docs,notebook]", ] -[tool.coverage.report] -omit = [ - "config.py", - "config-3.py", -] - [tool.ruff] include = ["specsanalyzer/*.py", "specsscan/*.py", "tests/*.py"] lint.select = [ From af1191b7d288077e19e552108f0a60e3693624b1 Mon Sep 17 00:00:00 2001 From: rettigl Date: Wed, 29 Jan 2025 20:44:11 +0100 Subject: [PATCH 21/44] fix matplotlib warnings --- pyproject.toml | 2 +- src/specsanalyzer/core.py | 4 ++-- src/specsscan/core.py | 3 +-- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index b10deec..22e74e2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -37,7 +37,7 @@ dependencies = [ "imutils>=0.5.4", "ipympl>=0.9.1", "ipywidgets>=7.7.1", - "matplotlib>=3.5.1,<3.10.0", + "matplotlib>=3.5.1", "numpy>=1.21.6", "opencv-python>=4.8.1.78", "pynxtools-mpes>=0.2.1", diff --git a/src/specsanalyzer/core.py b/src/specsanalyzer/core.py index 7e80ef1..be7d74b 100755 --- a/src/specsanalyzer/core.py +++ b/src/specsanalyzer/core.py @@ -753,8 +753,8 @@ def update(v_vals, pos_x, pos_y, sigma_x, sigma_y, amplitude): fft_filt.set_data(np.abs(fft_filtered_new.T)) nonlocal cont - for i in range(len(cont.collections)): - cont.collections[i].remove() + for path in cont.get_paths(): + path.remove() cont = ax.contour(msk.T) edc.set_ydata(np.sum(filtered_new, 0)) diff --git a/src/specsscan/core.py b/src/specsscan/core.py index 039d7b4..d257c05 100755 --- a/src/specsscan/core.py +++ b/src/specsscan/core.py @@ -5,7 +5,6 @@ import os import pathlib from importlib.util import find_spec -from logging import warn from pathlib import Path from typing import Any from typing import Sequence @@ -671,7 +670,7 @@ def process_sweep_scan( """ ekin_step = kinetic_energy[1] - kinetic_energy[0] if not (np.diff(kinetic_energy) == ekin_step).all(): - warn( + logger.warning( "Conversion of sweep scans with non-equidistant energy steps " "might produce wrong results!", ) From e3871b37d8530845d79bf35505f853def6ea87d7 Mon Sep 17 00:00:00 2001 From: rettigl Date: Wed, 29 Jan 2025 21:00:00 +0100 Subject: [PATCH 22/44] fix time-zone warning --- src/specsscan/helpers.py | 4 ++-- src/specsscan/metadata.py | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/specsscan/helpers.py b/src/specsscan/helpers.py index f67d580..8980d7c 100644 --- a/src/specsscan/helpers.py +++ b/src/specsscan/helpers.py @@ -423,10 +423,10 @@ def handle_meta( ts_from = dt.datetime.timestamp(datetime_list[0]) # POSIX timestamp ts_to = dt.datetime.timestamp(datetime_list[-1]) # POSIX timestamp metadata["timing"] = { - "acquisition_start": dt.datetime.utcfromtimestamp(ts_from) + "acquisition_start": dt.datetime.fromtimestamp(ts_from, dt.timezone.utc) .replace(tzinfo=dt.timezone.utc) .isoformat(), - "acquisition_stop": dt.datetime.utcfromtimestamp(ts_to) + "acquisition_stop": dt.datetime.fromtimestamp(ts_to, dt.timezone.utc) .replace(tzinfo=dt.timezone.utc) .isoformat(), "acquisition_duration": int(ts_to - ts_from), diff --git a/src/specsscan/metadata.py b/src/specsscan/metadata.py index e2c1776..fb8c56a 100644 --- a/src/specsscan/metadata.py +++ b/src/specsscan/metadata.py @@ -92,7 +92,7 @@ def fetch_epics_metadata(self, ts_from: float, ts_to: float, metadata: dict) -> Returns: dict: Updated metadata dictionary. """ - start = datetime.datetime.utcfromtimestamp(ts_from) + start = datetime.datetime.fromtimestamp(ts_from, datetime.timezone.utc).isoformat() # replace metadata names by epics channels try: @@ -315,8 +315,8 @@ def get_archiver_data( Returns: tuple[np.ndarray, np.ndarray]: The extracted time stamps and corresponding data """ - iso_from = datetime.datetime.utcfromtimestamp(ts_from).isoformat() - iso_to = datetime.datetime.utcfromtimestamp(ts_to).isoformat() + iso_from = datetime.datetime.fromtimestamp(ts_from, datetime.timezone.utc).isoformat() + iso_to = datetime.datetime.fromtimestamp(ts_to, datetime.timezone.utc).isoformat() req_str = archiver_url + archiver_channel + "&from=" + iso_from + "Z&to=" + iso_to + "Z" with urlopen(req_str) as req: data = json.load(req) From 20d53bd87b24afa0dd8940269bab4dcb02e63834 Mon Sep 17 00:00:00 2001 From: rettigl Date: Mon, 3 Feb 2025 10:11:52 +0100 Subject: [PATCH 23/44] properly remove contours --- src/specsanalyzer/core.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/specsanalyzer/core.py b/src/specsanalyzer/core.py index be7d74b..f8b07d9 100755 --- a/src/specsanalyzer/core.py +++ b/src/specsanalyzer/core.py @@ -753,8 +753,7 @@ def update(v_vals, pos_x, pos_y, sigma_x, sigma_y, amplitude): fft_filt.set_data(np.abs(fft_filtered_new.T)) nonlocal cont - for path in cont.get_paths(): - path.remove() + cont.remove() cont = ax.contour(msk.T) edc.set_ydata(np.sum(filtered_new, 0)) From d74b10a60e7921990cfe4be8b14b17f9636075e0 Mon Sep 17 00:00:00 2001 From: rettigl Date: Mon, 3 Feb 2025 10:42:00 +0100 Subject: [PATCH 24/44] correct time stamp formatting --- src/specsscan/helpers.py | 8 ++------ src/specsscan/metadata.py | 12 ++++++++++-- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/src/specsscan/helpers.py b/src/specsscan/helpers.py index 8980d7c..7a026fd 100644 --- a/src/specsscan/helpers.py +++ b/src/specsscan/helpers.py @@ -423,12 +423,8 @@ def handle_meta( ts_from = dt.datetime.timestamp(datetime_list[0]) # POSIX timestamp ts_to = dt.datetime.timestamp(datetime_list[-1]) # POSIX timestamp metadata["timing"] = { - "acquisition_start": dt.datetime.fromtimestamp(ts_from, dt.timezone.utc) - .replace(tzinfo=dt.timezone.utc) - .isoformat(), - "acquisition_stop": dt.datetime.fromtimestamp(ts_to, dt.timezone.utc) - .replace(tzinfo=dt.timezone.utc) - .isoformat(), + "acquisition_start": dt.datetime.fromtimestamp(ts_from, dt.timezone.utc).isoformat(), + "acquisition_stop": dt.datetime.fromtimestamp(ts_to, dt.timezone.utc).isoformat(), "acquisition_duration": int(ts_to - ts_from), "collection_time": float(ts_to - ts_from), } diff --git a/src/specsscan/metadata.py b/src/specsscan/metadata.py index fb8c56a..6e1f062 100644 --- a/src/specsscan/metadata.py +++ b/src/specsscan/metadata.py @@ -315,8 +315,16 @@ def get_archiver_data( Returns: tuple[np.ndarray, np.ndarray]: The extracted time stamps and corresponding data """ - iso_from = datetime.datetime.fromtimestamp(ts_from, datetime.timezone.utc).isoformat() - iso_to = datetime.datetime.fromtimestamp(ts_to, datetime.timezone.utc).isoformat() + iso_from = ( + datetime.datetime.fromtimestamp(ts_from, datetime.timezone.utc) + .replace(tzinfo=None) + .isoformat() + ) + iso_to = ( + datetime.datetime.fromtimestamp(ts_to, datetime.timezone.utc) + .replace(tzinfo=None) + .isoformat() + ) req_str = archiver_url + archiver_channel + "&from=" + iso_from + "Z&to=" + iso_to + "Z" with urlopen(req_str) as req: data = json.load(req) From 0d337da9d54901c666e695a95a7bf18557254b9b Mon Sep 17 00:00:00 2001 From: rettigl Date: Sat, 8 Feb 2025 18:15:14 +0100 Subject: [PATCH 25/44] add logging tests, and fix issue if cwd is not writable --- .cspell/custom-dictionary.txt | 1 + src/specsanalyzer/logging.py | 38 +++++++++------- tests/test_logging.py | 86 +++++++++++++++++++++++++++++++++++ 3 files changed, 108 insertions(+), 17 deletions(-) create mode 100644 tests/test_logging.py diff --git a/.cspell/custom-dictionary.txt b/.cspell/custom-dictionary.txt index 9ce01cb..6ac077a 100644 --- a/.cspell/custom-dictionary.txt +++ b/.cspell/custom-dictionary.txt @@ -20,6 +20,7 @@ basepath bitshift bysource calib +caplog checkscan clim codemirror diff --git a/src/specsanalyzer/logging.py b/src/specsanalyzer/logging.py index bcb1977..3b5a863 100644 --- a/src/specsanalyzer/logging.py +++ b/src/specsanalyzer/logging.py @@ -41,28 +41,32 @@ def setup_logging( # Create base logger base_logger = logging.getLogger("specsanalyzer") base_logger.setLevel(logging.DEBUG) # Set the minimum log level for the logger - if set_base_handler or not base_logger.hasHandlers(): - if base_logger.hasHandlers(): + if set_base_handler or len(base_logger.handlers) == 0: + if len(base_logger.handlers): base_logger.handlers.clear() # Determine log file path if user_log_path is None: user_log_path = DEFAULT_LOG_DIR - os.makedirs(user_log_path, exist_ok=True) - log_file = os.path.join(user_log_path, f"sed_{datetime.now().strftime('%Y-%m-%d')}.log") - - # Create file handler and set level to debug - file_handler = logging.FileHandler(log_file) - file_handler.setLevel(FILE_VERBOSITY) - - # Create formatter for file - file_formatter = logging.Formatter( - "%(asctime)s - %(name)s - %(levelname)s - %(message)s in %(filename)s:%(lineno)d", - ) - file_handler.setFormatter(file_formatter) - - # Add file handler to logger - base_logger.addHandler(file_handler) + try: + os.makedirs(user_log_path, exist_ok=True) + log_file = os.path.join(user_log_path, f"sed_{datetime.now().strftime('%Y-%m-%d')}.log") + + # Create file handler and set level to debug + file_handler = logging.FileHandler(log_file) + file_handler.setLevel(FILE_VERBOSITY) + + # Create formatter for file + file_formatter = logging.Formatter( + "%(asctime)s - %(name)s - %(levelname)s - %(message)s in %(filename)s:%(lineno)d", + ) + file_handler.setFormatter(file_formatter) + + # Add file handler to logger + base_logger.addHandler(file_handler) + except PermissionError: + logging.warning(f"Cannot create logfile in Folder {user_log_path}, disabling logfile.") + base_logger.addHandler(logging.NullHandler()) # create named logger logger = base_logger.getChild(name) diff --git a/tests/test_logging.py b/tests/test_logging.py new file mode 100644 index 0000000..e78aba0 --- /dev/null +++ b/tests/test_logging.py @@ -0,0 +1,86 @@ +import io +import logging +import os +from datetime import datetime + +import pytest + +from specsanalyzer.logging import set_verbosity +from specsanalyzer.logging import setup_logging + + +@pytest.fixture +def logger_(): + logger = setup_logging("test_logger") + log_capture_string = io.StringIO() + ch = logging.StreamHandler(log_capture_string) + ch.setLevel(logging.DEBUG) + logger.addHandler(ch) + yield logger, log_capture_string + + +def test_debug_logging(logger_): + logger, log_capture_string = logger_ + logger.debug("This is a debug message") + assert "This is a debug message" in log_capture_string.getvalue() + + +def test_info_logging(logger_): + logger, log_capture_string = logger_ + logger.info("This is an info message") + assert "This is an info message" in log_capture_string.getvalue() + + +def test_warning_logging(logger_): + logger, log_capture_string = logger_ + logger.warning("This is a warning message") + assert "This is a warning message" in log_capture_string.getvalue() + + +def test_error_logging(logger_): + logger, log_capture_string = logger_ + logger.error("This is an error message") + assert "This is an error message" in log_capture_string.getvalue() + + +def test_critical_logging(logger_): + logger, log_capture_string = logger_ + logger.critical("This is a critical message") + assert "This is a critical message" in log_capture_string.getvalue() + + +def test_set_verbosity(logger_): + logger, log_capture_string = logger_ + set_verbosity(logger, verbose=True) + assert logger.handlers[0].level == logging.INFO + set_verbosity(logger, verbose=False) + assert logger.handlers[0].level == logging.WARNING + + +def test_logger_has_base_logger(logger_): + logger, log_capture_string = logger_ + assert logger.name == "specsanalyzer.test_logger" + assert logger.parent.name == "specsanalyzer" + assert logger.parent.parent.name == "root" + assert logger.parent.level == logging.DEBUG + assert isinstance(logger.parent.handlers[0], logging.FileHandler) + + +def test_logger_creates_logfile(tmp_path): + logger = setup_logging("test_logger", set_base_handler=True, user_log_path=tmp_path) + log_file = os.path.join(tmp_path, f"sed_{datetime.now().strftime('%Y-%m-%d')}.log") + assert os.path.exists(log_file) + with open(log_file) as f: + assert f.read() == "" + logger.debug("This is a debug message") + with open(log_file) as f: + assert "This is a debug message" in f.read() + + +def test_readonly_path(tmp_path, caplog): + os.chmod(tmp_path, 0o444) + with caplog.at_level(logging.WARNING): + setup_logging("test_logger", set_base_handler=True, user_log_path=tmp_path) + assert f"Cannot create logfile in Folder {tmp_path}, disabling logfile." in caplog.messages[0] + log_file = os.path.join(tmp_path, f"sed_{datetime.now().strftime('%Y-%m-%d')}.log") + assert not os.path.exists(log_file) From 1a3f37294c6417b43a3458e7ef4d96a1d5375f99 Mon Sep 17 00:00:00 2001 From: rettigl Date: Sat, 8 Feb 2025 23:25:54 +0100 Subject: [PATCH 26/44] fix logfile name and example config --- src/specsanalyzer/logging.py | 5 ++++- src/specsscan/config/example_config_FHI.yaml | 4 ++-- tutorial/3_specsscan_conversion_to_NeXus.ipynb | 11 +++-------- 3 files changed, 9 insertions(+), 11 deletions(-) diff --git a/src/specsanalyzer/logging.py b/src/specsanalyzer/logging.py index 3b5a863..3698340 100644 --- a/src/specsanalyzer/logging.py +++ b/src/specsanalyzer/logging.py @@ -50,7 +50,10 @@ def setup_logging( user_log_path = DEFAULT_LOG_DIR try: os.makedirs(user_log_path, exist_ok=True) - log_file = os.path.join(user_log_path, f"sed_{datetime.now().strftime('%Y-%m-%d')}.log") + log_file = os.path.join( + user_log_path, + f"specsanalyzer_{datetime.now().strftime('%Y-%m-%d')}.log", + ) # Create file handler and set level to debug file_handler = logging.FileHandler(log_file) diff --git a/src/specsscan/config/example_config_FHI.yaml b/src/specsscan/config/example_config_FHI.yaml index bf4976b..a18a255 100644 --- a/src/specsscan/config/example_config_FHI.yaml +++ b/src/specsscan/config/example_config_FHI.yaml @@ -38,9 +38,9 @@ units: metadata: # URL of the elabFTW instance API interface - elab_url: "https://__elabftw_host__/api/v2" + # elab_url: "https://__elabftw_host__/api/v2" # URL of the epics archiver request engine - archiver_url: "http://__epicsarchiver_host__:17668/retrieval/data/getData.json?pv=" + # archiver_url: "http://__epicsarchiver_host__:17668/retrieval/data/getData.json?pv=" # dictionary containing axis names with Epics channels to request from the EPICS archiver epics_channels: tempa: "trARPES:Carving:TEMP_RBV" diff --git a/tutorial/3_specsscan_conversion_to_NeXus.ipynb b/tutorial/3_specsscan_conversion_to_NeXus.ipynb index 22d19fd..0e519d6 100755 --- a/tutorial/3_specsscan_conversion_to_NeXus.ipynb +++ b/tutorial/3_specsscan_conversion_to_NeXus.ipynb @@ -125,7 +125,7 @@ "outputs": [], "source": [ "config = {\"nexus\":{\"definition\": \"NXmpes_arpes\"}, \"spa_params\":{\"crop\":True, \"ek_range_min\":0.07597844332538181, \"ek_range_max\":0.9117413199045858, \"ang_range_min\":0.16453159041394336, \"ang_range_max\":0.8840087145969499,}}\n", - "sps = SpecsScan(config=config, user_config=\"../src/specsscan/config/example_config_FHI.yaml\")\n", + "sps = SpecsScan(config=config, user_config=\"../src/specsscan/config/example_config_FHI.yaml\", system_config={})\n", "path = \"../tests/data/\" # Path to the test data set" ] }, @@ -234,7 +234,7 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3", + "display_name": ".venv", "language": "python", "name": "python3" }, @@ -248,12 +248,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.8.12" - }, - "vscode": { - "interpreter": { - "hash": "a164666994e9db75450cd7016dd7e51d42ea6e7c1e5e8017af1f8068ca906367" - } + "version": "3.12.8" } }, "nbformat": 4, From fd2b3132716583b4146eab7b94190bb21c83716f Mon Sep 17 00:00:00 2001 From: rettigl Date: Sun, 9 Feb 2025 22:19:16 +0100 Subject: [PATCH 27/44] fix tests --- tests/test_logging.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_logging.py b/tests/test_logging.py index e78aba0..99f9d1b 100644 --- a/tests/test_logging.py +++ b/tests/test_logging.py @@ -68,7 +68,7 @@ def test_logger_has_base_logger(logger_): def test_logger_creates_logfile(tmp_path): logger = setup_logging("test_logger", set_base_handler=True, user_log_path=tmp_path) - log_file = os.path.join(tmp_path, f"sed_{datetime.now().strftime('%Y-%m-%d')}.log") + log_file = os.path.join(tmp_path, f"specsanalyzer_{datetime.now().strftime('%Y-%m-%d')}.log") assert os.path.exists(log_file) with open(log_file) as f: assert f.read() == "" From a988e6b5f14745c6a406195bd6f016dc4b02c53d Mon Sep 17 00:00:00 2001 From: rettigl Date: Sun, 9 Feb 2025 22:32:59 +0100 Subject: [PATCH 28/44] disable propagation of base_logger if logfile cannot be opened --- src/specsanalyzer/logging.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/specsanalyzer/logging.py b/src/specsanalyzer/logging.py index 3698340..d6915d1 100644 --- a/src/specsanalyzer/logging.py +++ b/src/specsanalyzer/logging.py @@ -70,6 +70,7 @@ def setup_logging( except PermissionError: logging.warning(f"Cannot create logfile in Folder {user_log_path}, disabling logfile.") base_logger.addHandler(logging.NullHandler()) + base_logger.propagate = False # create named logger logger = base_logger.getChild(name) From 4efb02dc2e0679d4a15b49938fbc92ba700545ad Mon Sep 17 00:00:00 2001 From: rettigl Date: Mon, 10 Feb 2025 23:44:48 +0100 Subject: [PATCH 29/44] fix beam status --- src/specsscan/metadata.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/src/specsscan/metadata.py b/src/specsscan/metadata.py index e2c1776..6b45809 100644 --- a/src/specsscan/metadata.py +++ b/src/specsscan/metadata.py @@ -286,6 +286,22 @@ def fetch_elab_metadata(self, scan: int, metadata: dict) -> dict: elif metadata["elabFTW"]["scan"]["pump2_polarization"] == "p": metadata["elabFTW"]["scan"]["pump2_polarization"] = 0 + # fix pump status + if "scan" in metadata["elabFTW"] and "pump_status" in metadata["elabFTW"]["scan"]: + try: + metadata["elabFTW"]["scan"]["pump_status"] = ( + "opened" if int(metadata["elabFTW"]["scan"]["pump_status"]) else "closed" + ) + except ValueError: + pass + if "scan" in metadata["elabFTW"] and "pump2_status" in metadata["elabFTW"]["scan"]: + try: + metadata["elabFTW"]["scan"]["pump2_status"] = ( + "opened" if int(metadata["elabFTW"]["scan"]["pump2_status"]) else "closed" + ) + except ValueError: + pass + # remove pump information if pump not applied: if metadata["elabFTW"]["scan"].get("pump_status", "closed") == "closed": if "pump_photon_energy" in metadata["elabFTW"].get("laser_status", {}): From f716b77554a169e56d1fd6eed5460794535c8a2f Mon Sep 17 00:00:00 2001 From: rettigl Date: Tue, 11 Feb 2025 12:11:47 +0100 Subject: [PATCH 30/44] fix polarization and pump status conversions --- src/specsscan/metadata.py | 25 +++++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/src/specsscan/metadata.py b/src/specsscan/metadata.py index 6b45809..b5052fe 100644 --- a/src/specsscan/metadata.py +++ b/src/specsscan/metadata.py @@ -265,6 +265,13 @@ def fetch_elab_metadata(self, scan: int, metadata: dict) -> dict: metadata["elabFTW"]["scan"]["pump_polarization"] = 90 elif metadata["elabFTW"]["scan"]["pump_polarization"] == "p": metadata["elabFTW"]["scan"]["pump_polarization"] = 0 + else: + try: + metadata["elabFTW"]["scan"]["pump_polarization"] = float( + metadata["elabFTW"]["scan"]["pump_polarization"], + ) + except ValueError: + pass if ( "scan" in metadata["elabFTW"] @@ -275,6 +282,13 @@ def fetch_elab_metadata(self, scan: int, metadata: dict) -> dict: metadata["elabFTW"]["scan"]["probe_polarization"] = 90 elif metadata["elabFTW"]["scan"]["probe_polarization"] == "p": metadata["elabFTW"]["scan"]["probe_polarization"] = 0 + else: + try: + metadata["elabFTW"]["scan"]["probe_polarization"] = float( + metadata["elabFTW"]["scan"]["probe_polarization"], + ) + except ValueError: + pass if ( "scan" in metadata["elabFTW"] @@ -285,19 +299,26 @@ def fetch_elab_metadata(self, scan: int, metadata: dict) -> dict: metadata["elabFTW"]["scan"]["pump2_polarization"] = 90 elif metadata["elabFTW"]["scan"]["pump2_polarization"] == "p": metadata["elabFTW"]["scan"]["pump2_polarization"] = 0 + else: + try: + metadata["elabFTW"]["scan"]["pump2_polarization"] = float( + metadata["elabFTW"]["scan"]["pump2_polarization"], + ) + except ValueError: + pass # fix pump status if "scan" in metadata["elabFTW"] and "pump_status" in metadata["elabFTW"]["scan"]: try: metadata["elabFTW"]["scan"]["pump_status"] = ( - "opened" if int(metadata["elabFTW"]["scan"]["pump_status"]) else "closed" + "open" if int(metadata["elabFTW"]["scan"]["pump_status"]) else "closed" ) except ValueError: pass if "scan" in metadata["elabFTW"] and "pump2_status" in metadata["elabFTW"]["scan"]: try: metadata["elabFTW"]["scan"]["pump2_status"] = ( - "opened" if int(metadata["elabFTW"]["scan"]["pump2_status"]) else "closed" + "open" if int(metadata["elabFTW"]["scan"]["pump2_status"]) else "closed" ) except ValueError: pass From 488209c82ce29243c6217aeb1816929b57233270 Mon Sep 17 00:00:00 2001 From: rettigl Date: Wed, 12 Feb 2025 13:51:47 +0100 Subject: [PATCH 31/44] add program name and version --- src/specsscan/helpers.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/specsscan/helpers.py b/src/specsscan/helpers.py index f67d580..934b8df 100644 --- a/src/specsscan/helpers.py +++ b/src/specsscan/helpers.py @@ -2,6 +2,7 @@ from __future__ import annotations import datetime as dt +import importlib import logging from pathlib import Path from typing import Any @@ -410,6 +411,11 @@ def handle_meta( complete_dictionary(scan_info, lut_meta), ) # merging dictionaries + # store program version + metadata["scan_info"]["program_name"] = "specsanalyzer" + metadata["scan_info"]["program_version"] = importlib.metadata.version("specsanalyzer") + + # timing logger.info("Collecting time stamps...") if "time" in metadata["scan_info"]: time_list = [metadata["scan_info"]["time"][0], metadata["scan_info"]["time"][-1]] From 2eef16aee1a683aec8c9bd222c685423d2573d17 Mon Sep 17 00:00:00 2001 From: rettigl Date: Wed, 12 Feb 2025 14:09:02 +0100 Subject: [PATCH 32/44] add option to shift energy axis to EF, and add axis labels --- src/specsscan/config/default.yaml | 3 ++ src/specsscan/config/example_config_FHI.yaml | 13 ++++++++ src/specsscan/core.py | 34 ++++++++++++++++++++ 3 files changed, 50 insertions(+) diff --git a/src/specsscan/config/default.yaml b/src/specsscan/config/default.yaml index 94d2250..652062a 100644 --- a/src/specsscan/config/default.yaml +++ b/src/specsscan/config/default.yaml @@ -2,6 +2,9 @@ data_path: "" # option to enable nested progress bars enable_nested_progress_bar: false +# option to shift by photon energy if available +shift_by_photon_energy: false + # dictionary containing parameters passed to the SpecsAnalyzer. Will be completed by the SpecsAnalyzer default config parameters spa_params: apply_fft_filter: false diff --git a/src/specsscan/config/example_config_FHI.yaml b/src/specsscan/config/example_config_FHI.yaml index a18a255..cb26071 100644 --- a/src/specsscan/config/example_config_FHI.yaml +++ b/src/specsscan/config/example_config_FHI.yaml @@ -2,6 +2,8 @@ data_path: "path/to/data" # option to enable nested progress bars enable_nested_progress_bar: false +# option to shift by photon energy if available +shift_by_photon_energy: true # dictionary containing renaming rules for axis names (to change the name in the xarrays) coordinate_mapping: @@ -25,6 +27,17 @@ coordinate_depends: Y: "/entry/sample/transformations/trans_y" Z: "/entry/sample/transformations/trans_z" +# dictionary of axis labels for the different axes +coordinate_labels: + Ekin: "Kinetic Energy" + Angle: "Analyzer Dispersion" + polar: "Sample Polar" + tilt: "Sample Tilt" + azimuth: "Sample Azimuth" + X: "X Translation" + Y: "Y Translation" + Z: "Z Translation" + # dictionary containing units for the respective axes units: angular0: "degree" diff --git a/src/specsscan/core.py b/src/specsscan/core.py index 039d7b4..8b88c6c 100755 --- a/src/specsscan/core.py +++ b/src/specsscan/core.py @@ -245,6 +245,7 @@ def load_scan( # rename coords and store mapping information, if available coordinate_mapping = self._config.get("coordinate_mapping", {}) coordinate_depends = self._config.get("coordinate_depends", {}) + coordinate_labels = self._config.get("coordinate_labels", {}) rename_dict = { k: coordinate_mapping[k] for k in coordinate_mapping.keys() if k in res_xarray.dims } @@ -253,6 +254,11 @@ def load_scan( for k in coordinate_depends.keys() if k in res_xarray.dims } + label_dict = { + rename_dict.get(k, k): coordinate_labels[k] + for k in coordinate_depends.keys() + if k in res_xarray.dims + } # store data for resolved axis coordinates for axis in res_xarray.dims: @@ -267,10 +273,12 @@ def load_scan( slow_axes.remove(k) slow_axes.add(v) self._scan_info["coordinate_depends"] = depends_dict + self._scan_info["coordinate_label"] = label_dict for name in res_xarray.dims: try: res_xarray[name].attrs["unit"] = self._config["units"][name] + res_xarray[name].attrs["long_name"] = label_dict[name] except KeyError: pass @@ -294,6 +302,32 @@ def load_scan( **{"conversion_parameters": conversion_metadata}, ) + # shift energy axis + photon_energy = 0.0 + try: + photon_energy = self.metadata["elabFTW"]["laser_status"]["probe_photon_energy"] + except KeyError: + try: + photon_energy = self.metadata["elabFTW"]["laser_status"]["probe_photon_energy"] + except KeyError: + pass + + self.metadata["scan_info"]["reference_energy"] = "vacuum level" + if photon_energy and self._config["shift_by_photon_energy"]: + logger.info(f"Shifting energy axis by photon energy: -{photon_energy} eV") + res_xarray = res_xarray.assign_coords( + { + rename_dict["Ekin"]: ( + rename_dict["Ekin"], + res_xarray[rename_dict["Ekin"]].data - photon_energy, + res_xarray[rename_dict["Ekin"]].attrs, + ), + }, + ) + self.metadata["scan_info"]["reference_energy"] = "Fermi edge" + self.metadata["scan_info"]["coordinate_label"][rename_dict["Ekin"]] = "E-E_F" + res_xarray[rename_dict["Ekin"]].attrs["long_name"] = "E-E_F" + res_xarray.attrs["metadata"] = self.metadata self._result = res_xarray From c9496db841387dd686d41f6c9f462e362cc2ef71 Mon Sep 17 00:00:00 2001 From: rettigl Date: Wed, 12 Feb 2025 16:09:19 +0100 Subject: [PATCH 33/44] fix labels --- src/specsscan/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/specsscan/core.py b/src/specsscan/core.py index 8b88c6c..88971f5 100755 --- a/src/specsscan/core.py +++ b/src/specsscan/core.py @@ -256,7 +256,7 @@ def load_scan( } label_dict = { rename_dict.get(k, k): coordinate_labels[k] - for k in coordinate_depends.keys() + for k in coordinate_labels.keys() if k in res_xarray.dims } From 5a8fe9536d26aba39a96e92b2048ed8738645b7d Mon Sep 17 00:00:00 2001 From: rettigl Date: Thu, 27 Feb 2025 22:52:17 +0100 Subject: [PATCH 34/44] remove also source_pump etc. if not applied --- src/specsscan/metadata.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/specsscan/metadata.py b/src/specsscan/metadata.py index b5052fe..37f2c9c 100644 --- a/src/specsscan/metadata.py +++ b/src/specsscan/metadata.py @@ -327,10 +327,14 @@ def fetch_elab_metadata(self, scan: int, metadata: dict) -> dict: if metadata["elabFTW"]["scan"].get("pump_status", "closed") == "closed": if "pump_photon_energy" in metadata["elabFTW"].get("laser_status", {}): del metadata["elabFTW"]["laser_status"]["pump_photon_energy"] + if "pump_repetition_rate" in metadata["elabFTW"].get("laser_status", {}): + del metadata["elabFTW"]["laser_status"]["pump_repetition_rate"] if metadata["elabFTW"]["scan"].get("pump2_status", "closed") == "closed": if "pump2_photon_energy" in metadata["elabFTW"].get("laser_status", {}): del metadata["elabFTW"]["laser_status"]["pump2_photon_energy"] + if "pump2_repetition_rate" in metadata["elabFTW"].get("laser_status", {}): + del metadata["elabFTW"]["laser_status"]["pump2_repetition_rate"] return metadata From 5bde7dc8e8c4b5aec71f3b9af9b715e34854bd55 Mon Sep 17 00:00:00 2001 From: rettigl Date: Mon, 3 Mar 2025 22:40:37 +0100 Subject: [PATCH 35/44] don't add entries if not found in epics archiver --- src/specsscan/metadata.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/specsscan/metadata.py b/src/specsscan/metadata.py index 37f2c9c..27755e3 100644 --- a/src/specsscan/metadata.py +++ b/src/specsscan/metadata.py @@ -119,7 +119,6 @@ def fetch_epics_metadata(self, ts_from: float, ts_to: float, metadata: dict) -> metadata["scan_info"][f"{channel}"] = np.mean(vals) except IndexError: - metadata["scan_info"][f"{channel}"] = np.nan logger.info( f"Data for channel {channel} doesn't exist for time {start}", ) From da8671a6f41f897b33f4ab78b4c48a70888661e1 Mon Sep 17 00:00:00 2001 From: rettigl Date: Mon, 24 Mar 2025 23:04:30 +0100 Subject: [PATCH 36/44] allow photon energy from manual metadata --- src/specsscan/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/specsscan/core.py b/src/specsscan/core.py index 88971f5..e943c8e 100755 --- a/src/specsscan/core.py +++ b/src/specsscan/core.py @@ -305,7 +305,7 @@ def load_scan( # shift energy axis photon_energy = 0.0 try: - photon_energy = self.metadata["elabFTW"]["laser_status"]["probe_photon_energy"] + photon_energy = self.metadata["instrument"]["beam"]["probe"]["incident_energy"] except KeyError: try: photon_energy = self.metadata["elabFTW"]["laser_status"]["probe_photon_energy"] From 7890bcabb30faedd358070fec346b74d7ab80c79 Mon Sep 17 00:00:00 2001 From: rettigl Date: Mon, 24 Mar 2025 12:59:41 +0100 Subject: [PATCH 37/44] add pulse energy --- src/specsscan/helpers.py | 2 +- src/specsscan/metadata.py | 18 ++++++++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/src/specsscan/helpers.py b/src/specsscan/helpers.py index 2be1ff1..371122e 100644 --- a/src/specsscan/helpers.py +++ b/src/specsscan/helpers.py @@ -456,7 +456,7 @@ def handle_meta( "angular dispersive" if projection == "reciprocal" else "spatial dispersive" ) - metadata["scan_info"]["slow_axes"] = slow_axes + metadata["scan_info"]["slow_axes"] = slow_axes if slow_axes else "" metadata["scan_info"]["fast_axes"] = fast_axes return metadata diff --git a/src/specsscan/metadata.py b/src/specsscan/metadata.py index 4224351..e1408f9 100644 --- a/src/specsscan/metadata.py +++ b/src/specsscan/metadata.py @@ -328,12 +328,30 @@ def fetch_elab_metadata(self, scan: int, metadata: dict) -> dict: del metadata["elabFTW"]["laser_status"]["pump_photon_energy"] if "pump_repetition_rate" in metadata["elabFTW"].get("laser_status", {}): del metadata["elabFTW"]["laser_status"]["pump_repetition_rate"] + else: + # add pulse energy if applicable + try: + metadata["elabFTW"]["scan"]["pump_pulse_energy"] = ( + metadata["scan_info"]["trARPES:Pump:Power.RBV"] + / metadata["elabFTW"]["laser_status"]["pump_repetition_rate"] + ) + except KeyError: + pass if metadata["elabFTW"]["scan"].get("pump2_status", "closed") == "closed": if "pump2_photon_energy" in metadata["elabFTW"].get("laser_status", {}): del metadata["elabFTW"]["laser_status"]["pump2_photon_energy"] if "pump2_repetition_rate" in metadata["elabFTW"].get("laser_status", {}): del metadata["elabFTW"]["laser_status"]["pump2_repetition_rate"] + else: + # add pulse energy if applicable + try: + metadata["elabFTW"]["scan"]["pump2_pulse_energy"] = ( + metadata["scan_info"]["trARPES:Pump2:Power.RBV"] + / metadata["elabFTW"]["laser_status"]["pump_repetition_rate"] + ) + except KeyError: + pass return metadata From ead6db7bd4db1235c8e95d83d6c0b0793b9a88b0 Mon Sep 17 00:00:00 2001 From: rettigl Date: Thu, 6 Feb 2025 23:37:10 +0100 Subject: [PATCH 38/44] estimate collection time for static exposures --- src/specsscan/helpers.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/specsscan/helpers.py b/src/specsscan/helpers.py index 371122e..4677bfe 100644 --- a/src/specsscan/helpers.py +++ b/src/specsscan/helpers.py @@ -426,8 +426,16 @@ def handle_meta( dt_list_iso = [time.replace(".", "-").replace(" ", "T") for time in time_list] datetime_list = [dt.datetime.fromisoformat(dt_iso) for dt_iso in dt_list_iso] - ts_from = dt.datetime.timestamp(datetime_list[0]) # POSIX timestamp - ts_to = dt.datetime.timestamp(datetime_list[-1]) # POSIX timestamp + ts_from = dt.datetime.timestamp(min(datetime_list)) # POSIX timestamp + ts_to = dt.datetime.timestamp(max(datetime_list)) # POSIX timestamp + if ts_from == ts_to: + try: + ts_to = ( + ts_from + + metadata["scan_info"]["Exposure"] / 1000 * metadata["scan_info"]["Averages"] + ) + except KeyError: + pass metadata["timing"] = { "acquisition_start": dt.datetime.fromtimestamp(ts_from, dt.timezone.utc).isoformat(), "acquisition_stop": dt.datetime.fromtimestamp(ts_to, dt.timezone.utc).isoformat(), From 9a8e27762297664da77200a064ad751c3f4715ef Mon Sep 17 00:00:00 2001 From: rettigl Date: Mon, 24 Mar 2025 13:00:42 +0100 Subject: [PATCH 39/44] use LUT data for delay as well --- pyproject.toml | 2 +- src/specsscan/helpers.py | 10 +++++++--- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 22e74e2..44e7fdb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -41,7 +41,7 @@ dependencies = [ "numpy>=1.21.6", "opencv-python>=4.8.1.78", "pynxtools-mpes>=0.2.1", - "pynxtools>=0.9.3", + "pynxtools>=0.10.0", "python-dateutil>=2.8.2", "pyyaml>=6.0", "xarray>=0.20.2", diff --git a/src/specsscan/helpers.py b/src/specsscan/helpers.py index 4677bfe..8dd17e3 100644 --- a/src/specsscan/helpers.py +++ b/src/specsscan/helpers.py @@ -348,9 +348,6 @@ def parse_info_to_dict(path: Path) -> dict: except FileNotFoundError as exc: raise FileNotFoundError("info.txt file not found.") from exc - if "DelayStage" in info_dict and "TimeZero" in info_dict: - info_dict["delay"] = mm_to_fs(info_dict["DelayStage"], info_dict["TimeZero"]) - return info_dict @@ -411,6 +408,13 @@ def handle_meta( complete_dictionary(scan_info, lut_meta), ) # merging dictionaries + # Store delay info + if "DelayStage" in metadata["scan_info"] and "TimeZero" in metadata["scan_info"]: + metadata["scan_info"]["delay"] = mm_to_fs( + metadata["scan_info"]["DelayStage"], + metadata["scan_info"]["TimeZero"], + ) + # store program version metadata["scan_info"]["program_name"] = "specsanalyzer" metadata["scan_info"]["program_version"] = importlib.metadata.version("specsanalyzer") From 9a153b3af5abe73649da443c8de503a91b422017 Mon Sep 17 00:00:00 2001 From: rettigl Date: Mon, 24 Mar 2025 13:03:53 +0100 Subject: [PATCH 40/44] update config and example --- .cspell/custom-dictionary.txt | 3 +- src/specsscan/config/NXmpes_arpes_config.json | 434 +++++++++--------- .../3_specsscan_conversion_to_NeXus.ipynb | 17 +- 3 files changed, 235 insertions(+), 219 deletions(-) mode change 100755 => 100644 src/specsscan/config/NXmpes_arpes_config.json diff --git a/.cspell/custom-dictionary.txt b/.cspell/custom-dictionary.txt index 6ac077a..c31708e 100644 --- a/.cspell/custom-dictionary.txt +++ b/.cspell/custom-dictionary.txt @@ -45,7 +45,7 @@ Ekin elab elabapi elabid -electronanalyser +electronanalyzer elems endstation energydispersion @@ -142,6 +142,7 @@ toctree tomlkit topfloor tqdm +trarpes typehints TZCYXS undoc diff --git a/src/specsscan/config/NXmpes_arpes_config.json b/src/specsscan/config/NXmpes_arpes_config.json old mode 100755 new mode 100644 index 6e6eb3f..2df10fb --- a/src/specsscan/config/NXmpes_arpes_config.json +++ b/src/specsscan/config/NXmpes_arpes_config.json @@ -1,28 +1,26 @@ { "/@default": "entry", "/ENTRY/@default": "data", - "/ENTRY/title": "['@eln:/ENTRY/title', '@attrs:metadata/entry_title']", + "/ENTRY/title": "['@eln:/ENTRY/title', '@attrs:metadata/entry_title', '@attrs:metadata/elabFTW/scan/title']", "/ENTRY/start_time": "@attrs:metadata/timing/acquisition_start", "/ENTRY/experiment_institution": "Fritz Haber Institute - Max Planck Society", "/ENTRY/experiment_facility": "Time Resolved ARPES", "/ENTRY/experiment_laboratory": "Clean Room 4", - "/ENTRY/entry_identifier": { - "identifier":"@attrs:metadata/loader/scan_path" - }, + "/ENTRY/identifierNAME[entry_identifier]": "@attrs:metadata/loader/scan_path", "/ENTRY/end_time": "@attrs:metadata/timing/acquisition_stop", "/ENTRY/duration": "@attrs:metadata/timing/acquisition_duration", "/ENTRY/duration/@units": "s", "/ENTRY/collection_time": "@attrs:metadata/timing/collection_time", "/ENTRY/collection_time/@units": "s", "/ENTRY/USER[user]": { - "name": "!['@eln:/ENTRY/User/name', '@attrs:metadata/user0/name']", - "role": "['@eln:/ENTRY/User/role', '@attrs:metadata/user0/role']", - "affiliation": "!['@eln:/ENTRY/User/affiliation', '@attrs:metadata/user0/affiliation']", - "address": "['@eln:/ENTRY/User/address', '@attrs:metadata/user0/address']", - "email": "['@eln:/ENTRY/User/email', '@attrs:metadata/user0/email']" + "name": "!['@eln:/ENTRY/User/name', '@attrs:metadata/user0/name', '@attrs:metadata/elabFTW/user/name']", + "role": "['@eln:/ENTRY/User/role', '@attrs:metadata/user0/role', 'Principal Investigator']", + "affiliation": "['@eln:/ENTRY/User/affiliation', '@attrs:metadata/user0/affiliation', 'Fritz Haber Institute of the Max Planck Society']", + "address": "['@eln:/ENTRY/User/address', '@attrs:metadata/user0/address', 'Faradayweg 4-6, 14195 Berlin, Germany']", + "email": "['@eln:/ENTRY/User/email', '@attrs:metadata/user0/email', '@attrs:metadata/elabFTW/user/email']" }, - "/ENTRY/geometries/COORDINATE_SYSTEM[arpes_geometry]": { - "depends_on": "/entry/geometries/arpes_geometry/transformations/rot_y", + "/ENTRY/COORDINATE_SYSTEM[arpes_geometry]": { + "depends_on": "/entry/arpes_geometry/transformations/rot_y", "TRANSFORMATIONS[transformations]": { "AXISNAME[rot_y]/@depends_on": ".", "AXISNAME[rot_y]": 140.0, @@ -35,19 +33,19 @@ "name": "Phoibos detector, at the endstation of the high rep-rate HHG source at FHI", "name/@short_name": "TR-ARPES @ FHI", "energy_resolution": { - "resolution": "!['@eln:/ENTRY/Instrument/Analyzer/energy_resolution', '@attrs:metadata/instrument/energy_resolution']", + "resolution": "!['@eln:/ENTRY/Instrument/Analyzer/energy_resolution', '@attrs:metadata/instrument/energy_resolution', '@attrs:metadata/elabFTW/trarpes_phoibos/energy_resolution']", "resolution/@units": "meV", "physical_quantity": "energy", "type": "estimated" }, "RESOLUTION[temporal_resolution]": { - "resolution": 35.0, + "resolution": "!['@attrs:metadata/elabFTW/laser_status/temporal_resolution', '35.0']", "resolution/@units": "fs", "physical_quantity": "time", "type": "estimated" }, "RESOLUTION[angular_resolution]": { - "resolution": "@link:/entry/instrument/electronanalyser/angular_resolution/resolution", + "resolution": "@link:/entry/instrument/electronanalyzer/angular_resolution/resolution", "resolution/@units": "deg", "physical_quantity": "angle", "type": "derived" @@ -57,207 +55,205 @@ "measurement": "pressure", "value": "!['@eln:/ENTRY/Sample/gas_pressure', '@attrs:metadata/scan_info/trARPES:XGS600:PressureAC:P_RD']", "value/@units": "mbar" - } - }, - "/ENTRY/INSTRUMENT[instrument]/ELECTRONANALYSER[electronanalyser]": { - "description": "SPECS Phoibos 150 Hemispherical Energy Analyzer", - "device_information": { - "vendor": "SPECS GmbH", - "model": "Phoibos 150 CCD Hemispherical Analyzer" }, - "work_function": 4.55, - "work_function/@units": "eV", - "fast_axes": "@attrs:metadata/scan_info/fast_axes", - "slow_axes": "@attrs:metadata/scan_info/slow_axes", - "energy_resolution": { - "resolution": "!['@eln:/ENTRY/Instrument/Analyzer/energy_resolution', '@attrs:metadata/instrument/electronanalyser/energy_resolution']", - "resolution/@units": "meV", - "physical_quantity": "energy", - "type": "derived" + "ELECTRONANALYZER[electronanalyzer]": { + "description": "SPECS Phoibos 150 Hemispherical Energy Analyzer", + "device_information": { + "vendor": "SPECS GmbH", + "model": "Phoibos 150 CCD Hemispherical Analyzer" + }, + "work_function": "@attrs:metadata/conversion_parameters/work_function", + "work_function/@units": "eV", + "fast_axes": "@attrs:metadata/scan_info/fast_axes", + "slow_axes": "@attrs:metadata/scan_info/slow_axes", + "energy_resolution": { + "resolution": "!['@eln:/ENTRY/Instrument/Analyzer/energy_resolution', '@attrs:metadata/instrument/electronanalyzer/energy_resolution', '@attrs:metadata/elabFTW/trarpes_phoibos/energy_resolution']", + "resolution/@units": "meV", + "physical_quantity": "energy", + "type": "derived" + }, + "angular_resolution": { + "resolution": "!['@eln:/ENTRY/Instrument/Analyzer/angular_resolution', '@attrs:metadata/instrument/electronanalyzer/angular_resolution', '@attrs:metadata/elabFTW/trarpes_phoibos/angular_resolution']", + "resolution/@units": "deg", + "physical_quantity": "angle", + "type": "derived" + }, + "spatial_resolution": { + "resolution": "!['@eln:/ENTRY/Instrument/Analyzer/spatial_resolution', '@attrs:metadata/instrument/electronanalyzer/spatial_resolution', '@attrs:metadata/elabFTW/trarpes_phoibos/spatial_resolution']", + "resolution/@units": "µm", + "physical_quantity": "length", + "type": "estimated" + }, + "depends_on": "/entry/instrument/electronanalyzer/transformations/trans_z", + "TRANSFORMATIONS[transformations]": { + "AXISNAME[trans_z]/@depends_on": "analyzer_elevation", + "AXISNAME[trans_z]": 40.0, + "AXISNAME[trans_z]/@transformation_type": "translation", + "AXISNAME[trans_z]/@units": "mm", + "AXISNAME[trans_z]/@vector": [0, 0, 1], + "analyzer_dispersion/@depends_on": "analyzer_rotation", + "analyzer_dispersion": "@data:angular1.data", + "analyzer_dispersion/@transformation_type": "rotation", + "analyzer_dispersion/@units": "degrees", + "analyzer_dispersion/@vector": [1, 0, 0], + "analyzer_elevation/@depends_on": "analyzer_dispersion", + "analyzer_elevation": 0, + "analyzer_elevation/@transformation_type": "rotation", + "analyzer_elevation/@units": "degrees", + "analyzer_elevation/@vector": [0, 1, 0], + "analyzer_rotation/@depends_on": "rot_y", + "analyzer_rotation": 0, + "analyzer_rotation/@transformation_type": "rotation", + "analyzer_rotation/@units": "degrees", + "analyzer_rotation/@vector": [0, 0, 1], + "AXISNAME[rot_y]/@depends_on": ".", + "AXISNAME[rot_y]": 140.0, + "AXISNAME[rot_y]/@transformation_type": "rotation", + "AXISNAME[rot_y]/@units": "degrees", + "AXISNAME[rot_y]/@vector": [0, 1, 0] + }, + "COLLECTIONCOLUMN[collectioncolumn]": { + "projection": "@attrs:metadata/scan_info/projection", + "scheme": "@attrs:metadata/scan_info/scheme", + "lens_mode": "@attrs:metadata/scan_info/LensMode", + "working_distance": 40.0, + "working_distance/@units": "mm" + }, + "ENERGYDISPERSION[energydispersion]": { + "energy_scan_mode": "@attrs:metadata/scan_info/energy_scan_mode", + "pass_energy": "@attrs:metadata/scan_info/PassEnergy", + "pass_energy/@units": "eV", + "center_energy": "@attrs:metadata/scan_info/KineticEnergy", + "center_energy/@units": "eV", + "scheme": "hemispherical", + "diameter": 300.0, + "diameter/@units": "mm", + "entrance_slit": { + "shape": "['@eln:/ENTRY/Instrument/Analyzer/slit_shape', '@attrs:metadata/instrument/electronanalyzer/slit_shape', '@attrs:metadata/elabFTW/trarpes_phoibos/entrance_slit_shape', 'curved slit']", + "size": "['@eln:/ENTRY/Instrument/Analyzer/slit_size', '@attrs:metadata/instrument/electronanalyzer/slit_size', '@attrs:metadata/elabFTW/trarpes_phoibos/entrance_slit_size', '1.0']", + "size/@units": "mm" + }, + "exit_slit": { + "shape": "['@attrs:metadata/elabFTW/trarpes_phoibos/exit_slit_type', 'grid']" + } + }, + "ELECTRON_DETECTOR[detector]": { + "amplifier_type": "MCP", + "detector_type": "Phosphor+CCD", + "amplifier_bias": "@attrs:metadata/scan_info/ConversionVoltage", + "amplifier_bias/@units": "V", + "amplifier_voltage": "@attrs:metadata/scan_info/DetectorVoltage", + "amplifier_voltage/@units": "V", + "detector_voltage": "@attrs:metadata/scan_info/ScreenVoltage", + "detector_voltage/@units": "V" + } }, - "angular_resolution": { - "resolution": "!['@eln:/ENTRY/Instrument/Analyzer/angular_resolution', '@attrs:metadata/instrument/electronanalyser/angular_resolution']", - "resolution/@units": "deg", - "physical_quantity": "angle", - "type": "derived" + "source_probe": { + "name": "HHG @ TR-ARPES @ FHI", + "probe": "photon", + "type": "HHG laser", + "mode": "Single Bunch", + "frequency": "!['@eln:/ENTRY/Instrument/Source/Probe/frequency', '@attrs:metadata/instrument/beam/probe/frequency', '@attrs:metadata/elabFTW/laser_status/probe_repetition_rate']", + "frequency/@units": "kHz", + "associated_beam": "/entry/instrument/beam_probe" }, - "spatial_resolution": { - "resolution": "!['@eln:/ENTRY/Instrument/Analyzer/spatial_resolution', '@attrs:metadata/instrument/electronanalyser/spatial_resolution']", - "resolution/@units": "µm", - "physical_quantity": "length", - "type": "estimated" + "beam_probe": { + "distance": 0.0, + "distance/@units": "mm", + "incident_energy": "!['@eln:/ENTRY/Instrument/Beam/Probe/incident_energy', '@attrs:metadata/instrument/beam/probe/incident_energy', '@attrs:metadata/elabFTW/laser_status/probe_photon_energy']", + "incident_energy/@units": "eV", + "incident_energy_spread": "['@eln:/ENTRY/Instrument/Beam/Probe/incident_energy_spread', '@attrs:metadata/instrument/beam/probe/incident_energy_spread', '@attrs:metadata/elabFTW/laser_status/probe_photon_energy_spread']", + "incident_energy_spread/@units": "eV", + "pulse_duration": "['@eln:/ENTRY/Instrument/Beam/Probe/pulse_duration', '@attrs:metadata/instrument/beam/probe/pulse_duration', '@attrs:metadata/elabFTW/laser_status/probe_pulse_duration']", + "pulse_duration/@units": "fs", + "incident_polarization": "['@eln:/ENTRY/Instrument/Beam/Probe/incident_polarization', '@attrs:metadata/instrument/beam/probe/incident_polarization', '@attrs:metadata/elabFTW/scan/probe_polarization', '@attrs:metadata/scan_info/trARPES:HHG:wp']", + "incident_polarization/@units": "V^2/mm^2", + "extent": "['@eln:/ENTRY/Instrument/Beam/Probe/extent', '@attrs:metadata/instrument/beam/probe/extent', '@attrs:metadata/elabFTW/laser_status/probe_profile']", + "extent/@units": "µm", + "associated_source": "/entry/instrument/source_probe" }, - - - "depends_on": "/entry/instrument/electronanalyser/transformations/trans_z", - "TRANSFORMATIONS[transformations]": { - "AXISNAME[trans_z]/@depends_on": "analyzer_elevation", - "AXISNAME[trans_z]": 40.0, - "AXISNAME[trans_z]/@transformation_type": "translation", - "AXISNAME[trans_z]/@units": "mm", - "AXISNAME[trans_z]/@vector": [0, 0, 1], - "analyzer_dispersion/@depends_on": "analyzer_rotation", - "analyzer_dispersion": "@data:angular1.data", - "analyzer_dispersion/@transformation_type": "rotation", - "analyzer_dispersion/@units": "degrees", - "analyzer_dispersion/@vector": [1, 0, 0], - "analyzer_elevation/@depends_on": "analyzer_dispersion", - "analyzer_elevation": 0, - "analyzer_elevation/@transformation_type": "rotation", - "analyzer_elevation/@units": "degrees", - "analyzer_elevation/@vector": [0, 1, 0], - "analyzer_rotation/@depends_on": "rot_y", - "analyzer_rotation": 0, - "analyzer_rotation/@transformation_type": "rotation", - "analyzer_rotation/@units": "degrees", - "analyzer_rotation/@vector": [0, 0, 1], - "AXISNAME[rot_y]/@depends_on": ".", - "AXISNAME[rot_y]": 140.0, - "AXISNAME[rot_y]/@transformation_type": "rotation", - "AXISNAME[rot_y]/@units": "degrees", - "AXISNAME[rot_y]/@vector": [0, 1, 0] - } - }, - "/ENTRY/INSTRUMENT[instrument]/ELECTRONANALYSER[electronanalyser]/COLLECTIONCOLUMN[collectioncolumn]": { - "projection": "@attrs:metadata/scan_info/projection", - "scheme": "@attrs:metadata/scan_info/scheme", - "lens_mode": "@attrs:metadata/scan_info/LensMode", - "working_distance": 40.0, - "working_distance/@units": "mm" - }, - "/ENTRY/INSTRUMENT[instrument]/ELECTRONANALYSER[electronanalyser]/ENERGYDISPERSION[energydispersion]": { - "energy_scan_mode": "@attrs:metadata/scan_info/energy_scan_mode", - "pass_energy": "@attrs:metadata/scan_info/PassEnergy", - "pass_energy/@units": "eV", - "center_energy": "@attrs:metadata/scan_info/KineticEnergy", - "center_energy/@units": "eV", - "scheme": "hemispherical", - "diameter": 300.0, - "diameter/@units": "mm", - "entrance_slit": { - "shape": "curved slit", - "size": 1.0, - "size/@units": "mm" - }, - "exit_slit": { - "shape": "grid" - } - }, - "/ENTRY/INSTRUMENT[instrument]/ELECTRONANALYSER[electronanalyser]/DETECTOR[detector]": { - "amplifier_type": "MCP", - "detector_type": "Phosphor+CCD", - "sensor_pixels": [ - 256, - 320 - ], - "amplifier_bias": "@attrs:metadata/scan_info/ConversionVoltage", - "amplifier_bias/@units": "V", - "amplifier_voltage": "@attrs:metadata/scan_info/DetectorVoltage", - "amplifier_voltage/@units": "V", - "detector_voltage": "@attrs:metadata/scan_info/ScreenVoltage", - "detector_voltage/@units": "V" - }, - "/ENTRY/INSTRUMENT[instrument]/sourceTYPE[source_probe]": { - "name": "HHG @ TR-ARPES @ FHI", - "probe": "photon", - "type": "HHG laser", - "mode": "Single Bunch", - "frequency": "['@eln:/ENTRY/Instrument/Source/Probe/frequency', '@attrs:metadata/instrument/beam/probe/frequency']", - "frequency/@units": "kHz", - "associated_beam": "/entry/instrument/beam_probe" - }, - "/ENTRY/INSTRUMENT[instrument]/beamTYPE[beam_probe]": { - "distance": 0.0, - "distance/@units": "mm", - "incident_energy": "!['@eln:/ENTRY/Instrument/Beam/Probe/incident_energy', '@attrs:metadata/instrument/beam/probe/incident_energy']", - "incident_energy/@units": "eV", - "incident_energy_spread": "['@eln:/ENTRY/Instrument/Beam/Probe/incident_energy_spread', '@attrs:metadata/instrument/beam/probe/incident_energy_spread']", - "incident_energy_spread/@units": "eV", - "pulse_duration": "['@eln:/ENTRY/Instrument/Beam/Probe/pulse_duration', '@attrs:metadata/instrument/beam/probe/pulse_duration']", - "pulse_duration/@units": "fs", - "incident_polarization": "['@eln:/ENTRY/Instrument/Beam/Probe/incident_polarization', '@attrs:metadata/instrument/beam/probe/incident_polarization']", - "incident_polarization/@units": "V^2/mm^2", - "extent": "['@eln:/ENTRY/Instrument/Beam/Probe/extent', '@attrs:metadata/instrument/beam/probe/extent']", - "extent/@units": "µm", - "associated_source": "/entry/instrument/source_probe" - }, - "/ENTRY/INSTRUMENT[instrument]/sourceTYPE[source_pump]": { - "name": "OPCPA @ TR-ARPES @ FHI", - "probe": "visible light", - "type": "Optical Laser", - "mode": "Single Bunch", - "frequency": "['@eln:/ENTRY/Instrument/Source/Pump/frequency', '@attrs:metadata/instrument/beam/pump/frequency']", - "frequency/@units": "kHz", - "associated_beam": "/entry/instrument/beam_pump" - }, - "/ENTRY/INSTRUMENT[instrument]/beamTYPE[beam_pump]": { - "distance": 0.0, - "distance/@units": "mm", - "incident_energy": "!['@eln:/ENTRY/Instrument/Beam/Pump/incident_energy', '@attrs:metadata/instrument/beam/pump/incident_energy']", - "incident_energy/@units": "eV", - "incident_energy_spread": "['@eln:/ENTRY/Instrument/Beam/Pump/incident_energy_spread', '@attrs:metadata/instrument/beam/pump/incident_energy_spread']", - "incident_energy_spread/@units": "eV", - "incident_wavelength": "['@eln:/ENTRY/Instrument/Beam/Pump/incident_wavelength', '@attrs:metadata/instrument/beam/pump/incident_wavelength']", - "incident_wavelength/@units": "nm", - "pulse_duration": "['@eln:/ENTRY/Instrument/Beam/Pump/pulse_duration', '@attrs:metadata/instrument/beam/pump/pulse_duration']", - "pulse_duration/@units": "fs", - "incident_polarization": "['@eln:/ENTRY/Instrument/Beam/Pump/incident_polarization', '@attrs:metadata/instrument/beam/pump/incident_polarization']", - "incident_polarization/@units": "V^2/mm^2", - "pulse_energy": "['@eln:/ENTRY/Instrument/Beam/Pump/pulse_energy', '@attrs:metadata/instrument/beam/pump/pulse_energy']", - "pulse_energy/@units": "µJ", - "average_power": "['@eln:/ENTRY/Instrument/Beam/Pump/average_power', '@attrs:metadata/instrument/beam/pump/average_power']", - "average_power/@units": "mW", - "extent": "['@eln:/ENTRY/Instrument/Beam/Pump/extent', '@attrs:metadata/instrument/beam/pump/extent']", - "extent/@units": "µm", - "fluence": "['@eln:/ENTRY/Instrument/Beam/Pump/fluence', '@attrs:metadata/instrument/beam/pump/fluence']", - "fluence/@units": "mJ/cm^2", - "associated_source": "/entry/instrument/source_pump" - }, - "/ENTRY/INSTRUMENT[instrument]/MANIPULATOR[manipulator]": { - "temperature_sensor": { - "name": "sample_temperature", - "measurement": "temperature", - "value": "!['@eln:/ENTRY/Instrument/Manipulator/sample_temperature', '@attrs:metadata/scan_info/trARPES:Carving:TEMP_RBV']", - "value/@units": "K" + "source_pump": { + "name": "OPCPA @ TR-ARPES @ FHI", + "probe": "visible light", + "type": "Optical Laser", + "mode": "Single Bunch", + "frequency": "!['@eln:/ENTRY/Instrument/Source/Pump/frequency', '@attrs:metadata/instrument/beam/pump/frequency', '@attrs:metadata/elabFTW/laser_status/pump_repetition_rate']", + "frequency/@units": "kHz", + "associated_beam": "/entry/instrument/beam_pump" }, - "sample_bias_voltmeter": { - "name": "sample_bias", - "measurement": "voltage", - "value": 0.0, - "value/@units": "V" + "beam_pump": { + "distance": 0.0, + "distance/@units": "mm", + "incident_energy": "!['@eln:/ENTRY/Instrument/Beam/Pump/incident_energy', '@attrs:metadata/instrument/beam/pump/incident_energy', '@attrs:metadata/elabFTW/laser_status/pump_photon_energy']", + "incident_energy/@units": "eV", + "incident_energy_spread": "['@eln:/ENTRY/Instrument/Beam/Pump/incident_energy_spread', '@attrs:metadata/instrument/beam/pump/incident_energy_spread', '@attrs:metadata/elabFTW/laser_status/pump_photon_energy_spread']", + "incident_energy_spread/@units": "eV", + "incident_wavelength": "['@eln:/ENTRY/Instrument/Beam/Pump/incident_wavelength', '@attrs:metadata/instrument/beam/pump/incident_wavelength', '@attrs:metadata/scan_info/trARPES:Orpheus:Wavelength']", + "incident_wavelength/@units": "nm", + "pulse_duration": "['@eln:/ENTRY/Instrument/Beam/Pump/pulse_duration', '@attrs:metadata/instrument/beam/pump/pulse_duration', '@attrs:metadata/elabFTW/laser_status/pump_pulse_duration']", + "pulse_duration/@units": "fs", + "pulse_delay": "@attrs:metadata/scan_info/delay", + "pulse_delay/@units": "fs", + "pulse_delay/@reference_beam": "/entry/instrument/beam_probe", + "incident_polarization": "['@eln:/ENTRY/Instrument/Beam/Pump/incident_polarization', '@attrs:metadata/instrument/beam/pump/incident_polarization', '@attrs:metadata/elabFTW/scan/pump_polarization', '@attrs:metadata/scan_info/trARPES:Pump:wp.RBV']", + "incident_polarization/@units": "V^2/mm^2", + "pulse_energy": "['@eln:/ENTRY/Instrument/Beam/Pump/pulse_energy', '@attrs:metadata/instrument/beam/pump/pulse_energy', '@attrs:metadata/elabFTW/scan/pump_pulse_energy']", + "pulse_energy/@units": "µJ", + "average_power": "['@eln:/ENTRY/Instrument/Beam/Pump/average_power', '@attrs:metadata/instrument/beam/pump/average_power', '@attrs:metadata/scan_info/trARPES:Pump:Power.RBV']", + "average_power/@units": "mW", + "extent": "['@eln:/ENTRY/Instrument/Beam/Pump/extent', '@attrs:metadata/instrument/beam/pump/extent', '@attrs:metadata/elabFTW/laser_status/pump_profile']", + "extent/@units": "µm", + "fluence": "['@eln:/ENTRY/Instrument/Beam/Pump/fluence', '@attrs:metadata/instrument/beam/pump/fluence', '@attrs:metadata/elabFTW/scan/pump_fluence', '@attrs:metadata/scan_info/trARPES:Pump:fluence.RBV']", + "fluence/@units": "mJ/cm^2", + "associated_source": "/entry/instrument/source_pump" }, - "drain_current_amperemeter": { - "name": "drain_current", - "measurement": "current", - "value": "!['@eln:/ENTRY/Instrument/Manipulator/drain_current', '@attrs:metadata/scan_info/trARPES:Sample:Measure']", - "value/@units": "A" - }, - "depends_on": "/entry/instrument/manipulator/transformations/trans_z", - "TRANSFORMATIONS[transformations]": { - "AXISNAME[trans_z]": -0.32, - "AXISNAME[trans_z]/@depends_on": "rot_z", - "AXISNAME[trans_z]/@transformation_type": "translation", - "AXISNAME[trans_z]/@units": "m", - "AXISNAME[trans_z]/@vector": [0, 0, 1], - "AXISNAME[rot_z]/@depends_on": "rot_x", - "AXISNAME[rot_z]": -115.0, - "AXISNAME[rot_z]/@transformation_type": "rotation", - "AXISNAME[rot_z]/@units": "degrees", - "AXISNAME[rot_z]/@vector": [0, 0, 1], - "AXISNAME[rot_x]/@depends_on": ".", - "AXISNAME[rot_x]": -90.0, - "AXISNAME[rot_x]/@transformation_type": "rotation", - "AXISNAME[rot_x]/@units": "degrees", - "AXISNAME[rot_x]/@vector": [1, 0, 0] + "MANIPULATOR[manipulator]": { + "temperature_sensor": { + "name": "sample_temperature", + "measurement": "temperature", + "value": "!['@eln:/ENTRY/Instrument/Manipulator/sample_temperature', '@attrs:metadata/scan_info/trARPES:Carving:TEMP_RBV']", + "value/@units": "K" + }, + "sample_bias_voltmeter": { + "name": "sample_bias", + "measurement": "voltage", + "value": 0.0, + "value/@units": "V" + }, + "drain_current_ammeter": { + "name": "drain_current", + "measurement": "current", + "value": "!['@eln:/ENTRY/Instrument/Manipulator/drain_current', '@attrs:metadata/scan_info/trARPES:Sample:Measure']", + "value/@units": "A" + }, + "depends_on": "/entry/instrument/manipulator/transformations/trans_z", + "TRANSFORMATIONS[transformations]": { + "AXISNAME[trans_z]": -0.32, + "AXISNAME[trans_z]/@depends_on": "rot_z", + "AXISNAME[trans_z]/@transformation_type": "translation", + "AXISNAME[trans_z]/@units": "m", + "AXISNAME[trans_z]/@vector": [0, 0, 1], + "AXISNAME[rot_z]/@depends_on": "rot_x", + "AXISNAME[rot_z]": -115.0, + "AXISNAME[rot_z]/@transformation_type": "rotation", + "AXISNAME[rot_z]/@units": "degrees", + "AXISNAME[rot_z]/@vector": [0, 0, 1], + "AXISNAME[rot_x]/@depends_on": ".", + "AXISNAME[rot_x]": -90.0, + "AXISNAME[rot_x]/@transformation_type": "rotation", + "AXISNAME[rot_x]/@units": "degrees", + "AXISNAME[rot_x]/@vector": [1, 0, 0] + } } }, "/ENTRY/SAMPLE[sample]": { - "preparation_date": "['@eln:/ENTRY/Sample/preparation_date', '@attrs:metadata/sample/preparation_date']", - "history/notes/description": "['@eln:/ENTRY/Sample/sample_history', '@attrs:metadata/sample/sample_history']", - "description": "['@eln:/ENTRY/Sample/description', '@attrs:metadata/sample/chemical_formula']", - "name": "!['@eln:/ENTRY/Sample/name', '@attrs:metadata/sample/name']", + "preparation_date": "['@eln:/ENTRY/Sample/preparation_date', '@attrs:metadata/sample/preparation_date', '@attrs:metadata/elabFTW/sample/preparation_date']", + "history/sample_preparation/description": "['@eln:/ENTRY/Sample/sample_history', '@attrs:metadata/sample/sample_history', '@attrs:metadata/elabFTW/sample/sample_history']", + "history/sample_preparation/start_time": "['@eln:/ENTRY/Sample/preparation_date', '@attrs:metadata/sample/preparation_date', '@attrs:metadata/elabFTW/sample/preparation_date']", + "description": "['@eln:/ENTRY/Sample/description', '@attrs:metadata/sample/chemical_formula', '@attrs:metadata/elabFTW/sample/summary']", + "name": "!['@eln:/ENTRY/Sample/name', '@attrs:metadata/sample/name', '@attrs:metadata/elabFTW/sample/title']", "situation": "vacuum", - "SUBSTANCE[substance]/molecular_formula_hill": "['@eln:/ENTRY/Sample/chemical_formula', '@attrs:metadata/sample/chemical_formula']", + "chemical_formula": "['@eln:/ENTRY/Sample/chemical_formula', '@attrs:metadata/sample/chemical_formula', '@attrs:metadata/elabFTW/sample/sample_formula']", "temperature_env": { "temperature_sensor": "@link:/entry/instrument/manipulator/temperature_sensor" }, @@ -268,7 +264,7 @@ "voltmeter": "@link:/entry/instrument/manipulator/sample_bias_voltmeter" }, "drain_current_env": { - "amperemeter": "@link:/entry/instrument/manipulator/drain_current_amperemeter" + "ammeter": "@link:/entry/instrument/manipulator/drain_current_ammeter" }, "depends_on": "/entry/sample/transformations/offset_azimuth", "TRANSFORMATIONS[transformations]": { @@ -329,11 +325,28 @@ "AXISNAME[trans_x]/@vector": [0, 1, 0] } }, - "/ENTRY/PROCESS[process]/CALIBRATION[energy_calibration]":{ - "calibrated_axis": "@data:energy.data" + "/ENTRY/CALIBRATION[energy_calibration]":{ + "program": "@attrs:metadata/scan_info/program_name", + "version": "@attrs:metadata/scan_info/program_version", + "calibrated_axis": "@attrs:metadata/scan_info/Ekin", + "calibrated_axis/@units": "energy", + "physical_quantity": "energy", + "PARAMETERS[conversion_parameters]":{ + "TERM[*{apply_fft_filter,binning,rotation_angle,lens_mode,kinetic_energy,pass_energy,work_function,a_inner,da_matrix,retardation_ratio,source,dims,e_shift,de1,e_range,a_range,pixel_size,magnification,angle_offset_px,energy_offset_px}]": "@attrs:metadata/conversion_parameters/*" + } + }, + "/ENTRY/CALIBRATION[angular0_calibration]":{ + "calibrated_axis": "@link:/entry/data/angular0", + "calibrated_axis/@units": "angle", + "physical_quantity": "angle" }, - "/ENTRY/PROCESS[process]/CALIBRATION[angular0_calibration]":{ - "calibrated_axis": "@data:angular0.data" + "/ENTRY/CALIBRATION[energy_referencing]":{ + "calibrated_axis": "@link:/entry/data/energy", + "calibrated_axis/@units": "energy", + "physical_quantity": "energy", + "reference_peak": "@attrs:metadata/scan_info/reference_energy", + "binding_energy": 0.0, + "binding_energy/@units": "energy" }, "/ENTRY/data": { "@axes": "@data:dims", @@ -344,6 +357,7 @@ "*": "@data:*.data", "*/@units": "@data:*.unit", "*/@reference": "@attrs:metadata/scan_info/coordinate_depends/*", + "*/@long_name": "@attrs:metadata/scan_info/coordinate_label/*", "energy/@type": "kinetic" } } diff --git a/tutorial/3_specsscan_conversion_to_NeXus.ipynb b/tutorial/3_specsscan_conversion_to_NeXus.ipynb index 0e519d6..52507e4 100755 --- a/tutorial/3_specsscan_conversion_to_NeXus.ipynb +++ b/tutorial/3_specsscan_conversion_to_NeXus.ipynb @@ -66,11 +66,12 @@ "metadata['instrument'] = {}\n", "# energy resolution\n", "metadata['instrument']['energy_resolution'] = 150.\n", - "metadata['instrument']['electronanalyser'] = {}\n", - "metadata['instrument']['electronanalyser']['energy_resolution'] = 120\n", - "metadata['instrument']['electronanalyser']['angular_resolution'] = 0.2\n", - "metadata['instrument']['electronanalyser']['spatial_resolution'] = 0.5\n", - "\n", + "metadata['instrument']['electronanalyzer'] = {}\n", + "metadata['instrument']['electronanalyzer']['energy_resolution'] = 120.\n", + "metadata['instrument']['electronanalyzer']['angular_resolution'] = 0.2\n", + "metadata['instrument']['electronanalyzer']['spatial_resolution'] = 0.5\n", + "metadata['instrument']['electronanalyzer']['slit_shape'] = \"curved slit\"\n", + "metadata['instrument']['electronanalyzer']['slit_size'] = 1.0\n", "#probe beam\n", "metadata['instrument']['beam']={}\n", "metadata['instrument']['beam']['probe']={}\n", @@ -105,8 +106,8 @@ "\n", "metadata[\"scan_info\"] = {}\n", "metadata[\"scan_info\"][\"trARPES:XGS600:PressureAC:P_RD\"] = 2.5E-11\n", - "metadata[\"scan_info\"][\"trARPES:Carving:TEMP_RBV\"] = 70\n", - "metadata[\"scan_info\"][\"trARPES:Sample:Measure\"] = 0\n" + "metadata[\"scan_info\"][\"trARPES:Carving:TEMP_RBV\"] = 70.\n", + "metadata[\"scan_info\"][\"trARPES:Sample:Measure\"] = 0.\n" ] }, { @@ -248,7 +249,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.12.8" + "version": "3.10.16" } }, "nbformat": 4, From dddc80c43b7e66f11a5d0730da3fc7e27cd4bdf6 Mon Sep 17 00:00:00 2001 From: rettigl Date: Mon, 24 Mar 2025 13:11:14 +0100 Subject: [PATCH 41/44] fix test --- tests/test_specsscan.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/tests/test_specsscan.py b/tests/test_specsscan.py index ed39670..80edf62 100755 --- a/tests/test_specsscan.py +++ b/tests/test_specsscan.py @@ -318,15 +318,20 @@ def test_conversion_and_save_to_nexus(): metadata["instrument"] = {} # energy resolution metadata["instrument"]["energy_resolution"] = 150.0 - metadata["instrument"]["electronanalyser"] = {} - metadata["instrument"]["electronanalyser"]["energy_resolution"] = 120 - metadata["instrument"]["electronanalyser"]["angular_resolution"] = 0.2 - metadata["instrument"]["electronanalyser"]["spatial_resolution"] = 0.5 + metadata["instrument"]["electronanalyzer"] = {} + metadata["instrument"]["electronanalyzer"]["energy_resolution"] = 120.0 + metadata["instrument"]["electronanalyzer"]["angular_resolution"] = 0.2 + metadata["instrument"]["electronanalyzer"]["spatial_resolution"] = 0.5 # probe beam metadata["instrument"]["beam"] = {} metadata["instrument"]["beam"]["probe"] = {} metadata["instrument"]["beam"]["probe"]["incident_energy"] = 21.7 + metadata["instrument"]["beam"]["probe"]["frequency"] = 500.0 + metadata["instrument"]["beam"]["probe"]["incident_energy_spread"] = 0.11 + metadata["instrument"]["beam"]["probe"]["pulse_duration"] = 20.0 + metadata["instrument"]["beam"]["probe"]["incident_polarization"] = [1, 1, 0, 0] + metadata["instrument"]["beam"]["probe"]["extent"] = [80.0, 80.0] # sample metadata["sample"] = {} @@ -334,8 +339,8 @@ def test_conversion_and_save_to_nexus(): metadata["scan_info"] = {} metadata["scan_info"]["trARPES:XGS600:PressureAC:P_RD"] = 2.5e-11 - metadata["scan_info"]["trARPES:Carving:TEMP_RBV"] = 70 - metadata["scan_info"]["trARPES:Sample:Measure"] = 0 + metadata["scan_info"]["trARPES:Carving:TEMP_RBV"] = 70.0 + metadata["scan_info"]["trARPES:Sample:Measure"] = 0.0 res_xarray = sps.load_scan( scan=1496, path=data_dir, From 1e9b52ab0e8d80cb6dd658f17c68b54f5970b593 Mon Sep 17 00:00:00 2001 From: rettigl Date: Mon, 24 Mar 2025 13:13:25 +0100 Subject: [PATCH 42/44] fix config --- src/specsscan/config/example_config_FHI.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/specsscan/config/example_config_FHI.yaml b/src/specsscan/config/example_config_FHI.yaml index cb26071..23e3538 100644 --- a/src/specsscan/config/example_config_FHI.yaml +++ b/src/specsscan/config/example_config_FHI.yaml @@ -18,8 +18,8 @@ coordinate_mapping: # dictionary of corresponding NeXus paths for the different axes coordinate_depends: - Ekin: "/entry/instrument/electronanalyser/energydispersion/kinetic_energy" - Angle: "/entry/instrument/electronanalyser/transformations/analyzer_dispersion" + Ekin: "/entry/instrument/electronanalyzer/energydispersion/kinetic_energy" + Angle: "/entry/instrument/electronanalyzer/transformations/analyzer_dispersion" polar: "/entry/sample/transformations/sample_polar" tilt: "/entry/sample/transformations/sample_tilt" azimuth: "/entry/sample/transformations/sample_azimuth" From fe3ce79a5ac72036bbdeebf56723de8045f48d38 Mon Sep 17 00:00:00 2001 From: rettigl Date: Mon, 24 Mar 2025 22:24:34 +0100 Subject: [PATCH 43/44] use newest pynxtools --- pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 44e7fdb..55403a3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -40,8 +40,8 @@ dependencies = [ "matplotlib>=3.5.1", "numpy>=1.21.6", "opencv-python>=4.8.1.78", - "pynxtools-mpes>=0.2.1", - "pynxtools>=0.10.0", + "pynxtools-mpes@git+https:///github.com/FAIRmat-NFDI/pynxtools-mpes", + "pynxtools@git+https:///github.com/FAIRmat-NFDI/pynxtools", "python-dateutil>=2.8.2", "pyyaml>=6.0", "xarray>=0.20.2", From 35d602b8f1a35fa38f529cfb9da2c2e77c376761 Mon Sep 17 00:00:00 2001 From: rettigl Date: Wed, 26 Mar 2025 12:50:49 +0100 Subject: [PATCH 44/44] update pynxtools --- pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 55403a3..1d6ae8f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -40,8 +40,8 @@ dependencies = [ "matplotlib>=3.5.1", "numpy>=1.21.6", "opencv-python>=4.8.1.78", - "pynxtools-mpes@git+https:///github.com/FAIRmat-NFDI/pynxtools-mpes", - "pynxtools@git+https:///github.com/FAIRmat-NFDI/pynxtools", + "pynxtools-mpes>=0.2.2", + "pynxtools>=0.10.1", "python-dateutil>=2.8.2", "pyyaml>=6.0", "xarray>=0.20.2",