diff --git a/.cspell/custom-dictionary.txt b/.cspell/custom-dictionary.txt index bf46422..c31708e 100644 --- a/.cspell/custom-dictionary.txt +++ b/.cspell/custom-dictionary.txt @@ -2,6 +2,8 @@ allclose ALLUSERSPROFILE amperemeter +appauthor +appname arange archiver argwhere @@ -18,6 +20,7 @@ basepath bitshift bysource calib +caplog checkscan clim codemirror @@ -32,16 +35,21 @@ dapolymatrix dataconverter dataframe delaystage +delenv dtype dxda dxde dyda dyde Ekin -electronanalyser +elab +elabapi +elabid +electronanalyzer elems endstation energydispersion +entityid eshift faddr Faradayweg @@ -72,6 +80,7 @@ kwds labview Laurenz lensmodes +levelname lineh linev listf @@ -102,6 +111,7 @@ Nxpix Nxpixels Nypixels OPCPA +orcid pcolormesh Phoibos polyfit @@ -120,6 +130,7 @@ rrvec rtol rtype scanvector +sharelink specsanalyzer Specslab specsscan @@ -131,9 +142,11 @@ toctree tomlkit topfloor tqdm +trarpes typehints TZCYXS undoc +userid venv viewcode vline diff --git a/docs/specsanalyzer/config.rst b/docs/specsanalyzer/config.rst index 4165f69..34ce2f1 100644 --- a/docs/specsanalyzer/config.rst +++ b/docs/specsanalyzer/config.rst @@ -4,8 +4,8 @@ The config module contains a mechanics to collect configuration parameters from It will load an (optional) provided config file, or alternatively use a passed python dictionary as initial config dictionary, and subsequently look for the following additional config files to load: * ``folder_config``: A config file of name :file:`specs_config.yaml` in the current working directory. This is mostly intended to pass calibration parameters of the workflow between different notebook instances. -* ``user_config``: A config file provided by the user, stored as :file:`.specsanalyzer/config.yaml` in the current user's home directly. This is intended to give a user the option for individual configuration modifications of system settings. -* ``system_config``: A config file provided by the system administrator, stored as :file:`/etc/specsanalyzer/config.yaml` on Linux-based systems, and :file:`%ALLUSERSPROFILE%/specsanalyzer/config.yaml` on Windows. This should provide all necessary default parameters for using the specsanalyzer processor with a given setup. For an example for the setup at the Fritz Haber Institute setup, see :ref:`example_config` +* ``user_config``: A config file provided by the user, stored as :file:`.config/specsanalyzer/config_v1.yaml` in the current user's home directly. This is intended to give a user the option for individual configuration modifications of system settings. +* ``system_config``: A config file provided by the system administrator, stored as :file:`/etc/specsanalyzer/config_v1.yaml` on Linux-based systems, and :file:`%ALLUSERSPROFILE%/specsanalyzer/config_v1.yaml` on Windows. This should provide all necessary default parameters for using the specsanalyzer processor with a given setup. For an example for the setup at the Fritz Haber Institute setup, see :ref:`example_config` * ``default_config``: The default configuration shipped with the package. Typically, all parameters here should be overwritten by any of the other configuration files. The config mechanism returns the combined dictionary, and reports the loaded configuration files. In order to disable or overwrite any of the configuration files, they can be also given as optional parameters (path to a file, or python dictionary). diff --git a/docs/specsscan/helpers.rst b/docs/specsscan/helpers.rst index aeca3c1..dc4ce92 100644 --- a/docs/specsscan/helpers.rst +++ b/docs/specsscan/helpers.rst @@ -3,3 +3,7 @@ Helpers .. automodule:: specsscan.helpers :members: :undoc-members: + +.. automodule:: specsscan.metadata + :members: + :undoc-members: diff --git a/pyproject.toml b/pyproject.toml index 8f1ce02..1d6ae8f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -32,15 +32,16 @@ classifiers = [ "Operating System :: OS Independent", ] dependencies = [ + "elabapi-python>=5.0", "h5py>=3.6.0", "imutils>=0.5.4", "ipympl>=0.9.1", "ipywidgets>=7.7.1", - "matplotlib>=3.5.1,<3.10.0", + "matplotlib>=3.5.1", "numpy>=1.21.6", "opencv-python>=4.8.1.78", - "pynxtools-mpes>=0.2.1", - "pynxtools>=0.9.3", + "pynxtools-mpes>=0.2.2", + "pynxtools>=0.10.1", "python-dateutil>=2.8.2", "pyyaml>=6.0", "xarray>=0.20.2", @@ -83,12 +84,6 @@ all = [ "specsanalyzer[dev,docs,notebook]", ] -[tool.coverage.report] -omit = [ - "config.py", - "config-3.py", -] - [tool.ruff] include = ["specsanalyzer/*.py", "specsscan/*.py", "tests/*.py"] lint.select = [ diff --git a/src/specsanalyzer/config.py b/src/specsanalyzer/config.py index f946744..edeb50f 100755 --- a/src/specsanalyzer/config.py +++ b/src/specsanalyzer/config.py @@ -1,6 +1,7 @@ """This module contains a config library for loading yaml/json files into dicts""" from __future__ import annotations +import copy import json import os import platform @@ -8,9 +9,27 @@ from pathlib import Path import yaml +from platformdirs import user_config_path + +from specsanalyzer.logging import setup_logging package_dir = os.path.dirname(find_spec("specsanalyzer").origin) +USER_CONFIG_PATH = user_config_path( + appname="specsanalyzer", + appauthor="OpenCOMPES", + ensure_exists=True, +) +SYSTEM_CONFIG_PATH = ( + Path(os.environ["ALLUSERSPROFILE"]).joinpath("specsanalyzer") + if platform.system() == "Windows" + else Path("/etc/").joinpath("specsanalyzer") +) +ENV_DIR = Path(".env") + +# Configure logging +logger = setup_logging("config") + def parse_config( config: dict | str = None, @@ -36,12 +55,13 @@ def parse_config( user_config (dict | str, optional): user-based config dictionary or file path. The loaded dictionary is completed with the user-based values, taking preference over system and default values. - Defaults to the file ".specsanalyzer/config.yaml" in the current user's home directory. + Defaults to the file ".config/specsanalyzer/config_v1.yaml" in the current user's home + directory. system_config (dict | str, optional): system-wide config dictionary or file path. The loaded dictionary is completed with the system-wide values, taking preference over default values. - Defaults to the file "/etc/specsanalyzer/config.yaml" on linux, - and "%ALLUSERSPROFILE%/specsanalyzer/config.yaml" on windows. + Defaults to the file "/etc/specsanalyzer/config_v1.yaml" on linux, + and "%ALLUSERSPROFILE%/specsanalyzer/config_v1.yaml" on windows. default_config (dict | str, optional): default config dictionary or file path. The loaded dictionary is completed with the default values. Defaults to *package_dir*/config/default.yaml". @@ -57,62 +77,51 @@ def parse_config( config = {} if isinstance(config, dict): - config_dict = config + config_dict = copy.deepcopy(config) else: config_dict = load_config(config) if verbose: - print(f"Configuration loaded from: [{str(Path(config).resolve())}]") + logger.info(f"Configuration loaded from: [{str(Path(config).resolve())}]") folder_dict: dict = None if isinstance(folder_config, dict): - folder_dict = folder_config + folder_dict = copy.deepcopy(folder_config) else: if folder_config is None: folder_config = "./specs_config.yaml" if Path(folder_config).exists(): folder_dict = load_config(folder_config) if verbose: - print(f"Folder config loaded from: [{str(Path(folder_config).resolve())}]") + logger.info(f"Folder config loaded from: [{str(Path(folder_config).resolve())}]") user_dict: dict = None if isinstance(user_config, dict): - user_dict = user_config + user_dict = copy.deepcopy(user_config) else: if user_config is None: - user_config = str( - Path.home().joinpath(".specsanalyzer").joinpath("config.yaml"), - ) + user_config = str(USER_CONFIG_PATH.joinpath("config_v1.yaml")) if Path(user_config).exists(): user_dict = load_config(user_config) if verbose: - print(f"User config loaded from: [{str(Path(user_config).resolve())}]") + logger.info(f"User config loaded from: [{str(Path(user_config).resolve())}]") system_dict: dict = None if isinstance(system_config, dict): - system_dict = system_config + system_dict = copy.deepcopy(system_config) else: if system_config is None: - if platform.system() in ["Linux", "Darwin"]: - system_config = str( - Path("/etc/").joinpath("specsanalyzer").joinpath("config.yaml"), - ) - elif platform.system() == "Windows": - system_config = str( - Path(os.environ["ALLUSERSPROFILE"]) - .joinpath("specsanalyzer") - .joinpath("config.yaml"), - ) + system_config = str(SYSTEM_CONFIG_PATH.joinpath("config_v1.yaml")) if Path(system_config).exists(): system_dict = load_config(system_config) if verbose: - print(f"System config loaded from: [{str(Path(system_config).resolve())}]") + logger.info(f"System config loaded from: [{str(Path(system_config).resolve())}]") if isinstance(default_config, dict): - default_dict = default_config + default_dict = copy.deepcopy(default_config) else: default_dict = load_config(default_config) if verbose: - print(f"Default config loaded from: [{str(Path(default_config).resolve())}]") + logger.info(f"Default config loaded from: [{str(Path(default_config).resolve())}]") if folder_dict is not None: config_dict = complete_dictionary( @@ -226,3 +235,85 @@ def complete_dictionary(dictionary: dict, base_dictionary: dict) -> dict: dictionary[k] = v return dictionary + + +def _parse_env_file(file_path: Path) -> dict: + """Helper function to parse a .env file into a dictionary. + + Args: + file_path (Path): Path to the .env file + + Returns: + dict: Dictionary of environment variables from the file + """ + env_content = {} + if file_path.exists(): + with open(file_path) as f: + for line in f: + line = line.strip() + if line and "=" in line: + key, val = line.split("=", 1) + env_content[key.strip()] = val.strip() + return env_content + + +def read_env_var(var_name: str) -> str | None: + """Read an environment variable from multiple locations in order: + 1. OS environment variables + 2. .env file in current directory + 3. .env file in user config directory + 4. .env file in system config directory + + Args: + var_name (str): Name of the environment variable to read + + Returns: + str | None: Value of the environment variable or None if not found + """ + # 1. check OS environment variables + value = os.getenv(var_name) + if value is not None: + logger.debug(f"Found {var_name} in OS environment variables") + return value + + # 2. check .env in current directory + local_vars = _parse_env_file(ENV_DIR) + if var_name in local_vars: + logger.debug(f"Found {var_name} in ./.env file") + return local_vars[var_name] + + # 3. check .env in user config directory + user_vars = _parse_env_file(USER_CONFIG_PATH / ".env") + if var_name in user_vars: + logger.debug(f"Found {var_name} in user config .env file") + return user_vars[var_name] + + # 4. check .env in system config directory + system_vars = _parse_env_file(SYSTEM_CONFIG_PATH / ".env") + if var_name in system_vars: + logger.debug(f"Found {var_name} in system config .env file") + return system_vars[var_name] + + logger.debug(f"Environment variable {var_name} not found in any location") + return None + + +def save_env_var(var_name: str, value: str) -> None: + """Save an environment variable to the .env file in the user config directory. + If the file exists, preserves other variables. If not, creates a new file. + + Args: + var_name (str): Name of the environment variable to save + value (str): Value to save for the environment variable + """ + env_path = USER_CONFIG_PATH / ".env" + env_content = _parse_env_file(env_path) + + # Update or add new variable + env_content[var_name] = value + + # Write all variables back to file + with open(env_path, "w") as f: + for key, val in env_content.items(): + f.write(f"{key}={val}\n") + logger.debug(f"Environment variable {var_name} saved to .env file") diff --git a/src/specsanalyzer/convert.py b/src/specsanalyzer/convert.py index aaa015f..0fa8de4 100755 --- a/src/specsanalyzer/convert.py +++ b/src/specsanalyzer/convert.py @@ -1,9 +1,14 @@ """Specsanalyzer image conversion module""" from __future__ import annotations +import logging + import numpy as np from scipy.ndimage import map_coordinates +# Configure logging +logger = logging.getLogger("specsanalyzer.specsscan") + def get_damatrix_from_calib2d( lens_mode: str, @@ -82,7 +87,7 @@ def get_damatrix_from_calib2d( elif lens_mode in supported_space_modes: # use the mode defaults - print("This is a spatial mode, using default " + lens_mode + " config") + logger.info("This is a spatial mode, using default " + lens_mode + " config") rr_vec, da_matrix_full = get_rr_da(lens_mode, calib2d_dict) a_inner = da_matrix_full[0][0] da_matrix = da_matrix_full[1:][:] diff --git a/src/specsanalyzer/core.py b/src/specsanalyzer/core.py index c461bcf..f8b07d9 100755 --- a/src/specsanalyzer/core.py +++ b/src/specsanalyzer/core.py @@ -3,7 +3,6 @@ import os from typing import Any -from typing import Generator import imutils import ipywidgets as ipw @@ -21,9 +20,14 @@ from specsanalyzer.convert import physical_unit_data from specsanalyzer.img_tools import crop_xarray from specsanalyzer.img_tools import fourier_filter_2d +from specsanalyzer.logging import set_verbosity +from specsanalyzer.logging import setup_logging package_dir = os.path.dirname(__file__) +# Configure logging +logger = setup_logging("specsanalyzer") + class SpecsAnalyzer: """SpecsAnalyzer: A class to convert photoemission data from a SPECS Phoibos analyzer from @@ -39,6 +43,7 @@ def __init__( self, metadata: dict[Any, Any] = {}, config: dict[Any, Any] | str = {}, + verbose: bool = True, **kwds, ): """SpecsAnalyzer constructor. @@ -46,12 +51,14 @@ def __init__( Args: metadata (dict, optional): Metadata dictionary. Defaults to {}. config (dict | str, optional): Metadata dictionary or file path. Defaults to {}. + verbose (bool, optional): Disable info logs if set to False. **kwds: Keyword arguments passed to ``parse_config``. """ self._config = parse_config( config, **kwds, ) + set_verbosity(logger, verbose) self.metadata = metadata self._data_array = None self.print_msg = True @@ -273,7 +280,7 @@ def convert_image( }, dims=conversion_parameters["dims"], attrs={"conversion_parameters": conversion_parameters}, - ) + ).astype(np.float32) # Handle cropping based on parameters stored in correction dictionary crop = kwds.pop("crop", self._config.get("crop", False)) @@ -287,7 +294,7 @@ def convert_image( ek_min = range_dict["ek_min"] ek_max = range_dict["ek_max"] if self.print_msg: - print("Using saved crop parameters...") + logger.info("Using saved crop parameters...") data_array = crop_xarray(data_array, ang_min, ang_max, ek_min, ek_max) except KeyError: try: @@ -344,11 +351,13 @@ def convert_image( + data_array.coords[data_array.dims[1]][0] ) if self.print_msg: - print("Cropping parameters not found, using cropping ranges from config...") + logger.info( + "Cropping parameters not found, using cropping ranges from config...", + ) data_array = crop_xarray(data_array, ang_min, ang_max, ek_min, ek_max) except KeyError: if self.print_msg: - print( + logger.warning( "Warning: Cropping parameters not found, " "use method crop_tool() after loading.", ) @@ -382,6 +391,8 @@ def crop_tool( - ek_range_max - ang_range_min - ang_range_max + - angle_offset_px + - rotation_angle Other parameters are passed to ``convert_image()``. """ @@ -401,7 +412,7 @@ def crop_tool( try: mesh_obj = data_array.plot(ax=ax) except AttributeError: - print("Load the scan first!") + logger.info("Load the scan first!") raise lineh1 = ax.axhline(y=data_array.Angle[0]) @@ -473,6 +484,15 @@ def crop_tool( vline_range = [ek_min, ek_max] hline_range = [ang_min, ang_max] + angle_offset_px = kwds.get("angle_offset_px", self._config.get("angle_offset_px", 0)) + rotation_angle = kwds.get("rotation_angle", self._config.get("rotation_angle", 0)) + + clim_slider = ipw.FloatRangeSlider( + description="colorbar limits", + value=[data_array.data.min(), data_array.data.max()], + min=data_array.data.min(), + max=data_array.data.max(), + ) vline_slider = ipw.FloatRangeSlider( description="Ekin", value=vline_range, @@ -487,14 +507,33 @@ def crop_tool( max=data_array.Angle[-1], step=0.1, ) - clim_slider = ipw.FloatRangeSlider( - description="colorbar limits", - value=[data_array.data.min(), data_array.data.max()], - min=data_array.data.min(), - max=data_array.data.max(), + ang_offset_slider = ipw.FloatSlider( + description="Angle offset", + value=angle_offset_px, + min=-20, + max=20, + step=1, + ) + rotation_slider = ipw.FloatSlider( + description="Rotation angle", + value=rotation_angle, + min=-5, + max=5, + step=0.1, ) - def update(hline, vline, v_vals): + def update(hline, vline, v_vals, angle_offset_px, rotation_angle): + data_array = self.convert_image( + raw_img=raw_img, + lens_mode=lens_mode, + kinetic_energy=kinetic_energy, + pass_energy=pass_energy, + work_function=work_function, + crop=False, + angle_offset_px=angle_offset_px, + rotation_angle=rotation_angle, + ) + mesh_obj.update({"array": data_array.data}) lineh1.set_ydata([hline[0]]) lineh2.set_ydata([hline[1]]) linev1.set_xdata([vline[0]]) @@ -507,9 +546,11 @@ def update(hline, vline, v_vals): hline=hline_slider, vline=vline_slider, v_vals=clim_slider, + angle_offset_px=ang_offset_slider, + rotation_angle=rotation_slider, ) - def cropit(val): # pylint: disable=unused-argument + def cropit(val): # noqa: ARG001 ang_min = min(hline_slider.value) ang_max = max(hline_slider.value) ek_min = min(vline_slider.value) @@ -552,6 +593,8 @@ def cropit(val): # pylint: disable=unused-argument ) ).item() self._config["crop"] = True + self._config["angle_offset_px"] = ang_offset_slider.value + self._config["rotation_angle"] = rotation_slider.value ax.cla() self._data_array.plot(ax=ax, add_colorbar=False) @@ -561,6 +604,8 @@ def cropit(val): # pylint: disable=unused-argument hline_slider.close() clim_slider.close() apply_button.close() + ang_offset_slider.close() + rotation_slider.close() apply_button = ipw.Button(description="Crop") display(apply_button) @@ -615,7 +660,7 @@ def fft_tool( filtered = fourier_filter_2d(raw_image, peaks=fft_filter_peaks, ret="filtered") except IndexError: - print("Load the scan first!") + logger.warning("Load the scan first!") raise fig = plt.figure() @@ -708,8 +753,7 @@ def update(v_vals, pos_x, pos_y, sigma_x, sigma_y, amplitude): fft_filt.set_data(np.abs(fft_filtered_new.T)) nonlocal cont - for i in range(len(cont.collections)): - cont.collections[i].remove() + cont.remove() cont = ax.contour(msk.T) edc.set_ydata(np.sum(filtered_new, 0)) @@ -727,7 +771,7 @@ def update(v_vals, pos_x, pos_y, sigma_x, sigma_y, amplitude): v_vals=clim_slider, ) - def apply_fft(apply: bool): # pylint: disable=unused-argument + def apply_fft(apply: bool): # noqa: ARG001 amplitude = amplitude_slider.value pos_x = pos_x_slider.value pos_y = pos_y_slider.value diff --git a/src/specsanalyzer/logging.py b/src/specsanalyzer/logging.py new file mode 100644 index 0000000..d6915d1 --- /dev/null +++ b/src/specsanalyzer/logging.py @@ -0,0 +1,109 @@ +""" +This module provides a function to set up logging for the application. It configures +both console and file logging handlers, allowing different log levels for each. The +log files are stored in a user-specific log directory. + +""" +from __future__ import annotations + +import logging +import os +import sys +from datetime import datetime + +# Default log directory +DEFAULT_LOG_DIR = os.path.join(os.getcwd(), "logs") +CONSOLE_VERBOSITY = logging.INFO +FILE_VERBOSITY = logging.DEBUG + + +def setup_logging( + name: str, + set_base_handler: bool = False, + user_log_path: str | None = None, +) -> logging.Logger: + """ + Configures and returns a logger with specified log levels for console and file handlers. + + Args: + name (str): The name of the logger. + set_base_handler (bool, optional): Option to re-initialize the base handler logging to the + logfile. Defaults to False. + user_log_path (str, optional): Path to the user-specific log directory. + Defaults to DEFAULT_LOG_DIR. + + Returns: + logging.Logger: The configured logger instance. + + The logger will always write DEBUG level messages to a file located in the user's log + directory, while the console log level can be adjusted based on the 'verbosity' parameter. + """ + # Create base logger + base_logger = logging.getLogger("specsanalyzer") + base_logger.setLevel(logging.DEBUG) # Set the minimum log level for the logger + if set_base_handler or len(base_logger.handlers) == 0: + if len(base_logger.handlers): + base_logger.handlers.clear() + + # Determine log file path + if user_log_path is None: + user_log_path = DEFAULT_LOG_DIR + try: + os.makedirs(user_log_path, exist_ok=True) + log_file = os.path.join( + user_log_path, + f"specsanalyzer_{datetime.now().strftime('%Y-%m-%d')}.log", + ) + + # Create file handler and set level to debug + file_handler = logging.FileHandler(log_file) + file_handler.setLevel(FILE_VERBOSITY) + + # Create formatter for file + file_formatter = logging.Formatter( + "%(asctime)s - %(name)s - %(levelname)s - %(message)s in %(filename)s:%(lineno)d", + ) + file_handler.setFormatter(file_formatter) + + # Add file handler to logger + base_logger.addHandler(file_handler) + except PermissionError: + logging.warning(f"Cannot create logfile in Folder {user_log_path}, disabling logfile.") + base_logger.addHandler(logging.NullHandler()) + base_logger.propagate = False + + # create named logger + logger = base_logger.getChild(name) + + if logger.hasHandlers(): + logger.handlers.clear() + + # Create console handler and set level + console_handler = logging.StreamHandler(sys.stdout) + console_handler.setLevel(CONSOLE_VERBOSITY) + + # Create formatter for console + console_formatter = logging.Formatter("%(levelname)s - %(message)s") + console_handler.setFormatter(console_formatter) + + # Add console handler to logger + logger.addHandler(console_handler) + + # Capture warnings with the logging system + logging.captureWarnings(True) + + return logger + + +def set_verbosity(logger: logging.Logger, verbose: bool) -> None: + """Sets log level for the given logger's default handler. + + Args: + logger (logging.Logger): The logger on which to set the log level. + verbose (bool): Sets loglevel to INFO if True, to WARNING otherwise. + """ + handler = logger.handlers[0] + if verbose: + handler.setLevel(logging.INFO) + else: + handler.setLevel(logging.WARNING) diff --git a/src/specsscan/config/NXmpes_arpes_config.json b/src/specsscan/config/NXmpes_arpes_config.json old mode 100755 new mode 100644 index 6e6eb3f..2df10fb --- a/src/specsscan/config/NXmpes_arpes_config.json +++ b/src/specsscan/config/NXmpes_arpes_config.json @@ -1,28 +1,26 @@ { "/@default": "entry", "/ENTRY/@default": "data", - "/ENTRY/title": "['@eln:/ENTRY/title', '@attrs:metadata/entry_title']", + "/ENTRY/title": "['@eln:/ENTRY/title', '@attrs:metadata/entry_title', '@attrs:metadata/elabFTW/scan/title']", "/ENTRY/start_time": "@attrs:metadata/timing/acquisition_start", "/ENTRY/experiment_institution": "Fritz Haber Institute - Max Planck Society", "/ENTRY/experiment_facility": "Time Resolved ARPES", "/ENTRY/experiment_laboratory": "Clean Room 4", - "/ENTRY/entry_identifier": { - "identifier":"@attrs:metadata/loader/scan_path" - }, + "/ENTRY/identifierNAME[entry_identifier]": "@attrs:metadata/loader/scan_path", "/ENTRY/end_time": "@attrs:metadata/timing/acquisition_stop", "/ENTRY/duration": "@attrs:metadata/timing/acquisition_duration", "/ENTRY/duration/@units": "s", "/ENTRY/collection_time": "@attrs:metadata/timing/collection_time", "/ENTRY/collection_time/@units": "s", "/ENTRY/USER[user]": { - "name": "!['@eln:/ENTRY/User/name', '@attrs:metadata/user0/name']", - "role": "['@eln:/ENTRY/User/role', '@attrs:metadata/user0/role']", - "affiliation": "!['@eln:/ENTRY/User/affiliation', '@attrs:metadata/user0/affiliation']", - "address": "['@eln:/ENTRY/User/address', '@attrs:metadata/user0/address']", - "email": "['@eln:/ENTRY/User/email', '@attrs:metadata/user0/email']" + "name": "!['@eln:/ENTRY/User/name', '@attrs:metadata/user0/name', '@attrs:metadata/elabFTW/user/name']", + "role": "['@eln:/ENTRY/User/role', '@attrs:metadata/user0/role', 'Principal Investigator']", + "affiliation": "['@eln:/ENTRY/User/affiliation', '@attrs:metadata/user0/affiliation', 'Fritz Haber Institute of the Max Planck Society']", + "address": "['@eln:/ENTRY/User/address', '@attrs:metadata/user0/address', 'Faradayweg 4-6, 14195 Berlin, Germany']", + "email": "['@eln:/ENTRY/User/email', '@attrs:metadata/user0/email', '@attrs:metadata/elabFTW/user/email']" }, - "/ENTRY/geometries/COORDINATE_SYSTEM[arpes_geometry]": { - "depends_on": "/entry/geometries/arpes_geometry/transformations/rot_y", + "/ENTRY/COORDINATE_SYSTEM[arpes_geometry]": { + "depends_on": "/entry/arpes_geometry/transformations/rot_y", "TRANSFORMATIONS[transformations]": { "AXISNAME[rot_y]/@depends_on": ".", "AXISNAME[rot_y]": 140.0, @@ -35,19 +33,19 @@ "name": "Phoibos detector, at the endstation of the high rep-rate HHG source at FHI", "name/@short_name": "TR-ARPES @ FHI", "energy_resolution": { - "resolution": "!['@eln:/ENTRY/Instrument/Analyzer/energy_resolution', '@attrs:metadata/instrument/energy_resolution']", + "resolution": "!['@eln:/ENTRY/Instrument/Analyzer/energy_resolution', '@attrs:metadata/instrument/energy_resolution', '@attrs:metadata/elabFTW/trarpes_phoibos/energy_resolution']", "resolution/@units": "meV", "physical_quantity": "energy", "type": "estimated" }, "RESOLUTION[temporal_resolution]": { - "resolution": 35.0, + "resolution": "!['@attrs:metadata/elabFTW/laser_status/temporal_resolution', '35.0']", "resolution/@units": "fs", "physical_quantity": "time", "type": "estimated" }, "RESOLUTION[angular_resolution]": { - "resolution": "@link:/entry/instrument/electronanalyser/angular_resolution/resolution", + "resolution": "@link:/entry/instrument/electronanalyzer/angular_resolution/resolution", "resolution/@units": "deg", "physical_quantity": "angle", "type": "derived" @@ -57,207 +55,205 @@ "measurement": "pressure", "value": "!['@eln:/ENTRY/Sample/gas_pressure', '@attrs:metadata/scan_info/trARPES:XGS600:PressureAC:P_RD']", "value/@units": "mbar" - } - }, - "/ENTRY/INSTRUMENT[instrument]/ELECTRONANALYSER[electronanalyser]": { - "description": "SPECS Phoibos 150 Hemispherical Energy Analyzer", - "device_information": { - "vendor": "SPECS GmbH", - "model": "Phoibos 150 CCD Hemispherical Analyzer" }, - "work_function": 4.55, - "work_function/@units": "eV", - "fast_axes": "@attrs:metadata/scan_info/fast_axes", - "slow_axes": "@attrs:metadata/scan_info/slow_axes", - "energy_resolution": { - "resolution": "!['@eln:/ENTRY/Instrument/Analyzer/energy_resolution', '@attrs:metadata/instrument/electronanalyser/energy_resolution']", - "resolution/@units": "meV", - "physical_quantity": "energy", - "type": "derived" + "ELECTRONANALYZER[electronanalyzer]": { + "description": "SPECS Phoibos 150 Hemispherical Energy Analyzer", + "device_information": { + "vendor": "SPECS GmbH", + "model": "Phoibos 150 CCD Hemispherical Analyzer" + }, + "work_function": "@attrs:metadata/conversion_parameters/work_function", + "work_function/@units": "eV", + "fast_axes": "@attrs:metadata/scan_info/fast_axes", + "slow_axes": "@attrs:metadata/scan_info/slow_axes", + "energy_resolution": { + "resolution": "!['@eln:/ENTRY/Instrument/Analyzer/energy_resolution', '@attrs:metadata/instrument/electronanalyzer/energy_resolution', '@attrs:metadata/elabFTW/trarpes_phoibos/energy_resolution']", + "resolution/@units": "meV", + "physical_quantity": "energy", + "type": "derived" + }, + "angular_resolution": { + "resolution": "!['@eln:/ENTRY/Instrument/Analyzer/angular_resolution', '@attrs:metadata/instrument/electronanalyzer/angular_resolution', '@attrs:metadata/elabFTW/trarpes_phoibos/angular_resolution']", + "resolution/@units": "deg", + "physical_quantity": "angle", + "type": "derived" + }, + "spatial_resolution": { + "resolution": "!['@eln:/ENTRY/Instrument/Analyzer/spatial_resolution', '@attrs:metadata/instrument/electronanalyzer/spatial_resolution', '@attrs:metadata/elabFTW/trarpes_phoibos/spatial_resolution']", + "resolution/@units": "µm", + "physical_quantity": "length", + "type": "estimated" + }, + "depends_on": "/entry/instrument/electronanalyzer/transformations/trans_z", + "TRANSFORMATIONS[transformations]": { + "AXISNAME[trans_z]/@depends_on": "analyzer_elevation", + "AXISNAME[trans_z]": 40.0, + "AXISNAME[trans_z]/@transformation_type": "translation", + "AXISNAME[trans_z]/@units": "mm", + "AXISNAME[trans_z]/@vector": [0, 0, 1], + "analyzer_dispersion/@depends_on": "analyzer_rotation", + "analyzer_dispersion": "@data:angular1.data", + "analyzer_dispersion/@transformation_type": "rotation", + "analyzer_dispersion/@units": "degrees", + "analyzer_dispersion/@vector": [1, 0, 0], + "analyzer_elevation/@depends_on": "analyzer_dispersion", + "analyzer_elevation": 0, + "analyzer_elevation/@transformation_type": "rotation", + "analyzer_elevation/@units": "degrees", + "analyzer_elevation/@vector": [0, 1, 0], + "analyzer_rotation/@depends_on": "rot_y", + "analyzer_rotation": 0, + "analyzer_rotation/@transformation_type": "rotation", + "analyzer_rotation/@units": "degrees", + "analyzer_rotation/@vector": [0, 0, 1], + "AXISNAME[rot_y]/@depends_on": ".", + "AXISNAME[rot_y]": 140.0, + "AXISNAME[rot_y]/@transformation_type": "rotation", + "AXISNAME[rot_y]/@units": "degrees", + "AXISNAME[rot_y]/@vector": [0, 1, 0] + }, + "COLLECTIONCOLUMN[collectioncolumn]": { + "projection": "@attrs:metadata/scan_info/projection", + "scheme": "@attrs:metadata/scan_info/scheme", + "lens_mode": "@attrs:metadata/scan_info/LensMode", + "working_distance": 40.0, + "working_distance/@units": "mm" + }, + "ENERGYDISPERSION[energydispersion]": { + "energy_scan_mode": "@attrs:metadata/scan_info/energy_scan_mode", + "pass_energy": "@attrs:metadata/scan_info/PassEnergy", + "pass_energy/@units": "eV", + "center_energy": "@attrs:metadata/scan_info/KineticEnergy", + "center_energy/@units": "eV", + "scheme": "hemispherical", + "diameter": 300.0, + "diameter/@units": "mm", + "entrance_slit": { + "shape": "['@eln:/ENTRY/Instrument/Analyzer/slit_shape', '@attrs:metadata/instrument/electronanalyzer/slit_shape', '@attrs:metadata/elabFTW/trarpes_phoibos/entrance_slit_shape', 'curved slit']", + "size": "['@eln:/ENTRY/Instrument/Analyzer/slit_size', '@attrs:metadata/instrument/electronanalyzer/slit_size', '@attrs:metadata/elabFTW/trarpes_phoibos/entrance_slit_size', '1.0']", + "size/@units": "mm" + }, + "exit_slit": { + "shape": "['@attrs:metadata/elabFTW/trarpes_phoibos/exit_slit_type', 'grid']" + } + }, + "ELECTRON_DETECTOR[detector]": { + "amplifier_type": "MCP", + "detector_type": "Phosphor+CCD", + "amplifier_bias": "@attrs:metadata/scan_info/ConversionVoltage", + "amplifier_bias/@units": "V", + "amplifier_voltage": "@attrs:metadata/scan_info/DetectorVoltage", + "amplifier_voltage/@units": "V", + "detector_voltage": "@attrs:metadata/scan_info/ScreenVoltage", + "detector_voltage/@units": "V" + } }, - "angular_resolution": { - "resolution": "!['@eln:/ENTRY/Instrument/Analyzer/angular_resolution', '@attrs:metadata/instrument/electronanalyser/angular_resolution']", - "resolution/@units": "deg", - "physical_quantity": "angle", - "type": "derived" + "source_probe": { + "name": "HHG @ TR-ARPES @ FHI", + "probe": "photon", + "type": "HHG laser", + "mode": "Single Bunch", + "frequency": "!['@eln:/ENTRY/Instrument/Source/Probe/frequency', '@attrs:metadata/instrument/beam/probe/frequency', '@attrs:metadata/elabFTW/laser_status/probe_repetition_rate']", + "frequency/@units": "kHz", + "associated_beam": "/entry/instrument/beam_probe" }, - "spatial_resolution": { - "resolution": "!['@eln:/ENTRY/Instrument/Analyzer/spatial_resolution', '@attrs:metadata/instrument/electronanalyser/spatial_resolution']", - "resolution/@units": "µm", - "physical_quantity": "length", - "type": "estimated" + "beam_probe": { + "distance": 0.0, + "distance/@units": "mm", + "incident_energy": "!['@eln:/ENTRY/Instrument/Beam/Probe/incident_energy', '@attrs:metadata/instrument/beam/probe/incident_energy', '@attrs:metadata/elabFTW/laser_status/probe_photon_energy']", + "incident_energy/@units": "eV", + "incident_energy_spread": "['@eln:/ENTRY/Instrument/Beam/Probe/incident_energy_spread', '@attrs:metadata/instrument/beam/probe/incident_energy_spread', '@attrs:metadata/elabFTW/laser_status/probe_photon_energy_spread']", + "incident_energy_spread/@units": "eV", + "pulse_duration": "['@eln:/ENTRY/Instrument/Beam/Probe/pulse_duration', '@attrs:metadata/instrument/beam/probe/pulse_duration', '@attrs:metadata/elabFTW/laser_status/probe_pulse_duration']", + "pulse_duration/@units": "fs", + "incident_polarization": "['@eln:/ENTRY/Instrument/Beam/Probe/incident_polarization', '@attrs:metadata/instrument/beam/probe/incident_polarization', '@attrs:metadata/elabFTW/scan/probe_polarization', '@attrs:metadata/scan_info/trARPES:HHG:wp']", + "incident_polarization/@units": "V^2/mm^2", + "extent": "['@eln:/ENTRY/Instrument/Beam/Probe/extent', '@attrs:metadata/instrument/beam/probe/extent', '@attrs:metadata/elabFTW/laser_status/probe_profile']", + "extent/@units": "µm", + "associated_source": "/entry/instrument/source_probe" }, - - - "depends_on": "/entry/instrument/electronanalyser/transformations/trans_z", - "TRANSFORMATIONS[transformations]": { - "AXISNAME[trans_z]/@depends_on": "analyzer_elevation", - "AXISNAME[trans_z]": 40.0, - "AXISNAME[trans_z]/@transformation_type": "translation", - "AXISNAME[trans_z]/@units": "mm", - "AXISNAME[trans_z]/@vector": [0, 0, 1], - "analyzer_dispersion/@depends_on": "analyzer_rotation", - "analyzer_dispersion": "@data:angular1.data", - "analyzer_dispersion/@transformation_type": "rotation", - "analyzer_dispersion/@units": "degrees", - "analyzer_dispersion/@vector": [1, 0, 0], - "analyzer_elevation/@depends_on": "analyzer_dispersion", - "analyzer_elevation": 0, - "analyzer_elevation/@transformation_type": "rotation", - "analyzer_elevation/@units": "degrees", - "analyzer_elevation/@vector": [0, 1, 0], - "analyzer_rotation/@depends_on": "rot_y", - "analyzer_rotation": 0, - "analyzer_rotation/@transformation_type": "rotation", - "analyzer_rotation/@units": "degrees", - "analyzer_rotation/@vector": [0, 0, 1], - "AXISNAME[rot_y]/@depends_on": ".", - "AXISNAME[rot_y]": 140.0, - "AXISNAME[rot_y]/@transformation_type": "rotation", - "AXISNAME[rot_y]/@units": "degrees", - "AXISNAME[rot_y]/@vector": [0, 1, 0] - } - }, - "/ENTRY/INSTRUMENT[instrument]/ELECTRONANALYSER[electronanalyser]/COLLECTIONCOLUMN[collectioncolumn]": { - "projection": "@attrs:metadata/scan_info/projection", - "scheme": "@attrs:metadata/scan_info/scheme", - "lens_mode": "@attrs:metadata/scan_info/LensMode", - "working_distance": 40.0, - "working_distance/@units": "mm" - }, - "/ENTRY/INSTRUMENT[instrument]/ELECTRONANALYSER[electronanalyser]/ENERGYDISPERSION[energydispersion]": { - "energy_scan_mode": "@attrs:metadata/scan_info/energy_scan_mode", - "pass_energy": "@attrs:metadata/scan_info/PassEnergy", - "pass_energy/@units": "eV", - "center_energy": "@attrs:metadata/scan_info/KineticEnergy", - "center_energy/@units": "eV", - "scheme": "hemispherical", - "diameter": 300.0, - "diameter/@units": "mm", - "entrance_slit": { - "shape": "curved slit", - "size": 1.0, - "size/@units": "mm" - }, - "exit_slit": { - "shape": "grid" - } - }, - "/ENTRY/INSTRUMENT[instrument]/ELECTRONANALYSER[electronanalyser]/DETECTOR[detector]": { - "amplifier_type": "MCP", - "detector_type": "Phosphor+CCD", - "sensor_pixels": [ - 256, - 320 - ], - "amplifier_bias": "@attrs:metadata/scan_info/ConversionVoltage", - "amplifier_bias/@units": "V", - "amplifier_voltage": "@attrs:metadata/scan_info/DetectorVoltage", - "amplifier_voltage/@units": "V", - "detector_voltage": "@attrs:metadata/scan_info/ScreenVoltage", - "detector_voltage/@units": "V" - }, - "/ENTRY/INSTRUMENT[instrument]/sourceTYPE[source_probe]": { - "name": "HHG @ TR-ARPES @ FHI", - "probe": "photon", - "type": "HHG laser", - "mode": "Single Bunch", - "frequency": "['@eln:/ENTRY/Instrument/Source/Probe/frequency', '@attrs:metadata/instrument/beam/probe/frequency']", - "frequency/@units": "kHz", - "associated_beam": "/entry/instrument/beam_probe" - }, - "/ENTRY/INSTRUMENT[instrument]/beamTYPE[beam_probe]": { - "distance": 0.0, - "distance/@units": "mm", - "incident_energy": "!['@eln:/ENTRY/Instrument/Beam/Probe/incident_energy', '@attrs:metadata/instrument/beam/probe/incident_energy']", - "incident_energy/@units": "eV", - "incident_energy_spread": "['@eln:/ENTRY/Instrument/Beam/Probe/incident_energy_spread', '@attrs:metadata/instrument/beam/probe/incident_energy_spread']", - "incident_energy_spread/@units": "eV", - "pulse_duration": "['@eln:/ENTRY/Instrument/Beam/Probe/pulse_duration', '@attrs:metadata/instrument/beam/probe/pulse_duration']", - "pulse_duration/@units": "fs", - "incident_polarization": "['@eln:/ENTRY/Instrument/Beam/Probe/incident_polarization', '@attrs:metadata/instrument/beam/probe/incident_polarization']", - "incident_polarization/@units": "V^2/mm^2", - "extent": "['@eln:/ENTRY/Instrument/Beam/Probe/extent', '@attrs:metadata/instrument/beam/probe/extent']", - "extent/@units": "µm", - "associated_source": "/entry/instrument/source_probe" - }, - "/ENTRY/INSTRUMENT[instrument]/sourceTYPE[source_pump]": { - "name": "OPCPA @ TR-ARPES @ FHI", - "probe": "visible light", - "type": "Optical Laser", - "mode": "Single Bunch", - "frequency": "['@eln:/ENTRY/Instrument/Source/Pump/frequency', '@attrs:metadata/instrument/beam/pump/frequency']", - "frequency/@units": "kHz", - "associated_beam": "/entry/instrument/beam_pump" - }, - "/ENTRY/INSTRUMENT[instrument]/beamTYPE[beam_pump]": { - "distance": 0.0, - "distance/@units": "mm", - "incident_energy": "!['@eln:/ENTRY/Instrument/Beam/Pump/incident_energy', '@attrs:metadata/instrument/beam/pump/incident_energy']", - "incident_energy/@units": "eV", - "incident_energy_spread": "['@eln:/ENTRY/Instrument/Beam/Pump/incident_energy_spread', '@attrs:metadata/instrument/beam/pump/incident_energy_spread']", - "incident_energy_spread/@units": "eV", - "incident_wavelength": "['@eln:/ENTRY/Instrument/Beam/Pump/incident_wavelength', '@attrs:metadata/instrument/beam/pump/incident_wavelength']", - "incident_wavelength/@units": "nm", - "pulse_duration": "['@eln:/ENTRY/Instrument/Beam/Pump/pulse_duration', '@attrs:metadata/instrument/beam/pump/pulse_duration']", - "pulse_duration/@units": "fs", - "incident_polarization": "['@eln:/ENTRY/Instrument/Beam/Pump/incident_polarization', '@attrs:metadata/instrument/beam/pump/incident_polarization']", - "incident_polarization/@units": "V^2/mm^2", - "pulse_energy": "['@eln:/ENTRY/Instrument/Beam/Pump/pulse_energy', '@attrs:metadata/instrument/beam/pump/pulse_energy']", - "pulse_energy/@units": "µJ", - "average_power": "['@eln:/ENTRY/Instrument/Beam/Pump/average_power', '@attrs:metadata/instrument/beam/pump/average_power']", - "average_power/@units": "mW", - "extent": "['@eln:/ENTRY/Instrument/Beam/Pump/extent', '@attrs:metadata/instrument/beam/pump/extent']", - "extent/@units": "µm", - "fluence": "['@eln:/ENTRY/Instrument/Beam/Pump/fluence', '@attrs:metadata/instrument/beam/pump/fluence']", - "fluence/@units": "mJ/cm^2", - "associated_source": "/entry/instrument/source_pump" - }, - "/ENTRY/INSTRUMENT[instrument]/MANIPULATOR[manipulator]": { - "temperature_sensor": { - "name": "sample_temperature", - "measurement": "temperature", - "value": "!['@eln:/ENTRY/Instrument/Manipulator/sample_temperature', '@attrs:metadata/scan_info/trARPES:Carving:TEMP_RBV']", - "value/@units": "K" + "source_pump": { + "name": "OPCPA @ TR-ARPES @ FHI", + "probe": "visible light", + "type": "Optical Laser", + "mode": "Single Bunch", + "frequency": "!['@eln:/ENTRY/Instrument/Source/Pump/frequency', '@attrs:metadata/instrument/beam/pump/frequency', '@attrs:metadata/elabFTW/laser_status/pump_repetition_rate']", + "frequency/@units": "kHz", + "associated_beam": "/entry/instrument/beam_pump" }, - "sample_bias_voltmeter": { - "name": "sample_bias", - "measurement": "voltage", - "value": 0.0, - "value/@units": "V" + "beam_pump": { + "distance": 0.0, + "distance/@units": "mm", + "incident_energy": "!['@eln:/ENTRY/Instrument/Beam/Pump/incident_energy', '@attrs:metadata/instrument/beam/pump/incident_energy', '@attrs:metadata/elabFTW/laser_status/pump_photon_energy']", + "incident_energy/@units": "eV", + "incident_energy_spread": "['@eln:/ENTRY/Instrument/Beam/Pump/incident_energy_spread', '@attrs:metadata/instrument/beam/pump/incident_energy_spread', '@attrs:metadata/elabFTW/laser_status/pump_photon_energy_spread']", + "incident_energy_spread/@units": "eV", + "incident_wavelength": "['@eln:/ENTRY/Instrument/Beam/Pump/incident_wavelength', '@attrs:metadata/instrument/beam/pump/incident_wavelength', '@attrs:metadata/scan_info/trARPES:Orpheus:Wavelength']", + "incident_wavelength/@units": "nm", + "pulse_duration": "['@eln:/ENTRY/Instrument/Beam/Pump/pulse_duration', '@attrs:metadata/instrument/beam/pump/pulse_duration', '@attrs:metadata/elabFTW/laser_status/pump_pulse_duration']", + "pulse_duration/@units": "fs", + "pulse_delay": "@attrs:metadata/scan_info/delay", + "pulse_delay/@units": "fs", + "pulse_delay/@reference_beam": "/entry/instrument/beam_probe", + "incident_polarization": "['@eln:/ENTRY/Instrument/Beam/Pump/incident_polarization', '@attrs:metadata/instrument/beam/pump/incident_polarization', '@attrs:metadata/elabFTW/scan/pump_polarization', '@attrs:metadata/scan_info/trARPES:Pump:wp.RBV']", + "incident_polarization/@units": "V^2/mm^2", + "pulse_energy": "['@eln:/ENTRY/Instrument/Beam/Pump/pulse_energy', '@attrs:metadata/instrument/beam/pump/pulse_energy', '@attrs:metadata/elabFTW/scan/pump_pulse_energy']", + "pulse_energy/@units": "µJ", + "average_power": "['@eln:/ENTRY/Instrument/Beam/Pump/average_power', '@attrs:metadata/instrument/beam/pump/average_power', '@attrs:metadata/scan_info/trARPES:Pump:Power.RBV']", + "average_power/@units": "mW", + "extent": "['@eln:/ENTRY/Instrument/Beam/Pump/extent', '@attrs:metadata/instrument/beam/pump/extent', '@attrs:metadata/elabFTW/laser_status/pump_profile']", + "extent/@units": "µm", + "fluence": "['@eln:/ENTRY/Instrument/Beam/Pump/fluence', '@attrs:metadata/instrument/beam/pump/fluence', '@attrs:metadata/elabFTW/scan/pump_fluence', '@attrs:metadata/scan_info/trARPES:Pump:fluence.RBV']", + "fluence/@units": "mJ/cm^2", + "associated_source": "/entry/instrument/source_pump" }, - "drain_current_amperemeter": { - "name": "drain_current", - "measurement": "current", - "value": "!['@eln:/ENTRY/Instrument/Manipulator/drain_current', '@attrs:metadata/scan_info/trARPES:Sample:Measure']", - "value/@units": "A" - }, - "depends_on": "/entry/instrument/manipulator/transformations/trans_z", - "TRANSFORMATIONS[transformations]": { - "AXISNAME[trans_z]": -0.32, - "AXISNAME[trans_z]/@depends_on": "rot_z", - "AXISNAME[trans_z]/@transformation_type": "translation", - "AXISNAME[trans_z]/@units": "m", - "AXISNAME[trans_z]/@vector": [0, 0, 1], - "AXISNAME[rot_z]/@depends_on": "rot_x", - "AXISNAME[rot_z]": -115.0, - "AXISNAME[rot_z]/@transformation_type": "rotation", - "AXISNAME[rot_z]/@units": "degrees", - "AXISNAME[rot_z]/@vector": [0, 0, 1], - "AXISNAME[rot_x]/@depends_on": ".", - "AXISNAME[rot_x]": -90.0, - "AXISNAME[rot_x]/@transformation_type": "rotation", - "AXISNAME[rot_x]/@units": "degrees", - "AXISNAME[rot_x]/@vector": [1, 0, 0] + "MANIPULATOR[manipulator]": { + "temperature_sensor": { + "name": "sample_temperature", + "measurement": "temperature", + "value": "!['@eln:/ENTRY/Instrument/Manipulator/sample_temperature', '@attrs:metadata/scan_info/trARPES:Carving:TEMP_RBV']", + "value/@units": "K" + }, + "sample_bias_voltmeter": { + "name": "sample_bias", + "measurement": "voltage", + "value": 0.0, + "value/@units": "V" + }, + "drain_current_ammeter": { + "name": "drain_current", + "measurement": "current", + "value": "!['@eln:/ENTRY/Instrument/Manipulator/drain_current', '@attrs:metadata/scan_info/trARPES:Sample:Measure']", + "value/@units": "A" + }, + "depends_on": "/entry/instrument/manipulator/transformations/trans_z", + "TRANSFORMATIONS[transformations]": { + "AXISNAME[trans_z]": -0.32, + "AXISNAME[trans_z]/@depends_on": "rot_z", + "AXISNAME[trans_z]/@transformation_type": "translation", + "AXISNAME[trans_z]/@units": "m", + "AXISNAME[trans_z]/@vector": [0, 0, 1], + "AXISNAME[rot_z]/@depends_on": "rot_x", + "AXISNAME[rot_z]": -115.0, + "AXISNAME[rot_z]/@transformation_type": "rotation", + "AXISNAME[rot_z]/@units": "degrees", + "AXISNAME[rot_z]/@vector": [0, 0, 1], + "AXISNAME[rot_x]/@depends_on": ".", + "AXISNAME[rot_x]": -90.0, + "AXISNAME[rot_x]/@transformation_type": "rotation", + "AXISNAME[rot_x]/@units": "degrees", + "AXISNAME[rot_x]/@vector": [1, 0, 0] + } } }, "/ENTRY/SAMPLE[sample]": { - "preparation_date": "['@eln:/ENTRY/Sample/preparation_date', '@attrs:metadata/sample/preparation_date']", - "history/notes/description": "['@eln:/ENTRY/Sample/sample_history', '@attrs:metadata/sample/sample_history']", - "description": "['@eln:/ENTRY/Sample/description', '@attrs:metadata/sample/chemical_formula']", - "name": "!['@eln:/ENTRY/Sample/name', '@attrs:metadata/sample/name']", + "preparation_date": "['@eln:/ENTRY/Sample/preparation_date', '@attrs:metadata/sample/preparation_date', '@attrs:metadata/elabFTW/sample/preparation_date']", + "history/sample_preparation/description": "['@eln:/ENTRY/Sample/sample_history', '@attrs:metadata/sample/sample_history', '@attrs:metadata/elabFTW/sample/sample_history']", + "history/sample_preparation/start_time": "['@eln:/ENTRY/Sample/preparation_date', '@attrs:metadata/sample/preparation_date', '@attrs:metadata/elabFTW/sample/preparation_date']", + "description": "['@eln:/ENTRY/Sample/description', '@attrs:metadata/sample/chemical_formula', '@attrs:metadata/elabFTW/sample/summary']", + "name": "!['@eln:/ENTRY/Sample/name', '@attrs:metadata/sample/name', '@attrs:metadata/elabFTW/sample/title']", "situation": "vacuum", - "SUBSTANCE[substance]/molecular_formula_hill": "['@eln:/ENTRY/Sample/chemical_formula', '@attrs:metadata/sample/chemical_formula']", + "chemical_formula": "['@eln:/ENTRY/Sample/chemical_formula', '@attrs:metadata/sample/chemical_formula', '@attrs:metadata/elabFTW/sample/sample_formula']", "temperature_env": { "temperature_sensor": "@link:/entry/instrument/manipulator/temperature_sensor" }, @@ -268,7 +264,7 @@ "voltmeter": "@link:/entry/instrument/manipulator/sample_bias_voltmeter" }, "drain_current_env": { - "amperemeter": "@link:/entry/instrument/manipulator/drain_current_amperemeter" + "ammeter": "@link:/entry/instrument/manipulator/drain_current_ammeter" }, "depends_on": "/entry/sample/transformations/offset_azimuth", "TRANSFORMATIONS[transformations]": { @@ -329,11 +325,28 @@ "AXISNAME[trans_x]/@vector": [0, 1, 0] } }, - "/ENTRY/PROCESS[process]/CALIBRATION[energy_calibration]":{ - "calibrated_axis": "@data:energy.data" + "/ENTRY/CALIBRATION[energy_calibration]":{ + "program": "@attrs:metadata/scan_info/program_name", + "version": "@attrs:metadata/scan_info/program_version", + "calibrated_axis": "@attrs:metadata/scan_info/Ekin", + "calibrated_axis/@units": "energy", + "physical_quantity": "energy", + "PARAMETERS[conversion_parameters]":{ + "TERM[*{apply_fft_filter,binning,rotation_angle,lens_mode,kinetic_energy,pass_energy,work_function,a_inner,da_matrix,retardation_ratio,source,dims,e_shift,de1,e_range,a_range,pixel_size,magnification,angle_offset_px,energy_offset_px}]": "@attrs:metadata/conversion_parameters/*" + } + }, + "/ENTRY/CALIBRATION[angular0_calibration]":{ + "calibrated_axis": "@link:/entry/data/angular0", + "calibrated_axis/@units": "angle", + "physical_quantity": "angle" }, - "/ENTRY/PROCESS[process]/CALIBRATION[angular0_calibration]":{ - "calibrated_axis": "@data:angular0.data" + "/ENTRY/CALIBRATION[energy_referencing]":{ + "calibrated_axis": "@link:/entry/data/energy", + "calibrated_axis/@units": "energy", + "physical_quantity": "energy", + "reference_peak": "@attrs:metadata/scan_info/reference_energy", + "binding_energy": 0.0, + "binding_energy/@units": "energy" }, "/ENTRY/data": { "@axes": "@data:dims", @@ -344,6 +357,7 @@ "*": "@data:*.data", "*/@units": "@data:*.unit", "*/@reference": "@attrs:metadata/scan_info/coordinate_depends/*", + "*/@long_name": "@attrs:metadata/scan_info/coordinate_label/*", "energy/@type": "kinetic" } } diff --git a/src/specsscan/config/default.yaml b/src/specsscan/config/default.yaml index 94d2250..652062a 100644 --- a/src/specsscan/config/default.yaml +++ b/src/specsscan/config/default.yaml @@ -2,6 +2,9 @@ data_path: "" # option to enable nested progress bars enable_nested_progress_bar: false +# option to shift by photon energy if available +shift_by_photon_energy: false + # dictionary containing parameters passed to the SpecsAnalyzer. Will be completed by the SpecsAnalyzer default config parameters spa_params: apply_fft_filter: false diff --git a/src/specsscan/config/example_config_FHI.yaml b/src/specsscan/config/example_config_FHI.yaml index 5f5aec9..23e3538 100644 --- a/src/specsscan/config/example_config_FHI.yaml +++ b/src/specsscan/config/example_config_FHI.yaml @@ -2,6 +2,8 @@ data_path: "path/to/data" # option to enable nested progress bars enable_nested_progress_bar: false +# option to shift by photon energy if available +shift_by_photon_energy: true # dictionary containing renaming rules for axis names (to change the name in the xarrays) coordinate_mapping: @@ -16,8 +18,8 @@ coordinate_mapping: # dictionary of corresponding NeXus paths for the different axes coordinate_depends: - Ekin: "/entry/instrument/electronanalyser/energydispersion/kinetic_energy" - Angle: "/entry/instrument/electronanalyser/transformations/analyzer_dispersion" + Ekin: "/entry/instrument/electronanalyzer/energydispersion/kinetic_energy" + Angle: "/entry/instrument/electronanalyzer/transformations/analyzer_dispersion" polar: "/entry/sample/transformations/sample_polar" tilt: "/entry/sample/transformations/sample_tilt" azimuth: "/entry/sample/transformations/sample_azimuth" @@ -25,6 +27,17 @@ coordinate_depends: Y: "/entry/sample/transformations/trans_y" Z: "/entry/sample/transformations/trans_z" +# dictionary of axis labels for the different axes +coordinate_labels: + Ekin: "Kinetic Energy" + Angle: "Analyzer Dispersion" + polar: "Sample Polar" + tilt: "Sample Tilt" + azimuth: "Sample Azimuth" + X: "X Translation" + Y: "Y Translation" + Z: "Z Translation" + # dictionary containing units for the respective axes units: angular0: "degree" @@ -36,19 +49,22 @@ units: spatial0: "mm" voltage: "V" -# URL of the epics archiver request engine -archiver_url: "http://__epicsarchiver_host__:17668/retrieval/data/getData.json?pv=" -# dictionary containing axis names with Epics channels to request from the EPICS archiver -epics_channels: - tempa: "trARPES:Carving:TEMP_RBV" - x: "trARPES:Carving:TRX.RBV" - y: "trARPES:Carving:TRY.RBV" - z: "trARPES:Carving:TRZ.RBV" - polar: "trARPES:Carving:THT.RBV" - tilt: "trARPES:Carving:PHI.RBV" - azimuth: "trARPES:Carving:OMG.RBV" - drain_current: "trARPES:Sample:Measure" - pressure: "trARPES:XGS600:PressureAC:P_RD" +metadata: + # URL of the elabFTW instance API interface + # elab_url: "https://__elabftw_host__/api/v2" + # URL of the epics archiver request engine + # archiver_url: "http://__epicsarchiver_host__:17668/retrieval/data/getData.json?pv=" + # dictionary containing axis names with Epics channels to request from the EPICS archiver + epics_channels: + tempa: "trARPES:Carving:TEMP_RBV" + x: "trARPES:Carving:TRX.RBV" + y: "trARPES:Carving:TRY.RBV" + z: "trARPES:Carving:TRZ.RBV" + polar: "trARPES:Carving:THT.RBV" + tilt: "trARPES:Carving:PHI.RBV" + azimuth: "trARPES:Carving:OMG.RBV" + drain_current: "trARPES:Sample:Measure" + pressure: "trARPES:XGS600:PressureAC:P_RD" # parameters for NeXus conversion nexus: diff --git a/src/specsscan/core.py b/src/specsscan/core.py index 81929c7..5e4545d 100755 --- a/src/specsscan/core.py +++ b/src/specsscan/core.py @@ -5,7 +5,6 @@ import os import pathlib from importlib.util import find_spec -from logging import warn from pathlib import Path from typing import Any from typing import Sequence @@ -17,9 +16,12 @@ from specsanalyzer import SpecsAnalyzer from specsanalyzer.config import parse_config +from specsanalyzer.config import save_config from specsanalyzer.io import to_h5 from specsanalyzer.io import to_nexus from specsanalyzer.io import to_tiff +from specsanalyzer.logging import set_verbosity +from specsanalyzer.logging import setup_logging from specsscan.helpers import get_coords from specsscan.helpers import get_scan_path from specsscan.helpers import handle_meta @@ -30,6 +32,9 @@ package_dir = os.path.dirname(find_spec("specsscan").origin) +# Configure logging +logger = setup_logging("specsscan") + class SpecsScan: """SpecsAnalyzer class for loading scans and data from SPECS Phoibos electron analyzers, @@ -45,6 +50,7 @@ def __init__( self, metadata: dict = {}, config: dict | str = {}, + verbose: bool = True, **kwds, ): """SpecsScan constructor. @@ -52,6 +58,7 @@ def __init__( Args: metadata (dict, optional): Metadata dictionary. Defaults to {}. config (Union[dict, str], optional): Metadata dictionary or file path. Defaults to {}. + verbose (bool, optional): Disable info logs if set to False. **kwds: Keyword arguments passed to ``parse_config``. """ self._config = parse_config( @@ -60,6 +67,8 @@ def __init__( **kwds, ) + set_verbosity(logger, verbose) + self.metadata = metadata self._scan_info: dict[Any, Any] = {} @@ -70,12 +79,14 @@ def __init__( folder_config={}, user_config={}, system_config={}, + verbose=verbose, ) except KeyError: self.spa = SpecsAnalyzer( folder_config={}, user_config={}, system_config={}, + verbose=verbose, ) self._result: xr.DataArray = None @@ -144,6 +155,7 @@ def load_scan( xr.DataArray: xarray DataArray object with kinetic energy, angle/position and optionally a third scanned axis (for ex., delay, temperature) as coordinates. """ + token = kwds.pop("token", None) scan_path = get_scan_path(path, scan, self._config["data_path"]) df_lut = parse_lut_to_df(scan_path) @@ -232,14 +244,25 @@ def load_scan( # rename coords and store mapping information, if available coordinate_mapping = self._config.get("coordinate_mapping", {}) coordinate_depends = self._config.get("coordinate_depends", {}) + coordinate_labels = self._config.get("coordinate_labels", {}) rename_dict = { k: coordinate_mapping[k] for k in coordinate_mapping.keys() if k in res_xarray.dims } depends_dict = { - rename_dict[k]: coordinate_depends[k] + rename_dict.get(k, k): coordinate_depends[k] for k in coordinate_depends.keys() if k in res_xarray.dims } + label_dict = { + rename_dict.get(k, k): coordinate_labels[k] + for k in coordinate_labels.keys() + if k in res_xarray.dims + } + + # store data for resolved axis coordinates + for axis in res_xarray.dims: + self._scan_info[axis] = res_xarray.coords[axis].data + res_xarray = res_xarray.rename(rename_dict) for k, v in coordinate_mapping.items(): if k in fast_axes: @@ -249,39 +272,61 @@ def load_scan( slow_axes.remove(k) slow_axes.add(v) self._scan_info["coordinate_depends"] = depends_dict - - axis_dict = { - "/entry/sample/transformations/sample_polar": "Polar", - "/entry/sample/transformations/sample_tilt": "Tilt", - "/entry/sample/transformations/sample_azimuth": "Azimuth", - } - - # store data for resolved axis coordinates - for k, v in depends_dict.items(): - if v in axis_dict: - self._scan_info[axis_dict[v]] = res_xarray.coords[k].data + self._scan_info["coordinate_label"] = label_dict for name in res_xarray.dims: try: res_xarray[name].attrs["unit"] = self._config["units"][name] + res_xarray[name].attrs["long_name"] = label_dict[name] except KeyError: pass + # reset metadata + self.metadata = {} + self.metadata.update( **handle_meta( - df_lut, - self._scan_info, - self.config, + df_lut=df_lut, + scan_info=self._scan_info, + config=self.config.get("metadata", {}), + scan=scan, fast_axes=list(fast_axes), # type: ignore slow_axes=list(slow_axes), projection=projection, metadata=copy.deepcopy(metadata), collect_metadata=collect_metadata, + token=token, ), **{"loader": loader_dict}, **{"conversion_parameters": conversion_metadata}, ) + # shift energy axis + photon_energy = 0.0 + try: + photon_energy = self.metadata["instrument"]["beam"]["probe"]["incident_energy"] + except KeyError: + try: + photon_energy = self.metadata["elabFTW"]["laser_status"]["probe_photon_energy"] + except KeyError: + pass + + self.metadata["scan_info"]["reference_energy"] = "vacuum level" + if photon_energy and self._config["shift_by_photon_energy"]: + logger.info(f"Shifting energy axis by photon energy: -{photon_energy} eV") + res_xarray = res_xarray.assign_coords( + { + rename_dict["Ekin"]: ( + rename_dict["Ekin"], + res_xarray[rename_dict["Ekin"]].data - photon_energy, + res_xarray[rename_dict["Ekin"]].attrs, + ), + }, + ) + self.metadata["scan_info"]["reference_energy"] = "Fermi edge" + self.metadata["scan_info"]["coordinate_label"][rename_dict["Ekin"]] = "E-E_F" + res_xarray[rename_dict["Ekin"]].attrs["long_name"] = "E-E_F" + res_xarray.attrs["metadata"] = self.metadata self._result = res_xarray @@ -323,6 +368,38 @@ def crop_tool(self, scan: int = None, path: Path | str = "", **kwds): **kwds, ) + def save_crop_params( + self, + filename: str = None, + overwrite: bool = False, + ): + """Save the generated crop parameters to the folder config file. + + Args: + filename (str, optional): Filename of the config dictionary to save to. + Defaults to "specs_config.yaml" in the current folder. + overwrite (bool, optional): Option to overwrite the present dictionary. + Defaults to False. + """ + if filename is None: + filename = "specs_config.yaml" + if "ek_range_min" not in self.spa.config: + raise ValueError("No crop parameters to save!") + + config = { + "spa_params": { + "crop": self.spa.config["crop"], + "ek_range_min": self.spa.config["ek_range_min"], + "ek_range_max": self.spa.config["ek_range_max"], + "ang_range_min": self.spa.config["ang_range_min"], + "ang_range_max": self.spa.config["ang_range_max"], + "angle_offset_px": self.spa.config["angle_offset_px"], + "rotation_angle": self.spa.config["rotation_angle"], + }, + } + save_config(config, filename, overwrite) + logger.info(f'Saved crop parameters to "{filename}".') + def fft_tool(self, scan: int = None, path: Path | str = "", **kwds): """FFT tool to play around with the peak parameters in the Fourier plane. Built to filter out the meshgrid appearing in the raw data images. The optimized parameters are stored in @@ -360,6 +437,33 @@ def fft_tool(self, scan: int = None, path: Path | str = "", **kwds): **kwds, ) + def save_fft_params( + self, + filename: str = None, + overwrite: bool = False, + ): + """Save the generated fft filter parameters to the folder config file. + + Args: + filename (str, optional): Filename of the config dictionary to save to. + Defaults to "specs_config.yaml" in the current folder. + overwrite (bool, optional): Option to overwrite the present dictionary. + Defaults to False. + """ + if filename is None: + filename = "specs_config.yaml" + if len(self.spa.config["fft_filter_peaks"]) == 0: + raise ValueError("No fft parameters to save!") + + config = { + "spa_params": { + "fft_filter_peaks": self.spa.config["fft_filter_peaks"], + "apply_fft_filter": self.spa.config["apply_fft_filter"], + }, + } + save_config(config, filename, overwrite) + logger.info(f'Saved fft parameters to "{filename}".') + def check_scan( self, scan: int, @@ -390,6 +494,7 @@ def check_scan( Returns: xr.DataArray: 3-D xarray of dimensions (Ekin, Angle, Iterations) """ + token = kwds.pop("token", None) scan_path = get_scan_path(path, scan, self._config["data_path"]) df_lut = parse_lut_to_df(scan_path) @@ -437,7 +542,7 @@ def check_scan( conversion_metadata = xr_list[0].attrs["conversion_parameters"] - dims = get_coords( + dims = get_coords( # noqa: F841 scan_path=scan_path, scan_type=scan_type, scan_info=self._scan_info, @@ -463,16 +568,21 @@ def check_scan( fast_axes = set(res_xarray.dims) - slow_axes projection = "reciprocal" if "Angle" in fast_axes else "real" + # reset metadata + self.metadata = {} + self.metadata.update( **handle_meta( - df_lut, - self._scan_info, - self.config, + df_lut=df_lut, + scan_info=self._scan_info, + config=self.config.get("metadata", {}), + scan=scan, fast_axes=list(fast_axes), # type: ignore slow_axes=list(slow_axes), projection=projection, - metadata=metadata, + metadata=copy.deepcopy(metadata), collect_metadata=collect_metadata, + token=token, ), **{"loader": loader_dict}, **{"conversion_parameters": conversion_metadata}, @@ -594,7 +704,7 @@ def process_sweep_scan( """ ekin_step = kinetic_energy[1] - kinetic_energy[0] if not (np.diff(kinetic_energy) == ekin_step).all(): - warn( + logger.warning( "Conversion of sweep scans with non-equidistant energy steps " "might produce wrong results!", ) @@ -616,7 +726,7 @@ def process_sweep_scan( ) or not self.spa.config["crop"] ): - warn("No valid cropping parameters found, consider using crop_tool() to set.") + logger.warning("No valid cropping parameters found, consider using crop_tool() to set.") e_step = converted.Ekin[1] - converted.Ekin[0] e0 = converted.Ekin[-1] - ekin_step diff --git a/src/specsscan/helpers.py b/src/specsscan/helpers.py index 02e396c..8dd17e3 100644 --- a/src/specsscan/helpers.py +++ b/src/specsscan/helpers.py @@ -2,19 +2,21 @@ from __future__ import annotations import datetime as dt -import json +import importlib +import logging from pathlib import Path from typing import Any from typing import Sequence -from urllib.error import HTTPError -from urllib.error import URLError -from urllib.request import urlopen import numpy as np import pandas as pd from tqdm.auto import tqdm from specsanalyzer.config import complete_dictionary +from specsscan.metadata import MetadataRetriever + +# Configure logging +logger = logging.getLogger("specsanalyzer.specsscan") def get_scan_path(path: Path | str, scan: int, basepath: Path | str) -> Path: @@ -126,7 +128,7 @@ def load_images( "load_scan method.", ) from exc - print(f"Averaging over {avg_dim}...") + logger.info(f"Averaging over {avg_dim}...") for dim in tqdm(raw_2d_sliced): avg_list = [] for image in tqdm(dim, leave=False, disable=not tqdm_enable_nested): @@ -213,14 +215,14 @@ def parse_lut_to_df(scan_path: Path) -> pd.DataFrame: df_lut.reset_index(inplace=True) new_cols = df_lut.columns.to_list()[1:] - new_cols[new_cols.index("delaystage")] = "Delay" + new_cols[new_cols.index("delaystage")] = "DelayStage" new_cols.insert(3, "delay (fs)") # Create label to drop the column later df_lut = df_lut.set_axis(new_cols, axis="columns") df_lut.drop(columns="delay (fs)", inplace=True) except FileNotFoundError: - print( + logger.info( "LUT.txt not found. Storing metadata from info.txt", ) return None @@ -268,7 +270,7 @@ def get_coords( return (np.array([]), "") if df_lut is not None: - print("scanvector.txt not found. Obtaining coordinates from LUT") + logger.info("scanvector.txt not found. Obtaining coordinates from LUT") df_new: pd.DataFrame = df_lut.loc[:, df_lut.columns[2:]] @@ -279,13 +281,18 @@ def get_coords( raise FileNotFoundError("scanvector.txt file not found!") from exc if scan_type == "delay": - t_0 = scan_info["TimeZero"] - coords -= t_0 - coords *= 2 / 3e11 * 1e15 + t0 = scan_info["TimeZero"] + coords = mm_to_fs(coords, t0) return coords, dim +def mm_to_fs(delaystage, t0): + delay = delaystage - t0 + delay *= 2 / 2.99792458e11 * 1e15 + return delay + + def compare_coords(axis_data: np.ndarray) -> tuple[np.ndarray, int]: """Identifies the most changing column in a given 2-D numpy array. @@ -348,11 +355,13 @@ def handle_meta( df_lut: pd.DataFrame, scan_info: dict, config: dict, + scan: int, fast_axes: list[str], slow_axes: list[str], projection: str, metadata: dict = None, collect_metadata: bool = False, + token: str = None, ) -> dict: """Helper function for the handling metadata from different files @@ -361,22 +370,24 @@ def handle_meta( from ``parse_lut_to_df()`` scan_info (dict): scan_info class dict containing containing the contents of info.txt file config (dict): config dictionary containing the contents of config.yaml file + scan (int): Scan number fast_axes (list[str]): The fast-axis dimensions of the scan slow_axes (list[str]): The slow-axis dimensions of the scan metadata (dict, optional): Metadata dictionary with additional metadata for the scan. Defaults to empty dictionary. collect_metadata (bool, optional): Option to collect further metadata e.g. from EPICS archiver needed for NeXus conversion. Defaults to False. + token (str, optional):: The elabFTW api token to use for fetching metadata Returns: dict: metadata dictionary containing additional metadata from the EPICS - archive. + archive and elabFTW. """ if metadata is None: metadata = {} - print("Gathering metadata from different locations") + logger.info("Gathering metadata from different locations") # get metadata from LUT dataframe lut_meta = {} energy_scan_mode = "snapshot" @@ -394,10 +405,22 @@ def handle_meta( metadata["scan_info"] = complete_dictionary( metadata.get("scan_info", {}), - complete_dictionary(lut_meta, scan_info), + complete_dictionary(scan_info, lut_meta), ) # merging dictionaries - print("Collecting time stamps...") + # Store delay info + if "DelayStage" in metadata["scan_info"] and "TimeZero" in metadata["scan_info"]: + metadata["scan_info"]["delay"] = mm_to_fs( + metadata["scan_info"]["DelayStage"], + metadata["scan_info"]["TimeZero"], + ) + + # store program version + metadata["scan_info"]["program_name"] = "specsanalyzer" + metadata["scan_info"]["program_version"] = importlib.metadata.version("specsanalyzer") + + # timing + logger.info("Collecting time stamps...") if "time" in metadata["scan_info"]: time_list = [metadata["scan_info"]["time"][0], metadata["scan_info"]["time"][-1]] elif "StartTime" in metadata["scan_info"]: @@ -407,67 +430,36 @@ def handle_meta( dt_list_iso = [time.replace(".", "-").replace(" ", "T") for time in time_list] datetime_list = [dt.datetime.fromisoformat(dt_iso) for dt_iso in dt_list_iso] - ts_from = dt.datetime.timestamp(datetime_list[0]) # POSIX timestamp - ts_to = dt.datetime.timestamp(datetime_list[-1]) # POSIX timestamp + ts_from = dt.datetime.timestamp(min(datetime_list)) # POSIX timestamp + ts_to = dt.datetime.timestamp(max(datetime_list)) # POSIX timestamp + if ts_from == ts_to: + try: + ts_to = ( + ts_from + + metadata["scan_info"]["Exposure"] / 1000 * metadata["scan_info"]["Averages"] + ) + except KeyError: + pass metadata["timing"] = { - "acquisition_start": dt.datetime.utcfromtimestamp(ts_from) - .replace(tzinfo=dt.timezone.utc) - .isoformat(), - "acquisition_stop": dt.datetime.utcfromtimestamp(ts_to) - .replace(tzinfo=dt.timezone.utc) - .isoformat(), + "acquisition_start": dt.datetime.fromtimestamp(ts_from, dt.timezone.utc).isoformat(), + "acquisition_stop": dt.datetime.fromtimestamp(ts_to, dt.timezone.utc).isoformat(), "acquisition_duration": int(ts_to - ts_from), "collection_time": float(ts_to - ts_from), } if collect_metadata: - # Get metadata from Epics archive if not present already - start = dt.datetime.utcfromtimestamp(ts_from).isoformat() + metadata_retriever = MetadataRetriever(config, token) - # replace metadata names by epics channels - try: - replace_dict = config["epics_channels"] - for key in list(metadata["scan_info"]): - if key.lower() in replace_dict: - metadata["scan_info"][replace_dict[key.lower()]] = metadata["scan_info"][key] - metadata["scan_info"].pop(key) - epics_channels = replace_dict.values() - except KeyError: - epics_channels = [] - - channels_missing = set(epics_channels) - set(metadata["scan_info"].keys()) - if channels_missing: - print("Collecting data from the EPICS archive...") - for channel in channels_missing: - try: - _, vals = get_archiver_data( - archiver_url=config.get("archiver_url"), - archiver_channel=channel, - ts_from=ts_from, - ts_to=ts_to, - ) - metadata["scan_info"][f"{channel}"] = np.mean(vals) + metadata = metadata_retriever.fetch_epics_metadata( + ts_from=ts_from, + ts_to=ts_to, + metadata=metadata, + ) - except IndexError: - metadata["scan_info"][f"{channel}"] = np.nan - print( - f"Data for channel {channel} doesn't exist for time {start}", - ) - except HTTPError as exc: - print( - f"Incorrect URL for the archive channel {channel}. " - "Make sure that the channel name and file start and end times are " - "correct.", - ) - print("Error code: ", exc) - except URLError as exc: - print( - f"Cannot access the archive URL for channel {channel}. " - f"Make sure that you are within the FHI network." - f"Skipping over channels {channels_missing}.", - ) - print("Error code: ", exc) - break + metadata = metadata_retriever.fetch_elab_metadata( + scan=scan, + metadata=metadata, + ) metadata["scan_info"]["energy_scan_mode"] = energy_scan_mode @@ -476,42 +468,12 @@ def handle_meta( "angular dispersive" if projection == "reciprocal" else "spatial dispersive" ) - metadata["scan_info"]["slow_axes"] = slow_axes + metadata["scan_info"]["slow_axes"] = slow_axes if slow_axes else "" metadata["scan_info"]["fast_axes"] = fast_axes - print("Done!") - return metadata -def get_archiver_data( - archiver_url: str, - archiver_channel: str, - ts_from: float, - ts_to: float, -) -> tuple[np.ndarray, np.ndarray]: - """Extract time stamps and corresponding data from and EPICS archiver instance - - Args: - archiver_url (str): URL of the archiver data extraction interface - archiver_channel (str): EPICS channel to extract data for - ts_from (float): starting time stamp of the range of interest - ts_to (float): ending time stamp of the range of interest - - Returns: - tuple[List, List]: The extracted time stamps and corresponding data - """ - iso_from = dt.datetime.utcfromtimestamp(ts_from).isoformat() - iso_to = dt.datetime.utcfromtimestamp(ts_to).isoformat() - req_str = archiver_url + archiver_channel + "&from=" + iso_from + "Z&to=" + iso_to + "Z" - with urlopen(req_str) as req: - data = json.load(req) - secs = [x["secs"] + x["nanos"] * 1e-9 for x in data[0]["data"]] - vals = [x["val"] for x in data[0]["data"]] - - return (np.asarray(secs), np.asarray(vals)) - - def find_scan(path: Path, scan: int) -> list[Path]: """Search function to locate the scan folder @@ -522,7 +484,7 @@ def find_scan(path: Path, scan: int) -> list[Path]: Returns: List[Path]: scan_path: Path object pointing to the scan folder """ - print("Scan path not provided, searching directories...") + logger.info("Scan path not provided, searching directories...") for file in path.iterdir(): if file.is_dir(): try: @@ -536,7 +498,7 @@ def find_scan(path: Path, scan: int) -> list[Path]: file.glob(f"*/*/Raw Data/{scan}"), ) if scan_path: - print("Scan found at path:", scan_path[0]) + logger.info(f"Scan found at path: {scan_path[0]}") break else: scan_path = [] diff --git a/src/specsscan/metadata.py b/src/specsscan/metadata.py new file mode 100644 index 0000000..e1408f9 --- /dev/null +++ b/src/specsscan/metadata.py @@ -0,0 +1,392 @@ +""" +The module provides a MetadataRetriever class for retrieving metadata +from an EPICS archiver and an elabFTW instance. +""" +from __future__ import annotations + +import datetime +import json +from copy import deepcopy +from urllib.error import HTTPError +from urllib.error import URLError +from urllib.request import urlopen + +import elabapi_python +import numpy as np +from urllib3.exceptions import MaxRetryError + +from specsanalyzer.config import read_env_var +from specsanalyzer.config import save_env_var +from specsanalyzer.logging import setup_logging + +logger = setup_logging("mpes_metadata_retriever") + + +class MetadataRetriever: + """ + A class for retrieving metadata from an EPICS archiver and an elabFTW instance. + """ + + def __init__(self, metadata_config: dict, token: str = None) -> None: + """ + Initializes the MetadataRetriever class. + + Args: + metadata_config (dict): Takes a dict containing at least url for the EPICS archiver and + elabFTW instance. + token (str, optional): The token to use for fetching metadata. If provided, + will be saved to .env file for future use. + """ + self._config = deepcopy(metadata_config) + + # Token handling + if token: + self.token = token + save_env_var("ELAB_TOKEN", self.token) + else: + # Try to load token from config or .env file + self.token = read_env_var("ELAB_TOKEN") + + if not self.token: + logger.warning( + "No valid token provided for elabFTW. Fetching elabFTW metadata will be skipped.", + ) + return + + self.url = metadata_config.get("elab_url") + if not self.url: + logger.warning( + "No URL provided for elabFTW. Fetching elabFTW metadata will be skipped.", + ) + return + + # Config + self.configuration = elabapi_python.Configuration() + self.configuration.api_key["api_key"] = self.token + self.configuration.api_key_prefix["api_key"] = "Authorization" + self.configuration.host = str(self.url) + self.configuration.debug = False + self.configuration.verify_ssl = False + + # create an instance of the API class + self.api_client = elabapi_python.ApiClient(self.configuration) + # fix issue with Authorization header not being properly set by the generated lib + self.api_client.set_default_header(header_name="Authorization", header_value=self.token) + + # create an instance of Items + self.itemsApi = elabapi_python.ItemsApi(self.api_client) + self.experimentsApi = elabapi_python.ExperimentsApi(self.api_client) + self.linksApi = elabapi_python.LinksToItemsApi(self.api_client) + self.experimentsLinksApi = elabapi_python.LinksToExperimentsApi(self.api_client) + self.usersApi = elabapi_python.UsersApi(self.api_client) + + def fetch_epics_metadata(self, ts_from: float, ts_to: float, metadata: dict) -> dict: + """Fetch metadata from an EPICS archiver instance for times between ts_from and ts_to. + Channels are defined in the config. + + Args: + ts_from (float): Start timestamp of the range to collect data from. + ts_to (float): End timestamp of the range to collect data from. + metadata (dict): Input metadata dictionary. Will be updated + + Returns: + dict: Updated metadata dictionary. + """ + start = datetime.datetime.fromtimestamp(ts_from, datetime.timezone.utc).isoformat() + + # replace metadata names by epics channels + try: + replace_dict = self._config["epics_channels"] + for key in list(metadata["scan_info"]): + if key.lower() in replace_dict: + metadata["scan_info"][replace_dict[key.lower()]] = metadata["scan_info"][key] + del metadata["scan_info"][key] + epics_channels = replace_dict.values() + except KeyError: + epics_channels = [] + + channels_missing = set(epics_channels) - set(metadata.get("scan_info", {}).keys()) + if channels_missing: + logger.info("Collecting data from the EPICS archive...") + for channel in channels_missing: + try: + _, vals = get_archiver_data( + archiver_url=str(self._config.get("archiver_url")), + archiver_channel=channel, + ts_from=ts_from, + ts_to=ts_to, + ) + metadata["scan_info"][f"{channel}"] = np.mean(vals) + + except IndexError: + logger.info( + f"Data for channel {channel} doesn't exist for time {start}", + ) + except HTTPError as exc: + logger.warning( + f"Incorrect URL for the archive channel {channel}. " + "Make sure that the channel name and file start and end times are " + "correct.", + ) + logger.warning(f"Error code: {exc}") + except URLError as exc: + logger.warning( + f"Cannot access the archive URL for channel {channel}. " + f"Make sure that you are within the FHI network." + f"Skipping over channels {channels_missing}.", + ) + logger.warning(f"Error code: {exc}") + break + + return metadata + + def fetch_elab_metadata(self, scan: int, metadata: dict) -> dict: + """Fetch metadata from an elabFTW instance + + Args: + scan (int): Scan number for which to fetch metadata + metadata (dict): Input metadata dictionary. Will be updated + + Returns: + dict: Updated metadata dictionary + """ + if not self.token: + logger.warning( + "No valid token found. Token is required for metadata collection. Either provide " + "a token parameter or set the ELAB_TOKEN environment variable.", + ) + return metadata + + if not self.url: + logger.warning( + "No URL provided for fetching metadata from elabFTW. " + "Fetching elabFTW metadata will be skipped.", + ) + return metadata + + logger.info("Collecting data from the elabFTW instance...") + # Get the experiment + try: + experiment = self.experimentsApi.read_experiments(q=f"'Phoibos scan {scan}'")[0] + except IndexError: + logger.warning(f"No elabFTW entry found for run {scan}") + return metadata + except MaxRetryError: + logger.warning("Connection to elabFTW could not be established. Check accessibility") + return metadata + + if "elabFTW" not in metadata: + metadata["elabFTW"] = {} + + exp_id = experiment.id + # Get user information + user = self.usersApi.read_user(experiment.userid) + metadata["elabFTW"]["user"] = {} + metadata["elabFTW"]["user"]["name"] = user.fullname + metadata["elabFTW"]["user"]["email"] = user.email + metadata["elabFTW"]["user"]["id"] = user.userid + if user.orcid: + metadata["elabFTW"]["user"]["orcid"] = user.orcid + # Get the links to items + links = self.linksApi.read_entity_items_links(entity_type="experiments", id=exp_id) + # Get the items + items = [self.itemsApi.get_item(link.entityid) for link in links] + items_dict = {item.category_title: item for item in items} + items_dict["scan"] = experiment + + # Sort the metadata + for category, item in items_dict.items(): + category = category.replace(":", "").replace(" ", "_").lower() + if category not in metadata["elabFTW"]: + metadata["elabFTW"][category] = {} + metadata["elabFTW"][category]["title"] = item.title + metadata["elabFTW"][category]["summary"] = item.body + metadata["elabFTW"][category]["id"] = item.id + metadata["elabFTW"][category]["elabid"] = item.elabid + if item.sharelink: + metadata["elabFTW"][category]["link"] = item.sharelink + if item.metadata is not None: + metadata_json = json.loads(item.metadata) + for key, val in metadata_json["extra_fields"].items(): + if val["value"] is not None and val["value"] != "" and val["value"] != ["None"]: + try: + metadata["elabFTW"][category][key] = float(val["value"]) + except (ValueError, TypeError): + metadata["elabFTW"][category][key] = val["value"] + + # group beam profiles: + if ( + "laser_status" in metadata["elabFTW"] + and "pump_profile_x" in metadata["elabFTW"]["laser_status"] + and "pump_profile_y" in metadata["elabFTW"]["laser_status"] + ): + metadata["elabFTW"]["laser_status"]["pump_profile"] = [ + float(metadata["elabFTW"]["laser_status"]["pump_profile_x"]), + float(metadata["elabFTW"]["laser_status"]["pump_profile_y"]), + ] + if ( + "laser_status" in metadata["elabFTW"] + and "probe_profile_x" in metadata["elabFTW"]["laser_status"] + and "probe_profile_y" in metadata["elabFTW"]["laser_status"] + ): + metadata["elabFTW"]["laser_status"]["probe_profile"] = [ + float(metadata["elabFTW"]["laser_status"]["probe_profile_x"]), + float(metadata["elabFTW"]["laser_status"]["probe_profile_y"]), + ] + if ( + "laser_status" in metadata["elabFTW"] + and "pump2_profile_x" in metadata["elabFTW"]["laser_status"] + and "pump2_profile_y" in metadata["elabFTW"]["laser_status"] + ): + metadata["elabFTW"]["laser_status"]["pump2_profile"] = [ + float(metadata["elabFTW"]["laser_status"]["pump2_profile_x"]), + float(metadata["elabFTW"]["laser_status"]["pump2_profile_y"]), + ] + + # fix preparation date + if "sample" in metadata["elabFTW"] and "preparation_date" in metadata["elabFTW"]["sample"]: + metadata["elabFTW"]["sample"]["preparation_date"] = ( + datetime.datetime.strptime( + metadata["elabFTW"]["sample"]["preparation_date"], + "%Y-%m-%d", + ) + .replace(tzinfo=datetime.timezone.utc) + .isoformat() + ) + + # fix polarizations + if ( + "scan" in metadata["elabFTW"] + and "pump_polarization" in metadata["elabFTW"]["scan"] + and isinstance(metadata["elabFTW"]["scan"]["pump_polarization"], str) + ): + if metadata["elabFTW"]["scan"]["pump_polarization"] == "s": + metadata["elabFTW"]["scan"]["pump_polarization"] = 90 + elif metadata["elabFTW"]["scan"]["pump_polarization"] == "p": + metadata["elabFTW"]["scan"]["pump_polarization"] = 0 + else: + try: + metadata["elabFTW"]["scan"]["pump_polarization"] = float( + metadata["elabFTW"]["scan"]["pump_polarization"], + ) + except ValueError: + pass + + if ( + "scan" in metadata["elabFTW"] + and "probe_polarization" in metadata["elabFTW"]["scan"] + and isinstance(metadata["elabFTW"]["scan"]["probe_polarization"], str) + ): + if metadata["elabFTW"]["scan"]["probe_polarization"] == "s": + metadata["elabFTW"]["scan"]["probe_polarization"] = 90 + elif metadata["elabFTW"]["scan"]["probe_polarization"] == "p": + metadata["elabFTW"]["scan"]["probe_polarization"] = 0 + else: + try: + metadata["elabFTW"]["scan"]["probe_polarization"] = float( + metadata["elabFTW"]["scan"]["probe_polarization"], + ) + except ValueError: + pass + + if ( + "scan" in metadata["elabFTW"] + and "pump2_polarization" in metadata["elabFTW"]["scan"] + and isinstance(metadata["elabFTW"]["scan"]["pump2_polarization"], str) + ): + if metadata["elabFTW"]["scan"]["pump2_polarization"] == "s": + metadata["elabFTW"]["scan"]["pump2_polarization"] = 90 + elif metadata["elabFTW"]["scan"]["pump2_polarization"] == "p": + metadata["elabFTW"]["scan"]["pump2_polarization"] = 0 + else: + try: + metadata["elabFTW"]["scan"]["pump2_polarization"] = float( + metadata["elabFTW"]["scan"]["pump2_polarization"], + ) + except ValueError: + pass + + # fix pump status + if "scan" in metadata["elabFTW"] and "pump_status" in metadata["elabFTW"]["scan"]: + try: + metadata["elabFTW"]["scan"]["pump_status"] = ( + "open" if int(metadata["elabFTW"]["scan"]["pump_status"]) else "closed" + ) + except ValueError: + pass + if "scan" in metadata["elabFTW"] and "pump2_status" in metadata["elabFTW"]["scan"]: + try: + metadata["elabFTW"]["scan"]["pump2_status"] = ( + "open" if int(metadata["elabFTW"]["scan"]["pump2_status"]) else "closed" + ) + except ValueError: + pass + + # remove pump information if pump not applied: + if metadata["elabFTW"]["scan"].get("pump_status", "closed") == "closed": + if "pump_photon_energy" in metadata["elabFTW"].get("laser_status", {}): + del metadata["elabFTW"]["laser_status"]["pump_photon_energy"] + if "pump_repetition_rate" in metadata["elabFTW"].get("laser_status", {}): + del metadata["elabFTW"]["laser_status"]["pump_repetition_rate"] + else: + # add pulse energy if applicable + try: + metadata["elabFTW"]["scan"]["pump_pulse_energy"] = ( + metadata["scan_info"]["trARPES:Pump:Power.RBV"] + / metadata["elabFTW"]["laser_status"]["pump_repetition_rate"] + ) + except KeyError: + pass + + if metadata["elabFTW"]["scan"].get("pump2_status", "closed") == "closed": + if "pump2_photon_energy" in metadata["elabFTW"].get("laser_status", {}): + del metadata["elabFTW"]["laser_status"]["pump2_photon_energy"] + if "pump2_repetition_rate" in metadata["elabFTW"].get("laser_status", {}): + del metadata["elabFTW"]["laser_status"]["pump2_repetition_rate"] + else: + # add pulse energy if applicable + try: + metadata["elabFTW"]["scan"]["pump2_pulse_energy"] = ( + metadata["scan_info"]["trARPES:Pump2:Power.RBV"] + / metadata["elabFTW"]["laser_status"]["pump_repetition_rate"] + ) + except KeyError: + pass + + return metadata + + +def get_archiver_data( + archiver_url: str, + archiver_channel: str, + ts_from: float, + ts_to: float, +) -> tuple[np.ndarray, np.ndarray]: + """Extract time stamps and corresponding data from and EPICS archiver instance + + Args: + archiver_url (str): URL of the archiver data extraction interface + archiver_channel (str): EPICS channel to extract data for + ts_from (float): starting time stamp of the range of interest + ts_to (float): ending time stamp of the range of interest + + Returns: + tuple[np.ndarray, np.ndarray]: The extracted time stamps and corresponding data + """ + iso_from = ( + datetime.datetime.fromtimestamp(ts_from, datetime.timezone.utc) + .replace(tzinfo=None) + .isoformat() + ) + iso_to = ( + datetime.datetime.fromtimestamp(ts_to, datetime.timezone.utc) + .replace(tzinfo=None) + .isoformat() + ) + req_str = archiver_url + archiver_channel + "&from=" + iso_from + "Z&to=" + iso_to + "Z" + with urlopen(req_str) as req: + data = json.load(req) + secs = [x["secs"] + x["nanos"] * 1e-9 for x in data[0]["data"]] + vals = [x["val"] for x in data[0]["data"]] + + return (np.asarray(secs), np.asarray(vals)) diff --git a/tests/data b/tests/data index e921fe9..bbede1b 160000 --- a/tests/data +++ b/tests/data @@ -1 +1 @@ -Subproject commit e921fe9c198d263c900d8568476e12e4dcef82b8 +Subproject commit bbede1bfb99020f3fb71d4b03570b83bfab07c83 diff --git a/tests/test_config.py b/tests/test_config.py index 20742d0..5b3b091 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -8,7 +8,9 @@ from specsanalyzer.config import complete_dictionary from specsanalyzer.config import load_config from specsanalyzer.config import parse_config +from specsanalyzer.config import read_env_var from specsanalyzer.config import save_config +from specsanalyzer.config import save_env_var test_dir = os.path.dirname(__file__) default_config_keys = [ @@ -88,3 +90,140 @@ def test_save_dict(): save_config(config_dict, filename, overwrite=True) config = load_config(filename) assert "test_entry" not in config.keys() + + +@pytest.fixture +def mock_env_file(tmp_path, monkeypatch): + """Mock the .env file for testing""" + monkeypatch.setattr("specsanalyzer.config.USER_CONFIG_PATH", tmp_path) + yield tmp_path + + +def test_env_var_read_write(mock_env_file): # noqa: ARG001 + """Test reading and writing environment variables.""" + # Test writing a new variable + save_env_var("TEST_VAR", "test_value") + assert read_env_var("TEST_VAR") == "test_value" + + # Test writing multiple variables + save_env_var("TEST_VAR2", "test_value2") + assert read_env_var("TEST_VAR") == "test_value" + assert read_env_var("TEST_VAR2") == "test_value2" + + # Test overwriting an existing variable + save_env_var("TEST_VAR", "new_value") + assert read_env_var("TEST_VAR") == "new_value" + assert read_env_var("TEST_VAR2") == "test_value2" # Other variables unchanged + + # Test reading non-existent variable + assert read_env_var("NON_EXISTENT_VAR") is None + + +def test_env_var_read_no_file(mock_env_file): # noqa: ARG001 + """Test reading environment variables when .env file doesn't exist.""" + # Test reading from non-existent file + assert read_env_var("TEST_VAR") is None + + +def test_env_var_special_characters(mock_env_file): # noqa: ARG001 + """Test reading and writing environment variables with special characters.""" + test_cases = { + "TEST_URL": "http://example.com/path?query=value", + "TEST_PATH": "/path/to/something/with/spaces and special=chars", + "TEST_QUOTES": "value with 'single' and \"double\" quotes", + } + + for var_name, value in test_cases.items(): + save_env_var(var_name, value) + assert read_env_var(var_name) == value + + +def test_env_var_precedence(mock_env_file, tmp_path, monkeypatch): # noqa: ARG001 + """Test that environment variables are read in correct order of precedence""" + # Create local .env directory if it doesn't exist + local_env_dir = tmp_path / "local" + local_env_dir.mkdir(exist_ok=True) + system_env_dir = tmp_path / "system" + system_env_dir.mkdir(exist_ok=True) + monkeypatch.setattr("specsanalyzer.config.ENV_DIR", local_env_dir / ".env") + monkeypatch.setattr("specsanalyzer.config.SYSTEM_CONFIG_PATH", system_env_dir) + + # Set up test values in different locations + os.environ["TEST_VAR"] = "os_value" + + # Save to system config first (4th precedence) + with open(system_env_dir / ".env", "w") as f: + f.write("TEST_VAR=system_value\n") + + # Save to user config first (3rd precedence) + save_env_var("TEST_VAR", "user_value") + + # Create local .env file (2nd precedence) + with open(local_env_dir / ".env", "w") as f: + f.write("TEST_VAR=local_value\n") + + assert read_env_var("TEST_VAR") == "os_value" + + # Remove from OS env to test other precedence levels + monkeypatch.delenv("TEST_VAR", raising=False) + assert read_env_var("TEST_VAR") == "local_value" + + # Remove local .env and should get user config value + (local_env_dir / ".env").unlink() + assert read_env_var("TEST_VAR") == "user_value" + + # Remove user config and should get system value + (mock_env_file / ".env").unlink() + assert read_env_var("TEST_VAR") == "system_value" + + # Remove system config and should get None + (system_env_dir / ".env").unlink() + assert read_env_var("TEST_VAR") is None + + +def test_env_var_save_and_load(mock_env_file, monkeypatch): # noqa: ARG001 + """Test saving and loading environment variables""" + # Clear any existing OS environment variables + monkeypatch.delenv("TEST_VAR", raising=False) + monkeypatch.delenv("OTHER_VAR", raising=False) + + # Save a variable + save_env_var("TEST_VAR", "test_value") + + # Should be able to read it back + assert read_env_var("TEST_VAR") == "test_value" + + # Save another variable - should preserve existing ones + save_env_var("OTHER_VAR", "other_value") + assert read_env_var("TEST_VAR") == "test_value" + assert read_env_var("OTHER_VAR") == "other_value" + + +def test_env_var_not_found(mock_env_file): # noqa: ARG001 + """Test behavior when environment variable is not found""" + assert read_env_var("NONEXISTENT_VAR") is None + + +def test_env_file_format(mock_env_file, monkeypatch): # noqa: ARG001 + """Test that .env file parsing handles different formats correctly""" + # Clear any existing OS environment variables + monkeypatch.delenv("TEST_VAR", raising=False) + monkeypatch.delenv("SPACES_VAR", raising=False) + monkeypatch.delenv("EMPTY_VAR", raising=False) + monkeypatch.delenv("COMMENT", raising=False) + + with open(mock_env_file / ".env", "w") as f: + f.write( + """ + TEST_VAR=value1 + SPACES_VAR = value2 + EMPTY_VAR= + #COMMENT=value3 + INVALID_LINE + """, + ) + + assert read_env_var("TEST_VAR") == "value1" + assert read_env_var("SPACES_VAR") == "value2" + assert read_env_var("EMPTY_VAR") == "" + assert read_env_var("COMMENT") is None diff --git a/tests/test_logging.py b/tests/test_logging.py new file mode 100644 index 0000000..99f9d1b --- /dev/null +++ b/tests/test_logging.py @@ -0,0 +1,86 @@ +import io +import logging +import os +from datetime import datetime + +import pytest + +from specsanalyzer.logging import set_verbosity +from specsanalyzer.logging import setup_logging + + +@pytest.fixture +def logger_(): + logger = setup_logging("test_logger") + log_capture_string = io.StringIO() + ch = logging.StreamHandler(log_capture_string) + ch.setLevel(logging.DEBUG) + logger.addHandler(ch) + yield logger, log_capture_string + + +def test_debug_logging(logger_): + logger, log_capture_string = logger_ + logger.debug("This is a debug message") + assert "This is a debug message" in log_capture_string.getvalue() + + +def test_info_logging(logger_): + logger, log_capture_string = logger_ + logger.info("This is an info message") + assert "This is an info message" in log_capture_string.getvalue() + + +def test_warning_logging(logger_): + logger, log_capture_string = logger_ + logger.warning("This is a warning message") + assert "This is a warning message" in log_capture_string.getvalue() + + +def test_error_logging(logger_): + logger, log_capture_string = logger_ + logger.error("This is an error message") + assert "This is an error message" in log_capture_string.getvalue() + + +def test_critical_logging(logger_): + logger, log_capture_string = logger_ + logger.critical("This is a critical message") + assert "This is a critical message" in log_capture_string.getvalue() + + +def test_set_verbosity(logger_): + logger, log_capture_string = logger_ + set_verbosity(logger, verbose=True) + assert logger.handlers[0].level == logging.INFO + set_verbosity(logger, verbose=False) + assert logger.handlers[0].level == logging.WARNING + + +def test_logger_has_base_logger(logger_): + logger, log_capture_string = logger_ + assert logger.name == "specsanalyzer.test_logger" + assert logger.parent.name == "specsanalyzer" + assert logger.parent.parent.name == "root" + assert logger.parent.level == logging.DEBUG + assert isinstance(logger.parent.handlers[0], logging.FileHandler) + + +def test_logger_creates_logfile(tmp_path): + logger = setup_logging("test_logger", set_base_handler=True, user_log_path=tmp_path) + log_file = os.path.join(tmp_path, f"specsanalyzer_{datetime.now().strftime('%Y-%m-%d')}.log") + assert os.path.exists(log_file) + with open(log_file) as f: + assert f.read() == "" + logger.debug("This is a debug message") + with open(log_file) as f: + assert "This is a debug message" in f.read() + + +def test_readonly_path(tmp_path, caplog): + os.chmod(tmp_path, 0o444) + with caplog.at_level(logging.WARNING): + setup_logging("test_logger", set_base_handler=True, user_log_path=tmp_path) + assert f"Cannot create logfile in Folder {tmp_path}, disabling logfile." in caplog.messages[0] + log_file = os.path.join(tmp_path, f"sed_{datetime.now().strftime('%Y-%m-%d')}.log") + assert not os.path.exists(log_file) diff --git a/tests/test_metadata.py b/tests/test_metadata.py new file mode 100644 index 0000000..064643f --- /dev/null +++ b/tests/test_metadata.py @@ -0,0 +1,150 @@ +"""Tests specific for metadata retrieval""" +from __future__ import annotations + +import datetime +import json +from unittest.mock import MagicMock +from unittest.mock import patch + +import numpy as np +import pytest + +from specsscan.metadata import get_archiver_data +from specsscan.metadata import MetadataRetriever +from tests.test_config import mock_env_file # noqa: F401 + + +@pytest.fixture +def metadata_config(): + return { + "elab_url": "http://example.com", + "epics_channels": {"channel1": "channel1"}, + "archiver_url": "http://archiver.example.com", + "aperture_config": { + datetime.datetime.fromisoformat("2023-01-01T00:00:00"): { + "fa_size": {"1.0": [(0, 1), (0, 1)]}, + "ca_size": {"1.0": (0, 1)}, + }, + }, + "lens_mode_config": {"mode1": {"lens1": 1.0, "lens2": 2.0}}, + "fa_in_channel": "fa_in", + "fa_hor_channel": "fa_hor", + "ca_in_channel": "ca_in", + } + + +@pytest.fixture +def metadata_retriever(metadata_config, mock_env_file): # noqa: ARG001 + return MetadataRetriever(metadata_config, "dummy_token") + + +def test_metadata_retriever_init(metadata_retriever): + assert metadata_retriever.token == "dummy_token" + assert metadata_retriever.url == "http://example.com" + + +def test_metadata_retriever_no_token(metadata_config, tmp_path, monkeypatch): + monkeypatch.setattr("specsanalyzer.config.ENV_DIR", tmp_path / ".dummy_env") + monkeypatch.setattr("specsanalyzer.config.SYSTEM_CONFIG_PATH", tmp_path / "system") + monkeypatch.setattr("specsanalyzer.config.USER_CONFIG_PATH", tmp_path / "user") + retriever = MetadataRetriever(metadata_config) + assert retriever.token is None + + metadata = {} + runs = ["run1"] + updated_metadata = retriever.fetch_elab_metadata(runs, metadata) + assert updated_metadata == metadata + + +def test_metadata_retriever_no_url(metadata_config, mock_env_file): # noqa: ARG001 + metadata_config.pop("elab_url") + retriever = MetadataRetriever(metadata_config, "dummy_token") + assert retriever.url is None + + metadata = {} + runs = ["run1"] + updated_metadata = retriever.fetch_elab_metadata(runs, metadata) + assert updated_metadata == metadata + + +@patch("specsscan.metadata.urlopen") +def test_get_archiver_data(mock_urlopen): + """Test get_archiver_data using a mock of urlopen.""" + mock_response = MagicMock() + mock_response.read.return_value = json.dumps( + [{"data": [{"secs": 1, "nanos": 500000000, "val": 10}]}], + ) + mock_urlopen.return_value.__enter__.return_value = mock_response + + ts_from = datetime.datetime(2023, 1, 1).timestamp() + ts_to = datetime.datetime(2023, 1, 2).timestamp() + archiver_url = "http://archiver.example.com" + archiver_channel = "channel1" + + secs, vals = get_archiver_data(archiver_url, archiver_channel, ts_from, ts_to) + + assert np.array_equal(secs, np.array([1.5])) + assert np.array_equal(vals, np.array([10])) + + +@patch("specsscan.metadata.get_archiver_data") +def test_fetch_epics_metadata(mock_get_archiver_data, metadata_retriever): + """Test fetch_epics_metadata using a mock of get_archiver_data.""" + mock_get_archiver_data.return_value = (np.array([1.5]), np.array([10])) + metadata = {"scan_info": {}} + ts_from = datetime.datetime(2023, 1, 1).timestamp() + ts_to = datetime.datetime(2023, 1, 2).timestamp() + + updated_metadata = metadata_retriever.fetch_epics_metadata(ts_from, ts_to, metadata) + + assert updated_metadata["scan_info"]["channel1"] == 10 + + +@patch("specsscan.metadata.get_archiver_data") +def test_fetch_epics_metadata_missing_channels(mock_get_archiver_data, metadata_retriever): + """Test fetch_epics_metadata with missing EPICS channels.""" + mock_get_archiver_data.return_value = (np.array([1.5]), np.array([10])) + metadata = {"file": {"channel1": 10}} + ts_from = datetime.datetime(2023, 1, 1).timestamp() + ts_to = datetime.datetime(2023, 1, 2).timestamp() + + updated_metadata = metadata_retriever.fetch_epics_metadata(ts_from, ts_to, metadata) + + assert "channel1" in updated_metadata["file"] + + +@patch("specsscan.metadata.elabapi_python") +def test_fetch_elab_metadata(mock_elabapi_python, metadata_config, mock_env_file): # noqa: ARG001 + """Test fetch_elab_metadata using a mock of elabapi_python.""" + mock_experiment = MagicMock() + mock_experiment.id = 1 + mock_experiment.userid = 1 + mock_experiment.title = "Test Experiment" + mock_experiment.body = "Test Body" + mock_experiment.metadata = json.dumps({"extra_fields": {"key": {"value": "value"}}}) + mock_elabapi_python.ExperimentsApi.return_value.read_experiments.return_value = [ + mock_experiment, + ] + mock_user = MagicMock() + mock_user.fullname = "Test User" + mock_user.email = "test@example.com" + mock_user.userid = 1 + mock_user.orcid = "0000-0000-0000-0000" + mock_elabapi_python.UsersApi.return_value.read_user.return_value = mock_user + mock_elabapi_python.LinksToItemsApi.return_value.read_entity_items_links.return_value = [] + + metadata_retriever = MetadataRetriever(metadata_config, "dummy_token") + + metadata = {} + runs = ["run1"] + + updated_metadata = metadata_retriever.fetch_elab_metadata(runs, metadata) + + assert updated_metadata["elabFTW"]["user"]["name"] == "Test User" + assert updated_metadata["elabFTW"]["user"]["email"] == "test@example.com" + assert updated_metadata["elabFTW"]["user"]["id"] == 1 + assert updated_metadata["elabFTW"]["user"]["orcid"] == "0000-0000-0000-0000" + assert updated_metadata["elabFTW"]["scan"]["title"] == "Test Experiment" + assert updated_metadata["elabFTW"]["scan"]["summary"] == "Test Body" + assert updated_metadata["elabFTW"]["scan"]["id"] == 1 + assert updated_metadata["elabFTW"]["scan"]["key"] == "value" diff --git a/tests/test_specsscan.py b/tests/test_specsscan.py index 3511b93..80edf62 100755 --- a/tests/test_specsscan.py +++ b/tests/test_specsscan.py @@ -68,7 +68,7 @@ def test_conversion_3d(): path=data_dir, iterations=np.s_[0:2], ) - np.testing.assert_allclose(res_xarray, res_xarray2) + np.testing.assert_allclose(res_xarray, res_xarray2, rtol=1e-6) with pytest.raises(IndexError): sps.load_scan( @@ -259,7 +259,7 @@ def test_fft_tool(): apply_fft_filter=False, ) - np.testing.assert_almost_equal(res_xarray.data.sum(), 62145561928.15108, decimal=3) + np.testing.assert_allclose(res_xarray.data.sum(), 62145556000.0) res_xarray = sps.load_scan( scan=3610, @@ -267,7 +267,7 @@ def test_fft_tool(): fft_filter_peaks=fft_filter_peaks, apply_fft_filter=True, ) - np.testing.assert_almost_equal(res_xarray.data.sum(), 62197237155.50347, decimal=3) + np.testing.assert_allclose(res_xarray.data.sum(), 62197240000.0) sps.fft_tool( amplitude=1, @@ -277,10 +277,9 @@ def test_fft_tool(): sigma_y=23, apply=True, ) - assert sps.config["spa_params"]["fft_filter_peaks"] == fft_filter_peaks assert sps.spa.config["fft_filter_peaks"] == fft_filter_peaks res_xarray = sps.load_scan(scan=3610, path=data_dir, apply_fft_filter=True) - np.testing.assert_almost_equal(res_xarray.data.sum(), 62197237155.50347, decimal=3) + np.testing.assert_allclose(res_xarray.data.sum(), 62197240000.0) def test_conversion_and_save_to_nexus(): @@ -319,15 +318,20 @@ def test_conversion_and_save_to_nexus(): metadata["instrument"] = {} # energy resolution metadata["instrument"]["energy_resolution"] = 150.0 - metadata["instrument"]["electronanalyser"] = {} - metadata["instrument"]["electronanalyser"]["energy_resolution"] = 120 - metadata["instrument"]["electronanalyser"]["angular_resolution"] = 0.2 - metadata["instrument"]["electronanalyser"]["spatial_resolution"] = 0.5 + metadata["instrument"]["electronanalyzer"] = {} + metadata["instrument"]["electronanalyzer"]["energy_resolution"] = 120.0 + metadata["instrument"]["electronanalyzer"]["angular_resolution"] = 0.2 + metadata["instrument"]["electronanalyzer"]["spatial_resolution"] = 0.5 # probe beam metadata["instrument"]["beam"] = {} metadata["instrument"]["beam"]["probe"] = {} metadata["instrument"]["beam"]["probe"]["incident_energy"] = 21.7 + metadata["instrument"]["beam"]["probe"]["frequency"] = 500.0 + metadata["instrument"]["beam"]["probe"]["incident_energy_spread"] = 0.11 + metadata["instrument"]["beam"]["probe"]["pulse_duration"] = 20.0 + metadata["instrument"]["beam"]["probe"]["incident_polarization"] = [1, 1, 0, 0] + metadata["instrument"]["beam"]["probe"]["extent"] = [80.0, 80.0] # sample metadata["sample"] = {} @@ -335,8 +339,8 @@ def test_conversion_and_save_to_nexus(): metadata["scan_info"] = {} metadata["scan_info"]["trARPES:XGS600:PressureAC:P_RD"] = 2.5e-11 - metadata["scan_info"]["trARPES:Carving:TEMP_RBV"] = 70 - metadata["scan_info"]["trARPES:Sample:Measure"] = 0 + metadata["scan_info"]["trARPES:Carving:TEMP_RBV"] = 70.0 + metadata["scan_info"]["trARPES:Sample:Measure"] = 0.0 res_xarray = sps.load_scan( scan=1496, path=data_dir, diff --git a/tutorial/3_specsscan_conversion_to_NeXus.ipynb b/tutorial/3_specsscan_conversion_to_NeXus.ipynb index 22d19fd..52507e4 100755 --- a/tutorial/3_specsscan_conversion_to_NeXus.ipynb +++ b/tutorial/3_specsscan_conversion_to_NeXus.ipynb @@ -66,11 +66,12 @@ "metadata['instrument'] = {}\n", "# energy resolution\n", "metadata['instrument']['energy_resolution'] = 150.\n", - "metadata['instrument']['electronanalyser'] = {}\n", - "metadata['instrument']['electronanalyser']['energy_resolution'] = 120\n", - "metadata['instrument']['electronanalyser']['angular_resolution'] = 0.2\n", - "metadata['instrument']['electronanalyser']['spatial_resolution'] = 0.5\n", - "\n", + "metadata['instrument']['electronanalyzer'] = {}\n", + "metadata['instrument']['electronanalyzer']['energy_resolution'] = 120.\n", + "metadata['instrument']['electronanalyzer']['angular_resolution'] = 0.2\n", + "metadata['instrument']['electronanalyzer']['spatial_resolution'] = 0.5\n", + "metadata['instrument']['electronanalyzer']['slit_shape'] = \"curved slit\"\n", + "metadata['instrument']['electronanalyzer']['slit_size'] = 1.0\n", "#probe beam\n", "metadata['instrument']['beam']={}\n", "metadata['instrument']['beam']['probe']={}\n", @@ -105,8 +106,8 @@ "\n", "metadata[\"scan_info\"] = {}\n", "metadata[\"scan_info\"][\"trARPES:XGS600:PressureAC:P_RD\"] = 2.5E-11\n", - "metadata[\"scan_info\"][\"trARPES:Carving:TEMP_RBV\"] = 70\n", - "metadata[\"scan_info\"][\"trARPES:Sample:Measure\"] = 0\n" + "metadata[\"scan_info\"][\"trARPES:Carving:TEMP_RBV\"] = 70.\n", + "metadata[\"scan_info\"][\"trARPES:Sample:Measure\"] = 0.\n" ] }, { @@ -125,7 +126,7 @@ "outputs": [], "source": [ "config = {\"nexus\":{\"definition\": \"NXmpes_arpes\"}, \"spa_params\":{\"crop\":True, \"ek_range_min\":0.07597844332538181, \"ek_range_max\":0.9117413199045858, \"ang_range_min\":0.16453159041394336, \"ang_range_max\":0.8840087145969499,}}\n", - "sps = SpecsScan(config=config, user_config=\"../src/specsscan/config/example_config_FHI.yaml\")\n", + "sps = SpecsScan(config=config, user_config=\"../src/specsscan/config/example_config_FHI.yaml\", system_config={})\n", "path = \"../tests/data/\" # Path to the test data set" ] }, @@ -234,7 +235,7 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3", + "display_name": ".venv", "language": "python", "name": "python3" }, @@ -248,12 +249,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.8.12" - }, - "vscode": { - "interpreter": { - "hash": "a164666994e9db75450cd7016dd7e51d42ea6e7c1e5e8017af1f8068ca906367" - } + "version": "3.10.16" } }, "nbformat": 4,