Feature/454 e2e process

src/hermes/commands/__init__.py

@@ -8,12 +8,12 @@
 # "unused import" errors.
 # flake8: noqa
 
-# from hermes.commands.base import HermesHelpCommand
-# from hermes.commands.base import HermesVersionCommand
-# from hermes.commands.clean.base import HermesCleanCommand
+from hermes.commands.base import HermesHelpCommand
+from hermes.commands.base import HermesVersionCommand
+from hermes.commands.clean.base import HermesCleanCommand
 # from hermes.commands.init.base import HermesInitCommand
 # from hermes.commands.curate.base import HermesCurateCommand
 from hermes.commands.harvest.base import HermesHarvestCommand
-# from hermes.commands.process.base import HermesProcessCommand
+from hermes.commands.process.base import HermesProcessCommand
 from hermes.commands.deposit.base import HermesDepositCommand
 # from hermes.commands.postprocess.base import HermesPostprocessCommand

src/hermes/commands/base.py

@@ -175,6 +175,7 @@ def __call__(self, command: HermesCommand) -> None:
 
 
 class HermesHelpSettings(BaseModel):
+    """Intentionally empty settings class for the help command."""
     pass
 
 
@@ -200,3 +201,23 @@ def __call__(self, args: argparse.Namespace) -> None:
             # Otherwise, simply show the general help and exit (cleanly).
             self.parser.print_help()
             self.parser.exit()
+
+
+class HermesVersionSettings(BaseModel):
+    """Intentionally empty settings class for the version command."""
+    pass
+
+
+class HermesVersionCommand(HermesCommand):
+    """Show HERMES version and exit."""
+
+    command_name = "version"
+    settings_class = HermesVersionSettings
+
+    def load_settings(self, args: argparse.Namespace):
+        """Pass loading settings as not necessary for this command."""
+        pass
+
+    def __call__(self, args: argparse.Namespace) -> None:
+        self.log.info(metadata.version("hermes"))
+        self.parser.exit()

src/hermes/commands/cli.py

@@ -16,7 +16,9 @@
 # from hermes.commands import (HermesHelpCommand, HermesVersionCommand, HermesCleanCommand,
 #                              HermesHarvestCommand, HermesProcessCommand, HermesCurateCommand,
 #                              HermesDepositCommand, HermesPostprocessCommand, HermesInitCommand)
-from hermes.commands import HermesDepositCommand, HermesHarvestCommand
+from hermes.commands import (
+    HermesDepositCommand, HermesHarvestCommand, HermesHelpCommand, HermesProcessCommand, HermesVersionCommand
+)
 from hermes.commands.base import HermesCommand
 
 
@@ -38,12 +40,12 @@ def main() -> None:
     setting_types = {}
 
     for command in (
-            # HermesHelpCommand(parser),
-            # HermesVersionCommand(parser),
+            HermesHelpCommand(parser),
+            HermesVersionCommand(parser),
             # HermesInitCommand(parser),
             # HermesCleanCommand(parser),
             HermesHarvestCommand(parser),
-            # HermesProcessCommand(parser),
+            HermesProcessCommand(parser),
             # HermesCurateCommand(parser),
             HermesDepositCommand(parser),
             # HermesPostprocessCommand(parser),

src/hermes/commands/deposit/invenio.py

@@ -513,7 +513,7 @@ def _codemeta_to_invenio_deposition(self) -> dict:
 
         creators = []
         for author in metadata.get("author", []):
-            if not "Person" in author.get("@type", []):
+            if "Person" not in author.get("@type", []):
                 continue
             creator = {}
             if len(
@@ -527,7 +527,7 @@ def _codemeta_to_invenio_deposition(self) -> dict:
                 raise HermesValidationError(f"Author has too many family names: {author}")
             if len(author.get("familyName", [])) == 1:
                 given_names_str = " ".join(author.get("givenName", []))
-                name = f"{author["familyName"][0]}, {given_names_str}"
+                name = f"{author['familyName'][0]}, {given_names_str}"
             elif len(author.get("name", [])) != 1:
                 raise HermesValidationError(f"Author has too many or no names: {author}")
             else:

src/hermes/commands/process/base.py

@@ -5,13 +5,14 @@
 # SPDX-FileContributor: Michael Meinel
 
 import argparse
-import json
-import sys
 
 from pydantic import BaseModel
 
 from hermes.commands.base import HermesCommand, HermesPlugin
-from hermes.model.context import HermesHarvestContext, CodeMetaContext
+from hermes.model.api import SoftwareMetadata
+from hermes.model.context_manager import HermesContext
+from hermes.model.error import HermesContextError
+from hermes.model.merge.container import ld_merge_dict
 
 
 class HermesProcessPlugin(HermesPlugin):
@@ -33,42 +34,27 @@ class HermesProcessCommand(HermesCommand):
 
     def __call__(self, args: argparse.Namespace) -> None:
         self.args = args
-        ctx = CodeMetaContext()
-
-        if not (ctx.hermes_dir / "harvest").exists():
-            self.log.error("You must run the harvest command before process")
-            sys.exit(1)
+        ctx = HermesContext()
+        merged_doc = ld_merge_dict([{}])
 
         # Get all harvesters
         harvester_names = self.root_settings.harvest.sources
-        harvester_names.reverse()   # Switch order for priority handling
 
+        ctx.prepare_step('harvest')
         for harvester in harvester_names:
             self.log.info("## Process data from %s", harvester)
-
-            harvest_context = HermesHarvestContext(ctx, harvester, {})
             try:
-                harvest_context.load_cache()
-            # when the harvest step ran, but there is no cache file, this is a serious flaw
-            except FileNotFoundError:
-                self.log.warning("No output data from harvester %s found, skipping", harvester)
+                metadata = SoftwareMetadata.load_from_cache(ctx, harvester)
+            except HermesContextError as e:
+                self.log.error("Error while trying to load data from harvest plugin '%s': %s", harvester, e)
+                self.errors.append(e)
                 continue
-
-            ctx.merge_from(harvest_context)
-            ctx.merge_contexts_from(harvest_context)
-
-        if ctx._errors:
-            self.log.error('Errors during merge')
-            self.errors.extend(ctx._errors)
-
-            for ep, error in ctx._errors:
-                self.log.info("    - %s: %s", ep.name, error)
-
-        tags_path = ctx.get_cache('process', 'tags', create=True)
-        with tags_path.open('w') as tags_file:
-            json.dump(ctx.tags, tags_file, indent=2)
-
-        ctx.prepare_codemeta()
-
-        with open(ctx.get_cache("process", ctx.hermes_name, create=True), 'w') as codemeta_file:
-            json.dump(ctx._data, codemeta_file, indent=2)
+            merged_doc.update(metadata)
+        ctx.finalize_step("harvest")
+
+        ctx.prepare_step("process")
+        with ctx["result"] as result_ctx:
+            result_ctx["codemeta"] = merged_doc.compact()
+            result_ctx["context"] = {"@context": merged_doc.full_context}
+            result_ctx["expanded"] = merged_doc.ld_value
+        ctx.finalize_step("process")

src/hermes/model/merge/__init__.py

@@ -0,0 +1,3 @@
+# SPDX-FileCopyrightText: 2022 German Aerospace Center (DLR)
+#
+# SPDX-License-Identifier: Apache-2.0

src/hermes/model/merge/action.py

@@ -0,0 +1,83 @@
+# SPDX-FileCopyrightText: 2025 German Aerospace Center (DLR)
+#
+# SPDX-License-Identifier: Apache-2.0
+
+# SPDX-FileContributor: Michael Meinel
+
+from hermes.model.types import ld_list
+
+
+class MergeError(ValueError):
+    pass
+
+
+class MergeAction:
+    def merge(self, target, key, value, update):
+        raise NotImplementedError()
+
+
+class Reject(MergeAction):
+    @classmethod
+    def merge(cls, target, key, value, update):
+        if value != update:
+            target.reject(key, update)
+        return value
+
+
+class Replace(MergeAction):
+    @classmethod
+    def merge(cls, target, key, value, update):
+        if value != update:
+            target.replace(key, value)
+        return update
+
+
+class Concat(MergeAction):
+    @classmethod
+    def merge(cls, target, key, value, update):
+        return cls.merge_to_list(value, update)
+
+    @classmethod
+    def merge_to_list(cls, head, tail):
+        if not isinstance(head, (list, ld_list)):
+            head = [head]
+        if not isinstance(tail, (list, ld_list)):
+            head.append(tail)
+        else:
+            head.extend(tail)
+        return head
+
+
+class Collect(MergeAction):
+    def __init__(self, match):
+        self.match = match
+
+    def merge(self, target, key, value, update):
+        if not isinstance(value, list):
+            value = [value]
+        if not isinstance(update, list):
+            update = [update]
+
+        for update_item in update:
+            if not any(self.match(item, update_item) for item in value):
+                value.append(update_item)
+
+        if len(value) == 1:
+            return value[0]
+        else:
+            return value
+
+
+class MergeSet(MergeAction):
+    def __init__(self, match, merge_items=True):
+        self.match = match
+        self.merge_items = merge_items
+
+    def merge(self, target, key, value, update):
+        for item in update:
+            target_item = target.match(key[-1], item, self.match)
+            if target_item and self.merge_items:
+                target_item.update(item)
+            else:
+                value.append(item)
+        return value