🐛 (manager) Make sure to emit Folders

Author: simonwoerpel

ingestors/manager.py

@@ -202,8 +202,9 @@ def ingest_entity(self, entity):
                 continue
             self.ingest(file_path, entity)
             return
-        # don't emit this entity if we didn't find a file to ingest
-        self.finalize(entity, emit=False)
+        # don't emit this entity if we didn't find a file to ingest and its not
+        # a folder
+        self.finalize(entity, emit=entity.schema.is_a("Folder"))
 
     def ingest(self, file_path, entity, **kwargs):
         """Main execution step of an ingestor."""

ingestors/tasks.py

@@ -1,5 +1,4 @@
 import gc
-import logging
 from pathlib import Path
 
 from anystore.logging import get_logger
@@ -21,8 +20,6 @@
 app = make_app(__loader__.name)
 sync_app = make_app(__loader__.name, sync=True)
 
-log = logging.getLogger(__name__)
-
 
 @task(app=app)
 def ingest(job: DatasetJob) -> None:
@@ -41,7 +38,7 @@ def ingest(job: DatasetJob) -> None:
 
     emitted = manager.get_emitted()
     if not len(emitted):
-        raise RuntimeError("No entities to be emitted!")
+        job.log.error("No entities to be emitted!")
 
     for entity in emitted:
         if entity.schema.is_a("Analyzable"):