Fix buildup of empty folders that eventually cause the worker to time out.

Author: Sean Sullivan

elastic_datashader/cache.py

@@ -2,6 +2,8 @@
 from collections import OrderedDict
 from datetime import datetime, timedelta, timezone
 from os import scandir
+import os
+from contextlib import suppress
 from pathlib import Path
 from shutil import rmtree
 from time import time
@@ -162,6 +164,15 @@ def age_off_cache(cache_path: Path, idx_name: str, max_age: timedelta) -> None:
             # set missing_ok=True in case another process deleted the same file
             file_path.unlink(missing_ok=True)
 
+    # clear all empty dirs and dirs that contain empty dirs to prevent build up of param hash directories
+    remove_empty_dirs(cache_path/idx_name)
+
+def remove_empty_dirs(path: Path):
+    for root,dirs,_ in os.walk(path, topdown=False):
+        for d in dirs:
+            with suppress(OSError):
+                os.rmdir(Path(root,d))
+
 def get_idx_names(cache_path: Path) -> Iterable[str]:
     for path in cache_path.glob("*"):
         if path.is_dir():

tests/test_cache.py

@@ -75,7 +75,7 @@ def test_clear_hash_cache(tmp_path):
 
 
 def test_age_off_cache(tmp_path):
-    xdir = tmp_path / "fooindex/somehash/3/1"
+    xdir = tmp_path / "fooindex/some_new_hash/3/1"
     xdir.mkdir(parents=True)
 
     yfile = xdir / "2.png"
@@ -90,6 +90,10 @@ def test_age_off_cache(tmp_path):
 
     assert not yfile.exists()
     assert yfile_after.exists()
+    sleep(2)
+    # clear again should remove all files and empty folders
+    cache.age_off_cache(tmp_path, "fooindex", timedelta(seconds=1))
+    assert not xdir.exists()
 
 
 def test_build_layer_info(tmp_path):