diff --git a/invokeai/backend/image_util/infill_methods/patchmatch.py b/invokeai/backend/image_util/infill_methods/patchmatch.py
index 7e9cdf8fa41..e9b090e9081 100644
--- a/invokeai/backend/image_util/infill_methods/patchmatch.py
+++ b/invokeai/backend/image_util/infill_methods/patchmatch.py
@@ -41,7 +41,8 @@ def _load_patch_match(cls):
 
     @classmethod
     def patchmatch_available(cls) -> bool:
-        cls._load_patch_match()
+        if not cls.tried_load:
+            cls._load_patch_match()
         if not cls.patch_match:
             return False
         return cls.patch_match.patchmatch_available