Tried working with connectors for datasets but just modified download for files

Author: Vismayak

pyclowder/connectors.py

@@ -309,6 +309,8 @@ def _download_file_metadata(self, host, secret_key, fileid, filepath):
         (fd, md_file) = tempfile.mkstemp(suffix=md_name, dir=md_dir)
 
         with os.fdopen(fd, "wb") as tmp_file:
+            print("Writing metadata to %s" % md_file)
+            print(file_md)
             tmp_file.write(json.dumps(file_md))
 
         return (md_dir, md_file)
@@ -327,12 +329,21 @@ def _prepare_dataset(self, host, secret_key, resource):
         tmp_dirs_created.append(temp_link_dir)
 
         # Check if miniomounted path is set and if the file is in the minio mounted path
-        if self.minio_mounted_path:
-            for file in resource['files']:
-                file_path = self._check_for_local_file(file, file['id'])
-                if file_path:
-                    # print("Found file locally: %s" % file_path)
-                    located_files.append(file_path)
+        # if self.minio_mounted_path:
+        #     for file in resource['files']:
+        #         file_path = self._check_for_local_file(file, file['id'])
+        #         if not file_path:
+        #             missing_files.append(file)
+        #         else:
+        #             md_file_path = file['name'].split('.')[0] + "_metadata.json"
+        #             (file_md_dir, file_md_tmp) = self._download_file_metadata(host, secret_key, file['id'], md_file_path)
+        #             located_files.append(file_path)
+        #             located_files.append(file_md_tmp)
+        #             tmp_files_created.append(file_md_tmp)
+        #             tmp_dirs_created.append(file_md_dir)
+                
+        # else:
+        
         #check if any files in dataset accessible locally
         ds_file_list = pyclowder.datasets.get_file_list(self, host, secret_key, resource["id"])
         for ds_file in ds_file_list:
@@ -349,7 +360,7 @@ def _prepare_dataset(self, host, secret_key, resource):
 
                 # Also get file metadata in format expected by extrator
                 (file_md_dir, file_md_tmp) = self._download_file_metadata(host, secret_key, ds_file['id'],
-                                                                          ds_file['filepath'])
+                                                                        ds_file['filepath'])
                 located_files.append(file_path)
                 located_files.append(file_md_tmp)
                 tmp_files_created.append(file_md_tmp)
@@ -393,6 +404,7 @@ def _prepare_dataset(self, host, secret_key, resource):
             except Exception as e:
                 logger.exception("No files found and download failed")
 
+        print("File paths: %s" % file_paths)
         return (file_paths, tmp_files_created, tmp_dirs_created)
 
     # pylint: disable=too-many-branches,too-many-statements

pyclowder/files.py

@@ -51,6 +51,14 @@ def download(connector, host, key, fileid, intermediatefileid=None, ext="", trac
     tracking -- should the download action be tracked
     """
     client = ClowderClient(host=host, key=key)
+    # Check if minio mounted path is set
+    minio_mounted_path = os.getenv("MINIO_MOUNTED_PATH", "")
+    if minio_mounted_path:
+        # Check if the file is stored in Minio mount path
+        minio_file_path = minio_mounted_path + "/" + fileid
+        if os.path.isfile(minio_file_path):
+            return minio_file_path
+    # Else download the file from Clowder
     inputfilename = files.download(connector, client, fileid, intermediatefileid, ext)
     return inputfilename