Update copy_from_local and copy_to_local for S3

Author: Thierry RAMORASOAVINA

khiops/core/internals/filesystems.py

@@ -23,6 +23,7 @@
 try:
     import boto3
     import boto3.session
+    from boto3.exceptions import S3UploadFailedError
     from botocore.exceptions import ClientError
 
     boto3_import_error = None
@@ -254,7 +255,7 @@ def copy_from_local(uri_or_path, local_path):
     Raises
     ------
     RuntimeError
-        If there was a problem when removing.
+        If there was a problem when copying.
     """
     create_resource(uri_or_path).copy_from_local(local_path)
 
@@ -272,7 +273,7 @@ def copy_to_local(uri_or_path, local_path):
     Raises
     ------
     RuntimeError
-        If there was a problem when removing.
+        If there was a problem when copying.
     """
     create_resource(uri_or_path).copy_to_local(local_path)
 
@@ -668,29 +669,22 @@ def remove(self):
             )
 
     def copy_from_local(self, local_path):
-        response = self.s3_client.Bucket(self.uri_info.netloc).upload_file(
-            local_path, self.uri_info.path[1:]
-        )
-        status_code = response["ResponseMetadata"]["HTTPStatusCode"]
-        copy_ok = 200 <= status_code <= 299
-        if not copy_ok:
-            raise RuntimeError(
-                f"S3 copy_from_local failed {self.uri} with code {status_code}: "
-                + json.dumps(response)
+        try:
+            self.s3_client.Bucket(self.uri_info.netloc).upload_file(
+                local_path, self.uri_info.path[1:]
             )
+        # normalize the raised exception
+        except S3UploadFailedError as exc:
+            raise RuntimeError(f"S3 copy_from_local failed {self.uri}") from exc
 
     def copy_to_local(self, local_path):
-        response = self.s3_client.Bucket(self.uri_info.netloc).download_file(
-            local_path, self.uri_info.path[1:]
-        )
-        status_code = response["ResponseMetadata"]["HTTPStatusCode"]
-        copy_ok = 200 <= status_code <= 299
-        if not copy_ok:
-            raise RuntimeError(
-                f"S3 download failed {self.uri} with code {status_code}: "
-                + json.dumps(response)
+        try:
+            self.s3_client.Bucket(self.uri_info.netloc).download_file(
+                self.uri_info.path[1:], local_path
             )
-        return copy_ok
+        # normalize the raised exception
+        except S3UploadFailedError as exc:
+            raise RuntimeError(f"S3 download failed {self.uri}") from exc
 
     def list_dir(self):
         # Add an extra slash to the path to treat it as a folder

tests/test_remote_access.py

@@ -84,6 +84,10 @@ def init_remote_bucket(cls, bucket_name=None, proto=None):
                     f"{proto}://{bucket_name}/khiops-cicd/samples/{file}",
                     os.path.join(kh.get_samples_dir(), file),
                 )
+                # symetric call to ensure the upload was OK
+                fs.copy_to_local(
+                    f"{proto}://{bucket_name}/khiops-cicd/samples/{file}", "/tmp/dummy"
+                )
 
         def results_dir_root(self):
             """To be overridden by descendants if needed