From f23d10565a2b3973064f3ea26f082777d5f5ab2c Mon Sep 17 00:00:00 2001 From: "IIPL\\14261" Date: Wed, 4 Sep 2024 21:33:05 +0530 Subject: [PATCH 1/5] Add functionality to upload files to S3 with optional public access Implement file download functionality from S3 as a stream --- pom.xml | 5 ++ src/main/java/com/app/config/AwsS3Config.java | 45 ++++++++++ .../java/com/app/controller/S3Controller.java | 41 ++++++++++ src/main/java/com/app/service/S3Service.java | 12 +++ .../java/com/app/service/S3ServiceImpl.java | 82 +++++++++++++++++++ src/main/resources/application-local.yml | 7 ++ 6 files changed, 192 insertions(+) create mode 100644 src/main/java/com/app/config/AwsS3Config.java create mode 100644 src/main/java/com/app/controller/S3Controller.java create mode 100644 src/main/java/com/app/service/S3Service.java create mode 100644 src/main/java/com/app/service/S3ServiceImpl.java diff --git a/pom.xml b/pom.xml index 2d8e109..0766dd1 100644 --- a/pom.xml +++ b/pom.xml @@ -31,6 +31,11 @@ org.springframework.boot spring-boot-starter-log4j2 + + software.amazon.awssdk + s3 + 2.25.70 + org.springframework.boot spring-boot-starter-test diff --git a/src/main/java/com/app/config/AwsS3Config.java b/src/main/java/com/app/config/AwsS3Config.java new file mode 100644 index 0000000..952544c --- /dev/null +++ b/src/main/java/com/app/config/AwsS3Config.java @@ -0,0 +1,45 @@ +package com.app.config; + + +import lombok.extern.log4j.Log4j2; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; +import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; +import software.amazon.awssdk.core.client.config.ClientOverrideConfiguration; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.s3.S3Client; + +@Configuration +@ConditionalOnProperty(name = "aws.s3.enabled", havingValue = "true") +@Log4j2 +public class AwsS3Config { + + @Value("${aws.s3.region:us-east-1}") + private String awsRegion; + @Value("${aws.s3.accessKeyId:us-east-1}") + private String accessKeyId; + @Value("${aws.s3.secretAccessKey}") + private String secretAccessKey; + + @Bean + public S3Client s3Client() { + try { + log.info("Trying to S3Client create."); + return S3Client.builder() + .region(Region.of(awsRegion)) + .credentialsProvider(StaticCredentialsProvider.create(AwsBasicCredentials.create(accessKeyId, secretAccessKey))) + .overrideConfiguration(ClientOverrideConfiguration.builder() + .retryPolicy(r -> r.numRetries(3)) + .build()) + .build(); + } catch (Exception e) { + log.error("Failed to create S3Client.", e); + throw e; + } finally { + log.info("S3Client created successfully."); + } + } +} diff --git a/src/main/java/com/app/controller/S3Controller.java b/src/main/java/com/app/controller/S3Controller.java new file mode 100644 index 0000000..f6c5ad6 --- /dev/null +++ b/src/main/java/com/app/controller/S3Controller.java @@ -0,0 +1,41 @@ +package com.app.controller; + +import com.app.service.S3Service; +import lombok.RequiredArgsConstructor; +import org.springframework.core.io.InputStreamResource; +import org.springframework.http.HttpHeaders; +import org.springframework.http.MediaType; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.*; +import org.springframework.web.multipart.MultipartFile; + +import java.io.InputStream; + +@RestController +@RequiredArgsConstructor +@RequestMapping("/api/files") +public class S3Controller { + + private final S3Service s3Service; + + @PostMapping("/upload") + public ResponseEntity uploadFile(@RequestPart("file") MultipartFile file, + @RequestParam(value = "isReadPublicly", defaultValue = "false") boolean isReadPublicly) { + boolean isUploaded = s3Service.uploadFile(file, isReadPublicly); + if (isUploaded) { + return ResponseEntity.ok("File uploaded successfully: " + file.getOriginalFilename()); + } else { + return ResponseEntity.status(500).body("Failed to upload file: " + file.getOriginalFilename()); + } + } + + @GetMapping("/download/{key}") + public ResponseEntity downloadFile(@PathVariable String key) { + InputStream fileStream = s3Service.downloadFileAsStream(key); + InputStreamResource resource = new InputStreamResource(fileStream); + return ResponseEntity.ok() + .header(HttpHeaders.CONTENT_DISPOSITION, "attachment; filename=" + key) + .contentType(MediaType.APPLICATION_OCTET_STREAM) + .body(resource); + } +} diff --git a/src/main/java/com/app/service/S3Service.java b/src/main/java/com/app/service/S3Service.java new file mode 100644 index 0000000..5f9d2f2 --- /dev/null +++ b/src/main/java/com/app/service/S3Service.java @@ -0,0 +1,12 @@ +package com.app.service; + +import org.springframework.web.multipart.MultipartFile; + +import java.io.InputStream; + +public interface S3Service { + + boolean uploadFile(MultipartFile file, boolean isReadPublicly); + + InputStream downloadFileAsStream(String key); +} diff --git a/src/main/java/com/app/service/S3ServiceImpl.java b/src/main/java/com/app/service/S3ServiceImpl.java new file mode 100644 index 0000000..7651f3a --- /dev/null +++ b/src/main/java/com/app/service/S3ServiceImpl.java @@ -0,0 +1,82 @@ +package com.app.service; + +import lombok.RequiredArgsConstructor; +import lombok.extern.log4j.Log4j2; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Service; +import org.springframework.web.multipart.MultipartFile; +import software.amazon.awssdk.core.ResponseBytes; +import software.amazon.awssdk.core.exception.SdkException; +import software.amazon.awssdk.core.sync.RequestBody; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.model.GetObjectRequest; +import software.amazon.awssdk.services.s3.model.GetObjectResponse; +import software.amazon.awssdk.services.s3.model.PutObjectRequest; +import software.amazon.awssdk.services.s3.model.S3Exception; + +import java.io.ByteArrayInputStream; +import java.io.InputStream; + +@Service +@Log4j2 +@RequiredArgsConstructor +public class S3ServiceImpl implements S3Service { + + private final S3Client s3Client; + + @Value("${aws.s3.bucket-name}") + private String bucketName; + + + @Override + public boolean uploadFile(MultipartFile file, boolean isReadPublicly) { + log.info("Started uploading file '{}' to S3 Bucket '{}'", file.getOriginalFilename(), bucketName); + PutObjectRequest putObjectRequest; + if (isReadPublicly) { + putObjectRequest = PutObjectRequest.builder() + .bucket(bucketName) + .key(file.getOriginalFilename()).acl("public-read") + .build(); + } else { + putObjectRequest = PutObjectRequest.builder() + .bucket(bucketName) + .key(file.getOriginalFilename()) + .build(); + } + try { + s3Client.putObject(putObjectRequest, RequestBody.fromBytes(file.getBytes())); + log.info("Successfully uploaded file to S3. Bucket: {}, Key: {}", bucketName, file.getOriginalFilename()); + return true; + } catch (Exception e) { + log.error("Failed to upload file to S3. Bucket: {}, Key: {}", bucketName, file.getOriginalFilename(), e); + return false; + } + } + + @Override + public InputStream downloadFileAsStream(String key) { + try { + GetObjectRequest getObjectRequest = GetObjectRequest.builder() + .bucket(bucketName) + .key(key) + .build(); + + ResponseBytes getObjectResponse = s3Client.getObjectAsBytes(getObjectRequest); + if (getObjectResponse == null) { + log.warn("Failed to get file from S3 bucket: Response is null"); + return new ByteArrayInputStream(new byte[0]); + } + + log.info("Successfully getting file in bytes from S3 bucket."); + byte[] fileBytes = getObjectResponse.asByteArray(); + return new ByteArrayInputStream(fileBytes); + + } catch (S3Exception e) { + log.error("Failed to fetch object from S3 Bucket: {}, Key: {}", bucketName, key, e); + throw e; + } catch (SdkException e) { + log.error("Error while downloading file from S3 Bucket: {}, Key: {}", bucketName, key, e); + throw e; + } + } +} diff --git a/src/main/resources/application-local.yml b/src/main/resources/application-local.yml index d2f8b2d..11b1e06 100644 --- a/src/main/resources/application-local.yml +++ b/src/main/resources/application-local.yml @@ -2,3 +2,10 @@ app: logs: path: C:/${spring.application.name}/logs +aws: + s3: + enabled: true + region: us-east-1 + accessKeyId: <> + secretAccessKey: <> + bucket-name: <> \ No newline at end of file From f37f953317f1bd06cd45e3cc6bed33adbd41e76b Mon Sep 17 00:00:00 2001 From: "IIPL\\14261" Date: Sat, 14 Sep 2024 23:49:45 +0530 Subject: [PATCH 2/5] Add S3 bucket creation service - Added AWS SDK dependency for S3. - Configured S3Client bean for AWS S3 interaction. - Implemented S3Service for creating S3 buckets. - Created S3Controller to handle bucket creation requests via REST API. --- .../java/com/app/controller/S3Controller.java | 5 +++++ src/main/java/com/app/service/S3Service.java | 2 ++ .../java/com/app/service/S3ServiceImpl.java | 17 +++++++++++++---- 3 files changed, 20 insertions(+), 4 deletions(-) diff --git a/src/main/java/com/app/controller/S3Controller.java b/src/main/java/com/app/controller/S3Controller.java index f6c5ad6..9561fe7 100644 --- a/src/main/java/com/app/controller/S3Controller.java +++ b/src/main/java/com/app/controller/S3Controller.java @@ -38,4 +38,9 @@ public ResponseEntity downloadFile(@PathVariable String key .contentType(MediaType.APPLICATION_OCTET_STREAM) .body(resource); } + + @GetMapping("/create-bucket") + public ResponseEntity createBucket(@RequestParam String bucketName) { + return ResponseEntity.ok(s3Service.createBucket(bucketName)); + } } diff --git a/src/main/java/com/app/service/S3Service.java b/src/main/java/com/app/service/S3Service.java index 5f9d2f2..ffd3790 100644 --- a/src/main/java/com/app/service/S3Service.java +++ b/src/main/java/com/app/service/S3Service.java @@ -9,4 +9,6 @@ public interface S3Service { boolean uploadFile(MultipartFile file, boolean isReadPublicly); InputStream downloadFileAsStream(String key); + + String createBucket(String bucketName); } diff --git a/src/main/java/com/app/service/S3ServiceImpl.java b/src/main/java/com/app/service/S3ServiceImpl.java index 7651f3a..c706dab 100644 --- a/src/main/java/com/app/service/S3ServiceImpl.java +++ b/src/main/java/com/app/service/S3ServiceImpl.java @@ -9,10 +9,7 @@ import software.amazon.awssdk.core.exception.SdkException; import software.amazon.awssdk.core.sync.RequestBody; import software.amazon.awssdk.services.s3.S3Client; -import software.amazon.awssdk.services.s3.model.GetObjectRequest; -import software.amazon.awssdk.services.s3.model.GetObjectResponse; -import software.amazon.awssdk.services.s3.model.PutObjectRequest; -import software.amazon.awssdk.services.s3.model.S3Exception; +import software.amazon.awssdk.services.s3.model.*; import java.io.ByteArrayInputStream; import java.io.InputStream; @@ -79,4 +76,16 @@ public InputStream downloadFileAsStream(String key) { throw e; } } + + public String createBucket(String bucketName) { + try { + CreateBucketRequest createBucketRequest = CreateBucketRequest.builder() + .bucket(bucketName) + .build(); + CreateBucketResponse createBucketResponse = s3Client.createBucket(createBucketRequest); + return "Bucket created successfully: " + createBucketResponse.location(); + } catch (S3Exception e) { + throw new RuntimeException("Failed to create bucket: " + e.awsErrorDetails().errorMessage(), e); + } + } } From 62d0f77f87d14f75dd654df2bdaae3b3b74c602f Mon Sep 17 00:00:00 2001 From: "IIPL\\14261" Date: Sun, 15 Sep 2024 22:44:28 +0530 Subject: [PATCH 3/5] Add functionality to list S3 buckets with regions - Implemented a new service method to fetch and display S3 bucket names with their respective regions. - Added a REST endpoint to return the list of buckets along with their regions. - Utilized AWS SDK to retrieve bucket location constraints and map them to AWS regions. - Included error handling for scenarios where the bucket region is not accessible or available. --- .../java/com/app/controller/S3Controller.java | 12 ++++ src/main/java/com/app/service/S3Service.java | 6 ++ .../java/com/app/service/S3ServiceImpl.java | 57 +++++++++++++++++++ 3 files changed, 75 insertions(+) diff --git a/src/main/java/com/app/controller/S3Controller.java b/src/main/java/com/app/controller/S3Controller.java index 9561fe7..501111b 100644 --- a/src/main/java/com/app/controller/S3Controller.java +++ b/src/main/java/com/app/controller/S3Controller.java @@ -10,6 +10,8 @@ import org.springframework.web.multipart.MultipartFile; import java.io.InputStream; +import java.util.List; +import java.util.Map; @RestController @RequiredArgsConstructor @@ -43,4 +45,14 @@ public ResponseEntity downloadFile(@PathVariable String key public ResponseEntity createBucket(@RequestParam String bucketName) { return ResponseEntity.ok(s3Service.createBucket(bucketName)); } + + @GetMapping("/bucket-list") + public ResponseEntity> getBucketList() { + return ResponseEntity.ok(s3Service.getBucketList()); + } + + @GetMapping("/list-buckets-with-regions") + public Map listBucketsWithRegions() { + return s3Service.listBucketsWithRegions(); + } } diff --git a/src/main/java/com/app/service/S3Service.java b/src/main/java/com/app/service/S3Service.java index ffd3790..ac42504 100644 --- a/src/main/java/com/app/service/S3Service.java +++ b/src/main/java/com/app/service/S3Service.java @@ -3,6 +3,8 @@ import org.springframework.web.multipart.MultipartFile; import java.io.InputStream; +import java.util.List; +import java.util.Map; public interface S3Service { @@ -11,4 +13,8 @@ public interface S3Service { InputStream downloadFileAsStream(String key); String createBucket(String bucketName); + + List getBucketList(); + + Map listBucketsWithRegions(); } diff --git a/src/main/java/com/app/service/S3ServiceImpl.java b/src/main/java/com/app/service/S3ServiceImpl.java index c706dab..addbdd8 100644 --- a/src/main/java/com/app/service/S3ServiceImpl.java +++ b/src/main/java/com/app/service/S3ServiceImpl.java @@ -8,11 +8,16 @@ import software.amazon.awssdk.core.ResponseBytes; import software.amazon.awssdk.core.exception.SdkException; import software.amazon.awssdk.core.sync.RequestBody; +import software.amazon.awssdk.regions.Region; import software.amazon.awssdk.services.s3.S3Client; import software.amazon.awssdk.services.s3.model.*; import java.io.ByteArrayInputStream; import java.io.InputStream; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; @Service @Log4j2 @@ -88,4 +93,56 @@ public String createBucket(String bucketName) { throw new RuntimeException("Failed to create bucket: " + e.awsErrorDetails().errorMessage(), e); } } + + @Override + public List getBucketList() { + try { + ListBucketsResponse listBucketsResponse = s3Client.listBuckets(); + return listBucketsResponse.buckets().stream() + .map(Bucket::name) + .toList(); + } catch (S3Exception e) { + throw new RuntimeException("Failed to list buckets: " + e.awsErrorDetails().errorMessage(), e); + } + } + + @Override + public Map listBucketsWithRegions() { + try { + ListBucketsResponse listBucketsResponse = s3Client.listBuckets(); + + // Create a map to store bucket names with their respective regions + Map bucketRegions = new HashMap<>(); + + for (var bucket : listBucketsResponse.buckets()) { + String bucketName = bucket.name(); + String bucketRegion = getBucketRegion(bucketName); + bucketRegions.put(bucketName, bucketRegion); + } + + return bucketRegions; + + } catch (S3Exception e) { + throw new RuntimeException("Failed to list buckets: " + e.awsErrorDetails().errorMessage(), e); + } + } + + private String getBucketRegion(String bucketName) { + try { + GetBucketLocationRequest locationRequest = GetBucketLocationRequest.builder() + .bucket(bucketName) + .build(); + + GetBucketLocationResponse locationResponse = s3Client.getBucketLocation(locationRequest); + + // Translate the bucket location constraint to a region name + Region region = locationResponse.locationConstraintAsString() == null ? Region.US_EAST_1 : + Region.of(locationResponse.locationConstraintAsString()); + + return region.id(); + } catch (S3Exception e) { + return "Unknown"; // Handle the case where the region is not accessible or available + } + } + } From 9bd1cffc5d5eeb1226c3585cb451fec49335537c Mon Sep 17 00:00:00 2001 From: "IIPL\\14261" Date: Tue, 17 Sep 2024 11:33:34 +0530 Subject: [PATCH 4/5] added --- src/main/java/com/app/service/S3Service.java | 2 +- src/main/java/com/app/service/S3ServiceImpl.java | 7 +++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/src/main/java/com/app/service/S3Service.java b/src/main/java/com/app/service/S3Service.java index ac42504..ea3c71f 100644 --- a/src/main/java/com/app/service/S3Service.java +++ b/src/main/java/com/app/service/S3Service.java @@ -14,7 +14,7 @@ public interface S3Service { String createBucket(String bucketName); - List getBucketList(); + List getBucketList() throws RuntimeException; Map listBucketsWithRegions(); } diff --git a/src/main/java/com/app/service/S3ServiceImpl.java b/src/main/java/com/app/service/S3ServiceImpl.java index addbdd8..b07a6c7 100644 --- a/src/main/java/com/app/service/S3ServiceImpl.java +++ b/src/main/java/com/app/service/S3ServiceImpl.java @@ -95,7 +95,7 @@ public String createBucket(String bucketName) { } @Override - public List getBucketList() { + public List getBucketList() throws RuntimeException { try { ListBucketsResponse listBucketsResponse = s3Client.listBuckets(); return listBucketsResponse.buckets().stream() @@ -115,9 +115,8 @@ public Map listBucketsWithRegions() { Map bucketRegions = new HashMap<>(); for (var bucket : listBucketsResponse.buckets()) { - String bucketName = bucket.name(); - String bucketRegion = getBucketRegion(bucketName); - bucketRegions.put(bucketName, bucketRegion); + String bucketRegion = getBucketRegion(bucket.name()); + bucketRegions.put(bucket.name(), bucketRegion); } return bucketRegions; From 52db244074537ea9d8052565ce426d4caa70b618 Mon Sep 17 00:00:00 2001 From: "IIPL\\14261" Date: Tue, 17 Sep 2024 16:02:30 +0530 Subject: [PATCH 5/5] provides S3 file management capabilities, including: - Downloading a single file from an S3 bucket. - Downloading all files from a specific bucket as a ZIP archive. - Moving files from one S3 bucket to another. --- .../java/com/app/controller/S3Controller.java | 30 +++++ src/main/java/com/app/service/S3Service.java | 5 + .../java/com/app/service/S3ServiceImpl.java | 109 +++++++++++++++++- 3 files changed, 143 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/app/controller/S3Controller.java b/src/main/java/com/app/controller/S3Controller.java index 501111b..f58a4b4 100644 --- a/src/main/java/com/app/controller/S3Controller.java +++ b/src/main/java/com/app/controller/S3Controller.java @@ -4,14 +4,18 @@ import lombok.RequiredArgsConstructor; import org.springframework.core.io.InputStreamResource; import org.springframework.http.HttpHeaders; +import org.springframework.http.HttpStatus; import org.springframework.http.MediaType; import org.springframework.http.ResponseEntity; import org.springframework.web.bind.annotation.*; import org.springframework.web.multipart.MultipartFile; +import org.springframework.web.servlet.mvc.method.annotation.StreamingResponseBody; +import java.io.IOException; import java.io.InputStream; import java.util.List; import java.util.Map; +import java.util.zip.ZipOutputStream; @RestController @RequiredArgsConstructor @@ -55,4 +59,30 @@ public ResponseEntity> getBucketList() { public Map listBucketsWithRegions() { return s3Service.listBucketsWithRegions(); } + + @GetMapping("/download-all-files-zip") + public ResponseEntity downloadAllFilesAsZip(@RequestParam String bucketName) { + + // Streaming response to handle large files efficiently + StreamingResponseBody responseBody = outputStream -> { + try (ZipOutputStream zos = new ZipOutputStream(outputStream)) { + s3Service.streamAllFilesAsZip(bucketName, zos); + } catch (IOException e) { + throw new RuntimeException("Error while streaming files to output stream", e); + } + }; + + // Set headers for ZIP file download + HttpHeaders headers = new HttpHeaders(); + headers.add("Content-Disposition", "attachment; filename=all-files.zip"); + headers.add("Content-Type", "application/zip"); + + return new ResponseEntity<>(responseBody, headers, HttpStatus.OK); + } + + @GetMapping("/move-files") + public String moveFiles(@RequestParam String sourceBucketName, @RequestParam String destinationBucketName) { + s3Service.moveFiles(sourceBucketName, destinationBucketName); + return "Files are being moved from " + sourceBucketName + " to " + destinationBucketName; + } } diff --git a/src/main/java/com/app/service/S3Service.java b/src/main/java/com/app/service/S3Service.java index ea3c71f..2a90f3b 100644 --- a/src/main/java/com/app/service/S3Service.java +++ b/src/main/java/com/app/service/S3Service.java @@ -5,6 +5,7 @@ import java.io.InputStream; import java.util.List; import java.util.Map; +import java.util.zip.ZipOutputStream; public interface S3Service { @@ -17,4 +18,8 @@ public interface S3Service { List getBucketList() throws RuntimeException; Map listBucketsWithRegions(); + + void streamAllFilesAsZip(String bucketName, ZipOutputStream zos); + + void moveFiles(String sourceBucketName, String destinationBucketName); } diff --git a/src/main/java/com/app/service/S3ServiceImpl.java b/src/main/java/com/app/service/S3ServiceImpl.java index b07a6c7..609d37c 100644 --- a/src/main/java/com/app/service/S3ServiceImpl.java +++ b/src/main/java/com/app/service/S3ServiceImpl.java @@ -6,6 +6,7 @@ import org.springframework.stereotype.Service; import org.springframework.web.multipart.MultipartFile; import software.amazon.awssdk.core.ResponseBytes; +import software.amazon.awssdk.core.ResponseInputStream; import software.amazon.awssdk.core.exception.SdkException; import software.amazon.awssdk.core.sync.RequestBody; import software.amazon.awssdk.regions.Region; @@ -13,11 +14,16 @@ import software.amazon.awssdk.services.s3.model.*; import java.io.ByteArrayInputStream; +import java.io.IOException; import java.io.InputStream; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.stream.Collectors; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.zip.ZipEntry; +import java.util.zip.ZipOutputStream; @Service @Log4j2 @@ -25,6 +31,8 @@ public class S3ServiceImpl implements S3Service { private final S3Client s3Client; + private static final int THREAD_POOL_SIZE = 10; // Number of threads for concurrent execution + @Value("${aws.s3.bucket-name}") private String bucketName; @@ -144,4 +152,103 @@ private String getBucketRegion(String bucketName) { } } + @Override + public void streamAllFilesAsZip(String bucketName, ZipOutputStream zos) { + ListObjectsV2Request listObjectsRequest = ListObjectsV2Request.builder() + .bucket(bucketName) + .build(); + + ListObjectsV2Response listObjectsResponse; + do { + listObjectsResponse = s3Client.listObjectsV2(listObjectsRequest); + List objects = listObjectsResponse.contents(); + + for (S3Object object : objects) { + addFileToZipStream(bucketName, object.key(), zos); + } + + } while (listObjectsResponse.isTruncated()); + } + + private void addFileToZipStream(String bucketName, String keyName, ZipOutputStream zos) { + GetObjectRequest getObjectRequest = GetObjectRequest.builder() + .bucket(bucketName) + .key(keyName) + .build(); + + try (ResponseInputStream s3ObjectStream = s3Client.getObject(getObjectRequest)) { + zos.putNextEntry(new ZipEntry(keyName)); + + byte[] buffer = new byte[1024]; + int length; + while ((length = s3ObjectStream.read(buffer)) > 0) { + zos.write(buffer, 0, length); + } + + zos.closeEntry(); + } catch (IOException | S3Exception e) { + throw new RuntimeException("Failed to add file to ZIP: " + keyName, e); + } + } + + @Override + public void moveFiles(String sourceBucketName, String destinationBucketName) { + ExecutorService executorService = Executors.newFixedThreadPool(THREAD_POOL_SIZE); + ListObjectsV2Request listObjectsRequest = ListObjectsV2Request.builder() + .bucket(sourceBucketName) + .build(); + + try { + ListObjectsV2Response listObjectsResponse; + do { + listObjectsResponse = s3Client.listObjectsV2(listObjectsRequest); + List objects = listObjectsResponse.contents(); + + for (S3Object object : objects) { + String keyName = object.key(); + // Submit the copy and delete tasks to be executed concurrently + executorService.submit(() -> copyAndDeleteObject(sourceBucketName, destinationBucketName, keyName)); + } + + } while (listObjectsResponse.isTruncated()); + + } catch (S3Exception e) { + log.error("Failed to list objects from bucket: {} - {}", sourceBucketName, e.getMessage()); + } finally { + executorService.shutdown(); + try { + if (!executorService.awaitTermination(60, TimeUnit.SECONDS)) { + executorService.shutdownNow(); + } + } catch (InterruptedException e) { + executorService.shutdownNow(); + Thread.currentThread().interrupt(); + } + } + } + + private void copyAndDeleteObject(String sourceBucketName, String destinationBucketName, String keyName) { + try { + // Copy file to the destination bucket + CopyObjectRequest copyRequest = CopyObjectRequest.builder() + .sourceBucket(sourceBucketName) + .sourceKey(keyName) + .destinationBucket(destinationBucketName) + .destinationKey(keyName) + .build(); + s3Client.copyObject(copyRequest); + log.info("Copied file: {} from {} to {}", keyName, sourceBucketName, destinationBucketName); + + // Delete file from the source bucket + DeleteObjectRequest deleteRequest = DeleteObjectRequest.builder() + .bucket(sourceBucketName) + .key(keyName) + .build(); + s3Client.deleteObject(deleteRequest); + log.info("Deleted file: {} from {}", keyName, sourceBucketName); + + } catch (S3Exception e) { + log.error("Error while moving file: {} - {}", keyName, e.getMessage()); + } + } }