diff --git a/.vale/styles/config/vocabularies/Utopia/accept.txt b/.vale/styles/config/vocabularies/Utopia/accept.txt index bd945e6b..84f36aba 100644 --- a/.vale/styles/config/vocabularies/Utopia/accept.txt +++ b/.vale/styles/config/vocabularies/Utopia/accept.txt @@ -89,6 +89,7 @@ JSON JWS JWT(?:s)? KaiOS +keepalive keypair(?:s)? libcurl Linode diff --git a/packages/storage/README.md b/packages/storage/README.md index cc749fbc..43fecb07 100644 --- a/packages/storage/README.md +++ b/packages/storage/README.md @@ -26,18 +26,19 @@ Devices are immutable value objects: construct one with its configuration and us require_once '../vendor/autoload.php'; +use Utopia\Psr7\Stream; use Utopia\Storage\Device\Local; $device = new Local('/path/to/storage'); -// Upload a file -$device->uploadData(file_get_contents('/local/path/to/file.png'), 'destination/path/file.png', 'image/png'); +// Upload a file — contents move as PSR-7 streams, so memory stays bounded +$device->upload(Stream::fromResource(fopen('/local/path/to/file.png', 'rb')), 'destination/path/file.png', 'image/png'); // Check if file exists $exists = $device->exists('destination/path/file.png'); -// Read file contents -$contents = $device->read('destination/path/file.png'); +// Read file contents as a stream +$contents = (string) $device->read('destination/path/file.png'); // Delete a file $device->delete('destination/path/file.png'); @@ -69,10 +70,13 @@ $device = new S3( 'YOUR_SECRET_KEY', 'YOUR_BUCKET_NAME.s3.us-east-1.amazonaws.com', // Host 'us-east-1', // Region - Acl::Private // Access control (default: private) + Acl::Private, // Access control (default: private) + bucket: 'YOUR_BUCKET_NAME', // Optional: enables server-side copy ); ``` +When the bucket name is known, a same-device `copy()` — and therefore `move()` — runs entirely server side (`CopyObject`, or `UploadPartCopy` above 5 GB), so no bytes move through PHP. The provider-specific adapters below pass their bucket automatically. Without it, `copy()` falls back to a streamed download and re-upload. + The provider-specific adapters below build the host for you from a bucket and region. Every S3-family adapter also accepts optional named constructor arguments: ```php @@ -185,8 +189,13 @@ $device = new Wasabi( All storage adapters provide a consistent API for working with files: ```php -// Upload file contents -$device->uploadData(file_get_contents('/path/to/local/file.jpg'), 'remote/path/file.jpg', 'image/jpeg'); +use Utopia\Psr7\Stream; + +// Upload a file from disk without loading it into memory +$device->upload(Stream::fromResource(fopen('/path/to/local/file.jpg', 'rb')), 'remote/path/file.jpg', 'image/jpeg'); + +// Contents already in memory wrap into a stream for free +$device->upload(new Stream($data), 'remote/path/file.jpg', 'image/jpeg'); // Check if file exists $exists = $device->exists('remote/path/file.jpg'); @@ -200,20 +209,20 @@ $mime = $device->getFileMimeType('remote/path/file.jpg'); // Get file MD5 hash $hash = $device->getFileHash('remote/path/file.jpg'); -// Read file contents -$contents = $device->read('remote/path/file.jpg'); +// Read file contents as a PSR-7 stream; casting to string buffers it +$stream = $device->read('remote/path/file.jpg'); // Read partial file contents -$chunk = $device->read('remote/path/file.jpg', 0, 1024); // Read first 1KB +$chunk = (string) $device->read('remote/path/file.jpg', 0, 1024); // Read first 1KB -// Multipart/chunked uploads +// Multipart/chunked uploads — part 1 of 3 $metadata = []; -$device->uploadData($firstChunk, 'remote/video.mp4', 'video/mp4', 1, 3, $metadata); // Part 1 of 3 +$device->upload(new Stream($firstChunk), 'remote/video.mp4', 'video/mp4', 1, 3, $metadata); // Resumable uploads: prepare, upload chunks in any order, finalize -$device->prepareUpload('remote/video.mp4', 'video/mp4', 3, $metadata); -$device->uploadChunk($secondChunk, 'remote/video.mp4', 2, 3, $metadata); -$device->finalizeUpload('remote/video.mp4', 3, $metadata); +$device->prepare('remote/video.mp4', 'video/mp4', 3, $metadata); +$device->upload(new Stream($secondChunk), 'remote/video.mp4', 'video/mp4', 2, 3, $metadata); +$device->finalize('remote/video.mp4', 3, $metadata); // List files under a prefix, one page at a time $list = $device->listFiles('remote/directory', 100); @@ -230,16 +239,20 @@ $device->delete('remote/path/file.jpg'); // Delete directory $device->deletePath('remote/directory'); -// Transfer files between storage devices -$sourceDevice->transfer('source/path.jpg', 'target/path.jpg', $targetDevice); +// Copy a file, on the same device or onto another one +$device->copy('source/path.jpg', 'target/path.jpg'); +$sourceDevice->copy('source/path.jpg', 'target/path.jpg', $targetDevice); -// Transfer with a custom chunk size (default: 20 MB) -$sourceDevice->transfer('source/path.jpg', 'target/path.jpg', $targetDevice, 10000000); +// Copy with a custom chunk size (default: 20 MB) +$sourceDevice->copy('source/path.jpg', 'target/path.jpg', $targetDevice, 10000000); + +// Move is copy plus delete +$device->move('source/path.jpg', 'target/path.jpg'); ``` ## Custom HTTP client -The S3-family adapters send requests through any [PSR-18](https://www.php-fig.org/psr/psr-18/) client. By default they use [utopia-php/client](https://github.com/utopia-php/client) with the cURL adapter, no request timeout, and the `Retry` decorator configured with `S3\RetryStrategy` — it retries transient S3 rate-limiting errors (`SlowDown`, `ServiceUnavailable`, `Throttling`, `RequestThrottled`, and plain 429/503 responses) three times with a 500 ms delay. +The S3-family adapters send requests through any [PSR-18](https://www.php-fig.org/psr/psr-18/) client. By default they use [utopia-php/client](https://github.com/utopia-php/client) with the cURL adapter — no overall request timeout, a stall watchdog that aborts once no bytes move for 60 seconds, TCP keepalive — and the `Retry` decorator configured with `S3\RetryStrategy`. It retries transient S3 rate-limiting errors (`SlowDown`, `ServiceUnavailable`, `Throttling`, `RequestThrottled`, and plain 429/503 responses) three times with exponential backoff and full jitter from a 0.5 second base delay. Inject your own client to change the transport or the retry policy — for example the Swoole coroutine adapter with more aggressive retries: @@ -300,6 +313,15 @@ use Utopia\Storage\Device\Telemetry; $device = new Telemetry($telemetryAdapter, new Local('/path/to/storage')); ``` +## Upgrading from 3.x + +Version 4.0 makes streaming the only I/O path: file contents move as PSR-7 streams end to end, so memory stays bounded regardless of file size. + +- `read()` returns a `Psr\Http\Message\StreamInterface` instead of a string. Cast with `(string)` or call `getContents()` where you want the bytes in memory — the cost is visible at the call site. +- `write()` and the upload methods take a `StreamInterface` instead of a string. Wrap a string with `new Utopia\Psr7\Stream($data)`; wrap an open file handle with `Utopia\Psr7\Stream::fromResource($handle)`. S3 requires seekable streams: the payload is hashed for signing, rewound and sent — the same approach the AWS SDK takes. +- `uploadData()` and `uploadChunk()` merged into one `upload($data, $path, $contentType, $chunk, $chunks, $metadata)` — a whole file is the single-chunk default. `prepareUpload()`/`finalizeUpload()` are now `prepare()`/`finalize()`. +- `transfer()` is replaced by `copy($source, $target, $to, $chunkSize)` where `$to` defaults to the same device, and `move()` now works across every adapter as copy plus delete (`Local` still renames in place). The `TRANSFER_CHUNK_SIZE` constant is now `COPY_CHUNK_SIZE`. + ## Upgrading from 2.x Version 3.0 makes every device immutable and safe to share across coroutines, and removes all global state: diff --git a/packages/storage/src/Storage/Device.php b/packages/storage/src/Storage/Device.php index a130259a..ac75a07e 100644 --- a/packages/storage/src/Storage/Device.php +++ b/packages/storage/src/Storage/Device.php @@ -4,6 +4,7 @@ namespace Utopia\Storage; +use Psr\Http\Message\StreamInterface; use Utopia\Storage\Exception\StorageException; use Utopia\Storage\Exception\UploadException; @@ -13,9 +14,9 @@ abstract class Device { /** - * Default max chunk size while transferring file from one device to another + * Default max chunk size while copying a file from one device to another */ - public const TRANSFER_CHUNK_SIZE = 20000000; // 20 MB + public const COPY_CHUNK_SIZE = 20000000; // 20 MB /** * Get Type. @@ -39,7 +40,7 @@ abstract public function getRoot(): string; abstract public function getPath(string $filename): string; /** - * Prepare Upload. + * Prepare. * * Initialize adapter-specific upload state without transferring a chunk body. * @@ -47,22 +48,22 @@ abstract public function getPath(string $filename): string; * * @throws StorageException */ - abstract public function prepareUpload(string $path, string $contentType, int $chunks = 1, array &$metadata = []): void; + abstract public function prepare(string $path, string $contentType, int $chunks = 1, array &$metadata = []): void; /** * Upload Chunk. * - * Upload exactly one chunk of file contents without finalizing the full upload. + * Store exactly one chunk of file contents without finalizing the full upload. * Returns the number of chunks received so far. * * @param UploadMetadata $metadata * * @throws StorageException */ - abstract public function uploadChunk(string $data, string $path, int $chunk = 1, int $chunks = 1, array &$metadata = []): int; + abstract protected function uploadChunk(StreamInterface $data, string $path, int $chunk, int $chunks, array &$metadata): int; /** - * Finalize Upload. + * Finalize. * * Complete a prepared upload once all chunks are known to be present. * @@ -70,24 +71,26 @@ abstract public function uploadChunk(string $data, string $path, int $chunk = 1, * * @throws StorageException */ - abstract public function finalizeUpload(string $path, int $chunks = 1, array &$metadata = []): bool; + abstract public function finalize(string $path, int $chunks = 1, array &$metadata = []): bool; /** - * Upload Data. + * Upload. * - * Upload file contents to desired destination in the selected disk. + * Upload one chunk of file contents to the desired destination, preparing the + * upload on first contact and finalizing it once every chunk has arrived. + * A whole file is the default single-chunk case. * Returns the number of chunks received so far. * * @param UploadMetadata $metadata * * @throws StorageException */ - public function uploadData(string $data, string $path, string $contentType, int $chunk = 1, int $chunks = 1, array &$metadata = []): int + public function upload(StreamInterface $data, string $path, string $contentType, int $chunk = 1, int $chunks = 1, array &$metadata = []): int { - $this->prepareUpload($path, $contentType, $chunks, $metadata); + $this->prepare($path, $contentType, $chunks, $metadata); $chunksReceived = $this->uploadChunk($data, $path, $chunk, $chunks, $metadata); - if ($chunks > 1 && $chunks === $chunksReceived && ! $this->finalizeUpload($path, $chunks, $metadata)) { + if ($chunks > 1 && $chunks === $chunksReceived && ! $this->finalize($path, $chunks, $metadata)) { throw new UploadException('Failed to finalize upload ' . $path); } @@ -100,24 +103,68 @@ public function uploadData(string $data, string $path, string $contentType, int abstract public function abort(string $path, string $uploadId = ''): bool; /** - * Read file by given path. + * Read file or part of file by given path, offset and length. + * + * The returned stream is positioned at the start of the requested window. * * @param int<0, max> $offset * @param int<0, max>|null $length */ - abstract public function read(string $path, int $offset = 0, ?int $length = null): string; + abstract public function read(string $path, int $offset = 0, ?int $length = null): StreamInterface; /** - * Transfer - * Transfer a file from current device to destination device. + * Write file by given path. * + * The stream is consumed in full: seekable streams are rewound and sent + * from the beginning on every adapter. */ - abstract public function transfer(string $path, string $destination, Device $device, int $chunkSize = self::TRANSFER_CHUNK_SIZE): bool; + abstract public function write(string $path, StreamInterface $data, string $contentType): bool; /** - * Write file by given path. + * Copy a file to another path, on this device or onto another one. + * + * Large files are piped chunk by chunk, so memory stays bounded regardless + * of file size. A started multipart upload is aborted on failure. + * + * @throws StorageException */ - abstract public function write(string $path, string $data, string $contentType): bool; + public function copy(string $source, string $target, ?Device $to = null, int $chunkSize = self::COPY_CHUNK_SIZE): bool + { + if ($chunkSize <= 0) { + throw new \InvalidArgumentException('Chunk size must be greater than zero'); + } + + $to ??= $this; + $size = $this->getFileSize($source); + $contentType = $this->getFileMimeType($source); + + if ($size <= $chunkSize) { + return $to->write($target, $this->read($source), $contentType); + } + + $totalChunks = (int) ceil($size / $chunkSize); + $metadata = ['content_type' => $contentType]; + try { + for ($counter = 0; $counter < $totalChunks; ++$counter) { + $data = $this->read($source, $counter * $chunkSize, $chunkSize); + $to->upload($data, $target, $contentType, $counter + 1, $totalChunks, $metadata); + } + } catch (\Throwable $e) { + // Best effort, and only once a multipart upload was actually started — + // its unclaimed parts are billed until aborted. Aborting without one + // could delete a pre-existing destination the copy never touched. + $uploadId = $metadata['uploadId'] ?? null; + if (\is_string($uploadId) && $uploadId !== '') { + try { + $to->abort($target, $uploadId); + } catch (\Throwable) { + } + } + throw $e; + } + + return true; + } /** * Move file from given source to given path, return true on success and false on failure. @@ -128,7 +175,7 @@ public function move(string $source, string $target): bool return false; } - if ($this->transfer($source, $target, $this)) { + if ($this->copy($source, $target)) { return $this->delete($source); } diff --git a/packages/storage/src/Storage/Device/AWS.php b/packages/storage/src/Storage/Device/AWS.php index 5bd82acb..786a7007 100644 --- a/packages/storage/src/Storage/Device/AWS.php +++ b/packages/storage/src/Storage/Device/AWS.php @@ -5,6 +5,7 @@ namespace Utopia\Storage\Device; use Psr\Http\Client\ClientInterface; +use Utopia\Psr18\StreamingClientInterface; use Utopia\Storage\Acl; use Utopia\Storage\DeviceType; @@ -76,13 +77,13 @@ public function __construct( string $bucket, string $region = self::US_EAST_1, Acl $acl = Acl::Private, - ?ClientInterface $client = null, + (ClientInterface&StreamingClientInterface)|null $client = null, ) { $host = match ($region) { self::CN_NORTH_1, self::CN_NORTH_4, self::CN_NORTHWEST_1 => $bucket . '.s3.' . $region . '.amazonaws.cn', default => $bucket . '.s3.' . $region . '.amazonaws.com' }; - parent::__construct($root, $accessKey, $secretKey, $host, $region, $acl, $client); + parent::__construct($root, $accessKey, $secretKey, $host, $region, $acl, $client, $bucket); } #[\Override] @@ -90,5 +91,4 @@ public function getType(): DeviceType { return DeviceType::AwsS3; } - } diff --git a/packages/storage/src/Storage/Device/Backblaze.php b/packages/storage/src/Storage/Device/Backblaze.php index 49f1372f..117bfc32 100644 --- a/packages/storage/src/Storage/Device/Backblaze.php +++ b/packages/storage/src/Storage/Device/Backblaze.php @@ -5,6 +5,7 @@ namespace Utopia\Storage\Device; use Psr\Http\Client\ClientInterface; +use Utopia\Psr18\StreamingClientInterface; use Utopia\Storage\Acl; use Utopia\Storage\DeviceType; @@ -37,10 +38,10 @@ public function __construct( string $bucket, string $region = self::US_WEST_004, Acl $acl = Acl::Private, - ?ClientInterface $client = null, + (ClientInterface&StreamingClientInterface)|null $client = null, ) { $host = $bucket . '.' . 's3' . '.' . $region . '.backblazeb2.com'; - parent::__construct($root, $accessKey, $secretKey, $host, $region, $acl, $client); + parent::__construct($root, $accessKey, $secretKey, $host, $region, $acl, $client, $bucket); } #[\Override] diff --git a/packages/storage/src/Storage/Device/DOSpaces.php b/packages/storage/src/Storage/Device/DOSpaces.php index a6aeea01..62fb2b94 100644 --- a/packages/storage/src/Storage/Device/DOSpaces.php +++ b/packages/storage/src/Storage/Device/DOSpaces.php @@ -5,6 +5,7 @@ namespace Utopia\Storage\Device; use Psr\Http\Client\ClientInterface; +use Utopia\Psr18\StreamingClientInterface; use Utopia\Storage\Acl; use Utopia\Storage\DeviceType; @@ -36,10 +37,10 @@ public function __construct( string $bucket, string $region = self::NYC3, Acl $acl = Acl::Private, - ?ClientInterface $client = null, + (ClientInterface&StreamingClientInterface)|null $client = null, ) { $host = $bucket . '.' . $region . '.digitaloceanspaces.com'; - parent::__construct($root, $accessKey, $secretKey, $host, $region, $acl, $client); + parent::__construct($root, $accessKey, $secretKey, $host, $region, $acl, $client, $bucket); } #[\Override] diff --git a/packages/storage/src/Storage/Device/Linode.php b/packages/storage/src/Storage/Device/Linode.php index 74519be0..57e5ada5 100644 --- a/packages/storage/src/Storage/Device/Linode.php +++ b/packages/storage/src/Storage/Device/Linode.php @@ -5,6 +5,7 @@ namespace Utopia\Storage\Device; use Psr\Http\Client\ClientInterface; +use Utopia\Psr18\StreamingClientInterface; use Utopia\Storage\Acl; use Utopia\Storage\DeviceType; @@ -32,10 +33,10 @@ public function __construct( string $bucket, string $region = self::EU_CENTRAL_1, Acl $acl = Acl::Private, - ?ClientInterface $client = null, + (ClientInterface&StreamingClientInterface)|null $client = null, ) { $host = $bucket . '.' . $region . '.' . 'linodeobjects.com'; - parent::__construct($root, $accessKey, $secretKey, $host, $region, $acl, $client); + parent::__construct($root, $accessKey, $secretKey, $host, $region, $acl, $client, $bucket); } #[\Override] diff --git a/packages/storage/src/Storage/Device/Local.php b/packages/storage/src/Storage/Device/Local.php index 9fff25d6..c97b9134 100644 --- a/packages/storage/src/Storage/Device/Local.php +++ b/packages/storage/src/Storage/Device/Local.php @@ -4,6 +4,8 @@ namespace Utopia\Storage\Device; +use Psr\Http\Message\StreamInterface; +use Utopia\Psr7\Stream; use Utopia\Storage\Device; use Utopia\Storage\DeviceType; use Utopia\Storage\Exception\NotFoundException; @@ -19,6 +21,8 @@ */ class Local extends Device { + private const int PIPE_CHUNK_SIZE = 524288; // 512 KB + /** * Local constructor. */ @@ -42,23 +46,24 @@ public function getPath(string $filename): string /** * @param UploadMetadata $metadata */ - public function prepareUpload(string $path, string $contentType, int $chunks = 1, array &$metadata = []): void + public function prepare(string $path, string $contentType, int $chunks = 1, array &$metadata = []): void { $this->createDirectory(\dirname($path)); $metadata['parts'] ??= []; $metadata['chunks'] ??= 0; } - public function uploadChunk(string $data, string $path, int $chunk = 1, int $chunks = 1, array &$metadata = []): int + /** + * @param UploadMetadata $metadata + */ + protected function uploadChunk(StreamInterface $data, string $path, int $chunk, int $chunks, array &$metadata): int { $this->createDirectory(\dirname($path)); $metadata['parts'] ??= []; $metadata['chunks'] ??= 0; if ($chunks === 1) { - if (file_put_contents($path, $data) === false) { - throw new StorageException('Can\'t write file ' . $path); - } + $this->writeFile($path, $data); $metadata['parts'][$chunk] = true; $metadata['chunks'] = 1; @@ -72,8 +77,8 @@ public function uploadChunk(string $data, string $path, int $chunk = 1, int $chu $chunkFilePath = $tmp . DIRECTORY_SEPARATOR . pathinfo($path, PATHINFO_FILENAME) . '.part.' . $chunk; // skip writing chunk if the chunk was re-uploaded - if (!file_exists($chunkFilePath) && file_put_contents($chunkFilePath, $data) === false) { - throw new StorageException('Failed to write chunk ' . $chunk); + if (! file_exists($chunkFilePath)) { + $this->writeFile($chunkFilePath, $data); } $chunksReceived = $this->countChunks($tmp, $path); @@ -83,7 +88,7 @@ public function uploadChunk(string $data, string $path, int $chunk = 1, int $chu return $chunksReceived; } - public function finalizeUpload(string $path, int $chunks = 1, array &$metadata = []): bool + public function finalize(string $path, int $chunks = 1, array &$metadata = []): bool { if (file_exists($path)) { return true; @@ -182,52 +187,6 @@ private function joinChunks(string $path, int $chunks): void } } - /** - * Transfer - */ - public function transfer(string $path, string $destination, Device $device, int $chunkSize = self::TRANSFER_CHUNK_SIZE): bool - { - if ($chunkSize <= 0) { - throw new \InvalidArgumentException('Chunk size must be greater than zero'); - } - - if (! $this->exists($path)) { - throw new NotFoundException('File not found'); - } - $size = $this->getFileSize($path); - $contentType = $this->getFileMimeType($path); - - if ($size <= $chunkSize) { - $source = $this->read($path); - - return $device->write($destination, $source, $contentType); - } - - $totalChunks = (int) ceil($size / $chunkSize); - $metadata = ['content_type' => $contentType]; - try { - for ($counter = 0; $counter < $totalChunks; ++$counter) { - $start = $counter * $chunkSize; - $data = $this->read($path, $start, $chunkSize); - $device->uploadData($data, $destination, $contentType, $counter + 1, $totalChunks, $metadata); - } - } catch (\Throwable $e) { - // Best effort, and only once a multipart upload was actually started — - // its unclaimed parts are billed until aborted. Aborting without one - // could delete a pre-existing destination the transfer never touched. - $uploadId = $metadata['uploadId'] ?? null; - if (\is_string($uploadId) && $uploadId !== '') { - try { - $device->abort($destination, $uploadId); - } catch (\Throwable) { - } - } - throw $e; - } - - return true; - } - /** * Abort Chunked Upload */ @@ -252,42 +211,98 @@ public function abort(string $path, string $uploadId = ''): bool } /** - * Read file by given path. + * Read file or part of file by given path, offset and length. * + * A full read returns a stream over the file itself; a bounded window is + * copied into a temporary stream so consumers can read to its end. * * @throws StorageException */ - public function read(string $path, int $offset = 0, ?int $length = null): string + public function read(string $path, int $offset = 0, ?int $length = null): StreamInterface { if (! $this->exists($path)) { throw new NotFoundException('File not found'); } - $contents = file_get_contents($path, use_include_path: false, offset: $offset, length: $length); - if ($contents === false) { + $handle = fopen($path, 'rb'); + if ($handle === false) { + throw new StorageException('Failed to read file ' . $path); + } + + if ($offset > 0 && fseek($handle, $offset) !== 0) { + fclose($handle); + throw new StorageException('Failed to seek file ' . $path); + } + + if ($length === null) { + return Stream::fromResource($handle); + } + + $window = fopen('php://temp', 'r+b'); + if ($window === false || stream_copy_to_stream($handle, $window, $length) === false) { + fclose($handle); throw new StorageException('Failed to read file ' . $path); } + fclose($handle); + rewind($window); - return $contents; + return Stream::fromResource($window); } /** * Write file by given path. */ - public function write(string $path, string $data, string $contentType = ''): bool + public function write(string $path, StreamInterface $data, string $contentType = ''): bool { // Checks if directory path to file exists - if (!file_exists(\dirname($path)) && ! @mkdir(\dirname($path), 0755, true)) { + if (! file_exists(\dirname($path)) && ! @mkdir(\dirname($path), 0755, true)) { throw new StorageException('Can\'t create directory ' . \dirname($path)); } - return (bool) file_put_contents($path, $data); + $this->writeFile($path, $data); + + return true; } /** - * Move file from given source to given path, Return true on success and false on failure. + * Pipe a stream into a file, chunk by chunk. * - * @see http://php.net/manual/en/function.filesize.php + * @throws StorageException + */ + private function writeFile(string $path, StreamInterface $data): void + { + if ($data->isSeekable()) { + $data->rewind(); + } + + $handle = fopen($path, 'wb'); + if ($handle === false) { + throw new StorageException('Can\'t write file ' . $path); + } + + try { + while (! $data->eof()) { + $chunk = $data->read(self::PIPE_CHUNK_SIZE); + $written = 0; + $length = \strlen($chunk); + if ($length === 0) { + break; + } + while ($written < $length) { + $bytes = fwrite($handle, substr($chunk, $written)); + if ($bytes === false || $bytes === 0) { + throw new StorageException('Can\'t write file ' . $path); + } + $written += $bytes; + } + } + } finally { + fclose($handle); + } + } + + /** + * Move file from given source to given path, Return true on success and false on failure. * * @throws StorageException */ @@ -299,16 +314,15 @@ public function move(string $source, string $target): bool } // Checks if directory path to file exists - if (!file_exists(\dirname($target)) && ! @mkdir(\dirname($target), 0755, true)) { + if (! file_exists(\dirname($target)) && ! @mkdir(\dirname($target), 0755, true)) { throw new StorageException('Can\'t create directory ' . \dirname($target)); } + return rename($source, $target); } /** * Delete file in given path, Return true on success and false on failure. - * - * @see http://php.net/manual/en/function.filesize.php */ public function delete(string $path, bool $recursive = false): bool { diff --git a/packages/storage/src/Storage/Device/S3.php b/packages/storage/src/Storage/Device/S3.php index 562a77ee..d172bee0 100644 --- a/packages/storage/src/Storage/Device/S3.php +++ b/packages/storage/src/Storage/Device/S3.php @@ -6,9 +6,11 @@ use Psr\Http\Client\ClientExceptionInterface; use Psr\Http\Client\ClientInterface; +use Psr\Http\Message\StreamInterface; use Utopia\Client\Adapter\Curl\Client as CurlAdapter; use Utopia\Client as HttpClient; use Utopia\Client\Decorator\Retry; +use Utopia\Psr18\StreamingClientInterface; use Utopia\Psr7\Method; use Utopia\Psr7\Request; use Utopia\Psr7\Stream; @@ -33,16 +35,23 @@ class S3 extends Device { protected const MAX_PAGE_SIZE = 1000; + private const int ERROR_BODY_BUFFER_SIZE = 16384; + + private const int PIPE_CHUNK_SIZE = 524288; // 512 KB + + private const int MAX_COPY_OBJECT_SIZE = 5368709120; // 5 GB — the CopyObject limit; larger objects copy server side part by part + private readonly string $fqdn; private readonly string $host; - private readonly ClientInterface $client; + private readonly ClientInterface&StreamingClientInterface $client; /** * S3 Constructor * - * @param ClientInterface|null $client PSR-18 client used for every request; defaults to `utopia-php/client` with the cURL adapter — no overall deadline (large transfers may take arbitrarily long), a stall watchdog that aborts once no bytes move for 60s, TCP keepalive, and transient-error retries via `S3\RetryStrategy` + * @param (ClientInterface&StreamingClientInterface)|null $client PSR-18 client used for every request; defaults to `utopia-php/client` with the cURL adapter — no overall deadline (large transfers may take arbitrarily long), a stall watchdog that aborts once no bytes move for 60s, TCP keepalive, and transient-error retries via `S3\RetryStrategy` + * @param string|null $bucket Bucket name, required by the `x-amz-copy-source` header. When set, same-device `copy()` (and therefore `move()`) runs server side without moving bytes through PHP; when null it falls back to a streamed download and re-upload. */ public function __construct( protected readonly string $root, @@ -52,7 +61,8 @@ public function __construct( string $host, protected readonly string $region, protected readonly Acl $acl = Acl::Private, - ?ClientInterface $client = null, + (ClientInterface&StreamingClientInterface)|null $client = null, + protected readonly ?string $bucket = null, ) { if (str_starts_with($host, 'http://') || str_starts_with($host, 'https://')) { $this->fqdn = $host; @@ -91,20 +101,20 @@ public function getPath(string $filename): string /** * @param UploadMetadata $metadata */ - public function prepareUpload(string $path, string $contentType, int $chunks = 1, array &$metadata = []): void + public function prepare(string $path, string $contentType, int $chunks = 1, array &$metadata = []): void { $metadata['parts'] ??= []; $metadata['chunks'] ??= 0; $metadata['content_type'] ??= $contentType; - if ($chunks === 1 || isset($metadata['uploadId']) && !\in_array($metadata['uploadId'], ['', '0', [], 0], true)) { + if ($chunks === 1 || isset($metadata['uploadId']) && ! \in_array($metadata['uploadId'], ['', '0', [], 0], true)) { return; } $metadata['uploadId'] = $this->createMultipartUpload($path, $contentType); } - public function finalizeUpload(string $path, int $chunks = 1, array &$metadata = []): bool + public function finalize(string $path, int $chunks = 1, array &$metadata = []): bool { if ($this->exists($path)) { return true; @@ -133,7 +143,7 @@ public function finalizeUpload(string $path, int $chunks = 1, array &$metadata = /** * @param UploadMetadata $metadata */ - public function uploadChunk(string $data, string $path, int $chunk = 1, int $chunks = 1, array &$metadata = []): int + protected function uploadChunk(StreamInterface $data, string $path, int $chunk, int $chunks, array &$metadata): int { $contentType = $metadata['content_type'] ?? ''; @@ -162,50 +172,6 @@ public function uploadChunk(string $data, string $path, int $chunk = 1, int $chu return $metadata['chunks']; } - /** - * Transfer - */ - public function transfer(string $path, string $destination, Device $device, int $chunkSize = self::TRANSFER_CHUNK_SIZE): bool - { - if ($chunkSize <= 0) { - throw new \InvalidArgumentException('Chunk size must be greater than zero'); - } - - $response = $this->getInfo($path); - $size = (int) ($response['content-length'] ?? 0); - $contentType = $response['content-type'] ?? ''; - - if ($size <= $chunkSize) { - $source = $this->read($path); - - return $device->write($destination, $source, $contentType); - } - - $totalChunks = (int) ceil($size / $chunkSize); - $metadata = ['content_type' => $contentType]; - try { - for ($counter = 0; $counter < $totalChunks; ++$counter) { - $start = $counter * $chunkSize; - $data = $this->read($path, $start, $chunkSize); - $device->uploadData($data, $destination, $contentType, $counter + 1, $totalChunks, $metadata); - } - } catch (\Throwable $e) { - // Best effort, and only once a multipart upload was actually started — - // its unclaimed parts are billed until aborted. Aborting without one - // could delete a pre-existing destination the transfer never touched. - $uploadId = $metadata['uploadId'] ?? null; - if (\is_string($uploadId) && $uploadId !== '') { - try { - $device->abort($destination, $uploadId); - } catch (\Throwable) { - } - } - throw $e; - } - - return true; - } - /** * Start Multipart Upload * @@ -241,7 +207,7 @@ protected function createMultipartUpload(string $path, string $contentType): str * * @throws StorageException */ - protected function uploadPart(string $data, string $path, string $contentType, int $chunk, string $uploadId): string + protected function uploadPart(StreamInterface $data, string $path, string $contentType, int $chunk, string $uploadId): string { $uri = $path !== '' ? '/' . str_replace(['%2F', '%3F'], ['/', '?'], rawurlencode($path)) : '/'; @@ -310,25 +276,46 @@ public function abort(string $path, string $uploadId = ''): bool /** * Read file or part of file by given path, offset and length. * + * The body streams into a temporary stream as it arrives, so memory stays + * bounded regardless of object size. * * @throws StorageException */ - public function read(string $path, int $offset = 0, ?int $length = null): string + public function read(string $path, int $offset = 0, ?int $length = null): StreamInterface { $uri = ($path !== '') ? '/' . str_replace('%2F', '/', rawurlencode($path)) : '/'; + if ($length === 0) { + return new Stream(''); + } + $headers = []; if ($length !== null) { $end = $offset + $length - 1; $headers['range'] = "bytes=$offset-$end"; + } elseif ($offset > 0) { + $headers['range'] = "bytes=$offset-"; } - $response = $this->call(Method::GET, $uri, headers: $headers, decode: false); - if (! \is_string($response->body)) { - throw new RemoteException('Unexpected S3 read response'); + $handle = fopen('php://temp', 'r+b'); + if ($handle === false) { + throw new StorageException('Failed to allocate read buffer'); } - return $response->body; + $this->call(Method::GET, $uri, headers: $headers, decode: false, sink: static function (string $chunk) use ($handle): void { + $written = 0; + $total = \strlen($chunk); + while ($written < $total) { + $bytes = fwrite($handle, substr($chunk, $written)); + if ($bytes === false || $bytes === 0) { + throw new StorageException('Failed to buffer S3 read response'); + } + $written += $bytes; + } + }); + rewind($handle); + + return Stream::fromResource($handle); } /** @@ -337,7 +324,7 @@ public function read(string $path, int $offset = 0, ?int $length = null): string * * @throws StorageException */ - public function write(string $path, string $data, string $contentType = ''): bool + public function write(string $path, StreamInterface $data, string $contentType = ''): bool { $uri = $path !== '' ? '/' . str_replace(['%2F', '%3F'], ['/', '?'], rawurlencode($path)) : '/'; @@ -383,7 +370,6 @@ protected function listObjects(string $prefix = '', int $maxKeys = self::MAX_PAG $uri = '/'; $prefix = ltrim($prefix, '/'); /** S3 specific requirement that prefix should never contain a leading slash */ - $parameters = [ 'list-type' => 2, 'prefix' => $prefix, @@ -403,6 +389,89 @@ protected function listObjects(string $prefix = '', int $maxKeys = self::MAX_PAG return $response->body; } + /** + * Copy a file to another path, on this device or onto another one. + * + * A same-device copy with a known bucket runs entirely server side — + * `CopyObject` up to 5 GB, `UploadPartCopy` beyond — so no bytes move + * through PHP. Everything else falls back to the streamed base copy. + * + * @throws StorageException + */ + #[\Override] + public function copy(string $source, string $target, ?Device $to = null, int $chunkSize = self::COPY_CHUNK_SIZE): bool + { + if ((!$to instanceof \Utopia\Storage\Device || $to === $this) && $this->bucket !== null) { + return $this->copyObject($source, $target); + } + + return parent::copy($source, $target, $to, $chunkSize); + } + + /** + * Copy an object server side within this bucket. + * + * @throws StorageException + */ + private function copyObject(string $source, string $target): bool + { + $info = $this->getInfo($source); + $size = (int) ($info['content-length'] ?? 0); + + $copySource = '/' . $this->bucket . '/' . ltrim(str_replace('%2F', '/', rawurlencode($source)), '/'); + $uri = $target !== '' ? '/' . str_replace(['%2F', '%3F'], ['/', '?'], rawurlencode($target)) : '/'; + + if ($size <= self::MAX_COPY_OBJECT_SIZE) { + $response = $this->call(Method::PUT, $uri, amzHeaders: [ + 'x-amz-copy-source' => $copySource, + 'x-amz-metadata-directive' => 'COPY', + 'x-amz-acl' => $this->acl->value, + ]); + + // S3 reports CopyObject failures in a 200 body, so success is the ETag. + if (! \is_array($response->body) || ! isset($response->body['ETag'])) { + throw new RemoteException('Unexpected S3 copy response'); + } + + return true; + } + + $uploadId = $this->createMultipartUpload($target, $info['content-type'] ?? ''); + try { + $parts = []; + $totalParts = (int) ceil($size / self::MAX_COPY_OBJECT_SIZE); + for ($part = 1; $part <= $totalParts; ++$part) { + $start = ($part - 1) * self::MAX_COPY_OBJECT_SIZE; + $end = min($size, $start + self::MAX_COPY_OBJECT_SIZE) - 1; + $response = $this->call( + Method::PUT, + $uri, + parameters: ['partNumber' => $part, 'uploadId' => $uploadId], + amzHeaders: [ + 'x-amz-copy-source' => $copySource, + 'x-amz-copy-source-range' => "bytes=$start-$end", + ], + ); + + $etag = \is_array($response->body) ? ($response->body['ETag'] ?? null) : null; + if (! \is_string($etag)) { + throw new RemoteException('Missing ETag in S3 response'); + } + $parts[$part] = $etag; + } + $this->completeMultipartUpload($target, $uploadId, $parts); + } catch (\Throwable $e) { + // Best effort — unclaimed multipart parts are billed until aborted. + try { + $this->abort($target, $uploadId); + } catch (\Throwable) { + } + throw $e; + } + + return true; + } + /** * Delete files in given path, path must be a directory. Return true on success and false on failure. * @@ -639,28 +708,42 @@ private function getSignatureV4(string $method, string $uri, array $parameters, * Headers are built per call — the instance holds no request state, so a * single device is safe to share across concurrent coroutines. * + * When a $sink is given, the response body is delivered to it chunk by + * chunk instead of being buffered; only its head is kept so an error + * response can still be parsed. + * * @param non-empty-string $method * @param array $parameters * @param array $headers * @param array $amzHeaders + * @param (callable(string): void)|null $sink * * @throws StorageException */ - protected function call(string $method, string $uri, string $data = '', array $parameters = [], array $headers = [], array $amzHeaders = [], bool $decode = true): S3\Response + protected function call(string $method, string $uri, StreamInterface|string $data = '', array $parameters = [], array $headers = [], array $amzHeaders = [], bool $decode = true, ?callable $sink = null): S3\Response { $uri = $this->getAbsolutePath($uri); $url = $this->fqdn . $uri . '?' . http_build_query($parameters, '', '&', PHP_QUERY_RFC3986); + if ($data instanceof StreamInterface) { + [$md5, $sha256] = $this->hashBody($data); + $body = $data; + } else { + $md5 = base64_encode(md5($data, true)); + $sha256 = hash('sha256', $data); + $body = new Stream($data); + } + $headers = array_filter($headers, fn(string $value): bool => $value !== ''); $headers['host'] = $this->host; $headers['date'] = gmdate('D, d M Y H:i:s T'); - $headers['content-md5'] = base64_encode(md5($data, true)); + $headers['content-md5'] = $md5; $amzHeaders = array_filter($amzHeaders, fn(string $value): bool => $value !== ''); $amzHeaders['x-amz-date'] = gmdate('Ymd\THis\Z'); - $amzHeaders['x-amz-content-sha256'] = hash('sha256', $data); + $amzHeaders['x-amz-content-sha256'] = $sha256; - $request = new Request($method, Uri::parse($url), new Stream($data)); + $request = new Request($method, Uri::parse($url), $body); foreach ([...$amzHeaders, ...$headers] as $header => $value) { $request = $request->withHeader($header, $value); } @@ -669,13 +752,25 @@ protected function call(string $method, string $uri, string $data = '', array $p ->withHeader('user-agent', 'utopia-php/storage'); try { - $response = $this->client->sendRequest($request); + if ($sink === null) { + $response = $this->client->sendRequest($request); + $responseBody = (string) $response->getBody(); + } else { + $head = ''; + $tee = static function (string $chunk) use ($sink, &$head): void { + if (\strlen($head) < self::ERROR_BODY_BUFFER_SIZE) { + $head .= substr($chunk, 0, self::ERROR_BODY_BUFFER_SIZE - \strlen($head)); + } + $sink($chunk); + }; + $response = $this->client->stream($request, $tee); + $responseBody = $head; + } } catch (ClientExceptionInterface $e) { throw new TransportException($e->getMessage(), $e->getCode(), $e); } $code = $response->getStatusCode(); - $responseBody = (string) $response->getBody(); $responseHeaders = []; foreach ($response->getHeaders() as $name => $values) { @@ -696,6 +791,40 @@ protected function call(string $method, string $uri, string $data = '', array $p ); } + /** + * Hash a request body for signing without materializing it as a string. + * + * SigV4 needs the payload digests before the first byte is sent, so the + * stream is read once for hashing and rewound for the actual send — the + * same approach the AWS SDK takes. This is why S3 requires seekable + * streams. Hashing starts from the beginning, matching the transport: + * the cURL adapter rewinds seekable bodies before sending, so the + * signature must cover the full stream. + * + * @return array{string, string} Base64 MD5 and hex SHA-256 of the full stream + */ + private function hashBody(StreamInterface $body): array + { + if (! $body->isSeekable()) { + throw new \InvalidArgumentException('S3 requires a seekable stream so the payload can be signed'); + } + + $body->rewind(); + $md5 = hash_init('md5'); + $sha256 = hash_init('sha256'); + while (! $body->eof()) { + $chunk = $body->read(self::PIPE_CHUNK_SIZE); + if ($chunk === '') { + break; + } + hash_update($md5, $chunk); + hash_update($sha256, $chunk); + } + $body->rewind(); + + return [base64_encode(hash_final($md5, true)), hash_final($sha256)]; + } + /** * Decode an XML response body into an associative array. * diff --git a/packages/storage/src/Storage/Device/Telemetry.php b/packages/storage/src/Storage/Device/Telemetry.php index 343ce40e..0113fc5b 100644 --- a/packages/storage/src/Storage/Device/Telemetry.php +++ b/packages/storage/src/Storage/Device/Telemetry.php @@ -4,6 +4,7 @@ namespace Utopia\Storage\Device; +use Psr\Http\Message\StreamInterface; use Utopia\Storage\Device; use Utopia\Storage\DeviceType; use Utopia\Storage\FileList; @@ -14,6 +15,7 @@ * Decorator that records a `storage.operation` histogram around every call to the decorated device. * * @phpstan-import-type UploadMetadata from Device + * * @see \Utopia\Tests\Storage\Device\TelemetryTest */ class Telemetry extends Device @@ -80,17 +82,20 @@ public function getPath(string $filename): string /** * @param UploadMetadata $metadata */ - public function prepareUpload(string $path, string $contentType, int $chunks = 1, array &$metadata = []): void + public function prepare(string $path, string $contentType, int $chunks = 1, array &$metadata = []): void { $this->measure(__FUNCTION__, function () use ($path, $contentType, $chunks, &$metadata): void { - $this->device->prepareUpload($path, $contentType, $chunks, $metadata); + $this->device->prepare($path, $contentType, $chunks, $metadata); }); } /** * @param UploadMetadata $metadata */ - public function uploadChunk(string $data, string $path, int $chunk = 1, int $chunks = 1, array &$metadata = []): int + /** + * @param UploadMetadata $metadata + */ + protected function uploadChunk(StreamInterface $data, string $path, int $chunk, int $chunks, array &$metadata): int { return $this->measure(__FUNCTION__, function () use ($data, $path, $chunk, $chunks, &$metadata): int { return $this->device->uploadChunk($data, $path, $chunk, $chunks, $metadata); @@ -100,10 +105,10 @@ public function uploadChunk(string $data, string $path, int $chunk = 1, int $chu /** * @param UploadMetadata $metadata */ - public function finalizeUpload(string $path, int $chunks = 1, array &$metadata = []): bool + public function finalize(string $path, int $chunks = 1, array &$metadata = []): bool { return $this->measure(__FUNCTION__, function () use ($path, $chunks, &$metadata): bool { - return $this->device->finalizeUpload($path, $chunks, $metadata); + return $this->device->finalize($path, $chunks, $metadata); }); } @@ -116,17 +121,18 @@ public function abort(string $path, string $uploadId = ''): bool * @param int<0, max> $offset * @param int<0, max>|null $length */ - public function read(string $path, int $offset = 0, ?int $length = null): string + public function read(string $path, int $offset = 0, ?int $length = null): StreamInterface { - return $this->measure(__FUNCTION__, fn(): string => $this->device->read($path, $offset, $length)); + return $this->measure(__FUNCTION__, fn(): StreamInterface => $this->device->read($path, $offset, $length)); } - public function transfer(string $path, string $destination, Device $device, int $chunkSize = self::TRANSFER_CHUNK_SIZE): bool + #[\Override] + public function copy(string $source, string $target, ?Device $to = null, int $chunkSize = self::COPY_CHUNK_SIZE): bool { - return $this->measure(__FUNCTION__, fn(): bool => $this->device->transfer($path, $destination, $device, $chunkSize)); + return $this->measure(__FUNCTION__, fn(): bool => $this->device->copy($source, $target, $to, $chunkSize)); } - public function write(string $path, string $data, string $contentType): bool + public function write(string $path, StreamInterface $data, string $contentType): bool { return $this->measure(__FUNCTION__, fn(): bool => $this->device->write($path, $data, $contentType)); } diff --git a/packages/storage/src/Storage/Device/Wasabi.php b/packages/storage/src/Storage/Device/Wasabi.php index cb3bfede..198ba30f 100644 --- a/packages/storage/src/Storage/Device/Wasabi.php +++ b/packages/storage/src/Storage/Device/Wasabi.php @@ -5,6 +5,7 @@ namespace Utopia\Storage\Device; use Psr\Http\Client\ClientInterface; +use Utopia\Psr18\StreamingClientInterface; use Utopia\Storage\Acl; use Utopia\Storage\DeviceType; @@ -44,10 +45,10 @@ public function __construct( string $bucket, string $region = self::EU_CENTRAL_1, Acl $acl = Acl::Private, - ?ClientInterface $client = null, + (ClientInterface&StreamingClientInterface)|null $client = null, ) { $host = $bucket . '.' . 's3' . '.' . $region . '.' . 'wasabisys' . '.' . 'com'; - parent::__construct($root, $accessKey, $secretKey, $host, $region, $acl, $client); + parent::__construct($root, $accessKey, $secretKey, $host, $region, $acl, $client, $bucket); } #[\Override] diff --git a/packages/storage/tests/E2E/S3Base.php b/packages/storage/tests/E2E/S3Base.php index 62b8ab87..729a1b39 100644 --- a/packages/storage/tests/E2E/S3Base.php +++ b/packages/storage/tests/E2E/S3Base.php @@ -4,9 +4,12 @@ namespace Utopia\Tests\Storage\E2E; +use PHPUnit\Framework\Attributes\Depends; use PHPUnit\Framework\TestCase; +use Utopia\Psr7\Stream; use Utopia\Storage\Device\Local; use Utopia\Storage\Device\S3; +use Utopia\Storage\DeviceType; use Utopia\Storage\Exception\NotFoundException; abstract class S3Base extends TestCase @@ -40,7 +43,7 @@ private function readBytes($handle, int $length): string abstract protected function init(): void; - abstract protected function getAdapterType(): \Utopia\Storage\DeviceType; + abstract protected function getAdapterType(): DeviceType; /** * @var S3 @@ -60,11 +63,8 @@ protected function setUp(): void private function uploadFile(string $source, string $path): void { - $data = file_get_contents($source); - if ($data === false) { - throw new \RuntimeException('Failed to read ' . $source); - } - $this->object->uploadData($data, $path, mime_content_type($source) ?: ''); + $handle = $this->openStream($source); + $this->object->upload(Stream::fromResource($handle), $path, mime_content_type($source) ?: ''); } private function uploadTestFiles(): void @@ -119,7 +119,7 @@ public function testListFilesPagination(): void public function testListFilesSingleObject(): void { $path = $this->object->getPath('single/'); - $this->object->write($path . 'only.txt', 'one', 'text/plain'); + $this->object->write($path . 'only.txt', new Stream('one'), 'text/plain'); $list = $this->object->listFiles($path); $this->assertCount(1, $list->files); @@ -146,15 +146,15 @@ public function testPath(): void public function testWrite(): void { - $this->assertEquals(true, $this->object->write($this->object->getPath('text.txt'), 'Hello World', 'text/plain')); + $this->assertEquals(true, $this->object->write($this->object->getPath('text.txt'), new Stream('Hello World'), 'text/plain')); $this->object->delete($this->object->getPath('text.txt')); } public function testRead(): void { - $this->assertEquals(true, $this->object->write($this->object->getPath('text-for-read.txt'), 'Hello World', 'text/plain')); - $this->assertEquals('Hello World', $this->object->read($this->object->getPath('text-for-read.txt'))); + $this->assertEquals(true, $this->object->write($this->object->getPath('text-for-read.txt'), new Stream('Hello World'), 'text/plain')); + $this->assertSame('Hello World', (string) $this->object->read($this->object->getPath('text-for-read.txt'))); $this->object->delete($this->object->getPath('text-for-read.txt')); } @@ -173,15 +173,27 @@ public function testFileExists(): void public function testMove(): void { - $this->assertEquals(true, $this->object->write($this->object->getPath('text-for-move.txt'), 'Hello World', 'text/plain')); + $this->assertEquals(true, $this->object->write($this->object->getPath('text-for-move.txt'), new Stream('Hello World'), 'text/plain')); $this->assertEquals(true, $this->object->exists($this->object->getPath('text-for-move.txt'))); $this->assertEquals(true, $this->object->move($this->object->getPath('text-for-move.txt'), $this->object->getPath('text-for-move-new.txt'))); - $this->assertEquals('Hello World', $this->object->read($this->object->getPath('text-for-move-new.txt'))); + $this->assertSame('Hello World', (string) $this->object->read($this->object->getPath('text-for-move-new.txt'))); $this->assertEquals(false, $this->object->exists($this->object->getPath('text-for-move.txt'))); $this->object->delete($this->object->getPath('text-for-move-new.txt')); } + public function testCopySameDevice(): void + { + $source = $this->object->getPath('testing/kitten-1.jpg'); + $target = $this->object->getPath('testing/kitten-copy.jpg'); + + $this->assertTrue($this->object->copy($source, $target)); + $this->assertSame($this->object->getFileHash($source), $this->object->getFileHash($target)); + $this->assertSame($this->object->getFileSize($source), $this->object->getFileSize($target)); + + $this->object->delete($target); + } + public function testMoveIdenticalName(): void { $file = '/kitten-1.jpg'; @@ -190,23 +202,23 @@ public function testMoveIdenticalName(): void public function testDelete(): void { - $this->assertEquals(true, $this->object->write($this->object->getPath('text-for-delete.txt'), 'Hello World', 'text/plain')); + $this->assertEquals(true, $this->object->write($this->object->getPath('text-for-delete.txt'), new Stream('Hello World'), 'text/plain')); $this->assertEquals(true, $this->object->exists($this->object->getPath('text-for-delete.txt'))); $this->assertEquals(true, $this->object->delete($this->object->getPath('text-for-delete.txt'))); } - public function testSVGUpload(): void + public function testSvgUpload(): void { $this->uploadFile(__DIR__ . '/../resources/disk-b/appwrite.svg', $this->object->getPath('testing/appwrite.svg')); - $this->assertEquals(file_get_contents(__DIR__ . '/../resources/disk-b/appwrite.svg'), $this->object->read($this->object->getPath('testing/appwrite.svg'))); + $this->assertEquals(file_get_contents(__DIR__ . '/../resources/disk-b/appwrite.svg'), (string) $this->object->read($this->object->getPath('testing/appwrite.svg'))); $this->assertEquals(true, $this->object->exists($this->object->getPath('testing/appwrite.svg'))); $this->assertEquals(true, $this->object->delete($this->object->getPath('testing/appwrite.svg'))); } - public function testXMLUpload(): void + public function testXmlUpload(): void { $this->uploadFile(__DIR__ . '/../resources/disk-a/config.xml', $this->object->getPath('testing/config.xml')); - $this->assertEquals(file_get_contents(__DIR__ . '/../resources/disk-a/config.xml'), $this->object->read($this->object->getPath('testing/config.xml'))); + $this->assertEquals(file_get_contents(__DIR__ . '/../resources/disk-a/config.xml'), (string) $this->object->read($this->object->getPath('testing/config.xml'))); $this->assertEquals(true, $this->object->exists($this->object->getPath('testing/config.xml'))); $this->assertEquals(true, $this->object->delete($this->object->getPath('testing/config.xml'))); } @@ -216,7 +228,7 @@ public function testDeletePath(): void // Test Single Object $path = $this->object->getPath('text-for-delete-path.txt'); $path = str_ireplace($this->object->getRoot(), $this->object->getRoot() . DIRECTORY_SEPARATOR . 'bucket', $path); - $this->assertEquals(true, $this->object->write($path, 'Hello World', 'text/plain')); + $this->assertEquals(true, $this->object->write($path, new Stream('Hello World'), 'text/plain')); $this->assertEquals(true, $this->object->exists($path)); $this->assertEquals(true, $this->object->deletePath('bucket')); $this->assertEquals(false, $this->object->exists($path)); @@ -224,12 +236,12 @@ public function testDeletePath(): void // Test Multiple Objects $path = $this->object->getPath('text-for-delete-path1.txt'); $path = str_ireplace($this->object->getRoot(), $this->object->getRoot() . DIRECTORY_SEPARATOR . 'bucket', $path); - $this->assertEquals(true, $this->object->write($path, 'Hello World', 'text/plain')); + $this->assertEquals(true, $this->object->write($path, new Stream('Hello World'), 'text/plain')); $this->assertEquals(true, $this->object->exists($path)); $path2 = $this->object->getPath('text-for-delete-path2.txt'); $path2 = str_ireplace($this->object->getRoot(), $this->object->getRoot() . DIRECTORY_SEPARATOR . 'bucket', $path2); - $this->assertEquals(true, $this->object->write($path2, 'Hello World', 'text/plain')); + $this->assertEquals(true, $this->object->write($path2, new Stream('Hello World'), 'text/plain')); $this->assertEquals(true, $this->object->exists($path2)); $this->assertEquals(true, $this->object->deletePath('bucket')); @@ -280,7 +292,7 @@ public function testPartUpload(): string $handle = $this->openStream($source); while ($start < $totalSize) { $contents = $this->readBytes($handle, $chunkSize); - $this->object->uploadData($contents, $dest, $metadata['content_type'] ?? '', $chunk, $chunks, $metadata); + $this->object->upload(new Stream($contents), $dest, $metadata['content_type'] ?? '', $chunk, $chunks, $metadata); $start += \strlen($contents); ++$chunk; fseek($handle, $start); @@ -320,7 +332,7 @@ public function testPartUploadRetry(): string $handle = $this->openStream($source); while ($start < $totalSize) { $contents = $this->readBytes($handle, $chunkSize); - $this->object->uploadData($contents, $dest, $metadata['content_type'] ?? '', $chunk, $chunks, $metadata); + $this->object->upload(new Stream($contents), $dest, $metadata['content_type'] ?? '', $chunk, $chunks, $metadata); $start += \strlen($contents); ++$chunk; break; @@ -333,7 +345,7 @@ public function testPartUploadRetry(): string $handle = $this->openStream($source); while ($start < $totalSize) { $contents = $this->readBytes($handle, $chunkSize); - $this->object->uploadData($contents, $dest, $metadata['content_type'] ?? '', $chunk, $chunks, $metadata); + $this->object->upload(new Stream($contents), $dest, $metadata['content_type'] ?? '', $chunk, $chunks, $metadata); $start += \strlen($contents); ++$chunk; fseek($handle, $start); @@ -380,7 +392,7 @@ public function testOutOfOrderPartUpload(): string // Upload chunks in reverse order for ($i = $chunks; $i >= 1; --$i) { - $this->object->uploadData($parts[$i], $dest, $metadata['content_type'] ?? '', $i, $chunks, $metadata); + $this->object->upload(new Stream($parts[$i]), $dest, $metadata['content_type'] ?? '', $i, $chunks, $metadata); } $this->assertEquals(filesize($source), $this->object->getFileSize($dest)); @@ -392,23 +404,23 @@ public function testOutOfOrderPartUpload(): string return $dest; } - #[\PHPUnit\Framework\Attributes\Depends('testPartUpload')] + #[Depends('testPartUpload')] public function testPartRead(string $path): void { $source = __DIR__ . '/../resources/disk-a/large_file.mp4'; $chunk = file_get_contents($source, false, null, 0, 500); - $readChunk = $this->object->read($path, 0, 500); + $readChunk = (string) $this->object->read($path, 0, 500); $this->assertEquals($chunk, $readChunk); } - #[\PHPUnit\Framework\Attributes\Depends('testPartUpload')] - public function testTransferLarge(string $path): void + #[Depends('testPartUpload')] + public function testCopyLarge(string $path): void { // chunked file $device = new Local(__DIR__ . '/../resources/disk-a'); $destination = $device->getPath('largefile.mp4'); - $this->assertTrue($this->object->transfer($path, $destination, $device, 10000000)); // 10 mb chunks + $this->assertTrue($this->object->copy($path, $destination, $device, 10000000)); // 10 mb chunks $this->assertTrue($device->exists($destination)); $this->assertSame('video/mp4', $device->getFileMimeType($destination)); @@ -416,23 +428,23 @@ public function testTransferLarge(string $path): void $this->object->delete($path); } - public function testTransferSmall(): void + public function testCopySmall(): void { $device = new Local(__DIR__ . '/../resources/disk-a'); $path = $this->object->getPath('text-for-read.txt'); - $this->object->write($path, 'Hello World', 'text/plain'); + $this->object->write($path, new Stream('Hello World'), 'text/plain'); $destination = $device->getPath('hello.txt'); - $this->assertTrue($this->object->transfer($path, $destination, $device, 10000000)); // 10 mb chunks + $this->assertTrue($this->object->copy($path, $destination, $device, 10000000)); // 10 mb chunks $this->assertTrue($device->exists($destination)); - $this->assertSame('Hello World', $device->read($destination)); + $this->assertSame('Hello World', (string) $device->read($destination)); $this->object->delete($path); $device->delete($destination); } - public function testTransferNonExistentFile(): void + public function testCopyNonExistentFile(): void { $device = new Local(__DIR__ . '/../resources/disk-a'); @@ -440,6 +452,6 @@ public function testTransferNonExistentFile(): void $destination = $device->getPath('hello.txt'); $this->expectException(NotFoundException::class); - $this->object->transfer($path, $destination, $device); + $this->object->copy($path, $destination, $device); } } diff --git a/packages/storage/tests/E2E/S3Test.php b/packages/storage/tests/E2E/S3Test.php index 38a3ec52..35b957e2 100644 --- a/packages/storage/tests/E2E/S3Test.php +++ b/packages/storage/tests/E2E/S3Test.php @@ -24,7 +24,7 @@ protected function init(): void $key = $this->env('S3_ACCESS_KEY', 'minioadmin'); $secret = $this->env('S3_SECRET', 'minioadmin'); - $this->object = new S3($this->root, $key, $secret, $host, 'us-east-1', Acl::Private); + $this->object = new S3($this->root, $key, $secret, $host, 'us-east-1', Acl::Private, bucket: 'utopia-storage-test'); } protected function getAdapterType(): DeviceType diff --git a/packages/storage/tests/Storage/Device/LocalTest.php b/packages/storage/tests/Storage/Device/LocalTest.php index 1b7da7a8..eecb338a 100644 --- a/packages/storage/tests/Storage/Device/LocalTest.php +++ b/packages/storage/tests/Storage/Device/LocalTest.php @@ -4,10 +4,15 @@ namespace Utopia\Tests\Storage\Device; +use PHPUnit\Framework\Attributes\Depends; use PHPUnit\Framework\TestCase; +use Psr\Http\Message\StreamInterface; +use Utopia\Psr7\Stream; use Utopia\Storage\Device\Local; +use Utopia\Storage\DeviceType; use Utopia\Storage\Exception\NotFoundException; use Utopia\Storage\Exception\UploadException; +use Utopia\Storage\FileInfo; final class LocalTest extends TestCase { @@ -38,7 +43,7 @@ private function readBytes($handle, int $length): string return $contents; } - private \Utopia\Storage\Device\Local $object; + private Local $object; protected function setUp(): void { @@ -60,7 +65,7 @@ public function testPaths(): void public function testType(): void { - $this->assertSame(\Utopia\Storage\DeviceType::Local, $this->object->getType()); + $this->assertSame(DeviceType::Local, $this->object->getType()); } public function testRoot(): void @@ -75,7 +80,7 @@ public function testPath(): void public function testWrite(): void { - $this->assertEquals(true, $this->object->write($this->object->getPath('text.txt'), 'Hello World')); + $this->assertEquals(true, $this->object->write($this->object->getPath('text.txt'), new Stream('Hello World'))); $this->assertFileExists($this->object->getPath('text.txt')); $this->assertIsReadable($this->object->getPath('text.txt')); @@ -84,12 +89,24 @@ public function testWrite(): void public function testRead(): void { - $this->assertEquals(true, $this->object->write($this->object->getPath('text-for-read.txt'), 'Hello World')); - $this->assertSame('Hello World', $this->object->read($this->object->getPath('text-for-read.txt'))); + $this->assertEquals(true, $this->object->write($this->object->getPath('text-for-read.txt'), new Stream('Hello World'))); + $this->assertSame('Hello World', (string) $this->object->read($this->object->getPath('text-for-read.txt'))); $this->object->delete($this->object->getPath('text-for-read.txt')); } + public function testWriteRewindsPartiallyConsumedStream(): void + { + $stream = new Stream('Hello World'); + $stream->read(6); // consume a prefix — seekable streams are sent from the beginning + + $path = $this->object->getPath('text-for-rewind.txt'); + $this->assertTrue($this->object->write($path, $stream)); + $this->assertSame('Hello World', (string) $this->object->read($path)); + + $this->object->delete($path); + } + public function testReadNonExistentFile(): void { $this->expectException(NotFoundException::class); @@ -98,7 +115,7 @@ public function testReadNonExistentFile(): void public function testFileExists(): void { - $this->assertEquals(true, $this->object->write($this->object->getPath('text-for-test-exists.txt'), 'Hello World')); + $this->assertEquals(true, $this->object->write($this->object->getPath('text-for-test-exists.txt'), new Stream('Hello World'))); $this->assertEquals(true, $this->object->exists($this->object->getPath('text-for-test-exists.txt'))); $this->assertEquals(false, $this->object->exists($this->object->getPath('text-for-test-doesnt-exist.txt'))); @@ -107,10 +124,10 @@ public function testFileExists(): void public function testMove(): void { - $this->assertEquals(true, $this->object->write($this->object->getPath('text-for-move.txt'), 'Hello World')); - $this->assertSame('Hello World', $this->object->read($this->object->getPath('text-for-move.txt'))); + $this->assertEquals(true, $this->object->write($this->object->getPath('text-for-move.txt'), new Stream('Hello World'))); + $this->assertSame('Hello World', (string) $this->object->read($this->object->getPath('text-for-move.txt'))); $this->assertEquals(true, $this->object->move($this->object->getPath('text-for-move.txt'), $this->object->getPath('text-for-move-new.txt'))); - $this->assertSame('Hello World', $this->object->read($this->object->getPath('text-for-move-new.txt'))); + $this->assertSame('Hello World', (string) $this->object->read($this->object->getPath('text-for-move-new.txt'))); $this->assertFileDoesNotExist($this->object->getPath('text-for-move.txt')); $this->assertIsNotReadable($this->object->getPath('text-for-move.txt')); $this->assertFileExists($this->object->getPath('text-for-move-new.txt')); @@ -119,20 +136,20 @@ public function testMove(): void $this->object->delete($this->object->getPath('text-for-move-new.txt')); } - public function testTransferRejectsInvalidChunkSize(): void + public function testCopyRejectsInvalidChunkSize(): void { $device = new Local(realpath(__DIR__ . '/../../resources/disk-b') ?: ''); $this->expectExceptionMessage('Chunk size must be greater than zero'); - $this->object->transfer($this->object->getPath('kitten-1.jpg'), $device->getPath('kitten-1.jpg'), $device, 0); + $this->object->copy($this->object->getPath('kitten-1.jpg'), $device->getPath('kitten-1.jpg'), $device, 0); } public function testListFiles(): void { $directory = $this->object->getPath('list-files'); - $this->object->write($directory . '/a.txt', 'aa'); - $this->object->write($directory . '/nested/b.txt', 'bb'); - $this->object->write($directory . '/.hidden', 'hh'); + $this->object->write($directory . '/a.txt', new Stream('aa')); + $this->object->write($directory . '/nested/b.txt', new Stream('bb')); + $this->object->write($directory . '/.hidden', new Stream('hh')); $list = $this->object->listFiles($directory, 2); $this->assertCount(2, $list->files); @@ -142,7 +159,7 @@ public function testListFiles(): void $this->assertCount(1, $rest->files); $this->assertNull($rest->cursor); - $paths = array_map(fn(\Utopia\Storage\FileInfo $file): string => $file->path, array_merge($list->files, $rest->files)); + $paths = array_map(fn(FileInfo $file): string => $file->path, array_merge($list->files, $rest->files)); $this->assertContains($directory . '/a.txt', $paths); $this->assertContains($directory . '/nested/b.txt', $paths); $this->assertContains($directory . '/.hidden', $paths); @@ -165,8 +182,8 @@ public function testMoveIdenticalName(): void public function testDelete(): void { - $this->assertEquals(true, $this->object->write($this->object->getPath('text-for-delete.txt'), 'Hello World')); - $this->assertSame('Hello World', $this->object->read($this->object->getPath('text-for-delete.txt'))); + $this->assertEquals(true, $this->object->write($this->object->getPath('text-for-delete.txt'), new Stream('Hello World'))); + $this->assertSame('Hello World', (string) $this->object->read($this->object->getPath('text-for-delete.txt'))); $this->assertEquals(true, $this->object->delete($this->object->getPath('text-for-delete.txt'))); $this->assertFileDoesNotExist($this->object->getPath('text-for-delete.txt')); $this->assertIsNotReadable($this->object->getPath('text-for-delete.txt')); @@ -177,8 +194,8 @@ public function testRecursiveDeleteRemovesHiddenFiles(): void $directory = $this->object->getPath('delete-hidden'); $this->assertTrue($this->object->createDirectory($directory)); - $this->assertTrue($this->object->write($directory . DIRECTORY_SEPARATOR . '.hidden', 'secret')); - $this->assertTrue($this->object->write($directory . DIRECTORY_SEPARATOR . 'visible', 'visible')); + $this->assertTrue($this->object->write($directory . DIRECTORY_SEPARATOR . '.hidden', new Stream('secret'))); + $this->assertTrue($this->object->write($directory . DIRECTORY_SEPARATOR . 'visible', new Stream('visible'))); $this->assertTrue($this->object->delete($directory, true)); $this->assertFalse($this->object->exists($directory)); @@ -229,22 +246,22 @@ public function testDirectorySize(): void } /** Without a started multipart upload there is nothing to reclaim — aborting could delete a pre-existing destination. */ - public function testFailedTransferDoesNotAbortOrDeleteExistingDestination(): void + public function testFailedCopyDoesNotAbortOrDeleteExistingDestination(): void { $source = $this->object->getPath(uniqid() . '.txt'); - $this->object->write($source, str_repeat('a', 30), 'text/plain'); + $this->object->write($source, new Stream(str_repeat('a', 30)), 'text/plain'); $device = new class (sys_get_temp_dir()) extends Local { public int $aborts = 0; #[\Override] - public function uploadData(string $data, string $path, string $contentType, int $chunk = 1, int $chunks = 1, array &$metadata = []): int + public function upload(StreamInterface $data, string $path, string $contentType, int $chunk = 1, int $chunks = 1, array &$metadata = []): int { if ($chunk === 2) { throw new UploadException('Injected chunk failure'); } - return parent::uploadData($data, $path, $contentType, $chunk, $chunks, $metadata); + return parent::upload($data, $path, $contentType, $chunk, $chunks, $metadata); } #[\Override] @@ -257,10 +274,10 @@ public function abort(string $path, string $uploadId = ''): bool }; $destination = $device->getPath(uniqid() . '-dest.txt'); - $device->write($destination, 'pre-existing', 'text/plain'); + $device->write($destination, new Stream('pre-existing'), 'text/plain'); try { - $this->object->transfer($source, $destination, $device, 10); + $this->object->copy($source, $destination, $device, 10); self::fail('Expected the injected chunk failure to surface'); } catch (UploadException $e) { $this->assertSame('Injected chunk failure', $e->getMessage()); @@ -269,7 +286,7 @@ public function abort(string $path, string $uploadId = ''): bool } $this->assertSame(0, $device->aborts); - $this->assertSame('pre-existing', $device->read($destination)); + $this->assertSame('pre-existing', (string) $device->read($destination)); $device->delete($destination); } @@ -288,7 +305,7 @@ public function testPartUpload(): string $handle = $this->openStream($source); while ($start < $totalSize) { $contents = $this->readBytes($handle, $chunkSize); - $this->object->uploadData($contents, $dest, '', $chunk, $chunks); + $this->object->upload(new Stream($contents), $dest, '', $chunk, $chunks); $start += \strlen($contents); ++$chunk; fseek($handle, $start); @@ -300,38 +317,33 @@ public function testPartUpload(): string return $dest; } - public function testUploadChunkDoesNotFinalizeUntilFinalizeUpload(): void + public function testUploadDoesNotFinalizeUntilLastChunk(): void { $dest = $this->object->getPath('chunked-phase-upload.txt'); $metadata = []; - $parts = [ - 2 => 'bbb', - 1 => 'aaa', - 3 => 'ccc', - ]; - - foreach ($parts as $chunk => $data) { - $this->object->uploadChunk($data, $dest, $chunk, 3, $metadata); - $this->assertFalse($this->object->exists($dest)); - } - $this->assertSame(3, $metadata['chunks'] ?? null); - $this->assertTrue($this->object->finalizeUpload($dest, 3, $metadata)); - $this->assertSame('aaabbbccc', $this->object->read($dest)); - $this->assertTrue($this->object->finalizeUpload($dest, 3, $metadata)); + $this->object->upload(new Stream('bbb'), $dest, '', 2, 3, $metadata); + $this->assertFalse($this->object->exists($dest)); + $this->object->upload(new Stream('aaa'), $dest, '', 1, 3, $metadata); + $this->assertFalse($this->object->exists($dest)); + + $this->assertSame(3, $this->object->upload(new Stream('ccc'), $dest, '', 3, 3, $metadata)); + $this->assertSame('aaabbbccc', (string) $this->object->read($dest)); + // Finalizing an already assembled upload is idempotent. + $this->assertTrue($this->object->finalize($dest, 3, $metadata)); $this->object->delete($dest); } - public function testFinalizeUploadRequiresAllLocalChunks(): void + public function testFinalizeRequiresAllLocalChunks(): void { $dest = $this->object->getPath('chunked-phase-missing.txt'); $metadata = []; - $this->object->uploadChunk('aaa', $dest, 1, 2, $metadata); + $this->object->upload(new Stream('aaa'), $dest, '', 1, 2, $metadata); try { - $this->object->finalizeUpload($dest, 2, $metadata); + $this->object->finalize($dest, 2, $metadata); $this->fail('Expected missing chunk exception'); } catch (UploadException $e) { $this->assertSame('Missing chunk 2', $e->getMessage()); @@ -355,7 +367,7 @@ public function testPartUploadRetry(): string $handle = $this->openStream($source); while ($start < $totalSize) { $contents = $this->readBytes($handle, $chunkSize); - $this->object->uploadData($contents, $dest, '', $chunk, $chunks); + $this->object->upload(new Stream($contents), $dest, '', $chunk, $chunks); $start += \strlen($contents); ++$chunk; break; @@ -368,7 +380,7 @@ public function testPartUploadRetry(): string $handle = $this->openStream($source); while ($start < $totalSize) { $contents = $this->readBytes($handle, $chunkSize); - $this->object->uploadData($contents, $dest, '', $chunk, $chunks); + $this->object->upload(new Stream($contents), $dest, '', $chunk, $chunks); $start += \strlen($contents); ++$chunk; fseek($handle, $start); @@ -395,7 +407,7 @@ public function testAbort(): void $handle = $this->openStream($source); while ($chunk < 3) { // only upload two chunks $contents = $this->readBytes($handle, $chunkSize); - $this->object->uploadData($contents, $dest, '', $chunk, $chunks); + $this->object->upload(new Stream($contents), $dest, '', $chunk, $chunks); $start += \strlen($contents); ++$chunk; fseek($handle, $start); @@ -415,7 +427,7 @@ public function testAbort(): void $handle = $this->openStream($source); while ($chunk < 3) { // only upload two chunks $contents = $this->readBytes($handle, $chunkSize); - $this->object->uploadData($contents, $dest1, '', $chunk, $chunks); + $this->object->upload(new Stream($contents), $dest1, '', $chunk, $chunks); $start += \strlen($contents); ++$chunk; fseek($handle, $start); @@ -426,12 +438,12 @@ public function testAbort(): void $this->assertTrue($this->object->abort($dest1)); } - #[\PHPUnit\Framework\Attributes\Depends('testPartUpload')] + #[Depends('testPartUpload')] public function testPartRead(string $path): void { $source = __DIR__ . '/../../resources/disk-a/large_file.mp4'; $chunk = file_get_contents($source, false, null, 0, 500); - $readChunk = $this->object->read($path, 0, 500); + $readChunk = (string) $this->object->read($path, 0, 500); $this->assertEquals($chunk, $readChunk); } @@ -445,14 +457,14 @@ public function testPartitionTotalSpace(): void $this->assertGreaterThan(0, $this->object->getPartitionTotalSpace()); } - #[\PHPUnit\Framework\Attributes\Depends('testPartUpload')] - public function testTransferLarge(string $path): void + #[Depends('testPartUpload')] + public function testCopyLarge(string $path): void { // chunked file $device = new Local(realpath(__DIR__ . '/../../resources/disk-b') ?: ''); $destination = $device->getPath('largefile.mp4'); - $this->assertTrue($this->object->transfer($path, $destination, $device, 10000000)); // 10 mb chunks + $this->assertTrue($this->object->copy($path, $destination, $device, 10000000)); // 10 mb chunks $this->assertTrue($device->exists($destination)); $this->assertSame('video/mp4', $device->getFileMimeType($destination)); @@ -460,17 +472,17 @@ public function testTransferLarge(string $path): void $this->object->delete($path); } - public function testTransferSmall(): void + public function testCopySmall(): void { $device = new Local(realpath(__DIR__ . '/../../resources/disk-b') ?: ''); $path = $this->object->getPath('text-for-read.txt'); - $this->object->write($path, 'Hello World'); + $this->object->write($path, new Stream('Hello World')); $destination = $device->getPath('hello.txt'); - $this->assertTrue($this->object->transfer($path, $destination, $device, 10000000)); // 10 mb chunks + $this->assertTrue($this->object->copy($path, $destination, $device, 10000000)); // 10 mb chunks $this->assertTrue($device->exists($destination)); - $this->assertSame('Hello World', $device->read($destination)); + $this->assertSame('Hello World', (string) $device->read($destination)); $this->object->delete($path); $device->delete($destination); @@ -481,7 +493,7 @@ public function testDeletePath(): void // Test Single Object $path = $this->object->getPath('text-for-delete-path.txt'); $path = str_ireplace($this->object->getRoot(), $this->object->getRoot() . DIRECTORY_SEPARATOR . 'bucket', $path); - $this->assertEquals(true, $this->object->write($path, 'Hello World', 'text/plain')); + $this->assertEquals(true, $this->object->write($path, new Stream('Hello World'), 'text/plain')); $this->assertEquals(true, $this->object->exists($path)); $this->assertEquals(true, $this->object->deletePath('bucket')); $this->assertEquals(false, $this->object->exists($path)); @@ -489,17 +501,17 @@ public function testDeletePath(): void // Test Multiple Objects $path = $this->object->getPath('text-for-delete-path1.txt'); $path = str_ireplace($this->object->getRoot(), $this->object->getRoot() . DIRECTORY_SEPARATOR . 'bucket', $path); - $this->assertEquals(true, $this->object->write($path, 'Hello World', 'text/plain')); + $this->assertEquals(true, $this->object->write($path, new Stream('Hello World'), 'text/plain')); $this->assertEquals(true, $this->object->exists($path)); $path2 = $this->object->getPath('text-for-delete-path2.txt'); $path2 = str_ireplace($this->object->getRoot(), $this->object->getRoot() . DIRECTORY_SEPARATOR . 'bucket', $path2); - $this->assertEquals(true, $this->object->write($path2, 'Hello World', 'text/plain')); + $this->assertEquals(true, $this->object->write($path2, new Stream('Hello World'), 'text/plain')); $this->assertEquals(true, $this->object->exists($path2)); $path3 = $this->object->getPath('.hidden.txt'); $path3 = str_ireplace($this->object->getRoot(), $this->object->getRoot() . DIRECTORY_SEPARATOR . 'bucket', $path3); - $this->assertEquals(true, $this->object->write($path3, 'Hello World', 'text/plain')); + $this->assertEquals(true, $this->object->write($path3, new Stream('Hello World'), 'text/plain')); $this->assertEquals(true, $this->object->exists($path3)); $this->assertEquals(true, $this->object->deletePath('bucket/')); @@ -516,8 +528,8 @@ public function testListFilesEmptyAndGrowingDirectory(): void $this->assertCount(0, $this->object->listFiles($dir)->files); - $this->object->write($dir . DIRECTORY_SEPARATOR . 'new-file.txt', 'Hello World'); - $this->object->write($dir . DIRECTORY_SEPARATOR . 'new-file-two.txt', 'Hello World'); + $this->object->write($dir . DIRECTORY_SEPARATOR . 'new-file.txt', new Stream('Hello World')); + $this->object->write($dir . DIRECTORY_SEPARATOR . 'new-file-two.txt', new Stream('Hello World')); $this->assertCount(2, $this->object->listFiles($dir)->files); @@ -531,11 +543,11 @@ public function testNestedDeletePath(): void $dir3 = $dir2 . DIRECTORY_SEPARATOR . 'dir3'; $this->assertTrue($this->object->createDirectory($dir)); - $this->object->write($dir . DIRECTORY_SEPARATOR . 'new-file.txt', 'Hello World'); + $this->object->write($dir . DIRECTORY_SEPARATOR . 'new-file.txt', new Stream('Hello World')); $this->assertTrue($this->object->createDirectory($dir2)); - $this->object->write($dir2 . DIRECTORY_SEPARATOR . 'new-file-2.txt', 'Hello World'); + $this->object->write($dir2 . DIRECTORY_SEPARATOR . 'new-file-2.txt', new Stream('Hello World')); $this->assertTrue($this->object->createDirectory($dir3)); - $this->object->write($dir3 . DIRECTORY_SEPARATOR . 'new-file-3.txt', 'Hello World'); + $this->object->write($dir3 . DIRECTORY_SEPARATOR . 'new-file-3.txt', new Stream('Hello World')); $this->assertTrue($this->object->deletePath('nested-delete-path-test')); $this->assertFalse($this->object->exists($dir)); @@ -562,9 +574,9 @@ public function testJoinChunksAssemblesContentCorrectly(): void $storage = $this->makeJoinTestStorage(); $dest = $storage->getRoot() . DIRECTORY_SEPARATOR . 'test.dat'; - $storage->uploadData('AAAA', $dest, 'application/octet-stream', 1, 3); - $storage->uploadData('BBBB', $dest, 'application/octet-stream', 2, 3); - $storage->uploadData('CCCC', $dest, 'application/octet-stream', 3, 3); + $storage->upload(new Stream('AAAA'), $dest, 'application/octet-stream', 1, 3); + $storage->upload(new Stream('BBBB'), $dest, 'application/octet-stream', 2, 3); + $storage->upload(new Stream('CCCC'), $dest, 'application/octet-stream', 3, 3); $this->assertFileExists($dest); $this->assertSame('AAAABBBBCCCC', file_get_contents($dest)); @@ -579,9 +591,9 @@ public function testJoinChunksCleansUpTempFilesOnSuccess(): void $tmpDir = $storage->getRoot() . DIRECTORY_SEPARATOR . 'tmp_test.dat'; $tmpAssemble = $storage->getRoot() . DIRECTORY_SEPARATOR . 'tmp_assemble_test.dat'; - $storage->uploadData('AAAA', $dest, 'application/octet-stream', 1, 3); - $storage->uploadData('BBBB', $dest, 'application/octet-stream', 2, 3); - $storage->uploadData('CCCC', $dest, 'application/octet-stream', 3, 3); + $storage->upload(new Stream('AAAA'), $dest, 'application/octet-stream', 1, 3); + $storage->upload(new Stream('BBBB'), $dest, 'application/octet-stream', 2, 3); + $storage->upload(new Stream('CCCC'), $dest, 'application/octet-stream', 3, 3); $this->assertDirectoryDoesNotExist($tmpDir, 'Temp chunk directory should be removed after assembly'); $this->assertFileDoesNotExist($tmpAssemble, 'Temp assembly file should be removed after rename'); @@ -602,8 +614,8 @@ public function testJoinChunksMissingPartDoesNotFinalize(): void $tmpDir = $storage->getRoot() . DIRECTORY_SEPARATOR . 'tmp_test.dat'; $tmpAssemble = $storage->getRoot() . DIRECTORY_SEPARATOR . 'tmp_assemble_test.dat'; - $storage->uploadData('AAAA', $dest, 'application/octet-stream', 1, 3); - $storage->uploadData('BBBB', $dest, 'application/octet-stream', 2, 3); + $storage->upload(new Stream('AAAA'), $dest, 'application/octet-stream', 1, 3); + $storage->upload(new Stream('BBBB'), $dest, 'application/octet-stream', 2, 3); // Simulate a missing/corrupted chunk by deleting part 1 before the // final upload triggers assembly. @@ -611,7 +623,7 @@ public function testJoinChunksMissingPartDoesNotFinalize(): void // Uploading the final chunk should NOT throw or finalize, // because part 1 is missing and the part-file count is only 2. - $storage->uploadData('CCCC', $dest, 'application/octet-stream', 3, 3); + $storage->upload(new Stream('CCCC'), $dest, 'application/octet-stream', 3, 3); $this->assertFileDoesNotExist($dest, 'Final file must not be created when a chunk is missing'); $this->assertFileDoesNotExist($tmpAssemble, 'Temp assembly file must not be created'); @@ -626,7 +638,7 @@ public function testJoinChunksMissingPartDoesNotFinalize(): void ); // Re-upload the missing chunk — assembly should now succeed. - $storage->uploadData('AAAA', $dest, 'application/octet-stream', 1, 3); + $storage->upload(new Stream('AAAA'), $dest, 'application/octet-stream', 1, 3); $this->assertFileExists($dest, 'Final file should be created after missing chunk is re-uploaded'); $this->assertSame('AAAABBBBCCCC', file_get_contents($dest), 'Re-uploaded chunk must allow correct assembly'); @@ -638,8 +650,8 @@ public function testJoinChunksStaleAssemblyFileIsOverwritten(): void $storage = $this->makeJoinTestStorage(); $dest = $storage->getRoot() . DIRECTORY_SEPARATOR . 'test.dat'; - $storage->uploadData('AAAA', $dest, 'application/octet-stream', 1, 3); - $storage->uploadData('BBBB', $dest, 'application/octet-stream', 2, 3); + $storage->upload(new Stream('AAAA'), $dest, 'application/octet-stream', 1, 3); + $storage->upload(new Stream('BBBB'), $dest, 'application/octet-stream', 2, 3); // Simulate a stale assembly file left by a previously crashed attempt. // With unique temp paths (tempnam), stale files at old hardcoded paths @@ -647,7 +659,7 @@ public function testJoinChunksStaleAssemblyFileIsOverwritten(): void $staleFile = $storage->getRoot() . DIRECTORY_SEPARATOR . 'tmp_assemble_test.dat'; file_put_contents($staleFile, 'STALE_GARBAGE_DATA'); - $storage->uploadData('CCCC', $dest, 'application/octet-stream', 3, 3); + $storage->upload(new Stream('CCCC'), $dest, 'application/octet-stream', 3, 3); $this->assertFileExists($dest); $this->assertSame('AAAABBBBCCCC', file_get_contents($dest), 'Stale assembly file must not corrupt output'); @@ -660,13 +672,13 @@ public function testOutOfOrderUpload(): void $storage = $this->makeJoinTestStorage(); $dest = $storage->getRoot() . DIRECTORY_SEPARATOR . 'out-of-order.dat'; - $storage->uploadData('CCCC', $dest, 'application/octet-stream', 3, 3); + $storage->upload(new Stream('CCCC'), $dest, 'application/octet-stream', 3, 3); $this->assertFileDoesNotExist($dest, 'File should not be assembled after chunk 3'); - $storage->uploadData('AAAA', $dest, 'application/octet-stream', 1, 3); + $storage->upload(new Stream('AAAA'), $dest, 'application/octet-stream', 1, 3); $this->assertFileDoesNotExist($dest, 'File should not be assembled after chunk 1'); - $storage->uploadData('BBBB', $dest, 'application/octet-stream', 2, 3); + $storage->upload(new Stream('BBBB'), $dest, 'application/octet-stream', 2, 3); $this->assertFileExists($dest, 'File should be assembled after final chunk'); $this->assertSame('AAAABBBBCCCC', file_get_contents($dest), 'Chunks must be assembled in correct order'); @@ -678,14 +690,14 @@ public function testOutOfOrderUploadWithRetry(): void $storage = $this->makeJoinTestStorage(); $dest = $storage->getRoot() . DIRECTORY_SEPARATOR . 'out-of-order-retry.dat'; - $storage->uploadData('BBBB', $dest, 'application/octet-stream', 2, 3); - $storage->uploadData('AAAA', $dest, 'application/octet-stream', 1, 3); + $storage->upload(new Stream('BBBB'), $dest, 'application/octet-stream', 2, 3); + $storage->upload(new Stream('AAAA'), $dest, 'application/octet-stream', 1, 3); // Re-upload chunk 2 (duplicate) — should be silently ignored - $storage->uploadData('BBBB', $dest, 'application/octet-stream', 2, 3); + $storage->upload(new Stream('BBBB'), $dest, 'application/octet-stream', 2, 3); $this->assertFileDoesNotExist($dest, 'File should not be assembled after duplicate retry'); - $storage->uploadData('CCCC', $dest, 'application/octet-stream', 3, 3); + $storage->upload(new Stream('CCCC'), $dest, 'application/octet-stream', 3, 3); $this->assertFileExists($dest, 'File should be assembled after final chunk'); $this->assertSame('AAAABBBBCCCC', file_get_contents($dest), 'Duplicate retry must not corrupt final file'); @@ -698,10 +710,10 @@ public function testParallelChunkUpload(): void $dest = $storage->getRoot() . DIRECTORY_SEPARATOR . 'parallel.dat'; // Upload chunk 1 (creates temp directory) - $storage->uploadData('AAAA', $dest, 'application/octet-stream', 1, 2); + $storage->upload(new Stream('AAAA'), $dest, 'application/octet-stream', 1, 2); // Upload chunk 2 (assembles the file) - $storage->uploadData('BBBB', $dest, 'application/octet-stream', 2, 2); + $storage->upload(new Stream('BBBB'), $dest, 'application/octet-stream', 2, 2); // Verify file exists and is correct $this->assertFileExists($dest); diff --git a/packages/storage/tests/Storage/Device/S3Test.php b/packages/storage/tests/Storage/Device/S3Test.php index 4952de91..735b775a 100644 --- a/packages/storage/tests/Storage/Device/S3Test.php +++ b/packages/storage/tests/Storage/Device/S3Test.php @@ -5,8 +5,10 @@ namespace Utopia\Tests\Storage\Device; use PHPUnit\Framework\TestCase; +use Psr\Http\Client\ClientExceptionInterface; use Psr\Http\Message\RequestInterface; use Psr\Http\Message\ResponseInterface; +use Psr\Http\Message\StreamInterface; use Utopia\Client\Adapter; use Utopia\Client\Decorator\Retry; use Utopia\Client\Exception\NetworkException; @@ -44,14 +46,23 @@ class TestableS3 extends S3 public ?int $failPart = null; - private bool $objectExists = false; + public bool $objectExists = false; + + public string $infoContentLength = '1'; + + /** + * @var array>> + */ + public array $amzHeadersByOperation = []; #[\Override] - protected function call(string $method, string $uri, string $data = '', array $parameters = [], array $headers = [], array $amzHeaders = [], bool $decode = true): S3Response + protected function call(string $method, string $uri, StreamInterface|string $data = '', array $parameters = [], array $headers = [], array $amzHeaders = [], bool $decode = true, ?callable $sink = null): S3Response { $operation = match (true) { $method === 'HEAD' => 's3:info', $method === 'POST' && \array_key_exists('uploads', $parameters) => 's3:createMultipartUpload', + $method === 'PUT' && isset($parameters['partNumber'], $amzHeaders['x-amz-copy-source']) => 's3:uploadPartCopy', + $method === 'PUT' && isset($amzHeaders['x-amz-copy-source']) => 's3:copyObject', $method === 'PUT' && isset($parameters['partNumber']) => 's3:uploadPart', $method === 'POST' && isset($parameters['uploadId']) => 's3:completeMultipartUpload', $method === 'DELETE' && isset($parameters['uploadId']) => 's3:abort', @@ -60,13 +71,22 @@ protected function call(string $method, string $uri, string $data = '', array $p }; $this->calls[] = $operation; $this->headersByOperation[$operation] = $headers; + $this->amzHeadersByOperation[$operation][] = $amzHeaders; if ($operation === 's3:info') { if (! $this->objectExists) { throw new NotFoundException('Not found'); } - return new S3Response(code: 200, headers: ['content-length' => '1'], body: ''); + return new S3Response(code: 200, headers: ['content-length' => $this->infoContentLength], body: ''); + } + + if ($operation === 's3:copyObject') { + return new S3Response(code: 200, headers: [], body: ['ETag' => '"etag-copy"']); + } + + if ($operation === 's3:uploadPartCopy') { + return new S3Response(code: 200, headers: [], body: ['ETag' => '"etag-' . $parameters['partNumber'] . '"']); } if ($operation === 's3:createMultipartUpload') { @@ -82,7 +102,7 @@ protected function call(string $method, string $uri, string $data = '', array $p } if ($operation === 's3:completeMultipartUpload') { - $this->completedBody = $data; + $this->completedBody = (string) $data; $this->objectExists = true; return new S3Response(code: 200, headers: [], body: ''); @@ -104,7 +124,7 @@ final class ScriptedClient implements Adapter public array $requests = []; /** - * @param array $responses + * @param array $responses */ public function __construct(private array $responses) {} @@ -112,7 +132,7 @@ public function sendRequest(RequestInterface $request): ResponseInterface { $this->requests[] = $request; $response = array_shift($this->responses); - if ($response instanceof \Psr\Http\Client\ClientExceptionInterface) { + if ($response instanceof ClientExceptionInterface) { throw $response; } if (! $response instanceof ResponseInterface) { @@ -181,11 +201,11 @@ protected function setUp(): void ); } - public function testPrepareUploadCreatesMultipartMetadata(): void + public function testPrepareCreatesMultipartMetadata(): void { $metadata = []; - $this->s3->prepareUpload('/root/file.txt', 'text/plain', 2, $metadata); + $this->s3->prepare('/root/file.txt', 'text/plain', 2, $metadata); $this->assertSame('upload-123', $metadata['uploadId'] ?? null); $this->assertSame([], $metadata['parts'] ?? null); @@ -193,29 +213,28 @@ public function testPrepareUploadCreatesMultipartMetadata(): void $this->assertSame(['s3:createMultipartUpload'], $this->s3->calls); } - public function testUploadChunkRecordsPartWithoutCompleting(): void + public function testUploadRecordsPartWithoutCompleting(): void { $metadata = []; - $this->s3->prepareUpload('/root/file.txt', 'text/plain', 2, $metadata); - $chunks = $this->s3->uploadChunk('aaa', '/root/file.txt', 1, 2, $metadata); + $chunks = $this->s3->upload(new Stream('aaa'), '/root/file.txt', 'text/plain', 1, 2, $metadata); $this->assertSame(1, $chunks); $this->assertSame('etag-1', ($metadata['parts'] ?? [])[1] ?? null); $this->assertNotContains('s3:completeMultipartUpload', $this->s3->calls); } - public function testSingleChunkUploadDataDoesNotFinalizeOrCheckExists(): void + public function testSingleChunkUploadDoesNotFinalizeOrCheckExists(): void { $metadata = []; - $this->assertSame(1, $this->s3->uploadData('aaa', '/root/file.txt', 'text/plain', 1, 1, $metadata)); + $this->assertSame(1, $this->s3->upload(new Stream('aaa'), '/root/file.txt', 'text/plain', 1, 1, $metadata)); $this->assertSame(['s3:write'], $this->s3->calls); $this->assertSame([1 => true], $metadata['parts'] ?? null); $this->assertSame(1, $metadata['chunks'] ?? null); } - public function testFinalizeUploadRequiresAllS3Parts(): void + public function testFinalizeRequiresAllS3Parts(): void { $metadata = [ 'uploadId' => 'upload-123', @@ -225,10 +244,10 @@ public function testFinalizeUploadRequiresAllS3Parts(): void $this->expectException(UploadException::class); $this->expectExceptionMessage('Missing chunk 2'); - $this->s3->finalizeUpload('/root/file.txt', 2, $metadata); + $this->s3->finalize('/root/file.txt', 2, $metadata); } - public function testFinalizeUploadCompletesS3PartsInNumericOrder(): void + public function testFinalizeCompletesS3PartsInNumericOrder(): void { $metadata = [ 'uploadId' => 'upload-123', @@ -247,7 +266,7 @@ public function testFinalizeUploadCompletesS3PartsInNumericOrder(): void 'chunks' => 10, ]; - $this->assertTrue($this->s3->finalizeUpload('/root/file.txt', 10, $metadata)); + $this->assertTrue($this->s3->finalize('/root/file.txt', 10, $metadata)); $part1 = strpos($this->s3->completedBody, '1'); $part2 = strpos($this->s3->completedBody, '2'); @@ -260,7 +279,7 @@ public function testFinalizeUploadCompletesS3PartsInNumericOrder(): void $this->assertLessThan($part10, $part2); } - public function testFinalizeUploadSendsCompleteBodyAsXml(): void + public function testFinalizeSendsCompleteBodyAsXml(): void { $metadata = [ 'uploadId' => 'upload-123', @@ -268,7 +287,7 @@ public function testFinalizeUploadSendsCompleteBodyAsXml(): void 'chunks' => 2, ]; - $this->assertTrue($this->s3->finalizeUpload('/root/file.txt', 2, $metadata)); + $this->assertTrue($this->s3->finalize('/root/file.txt', 2, $metadata)); $this->assertSame('application/xml', $this->s3->headersByOperation['s3:completeMultipartUpload']['content-type']); } @@ -295,7 +314,7 @@ public function testWriteSendsSignedRequest(): void { $client = new ScriptedClient([new Response(200)]); - $this->assertTrue($this->device($client)->write('/root/file.txt', 'Hello World', 'text/plain')); + $this->assertTrue($this->device($client)->write('/root/file.txt', new Stream('Hello World'), 'text/plain')); $this->assertCount(1, $client->requests); $request = $client->requests[0]; @@ -315,7 +334,7 @@ public function testTransientErrorIsRetriedUntilSuccess(): void { $client = new ScriptedClient([$this->slowDown(), $this->slowDown(), new Response(200)]); - $this->assertTrue($this->device($client)->write('/root/file.txt', 'Hello World', 'text/plain')); + $this->assertTrue($this->device($client)->write('/root/file.txt', new Stream('Hello World'), 'text/plain')); $this->assertCount(3, $client->requests); } @@ -324,7 +343,7 @@ public function testTransientErrorRetriesAreExhausted(): void $client = new ScriptedClient([$this->slowDown(), $this->slowDown(), $this->slowDown(), $this->slowDown()]); try { - $this->device($client)->write('/root/file.txt', 'Hello World', 'text/plain'); + $this->device($client)->write('/root/file.txt', new Stream('Hello World'), 'text/plain'); self::fail('Expected exception after exhausting retries'); } catch (RemoteException $e) { $this->assertSame(503, $e->getCode()); @@ -359,7 +378,7 @@ public function testTransportFailureIsWrapped(): void $client = new ScriptedClient([$psrError]); try { - $this->device($client)->write('/root/file.txt', 'Hello World', 'text/plain'); + $this->device($client)->write('/root/file.txt', new Stream('Hello World'), 'text/plain'); self::fail('Expected transport exception'); } catch (TransportException $e) { $this->assertSame($psrError, $e->getPrevious()); @@ -381,6 +400,70 @@ public function testEveryFailureIsCatchableAsStorageException(): void } } + private function bucketDevice(): TestableS3 + { + $s3 = new TestableS3( + root: '/root', + accessKey: 'test-key', + secretKey: 'test-secret', + host: 'https://s3.example.com', + region: 'us-east-1', + bucket: 'my-bucket', + ); + $s3->objectExists = true; + + return $s3; + } + + public function testCopySameDeviceRunsServerSide(): void + { + $s3 = $this->bucketDevice(); + + $this->assertTrue($s3->copy('/root/a.jpg', '/root/b.jpg')); + $this->assertSame(['s3:info', 's3:copyObject'], $s3->calls); + + $amzHeaders = $s3->amzHeadersByOperation['s3:copyObject'][0]; + $this->assertSame('/my-bucket/root/a.jpg', $amzHeaders['x-amz-copy-source']); + $this->assertSame('COPY', $amzHeaders['x-amz-metadata-directive']); + } + + public function testCopyLargeObjectUsesMultipartServerSideCopy(): void + { + $s3 = $this->bucketDevice(); + $s3->infoContentLength = (string) (6 * 1024 * 1024 * 1024); // 6 GB — above the CopyObject limit + + $this->assertTrue($s3->copy('/root/a.bin', '/root/b.bin')); + $this->assertSame([ + 's3:info', + 's3:createMultipartUpload', + 's3:uploadPartCopy', + 's3:uploadPartCopy', + 's3:completeMultipartUpload', + ], $s3->calls); + + $ranges = array_column($s3->amzHeadersByOperation['s3:uploadPartCopy'], 'x-amz-copy-source-range'); + $this->assertSame(['bytes=0-5368709119', 'bytes=5368709120-6442450943'], $ranges); + $this->assertStringContainsString('etag-2', $s3->completedBody); + } + + public function testCopyWithoutBucketFallsBackToStreaming(): void + { + $this->s3->objectExists = true; + + $this->assertTrue($this->s3->copy('/root/a.jpg', '/root/b.jpg')); + $this->assertNotContains('s3:copyObject', $this->s3->calls); + $this->assertContains('s3:get', $this->s3->calls); + $this->assertContains('s3:write', $this->s3->calls); + } + + public function testZeroLengthReadReturnsEmptyStreamWithoutARequest(): void + { + $client = new ScriptedClient([]); + + $this->assertSame('', (string) $this->device($client)->read('/root/file.txt', 5, 0)); + $this->assertCount(0, $client->requests); + } + public function testExistsReturnsFalseOnlyForNotFound(): void { $client = new ScriptedClient([new Response(404)]); @@ -415,7 +498,7 @@ public function testErrorMessageIncludesAmzRequestIds(): void } } - public function testTransferAbortsMultipartUploadOnFailure(): void + public function testCopyAbortsMultipartUploadOnFailure(): void { $dir = sys_get_temp_dir() . DIRECTORY_SEPARATOR . 'utopia-storage-' . uniqid(); mkdir($dir); @@ -425,7 +508,7 @@ public function testTransferAbortsMultipartUploadOnFailure(): void $this->s3->failPart = 2; try { - new Local($dir)->transfer($sourcePath, '/root/dest.bin', $this->s3, 10); + new Local($dir)->copy($sourcePath, '/root/dest.bin', $this->s3, 10); self::fail('Expected the injected part failure to surface'); } catch (RemoteException $e) { $this->assertSame('Injected part failure', $e->getMessage());