storage 4.0: streams-only I/O, one-word upload verbs, copy replaces transfer

.vale/styles/config/vocabularies/Utopia/accept.txt

@@ -89,6 +89,7 @@ JSON
 JWS
 JWT(?:s)?
 KaiOS
+keepalive
 keypair(?:s)?
 libcurl
 Linode

packages/storage/README.md

@@ -26,18 +26,19 @@ Devices are immutable value objects: construct one with its configuration and us
 
 require_once '../vendor/autoload.php';
 
+use Utopia\Psr7\Stream;
 use Utopia\Storage\Device\Local;
 
 $device = new Local('/path/to/storage');
 
-// Upload a file
-$device->uploadData(file_get_contents('/local/path/to/file.png'), 'destination/path/file.png', 'image/png');
+// Upload a file — contents move as PSR-7 streams, so memory stays bounded
+$device->upload(Stream::fromResource(fopen('/local/path/to/file.png', 'rb')), 'destination/path/file.png', 'image/png');
 
 // Check if file exists
 $exists = $device->exists('destination/path/file.png');
 
-// Read file contents
-$contents = $device->read('destination/path/file.png');
+// Read file contents as a stream
+$contents = (string) $device->read('destination/path/file.png');
 
 // Delete a file
 $device->delete('destination/path/file.png');
@@ -69,10 +70,13 @@ $device = new S3(
     'YOUR_SECRET_KEY',
     'YOUR_BUCKET_NAME.s3.us-east-1.amazonaws.com', // Host
     'us-east-1', // Region
-    Acl::Private // Access control (default: private)
+    Acl::Private, // Access control (default: private)
+    bucket: 'YOUR_BUCKET_NAME', // Optional: enables server-side copy
 );
 ```
 
+When the bucket name is known, a same-device `copy()` — and therefore `move()` — runs entirely server side (`CopyObject`, or `UploadPartCopy` above 5 GB), so no bytes move through PHP. The provider-specific adapters below pass their bucket automatically. Without it, `copy()` falls back to a streamed download and re-upload.
+
 The provider-specific adapters below build the host for you from a bucket and region. Every S3-family adapter also accepts optional named constructor arguments:
 
 ```php
@@ -185,8 +189,13 @@ $device = new Wasabi(
 All storage adapters provide a consistent API for working with files:
 
 ```php
-// Upload file contents
-$device->uploadData(file_get_contents('/path/to/local/file.jpg'), 'remote/path/file.jpg', 'image/jpeg');
+use Utopia\Psr7\Stream;
+
+// Upload a file from disk without loading it into memory
+$device->upload(Stream::fromResource(fopen('/path/to/local/file.jpg', 'rb')), 'remote/path/file.jpg', 'image/jpeg');
+
+// Contents already in memory wrap into a stream for free
+$device->upload(new Stream($data), 'remote/path/file.jpg', 'image/jpeg');
 
 // Check if file exists
 $exists = $device->exists('remote/path/file.jpg');
@@ -200,20 +209,20 @@ $mime = $device->getFileMimeType('remote/path/file.jpg');
 // Get file MD5 hash
 $hash = $device->getFileHash('remote/path/file.jpg');
 
-// Read file contents
-$contents = $device->read('remote/path/file.jpg');
+// Read file contents as a PSR-7 stream; casting to string buffers it
+$stream = $device->read('remote/path/file.jpg');
 
 // Read partial file contents
-$chunk = $device->read('remote/path/file.jpg', 0, 1024); // Read first 1KB
+$chunk = (string) $device->read('remote/path/file.jpg', 0, 1024); // Read first 1KB
 
-// Multipart/chunked uploads
+// Multipart/chunked uploads — part 1 of 3
 $metadata = [];
-$device->uploadData($firstChunk, 'remote/video.mp4', 'video/mp4', 1, 3, $metadata); // Part 1 of 3
+$device->upload(new Stream($firstChunk), 'remote/video.mp4', 'video/mp4', 1, 3, $metadata);
 
 // Resumable uploads: prepare, upload chunks in any order, finalize
-$device->prepareUpload('remote/video.mp4', 'video/mp4', 3, $metadata);
-$device->uploadChunk($secondChunk, 'remote/video.mp4', 2, 3, $metadata);
-$device->finalizeUpload('remote/video.mp4', 3, $metadata);
+$device->prepare('remote/video.mp4', 'video/mp4', 3, $metadata);
+$device->upload(new Stream($secondChunk), 'remote/video.mp4', 'video/mp4', 2, 3, $metadata);
+$device->finalize('remote/video.mp4', 3, $metadata);
 
 // List files under a prefix, one page at a time
 $list = $device->listFiles('remote/directory', 100);
@@ -230,16 +239,20 @@ $device->delete('remote/path/file.jpg');
 // Delete directory
 $device->deletePath('remote/directory');
 
-// Transfer files between storage devices
-$sourceDevice->transfer('source/path.jpg', 'target/path.jpg', $targetDevice);
+// Copy a file, on the same device or onto another one
+$device->copy('source/path.jpg', 'target/path.jpg');
+$sourceDevice->copy('source/path.jpg', 'target/path.jpg', $targetDevice);
 
-// Transfer with a custom chunk size (default: 20 MB)
-$sourceDevice->transfer('source/path.jpg', 'target/path.jpg', $targetDevice, 10000000);
+// Copy with a custom chunk size (default: 20 MB)
+$sourceDevice->copy('source/path.jpg', 'target/path.jpg', $targetDevice, 10000000);
+
+// Move is copy plus delete
+$device->move('source/path.jpg', 'target/path.jpg');
 ```
 
 ## Custom HTTP client
 
-The S3-family adapters send requests through any [PSR-18](https://www.php-fig.org/psr/psr-18/) client. By default they use [utopia-php/client](https://github.com/utopia-php/client) with the cURL adapter, no request timeout, and the `Retry` decorator configured with `S3\RetryStrategy` — it retries transient S3 rate-limiting errors (`SlowDown`, `ServiceUnavailable`, `Throttling`, `RequestThrottled`, and plain 429/503 responses) three times with a 500 ms delay.
+The S3-family adapters send requests through any [PSR-18](https://www.php-fig.org/psr/psr-18/) client. By default they use [utopia-php/client](https://github.com/utopia-php/client) with the cURL adapter — no overall request timeout, a stall watchdog that aborts once no bytes move for 60 seconds, TCP keepalive — and the `Retry` decorator configured with `S3\RetryStrategy`. It retries transient S3 rate-limiting errors (`SlowDown`, `ServiceUnavailable`, `Throttling`, `RequestThrottled`, and plain 429/503 responses) three times with exponential backoff and full jitter from a 0.5 second base delay.
 
 Inject your own client to change the transport or the retry policy — for example the Swoole coroutine adapter with more aggressive retries:
 
@@ -300,6 +313,15 @@ use Utopia\Storage\Device\Telemetry;
 $device = new Telemetry($telemetryAdapter, new Local('/path/to/storage'));
 ```
 
+## Upgrading from 3.x
+
+Version 4.0 makes streaming the only I/O path: file contents move as PSR-7 streams end to end, so memory stays bounded regardless of file size.
+
+- `read()` returns a `Psr\Http\Message\StreamInterface` instead of a string. Cast with `(string)` or call `getContents()` where you want the bytes in memory — the cost is visible at the call site.
+- `write()` and the upload methods take a `StreamInterface` instead of a string. Wrap a string with `new Utopia\Psr7\Stream($data)`; wrap an open file handle with `Utopia\Psr7\Stream::fromResource($handle)`. S3 requires seekable streams: the payload is hashed for signing, rewound and sent — the same approach the AWS SDK takes.
+- `uploadData()` and `uploadChunk()` merged into one `upload($data, $path, $contentType, $chunk, $chunks, $metadata)` — a whole file is the single-chunk default. `prepareUpload()`/`finalizeUpload()` are now `prepare()`/`finalize()`.
+- `transfer()` is replaced by `copy($source, $target, $to, $chunkSize)` where `$to` defaults to the same device, and `move()` now works across every adapter as copy plus delete (`Local` still renames in place). The `TRANSFER_CHUNK_SIZE` constant is now `COPY_CHUNK_SIZE`.
+
 ## Upgrading from 2.x
 
 Version 3.0 makes every device immutable and safe to share across coroutines, and removes all global state:

packages/storage/src/Storage/Device.php

@@ -4,6 +4,7 @@
 
 namespace Utopia\Storage;
 
+use Psr\Http\Message\StreamInterface;
 use Utopia\Storage\Exception\StorageException;
 use Utopia\Storage\Exception\UploadException;
 
@@ -13,9 +14,9 @@
 abstract class Device
 {
     /**
-     * Default max chunk size while transferring file from one device to another
+     * Default max chunk size while copying a file from one device to another
      */
-    public const TRANSFER_CHUNK_SIZE = 20000000; // 20 MB
+    public const COPY_CHUNK_SIZE = 20000000; // 20 MB
 
     /**
      * Get Type.
@@ -39,55 +40,57 @@ abstract public function getRoot(): string;
     abstract public function getPath(string $filename): string;
 
     /**
-     * Prepare Upload.
+     * Prepare.
      *
      * Initialize adapter-specific upload state without transferring a chunk body.
      *
      * @param  UploadMetadata  $metadata
      *
      * @throws StorageException
      */
-    abstract public function prepareUpload(string $path, string $contentType, int $chunks = 1, array &$metadata = []): void;
+    abstract public function prepare(string $path, string $contentType, int $chunks = 1, array &$metadata = []): void;
 
     /**
      * Upload Chunk.
      *
-     * Upload exactly one chunk of file contents without finalizing the full upload.
+     * Store exactly one chunk of file contents without finalizing the full upload.
      * Returns the number of chunks received so far.
      *
      * @param  UploadMetadata  $metadata
      *
      * @throws StorageException
      */
-    abstract public function uploadChunk(string $data, string $path, int $chunk = 1, int $chunks = 1, array &$metadata = []): int;
+    abstract protected function uploadChunk(StreamInterface $data, string $path, int $chunk, int $chunks, array &$metadata): int;
 
     /**
-     * Finalize Upload.
+     * Finalize.
      *
      * Complete a prepared upload once all chunks are known to be present.
      *
      * @param  UploadMetadata  $metadata
      *
      * @throws StorageException
      */
-    abstract public function finalizeUpload(string $path, int $chunks = 1, array &$metadata = []): bool;
+    abstract public function finalize(string $path, int $chunks = 1, array &$metadata = []): bool;
 
     /**
-     * Upload Data.
+     * Upload.
      *
-     * Upload file contents to desired destination in the selected disk.
+     * Upload one chunk of file contents to the desired destination, preparing the
+     * upload on first contact and finalizing it once every chunk has arrived.
+     * A whole file is the default single-chunk case.
      * Returns the number of chunks received so far.
      *
      * @param  UploadMetadata  $metadata
      *
      * @throws StorageException
      */
-    public function uploadData(string $data, string $path, string $contentType, int $chunk = 1, int $chunks = 1, array &$metadata = []): int
+    public function upload(StreamInterface $data, string $path, string $contentType, int $chunk = 1, int $chunks = 1, array &$metadata = []): int
     {
-        $this->prepareUpload($path, $contentType, $chunks, $metadata);
+        $this->prepare($path, $contentType, $chunks, $metadata);
         $chunksReceived = $this->uploadChunk($data, $path, $chunk, $chunks, $metadata);
 
-        if ($chunks > 1 && $chunks === $chunksReceived && ! $this->finalizeUpload($path, $chunks, $metadata)) {
+        if ($chunks > 1 && $chunks === $chunksReceived && ! $this->finalize($path, $chunks, $metadata)) {
             throw new UploadException('Failed to finalize upload ' . $path);
         }
 
@@ -100,24 +103,68 @@ public function uploadData(string $data, string $path, string $contentType, int
     abstract public function abort(string $path, string $uploadId = ''): bool;
 
     /**
-     * Read file by given path.
+     * Read file or part of file by given path, offset and length.
+     *
+     * The returned stream is positioned at the start of the requested window.
      *
      * @param  int<0, max>  $offset
      * @param  int<0, max>|null  $length
      */
-    abstract public function read(string $path, int $offset = 0, ?int $length = null): string;
+    abstract public function read(string $path, int $offset = 0, ?int $length = null): StreamInterface;
 
     /**
-     * Transfer
-     * Transfer a file from current device to destination device.
+     * Write file by given path.
      *
+     * The stream is consumed in full: seekable streams are rewound and sent
+     * from the beginning on every adapter.
      */
-    abstract public function transfer(string $path, string $destination, Device $device, int $chunkSize = self::TRANSFER_CHUNK_SIZE): bool;
+    abstract public function write(string $path, StreamInterface $data, string $contentType): bool;
 
     /**
-     * Write file by given path.
+     * Copy a file to another path, on this device or onto another one.
+     *
+     * Large files are piped chunk by chunk, so memory stays bounded regardless
+     * of file size. A started multipart upload is aborted on failure.
+     *
+     * @throws StorageException
      */
-    abstract public function write(string $path, string $data, string $contentType): bool;
+    public function copy(string $source, string $target, ?Device $to = null, int $chunkSize = self::COPY_CHUNK_SIZE): bool
+    {
+        if ($chunkSize <= 0) {
+            throw new \InvalidArgumentException('Chunk size must be greater than zero');
+        }
+
+        $to ??= $this;
+        $size = $this->getFileSize($source);
+        $contentType = $this->getFileMimeType($source);
+
+        if ($size <= $chunkSize) {
+            return $to->write($target, $this->read($source), $contentType);
+        }
+
+        $totalChunks = (int) ceil($size / $chunkSize);
+        $metadata = ['content_type' => $contentType];
+        try {
+            for ($counter = 0; $counter < $totalChunks; ++$counter) {
+                $data = $this->read($source, $counter * $chunkSize, $chunkSize);
+                $to->upload($data, $target, $contentType, $counter + 1, $totalChunks, $metadata);
+            }
+        } catch (\Throwable $e) {
+            // Best effort, and only once a multipart upload was actually started —
+            // its unclaimed parts are billed until aborted. Aborting without one
+            // could delete a pre-existing destination the copy never touched.
+            $uploadId = $metadata['uploadId'] ?? null;
+            if (\is_string($uploadId) && $uploadId !== '') {
+                try {
+                    $to->abort($target, $uploadId);
+                } catch (\Throwable) {
+                }
+            }
+            throw $e;
+        }
+
+        return true;
+    }
 
     /**
      * Move file from given source to given path, return true on success and false on failure.
@@ -128,7 +175,7 @@ public function move(string $source, string $target): bool
             return false;
         }
 
-        if ($this->transfer($source, $target, $this)) {
+        if ($this->copy($source, $target)) {
             return $this->delete($source);
         }
 

packages/storage/src/Storage/Device/AWS.php

@@ -5,6 +5,7 @@
 namespace Utopia\Storage\Device;
 
 use Psr\Http\Client\ClientInterface;
+use Utopia\Psr18\StreamingClientInterface;
 use Utopia\Storage\Acl;
 use Utopia\Storage\DeviceType;
 
@@ -76,19 +77,18 @@ public function __construct(
         string $bucket,
         string $region = self::US_EAST_1,
         Acl $acl = Acl::Private,
-        ?ClientInterface $client = null,
+        (ClientInterface&StreamingClientInterface)|null $client = null,
     ) {
         $host = match ($region) {
             self::CN_NORTH_1, self::CN_NORTH_4, self::CN_NORTHWEST_1 => $bucket . '.s3.' . $region . '.amazonaws.cn',
             default => $bucket . '.s3.' . $region . '.amazonaws.com'
         };
-        parent::__construct($root, $accessKey, $secretKey, $host, $region, $acl, $client);
+        parent::__construct($root, $accessKey, $secretKey, $host, $region, $acl, $client, $bucket);
     }
 
     #[\Override]
     public function getType(): DeviceType
     {
         return DeviceType::AwsS3;
     }
-
 }

packages/storage/src/Storage/Device/Backblaze.php

@@ -5,6 +5,7 @@
 namespace Utopia\Storage\Device;
 
 use Psr\Http\Client\ClientInterface;
+use Utopia\Psr18\StreamingClientInterface;
 use Utopia\Storage\Acl;
 use Utopia\Storage\DeviceType;
 
@@ -37,10 +38,10 @@ public function __construct(
         string $bucket,
         string $region = self::US_WEST_004,
         Acl $acl = Acl::Private,
-        ?ClientInterface $client = null,
+        (ClientInterface&StreamingClientInterface)|null $client = null,
     ) {
         $host = $bucket . '.' . 's3' . '.' . $region . '.backblazeb2.com';
-        parent::__construct($root, $accessKey, $secretKey, $host, $region, $acl, $client);
+        parent::__construct($root, $accessKey, $secretKey, $host, $region, $acl, $client, $bucket);
     }
 
     #[\Override]

packages/storage/src/Storage/Device/DOSpaces.php

@@ -5,6 +5,7 @@
 namespace Utopia\Storage\Device;
 
 use Psr\Http\Client\ClientInterface;
+use Utopia\Psr18\StreamingClientInterface;
 use Utopia\Storage\Acl;
 use Utopia\Storage\DeviceType;
 
@@ -36,10 +37,10 @@ public function __construct(
         string $bucket,
         string $region = self::NYC3,
         Acl $acl = Acl::Private,
-        ?ClientInterface $client = null,
+        (ClientInterface&StreamingClientInterface)|null $client = null,
     ) {
         $host = $bucket . '.' . $region . '.digitaloceanspaces.com';
-        parent::__construct($root, $accessKey, $secretKey, $host, $region, $acl, $client);
+        parent::__construct($root, $accessKey, $secretKey, $host, $region, $acl, $client, $bucket);
     }
 
     #[\Override]