diff --git a/.gitignore b/.gitignore
index aa7158a2..28a0fa45 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,4 +1,6 @@
/vendor/
/.vscode/
.phpunit.result.cache
-tests/chunk.php
\ No newline at end of file
+tests/chunk.php
+/.idea
+
diff --git a/src/Storage/Device.php b/src/Storage/Device.php
index 619e401a..cc43b7fd 100644
--- a/src/Storage/Device.php
+++ b/src/Storage/Device.php
@@ -39,7 +39,7 @@ abstract public function getRoot(): string;
* Each device hold a complex directory structure that is being build in this method.
*
* @param string $filename
- * @param string $prefix
+ * @param string|null $prefix
*
* @return string
*/
diff --git a/src/Storage/Device/BackBlaze.php b/src/Storage/Device/Backblaze.php
similarity index 81%
rename from src/Storage/Device/BackBlaze.php
rename to src/Storage/Device/Backblaze.php
index b0fcd597..7f7d9b7d 100644
--- a/src/Storage/Device/BackBlaze.php
+++ b/src/Storage/Device/Backblaze.php
@@ -4,7 +4,8 @@
use Utopia\Storage\Device\S3;
-class BackBlaze extends S3
+
+class Backblaze extends S3
{
/**
* Regions constants
@@ -20,7 +21,7 @@ class BackBlaze extends S3
const EU_CENTRAL_004 = 'eu-central-004';
/**
- * BackBlaze Constructor
+ * Backblaze Constructor
*
* @param string $root
* @param string $accessKey
@@ -31,8 +32,8 @@ class BackBlaze extends S3
*/
public function __construct(string $root, string $accessKey, string $secretKey, string $bucket, string $region = self::US_WEST_004, string $acl = self::ACL_PRIVATE)
{
- parent::__construct($root, $accessKey, $secretKey, $bucket, $region, $acl);
- $this->headers['host'] = $bucket . '.' . 's3' . '.' . $region . '.backblazeb2.com';
+ $hostName = $bucket . '.' . 's3' . '.' . $region . '.backblazeb2.com';
+ parent::__construct($root, $accessKey, $secretKey, $bucket, $region, $acl, $hostName);
}
/**
@@ -40,7 +41,7 @@ public function __construct(string $root, string $accessKey, string $secretKey,
*/
public function getName(): string
{
- return 'BackBlaze B2 Storage';
+ return 'Backblaze B2 Storage';
}
/**
@@ -48,6 +49,6 @@ public function getName(): string
*/
public function getDescription(): string
{
- return 'BackBlaze B2 Storage';
+ return 'Backblaze B2 Storage';
}
}
diff --git a/src/Storage/Device/DOSpaces.php b/src/Storage/Device/DOSpaces.php
index f2f9ebc5..f5051b0e 100644
--- a/src/Storage/Device/DOSpaces.php
+++ b/src/Storage/Device/DOSpaces.php
@@ -4,6 +4,7 @@
use Utopia\Storage\Device\S3;
+
class DOSpaces extends S3
{
/**
@@ -29,8 +30,8 @@ class DOSpaces extends S3
*/
public function __construct(string $root, string $accessKey, string $secretKey, string $bucket, string $region = self::NYC3, string $acl = self::ACL_PRIVATE)
{
- parent::__construct($root, $accessKey, $secretKey, $bucket, $region, $acl);
- $this->headers['host'] = $bucket . '.' . $region . '.digitaloceanspaces.com';
+ $hostName = $bucket . '.' . $region . '.digitaloceanspaces.com';
+ parent::__construct($root, $accessKey, $secretKey, $bucket, $region, $acl, $hostName);
}
/**
diff --git a/src/Storage/Device/Linode.php b/src/Storage/Device/Linode.php
index affd723b..502363b2 100644
--- a/src/Storage/Device/Linode.php
+++ b/src/Storage/Device/Linode.php
@@ -4,16 +4,17 @@
use Utopia\Storage\Device\S3;
+
class Linode extends S3
{
/**
* Regions constants
*
*/
- const EU_CENTRAL_1='eu-central-1';
- const US_SOUTHEAST_1='us-southeast-1';
- const US_EAST_1='us-east-1';
- const AP_SOUTH_1='ap-south-1';
+ const EU_CENTRAL_1 = 'eu-central-1';
+ const US_SOUTHEAST_1 = 'us-southeast-1';
+ const US_EAST_1 = 'us-east-1';
+ const AP_SOUTH_1 = 'ap-south-1';
/**
@@ -28,8 +29,8 @@ class Linode extends S3
*/
public function __construct(string $root, string $accessKey, string $secretKey, string $bucket, string $region = self::EU_CENTRAL_1, string $acl = self::ACL_PRIVATE)
{
- parent::__construct($root, $accessKey, $secretKey, $bucket, $region, $acl);
- $this->headers['host'] = $bucket.'.'.$region.'.'.'linodeobjects.com';
+ $hostName = $bucket.'.'.$region.'.linodeobjects.com';
+ parent::__construct($root, $accessKey, $secretKey, $bucket, $region, $acl, $hostName);
}
/**
diff --git a/src/Storage/Device/S3.php b/src/Storage/Device/S3.php
index 8f33a9da..e882411b 100644
--- a/src/Storage/Device/S3.php
+++ b/src/Storage/Device/S3.php
@@ -46,6 +46,7 @@ class S3 extends Device
const US_GOV_EAST_1 = 'us-gov-east-1';
const US_GOV_WEST_1 = 'us-gov-west-1';
+
/**
* AWS ACL Flag constants
*/
@@ -57,44 +58,38 @@ class S3 extends Device
/**
* @var string
*/
- protected $accessKey;
+ protected string $accessKey;
/**
* @var string
*/
- protected $secretKey;
+ protected string $secretKey;
/**
* @var string
*/
- protected $bucket;
-
+ protected string $bucket;
+
/**
* @var string
*/
- protected $region;
-
+ protected string $region;
+
/**
* @var string
*/
- protected $acl = self::ACL_PRIVATE;
-
+ protected string $acl = self::ACL_PRIVATE;
+
/**
* @var string
*/
- protected $root = 'temp';
-
- /**
- * @var array
- */
- protected $headers = [
- 'host' => '', 'date' => '', 'content-md5' => '', 'content-type' => '',
- ];
+ protected string $hostName;
/**
- * @var array
+ * @var string
*/
- protected $amzHeaders;
+ protected string $root = 'temp';
+
/**
* S3 Constructor
@@ -105,8 +100,9 @@ class S3 extends Device
* @param string $bucket
* @param string $region
* @param string $acl
+ * @param string $hostName
*/
- public function __construct(string $root, string $accessKey, string $secretKey, string $bucket, string $region = self::US_EAST_1, string $acl = self::ACL_PRIVATE)
+ public function __construct(string $root, string $accessKey, string $secretKey, string $bucket, string $region, string $acl = self::ACL_PRIVATE, string $hostName = '')
{
$this->accessKey = $accessKey;
$this->secretKey = $secretKey;
@@ -114,8 +110,7 @@ public function __construct(string $root, string $accessKey, string $secretKey,
$this->region = $region;
$this->root = $root;
$this->acl = $acl;
- $this->headers['host'] = $this->bucket . '.s3.'.$this->region.'.amazonaws.com';
- $this->amzHeaders = [];
+ $this->hostName = $hostName;
}
/**
@@ -131,20 +126,12 @@ public function getName(): string
*/
public function getDescription(): string
{
- return 'S3 Bucket Storage drive for AWS or on premise solution';
- }
-
- /**
- * @return string
- */
- public function getRoot(): string
- {
- return $this->root;
+ return 'S3 Generic Bucket Storage drive for AWS compatible solutions';
}
/**
* @param string $filename
- * @param string $prefix
+ * @param string|null $prefix
*
* @return string
*/
@@ -156,13 +143,29 @@ public function getPath(string $filename, string $prefix = null): string
$path = ($i < \strlen($filename)) ? $path . DIRECTORY_SEPARATOR . $filename[$i] : $path . DIRECTORY_SEPARATOR . 'x';
}
- if(!is_null($prefix)) {
+ if (!is_null($prefix)) {
$path = $prefix . DIRECTORY_SEPARATOR . $path;
}
return $this->getRoot() . $path . DIRECTORY_SEPARATOR . $filename;
}
+ /**
+ * @return string
+ */
+ public function getRoot(): string
+ {
+ return $this->root;
+ }
+
+ /**
+ * @return string
+ */
+ public function getHostName(): string
+ {
+ return !empty($this->hostName)? $this->hostName : $this->bucket . '.s3.' . $this->region.'.amazonaws.com';
+ }
+
/**
* Upload.
*
@@ -175,17 +178,17 @@ public function getPath(string $filename, string $prefix = null): string
* @param int chunks
* @param array $metadata
*
+ * @return int
* @throws \Exception
*
- * @return int
*/
public function upload(string $source, string $path, int $chunk = 1, int $chunks = 1, array &$metadata = []): int
{
- if($chunk == 1 && $chunks == 1) {
+ if ($chunk == 1 && $chunks == 1) {
return $this->write($path, \file_get_contents($source), \mime_content_type($source));
}
$uploadId = $metadata['uploadId'] ?? null;
- if(empty($uploadId)) {
+ if (empty($uploadId)) {
$uploadId = $this->createMultipartUpload($path, $metadata['content_type']);
$metadata['uploadId'] = $uploadId;
}
@@ -195,61 +198,280 @@ public function upload(string $source, string $path, int $chunk = 1, int $chunks
$metadata['parts'][] = ['partNumber' => $chunk, 'etag' => $etag];
$metadata['chunks'] ??= 0;
$metadata['chunks']++;
- if($metadata['chunks'] == $chunks) {
- $this->completeMultipartUpload($path, $uploadId, $metadata['parts']);
+ if ($metadata['chunks'] == $chunks) {
+ $this->completeMultipartUpload($path, $uploadId, $metadata['parts'], $source);
}
return $metadata['chunks'];
}
+ /**
+ * Write file by given path.
+ *
+ * @param string $path
+ * @param string $data
+ *
+ * @return bool
+ * @throws \Exception
+ *
+ */
+ public function write(string $path, string $data, string $contentType = ''): bool
+ {
+
+ $uri = '/';
+
+ if($path !== '') {
+
+ $uri = \str_replace(['%2F', '%3F'], ['/', '?'], \rawurlencode($path));
+
+ if (!str_starts_with($uri, '/')) {
+ $uri .= '/' . $uri;
+ }
+ }
+
+ $this->call(self::METHOD_PUT, $uri, $data, [], [
+ 'content-type' => $contentType,
+ 'content-md5' => \base64_encode(md5($data, true)),
+ 'x-amz-content-sha256' => \hash('sha256', $data),
+ 'x-amz-acl' => $this->acl
+ ]);
+
+ return true;
+ }
+
+ /**
+ * Get the S3 response
+ *
+ * @param string $method
+ * @param string $uri
+ * @param string $data
+ * @param array $parameters
+ * @param array $headers
+ *
+ * @return object
+ * @throws \Exception
+ *
+ */
+ private function call(string $method, string $uri, string $data = '', array $parameters = [], array $headers = [])
+ {
+
+ $url = 'https://' . $this->getHostName() . $uri . '?' . \http_build_query($parameters, '', '&', PHP_QUERY_RFC3986);
+ $response = new \stdClass;
+ $response->body = '';
+ $response->headers = [];
+
+ // Basic setup
+ $curl = \curl_init();
+ \curl_setopt($curl, CURLOPT_USERAGENT, 'utopia-php/storage');
+ \curl_setopt($curl, CURLOPT_URL, $url);
+
+ // Headers
+ $httpHeaders = [];
+ $headers['x-amz-date'] = \gmdate('Ymd\THis\Z');
+ $headers['date'] = \gmdate('D, d M Y H:i:s T');
+ $headers['host'] = $this->getHostName();
+
+ if (!isset($headers['x-amz-content-sha256'])) {
+ $headers['x-amz-content-sha256'] = \hash('sha256', $data);
+ }
+
+ foreach ($headers as $header => $value) {
+ if (\strlen($value) > 0) {
+ $httpHeaders[] = $header . ': ' . $value;
+ }
+ }
+
+ $httpHeaders[] = 'Authorization: ' . $this->getSignatureV4($method, $uri, $parameters, $headers);
+ \curl_setopt($curl, CURLOPT_HTTPHEADER, $httpHeaders);
+ \curl_setopt($curl, CURLOPT_HEADER, false);
+ \curl_setopt($curl, CURLOPT_RETURNTRANSFER, false);
+ \curl_setopt($curl, CURLOPT_WRITEFUNCTION, function ($curl, string $data) use ($response) {
+ $response->body .= $data;
+ return \strlen($data);
+ });
+ curl_setopt($curl, CURLOPT_HEADERFUNCTION, function ($curl, string $header) use (&$response) {
+ $len = strlen($header);
+ $header = explode(':', $header, 2);
+
+ if (count($header) < 2) { // ignore invalid headers
+ return $len;
+ }
+
+ $response->headers[strtolower(trim($header[0]))] = trim($header[1]);
+
+ return $len;
+ });
+ \curl_setopt($curl, CURLOPT_FOLLOWLOCATION, true);
+ \curl_setopt($curl, CURLOPT_CUSTOMREQUEST, $method);
+
+ // Request types
+ switch ($method) {
+ case self::METHOD_PUT:
+ case self::METHOD_POST: // POST only used for CloudFront
+ \curl_setopt($curl, CURLOPT_POSTFIELDS, $data);
+ break;
+ case self::METHOD_HEAD:
+ case self::METHOD_DELETE:
+ \curl_setopt($curl, CURLOPT_NOBODY, true);
+ break;
+ }
+
+ $result = \curl_exec($curl);
+
+ if (!$result) {
+ throw new Exception(\curl_error($curl));
+ }
+
+ $response->code = \curl_getinfo($curl, CURLINFO_HTTP_CODE);
+ if ($response->code >= 400) {
+ throw new Exception($response->body, $response->code);
+ }
+
+ \curl_close($curl);
+
+ // Parse body into XML
+ if ((isset($response->headers['content-type']) && $response->headers['content-type'] == 'application/xml') || (str_starts_with($response->body, 'headers['content-type'] ?? '') !== 'image/svg+xml')) {
+ $response->body = \simplexml_load_string($response->body);
+ $response->body = json_decode(json_encode($response->body), true);
+ }
+
+ return $response;
+ }
+
+ /**
+ * Generate the headers for AWS Signature V4
+ * @param string $method
+ * @param string $uri
+ * @param array parameters
+ * @param array headers
+ *
+ * @return string
+ */
+ private function getSignatureV4(string $method, string $uri, array $parameters = [], $headers = []): string
+ {
+ $service = 's3';
+ $region = $this->region;
+ $algorithm = 'AWS4-HMAC-SHA256';
+ $combinedHeaders = [];
+ $amzDateStamp = \substr($headers['x-amz-date'], 0, 8);
+
+ // CanonicalHeaders
+ foreach ($headers as $k => $v) {
+ $combinedHeaders[\strtolower($k)] = \trim($v);
+ }
+
+ uksort($combinedHeaders, [& $this, 'sortMetaHeadersCmp']);
+
+ // Convert null query string parameters to strings and sort
+ uksort($parameters, [& $this, 'sortMetaHeadersCmp']);
+ $queryString = \http_build_query($parameters, '', '&', PHP_QUERY_RFC3986);
+
+ // Payload
+ $amzPayload = [$method];
+
+ $qsPos = \strpos($uri, '?');
+ $amzPayload[] = ($qsPos === false ? $uri : \substr($uri, 0, $qsPos));
+ $amzPayload[] = $queryString;
+
+ foreach ($combinedHeaders as $k => $v) { // add header as string to requests
+ $amzPayload[] = $k . ':' . $v;
+ }
+
+ $amzPayload[] = ''; // add a blank entry so we end up with an extra line break
+ $amzPayload[] = \implode(';', \array_keys($combinedHeaders)); // SignedHeaders
+ $amzPayload[] = $headers['x-amz-content-sha256']; // payload hash
+
+ $amzPayloadStr = \implode("\n", $amzPayload); // request as string
+
+ // CredentialScope
+ $credentialScope = [$amzDateStamp, $region, $service, 'aws4_request'];
+
+ // stringToSign
+ $stringToSignStr = \implode("\n", [$algorithm, $headers['x-amz-date'],
+ \implode('/', $credentialScope), \hash('sha256', $amzPayloadStr)]);
+
+ // Make Signature
+ $kSecret = 'AWS4' . $this->secretKey;
+ $kDate = \hash_hmac('sha256', $amzDateStamp, $kSecret, true);
+ $kRegion = \hash_hmac('sha256', $region, $kDate, true);
+ $kService = \hash_hmac('sha256', $service, $kRegion, true);
+ $kSigning = \hash_hmac('sha256', 'aws4_request', $kService, true);
+
+ $signature = \hash_hmac('sha256', \utf8_encode($stringToSignStr), $kSigning);
+
+ return $algorithm . ' ' . \implode(',', [
+ 'Credential=' . $this->accessKey . '/' . \implode('/', $credentialScope),
+ 'SignedHeaders=' . \implode(';', \array_keys($combinedHeaders)),
+ 'Signature=' . $signature,
+ ]);
+ }
+
/**
* Start Multipart Upload
- *
+ *
* Initiate a multipart upload and return an upload ID.
- *
+ *
* @param string $path
* @param string $contentType
- *
- * @throws \Exception
- *
+ *
* @return string
+ * @throws \Exception
+ *
*/
protected function createMultipartUpload(string $path, string $contentType): string
{
- $uri = $path !== '' ? '/' . \str_replace(['%2F', '%3F'], ['/', '?'], \rawurlencode($path)) : '/';
+ $uri = '/';
+
+ if($path !== '') {
+
+ $uri = \str_replace(['%2F', '%3F'], ['/', '?'], \rawurlencode($path));
+
+ if (!str_starts_with($uri, '/')) {
+ $uri .= '/' . $uri;
+ }
+ }
+
+ $response = $this->call(self::METHOD_POST, $uri, '', ['uploads' => ''], [
+ 'content-type' => $contentType,
+ 'content-md5' => \base64_encode(md5('', true)),
+ 'x-amz-acl' => $this->acl
+ ]);
- $this->headers['content-md5'] = \base64_encode(md5('', true));
- unset($this->amzHeaders['x-amz-content-sha256']);
- $this->headers['content-type'] = $contentType;
- $this->amzHeaders['x-amz-acl'] = $this->acl;
- $response = $this->call(self::METHOD_POST, $uri, '', ['uploads' => '']);
return $response->body['UploadId'];
}
/**
* Upload Part
- *
+ *
* @param string $source
* @param string $path
* @param int $chunk
* @param string $uploadId
- *
- * @throws \Exception
- *
+ *
* @return string
+ * @throws \Exception
+ *
*/
- protected function uploadPart(string $source, string $path, int $chunk, string $uploadId) : string
+ protected function uploadPart(string $source, string $path, int $chunk, string $uploadId): string
{
- $uri = $path !== '' ? '/' . \str_replace(['%2F', '%3F'], ['/', '?'], \rawurlencode($path)) : '/';
-
- $data = \file_get_contents($source);
- $this->headers['content-type'] = \mime_content_type($source);
- $this->headers['content-md5'] = \base64_encode(md5($data, true));
- $this->amzHeaders['x-amz-content-sha256'] = \hash('sha256', $data);
- unset($this->amzHeaders['x-amz-acl']); // ACL header is not allowed in parts, only createMultipartUpload accepts this header.
+ $uri = '/';
+
+ if($path !== '') {
+ $uri = \str_replace(['%2F', '%3F'], ['/', '?'], \rawurlencode($path));
+
+ if (!str_starts_with($uri, '/')) {
+ $uri .= '/' . $uri;
+ }
+ }
+
+ $data = \file_get_contents($source);
$response = $this->call(self::METHOD_PUT, $uri, $data, [
- 'partNumber'=>$chunk,
+ 'partNumber' => $chunk,
'uploadId' => $uploadId
+ ], [
+ 'content-type' => \mime_content_type($source),
+ 'content-md5' => \base64_encode(md5($data, true)),
+ 'x-amz-content-sha256' => \hash('sha256', $data)
]);
return $response->headers['etag'];
@@ -257,18 +479,28 @@ protected function uploadPart(string $source, string $path, int $chunk, string $
/**
* Complete Multipart Upload
- *
+ *
* @param string $path
* @param string $uploadId
* @param array $parts
- *
- * @throws \Exception
- *
+ *
* @return bool
+ * @throws \Exception
+ *
*/
- protected function completeMultipartUpload(string $path, string $uploadId, array $parts): bool
+ protected function completeMultipartUpload(string $path, string $uploadId, array $parts, $source): bool
{
- $uri = $path !== '' ? '/' . \str_replace(['%2F', '%3F'], ['/', '?'], \rawurlencode($path)) : '/';
+
+ $uri = '/';
+
+ if($path !== '') {
+
+ $uri = \str_replace(['%2F', '%3F'], ['/', '?'], \rawurlencode($path));
+
+ if (!str_starts_with($uri, '/')) {
+ $uri .= '/' . $uri;
+ }
+ }
$body = '';
foreach ($parts as $part) {
@@ -276,77 +508,45 @@ protected function completeMultipartUpload(string $path, string $uploadId, array
}
$body .= '';
- $this->amzHeaders['x-amz-content-sha256'] = \hash('sha256', $body);
- $this->headers['content-md5'] = \base64_encode(md5($body, true));
- $this->call(self::METHOD_POST, $uri, $body , ['uploadId' => $uploadId]);
+ $this->call(self::METHOD_POST, $uri, $body, [
+ 'uploadId' => $uploadId
+ ], [
+ 'content-md5' => \base64_encode(md5($body, true)),
+ 'content-type' => \mime_content_type($source),
+ 'x-amz-content-sha256' => \hash('sha256', $body)
+ ]);
+
return true;
}
/**
* Abort Chunked Upload
- *
+ *
* @param string $path
* @param string $extra
- *
- * @throws \Exception
- *
+ *
* @return bool
+ * @throws \Exception
+ *
*/
public function abort(string $path, string $extra = ''): bool
{
- $uri = $path !== '' ? '/' . \str_replace(['%2F', '%3F'], ['/', '?'], \rawurlencode($path)) : '/';
- unset($this->headers['content-type']);
- $this->headers['content-md5'] = \base64_encode(md5('', true));
- $this->call(self::METHOD_DELETE, $uri, '', ['uploadId' => $extra]);
- return true;
- }
+ $uri = '/';
- /**
- * Read file or part of file by given path, offset and length.
- *
- * @param string $path
- * @param int offset
- * @param int length
- *
- * @throws \Exception
- *
- * @return string
- */
- public function read(string $path, int $offset = 0, int $length = null): string
- {
- unset($this->amzHeaders['x-amz-acl']);
- unset($this->amzHeaders['x-amz-content-sha256']);
- unset($this->headers['content-type']);
- $this->headers['content-md5'] = \base64_encode(md5('', true));
- $uri = ($path !== '') ? '/' . \str_replace('%2F', '/', \rawurlencode($path)) : '/';
- if($length !== null) {
- $end = $offset + $length - 1;
- $this->headers['range'] = "bytes=$offset-$end";
- }
- $response = $this->call(self::METHOD_GET, $uri);
- return $response->body;
- }
+ if($path !== '') {
- /**
- * Write file by given path.
- *
- * @param string $path
- * @param string $data
- *
- * @throws \Exception
- *
- * @return bool
- */
- public function write(string $path, string $data, string $contentType = ''): bool
- {
- $uri = $path !== '' ? '/' . \str_replace(['%2F', '%3F'], ['/', '?'], \rawurlencode($path)) : '/';
-
- $this->headers['content-type'] = $contentType;
- $this->headers['content-md5'] = \base64_encode(md5($data, true)); //TODO whould this work well with big file? can we skip it?
- $this->amzHeaders['x-amz-content-sha256'] = \hash('sha256', $data);
- $this->amzHeaders['x-amz-acl'] = $this->acl;
+ $uri = \str_replace(['%2F', '%3F'], ['/', '?'], \rawurlencode($path));
+
+ if (!str_starts_with($uri, '/')) {
+ $uri .= '/' . $uri;
+ }
+ }
- $this->call(self::METHOD_PUT, $uri, $data);
+ $this->call(self::METHOD_DELETE, $uri, '', [
+ 'uploadId' => $extra
+ ], [
+ 'content-md5' => \base64_encode(md5('', true))
+ ]);
return true;
}
@@ -358,7 +558,7 @@ public function write(string $path, string $data, string $contentType = ''): boo
*
* @param string $source
* @param string $target
- *
+ *
* @throw \Exception
*
* @return bool
@@ -375,95 +575,192 @@ public function move(string $source, string $target): bool
}
/**
- * Delete file in given path, Return true on success and false on failure.
+ * Returns given file path its mime type.
*
- * @see http://php.net/manual/en/function.filesize.php
+ * @see http://php.net/manual/en/function.mime-content-type.php
*
* @param string $path
- *
- * @throws \Exception
*
- * @return bool
+ * @return string
*/
- public function delete(string $path, bool $recursive = false): bool
+ public function getFileMimeType(string $path): string
{
- $uri = ($path !== '') ? '/' . \str_replace('%2F', '/', \rawurlencode($path)) : '/';
-
- unset($this->headers['content-type']);
- unset($this->amzHeaders['x-amz-acl']);
- unset($this->amzHeaders['x-amz-content-sha256']);
- $this->headers['content-md5'] = \base64_encode(md5('', true));
- $this->call(self::METHOD_DELETE, $uri);
+ $response = $this->getInfo($path);
+ return $response['content-type'] ?? '';
+ }
- return true;
+ /**
+ * Get file info
+ * @return array
+ */
+ private function getInfo(string $path): array
+ {
+ $uri = '/';
+
+ if($path !== '') {
+
+ $uri = \str_replace(['%2F', '%3F'], ['/', '?'], \rawurlencode($path));
+
+ if (!str_starts_with($uri, '/')) {
+ $uri .= '/' . $uri;
+ }
+ }
+
+ $response = $this->call(self::METHOD_HEAD, $uri, '', [], ['content-md5' => \base64_encode(md5('', true))]);
+
+ return $response->headers;
}
/**
- * Get list of objects in the given path.
+ * Read file or part of file by given path, offset and length.
*
* @param string $path
- *
+ * @param int offset
+ * @param int length
+ *
+ * @return string
* @throws \Exception
*
- * @return array
*/
- private function listObjects($prefix = '', $maxKeys = 1000, $continuationToken = '')
+ public function read(string $path, int $offset = 0, int $length = null): string
{
$uri = '/';
- $this->headers['content-type'] = 'text/plain';
- $this->headers['content-md5'] = \base64_encode(md5('', true));
- $parameters = [
- 'list-type' => 2,
- 'prefix' => $prefix,
- 'max-keys' => $maxKeys,
+ if($path !== '') {
+
+ $uri = \str_replace(['%2F', '%3F'], ['/', '?'], \rawurlencode($path));
+
+ if (!str_starts_with($uri, '/')) {
+ $uri .= '/' . $uri;
+ }
+ }
+
+ $headers = [
+ 'content-md5' => \base64_encode(md5('', true))
];
- if(!empty($continuationToken)) {
- $parameters['continuation-token'] = $continuationToken;
+
+ if ($length !== null) {
+ $end = $offset + $length - 1;
+ $this->headers['range'] = "bytes=$offset-$end";
+ $headers['range'] = "bytes=$offset-$end";
}
- $response = $this->call(self::METHOD_GET, $uri, '', $parameters);
+
+ $response = $this->call(self::METHOD_GET, $uri, '', [], $headers);
return $response->body;
}
/**
- * Delete files in given path, path must be a directory. Return true on success and false on failure.
+ * Delete file in given path, Return true on success and false on failure.
+ *
+ * @see http://php.net/manual/en/function.filesize.php
*
* @param string $path
- *
+ *
* @throws \Exception
*
* @return bool
*/
+ public function delete(string $path, bool $recursive = false): bool
+ {
+ $uri = '/';
+
+ if($path !== '') {
+
+ $uri = \str_replace(['%2F', '%3F'], ['/', '?'], \rawurlencode($path));
+
+ if (!str_starts_with($uri, '/')) {
+ $uri .= '/' . $uri;
+ }
+ }
+
+ $this->call(self::METHOD_DELETE, $uri, '', [], [
+ 'content-md5' => \base64_encode(md5('', true)),
+ ]);
+
+ return true;
+ }
+
+ /**
+ * Delete files in given path, path must be a directory. Return true on success and false on failure.
+ *
+ * @param string $path
+ *
+ * @return bool
+ * @throws \Exception
+ *
+ */
public function deletePath(string $path): bool
{
- $path = $this->getRoot() . '/' . $path;
+ $root = $this->getRoot();
+
+ if(str_starts_with($root, '/')){
+ $root = substr($root, 1);
+ }
+
+ $path = $root . '/' . $path;
$uri = '/';
+
$continuationToken = '';
do {
$objects = $this->listObjects($path, continuationToken: $continuationToken);
- $count = (int) ($objects['KeyCount'] ?? 1);
- if($count < 1) {
+ $count = (int)($objects['KeyCount'] ?? 1);
+ if ($count < 1) {
break;
}
$continuationToken = $objects['NextContinuationToken'] ?? '';
$body = '';
- if($count > 1) {
+ if ($count > 1) {
foreach ($objects['Contents'] as $object) {
$body .= "";
}
} else {
- $body .= "";
+ $body .= "";
}
$body .= 'true';
$body .= '';
- $this->amzHeaders['x-amz-content-sha256'] = \hash('sha256', $body);
- $this->headers['content-md5'] = \base64_encode(md5($body, true));
- $this->call(self::METHOD_POST, $uri, $body, ['delete'=>'']);
- } while(!empty($continuationToken));
+
+ $this->call(self::METHOD_POST, $uri, $body, [
+ 'delete' => ''
+ ], [
+ 'content-md5' => \base64_encode(md5($body, true)),
+ 'content-type' => 'text/plain',
+ 'x-amz-content-sha256' => \hash('sha256', $body)
+ ]);
+ } while (!empty($continuationToken));
return true;
}
+ /**
+ * Get list of objects in the given path.
+ *
+ * @param string $path
+ *
+ * @return array
+ * @throws \Exception
+ *
+ */
+ private function listObjects($prefix = '', $maxKeys = 1000, $continuationToken = '')
+ {
+ $uri = '/';
+
+ $parameters = [
+ 'list-type' => 2,
+ 'prefix' => $prefix,
+ 'max-keys' => $maxKeys,
+ ];
+
+ if (!empty($continuationToken)) {
+ $parameters['continuation-token'] = $continuationToken;
+ }
+
+ $response = $this->call(self::METHOD_GET, $uri, '', $parameters, [
+ 'content-type' => 'text/plain',
+ 'content-md5' => \base64_encode(md5('', true))
+ ]);
+ return $response->body;
+ }
+
/**
* Check if file exists
*
@@ -497,21 +794,6 @@ public function getFileSize(string $path): int
return (int)($response['content-length'] ?? 0);
}
- /**
- * Returns given file path its mime type.
- *
- * @see http://php.net/manual/en/function.mime-content-type.php
- *
- * @param string $path
- *
- * @return string
- */
- public function getFileMimeType(string $path): string
- {
- $response = $this->getInfo($path);
- return $response['content-type'] ?? '';
- }
-
/**
* Returns given file path its MD5 hash value.
*
@@ -524,7 +806,7 @@ public function getFileMimeType(string $path): string
public function getFileHash(string $path): string
{
$etag = $this->getInfo($path)['etag'] ?? '';
- return (!empty($etag)) ? substr($etag, 1, -1) : $etag;
+ return (!empty($etag)) ? substr($etag, 1, -1) : $etag;
}
/**
@@ -567,206 +849,13 @@ public function getPartitionTotalSpace(): float
return -1;
}
- /**
- * Get file info
- * @return array
- */
- private function getInfo(string $path): array
- {
- unset($this->headers['content-type']);
- unset($this->amzHeaders['x-amz-acl']);
- unset($this->amzHeaders['x-amz-content-sha256']);
- $this->headers['content-md5'] = \base64_encode(md5('', true));
- $uri = $path !== '' ? '/' . \str_replace('%2F', '/', \rawurlencode($path)) : '/';
- $response = $this->call(self::METHOD_HEAD, $uri);
-
- return $response->headers;
- }
-
- /**
- * Generate the headers for AWS Signature V4
- * @param string $method
- * @param string $uri
- * @param array parameters
- *
- * @return string
- */
- private function getSignatureV4(string $method, string $uri, array $parameters = []): string
- {
- $service = 's3';
- $region = $this->region;
-
- $algorithm = 'AWS4-HMAC-SHA256';
- $combinedHeaders = [];
-
- $amzDateStamp = \substr($this->amzHeaders['x-amz-date'], 0, 8);
-
- // CanonicalHeaders
- foreach ($this->headers as $k => $v) {
- $combinedHeaders[\strtolower($k)] = \trim($v);
- }
-
- foreach ($this->amzHeaders as $k => $v) {
- $combinedHeaders[\strtolower($k)] = \trim($v);
- }
-
- uksort($combinedHeaders, [ & $this, 'sortMetaHeadersCmp']);
-
- // Convert null query string parameters to strings and sort
- uksort($parameters, [ & $this, 'sortMetaHeadersCmp']);
- $queryString = \http_build_query($parameters, '', '&', PHP_QUERY_RFC3986);
-
- // Payload
- $amzPayload = [$method];
-
- $qsPos = \strpos($uri, '?');
- $amzPayload[] = ($qsPos === false ? $uri : \substr($uri, 0, $qsPos));
-
- $amzPayload[] = $queryString;
-
- foreach ($combinedHeaders as $k => $v) { // add header as string to requests
- $amzPayload[] = $k . ':' . $v;
- }
-
- $amzPayload[] = ''; // add a blank entry so we end up with an extra line break
- $amzPayload[] = \implode(';', \array_keys($combinedHeaders)); // SignedHeaders
- $amzPayload[] = $this->amzHeaders['x-amz-content-sha256']; // payload hash
-
- $amzPayloadStr = \implode("\n", $amzPayload); // request as string
-
- // CredentialScope
- $credentialScope = [$amzDateStamp, $region, $service, 'aws4_request'];
-
- // stringToSign
- $stringToSignStr = \implode("\n", [$algorithm, $this->amzHeaders['x-amz-date'],
- \implode('/', $credentialScope), \hash('sha256', $amzPayloadStr)]);
-
- // Make Signature
- $kSecret = 'AWS4' . $this->secretKey;
- $kDate = \hash_hmac('sha256', $amzDateStamp, $kSecret, true);
- $kRegion = \hash_hmac('sha256', $region, $kDate, true);
- $kService = \hash_hmac('sha256', $service, $kRegion, true);
- $kSigning = \hash_hmac('sha256', 'aws4_request', $kService, true);
-
- $signature = \hash_hmac('sha256', \utf8_encode($stringToSignStr), $kSigning);
-
- return $algorithm . ' ' . \implode(',', [
- 'Credential=' . $this->accessKey . '/' . \implode('/', $credentialScope),
- 'SignedHeaders=' . \implode(';', \array_keys($combinedHeaders)),
- 'Signature=' . $signature,
- ]);
- }
-
- /**
- * Get the S3 response
- *
- * @param string $method
- * @param string $uri
- * @param string $data
- * @param array $parameters
- *
- * @throws \Exception
- *
- * @return object
- */
- private function call(string $method, string $uri, string $data = '', array $parameters=[])
- {
- $url = 'https://' . $this->headers['host'] . $uri . '?' . \http_build_query($parameters, '', '&', PHP_QUERY_RFC3986);
- $response = new \stdClass;
- $response->body = '';
- $response->headers = [];
-
- // Basic setup
- $curl = \curl_init();
- \curl_setopt($curl, CURLOPT_USERAGENT, 'utopia-php/storage');
- \curl_setopt($curl, CURLOPT_URL, $url);
-
- // Headers
- $httpHeaders = [];
- $this->amzHeaders['x-amz-date'] = \gmdate('Ymd\THis\Z');
-
- if (!isset($this->amzHeaders['x-amz-content-sha256'])) {
- $this->amzHeaders['x-amz-content-sha256'] = \hash('sha256', $data);
- }
-
- foreach ($this->amzHeaders as $header => $value) {
- if (\strlen($value) > 0) {
- $httpHeaders[] = $header . ': ' . $value;
- }
- }
-
- $this->headers['date'] = \gmdate('D, d M Y H:i:s T');
- foreach ($this->headers as $header => $value) {
- if (\strlen($value) > 0) {
- $httpHeaders[] = $header . ': ' . $value;
- }
- }
-
- $httpHeaders[] = 'Authorization: ' . $this->getSignatureV4($method, $uri, $parameters);
-
- \curl_setopt($curl, CURLOPT_HTTPHEADER, $httpHeaders);
- \curl_setopt($curl, CURLOPT_HEADER, false);
- \curl_setopt($curl, CURLOPT_RETURNTRANSFER, false);
- \curl_setopt($curl, CURLOPT_WRITEFUNCTION, function ($curl, string $data) use ($response) {
- $response->body .= $data;
- return \strlen($data);
- });
- curl_setopt($curl, CURLOPT_HEADERFUNCTION, function ($curl, string $header) use (&$response) {
- $len = strlen($header);
- $header = explode(':', $header, 2);
-
- if (count($header) < 2) { // ignore invalid headers
- return $len;
- }
-
- $response->headers[strtolower(trim($header[0]))] = trim($header[1]);
-
- return $len;
- });
- \curl_setopt($curl, CURLOPT_FOLLOWLOCATION, true);
- \curl_setopt($curl, CURLOPT_CUSTOMREQUEST, $method);
-
- // Request types
- switch ($method) {
- case self::METHOD_PUT:
- case self::METHOD_POST: // POST only used for CloudFront
- \curl_setopt($curl, CURLOPT_POSTFIELDS, $data);
- break;
- case self::METHOD_HEAD:
- case self::METHOD_DELETE:
- \curl_setopt($curl, CURLOPT_NOBODY, true);
- break;
- }
-
- $result = \curl_exec($curl);
-
- if (!$result) {
- throw new Exception(\curl_error($curl));
- }
-
- $response->code = \curl_getinfo($curl, CURLINFO_HTTP_CODE);
- if ($response->code >= 400) {
- throw new Exception($response->body, $response->code);
- }
-
- \curl_close($curl);
-
- // Parse body into XML
- if ((isset($response->headers['content-type']) && $response->headers['content-type'] == 'application/xml') || (str_starts_with($response->body, 'headers['content-type'] ?? '') !== 'image/svg+xml')) {
- $response->body = \simplexml_load_string($response->body);
- $response->body = json_decode(json_encode($response->body), true);
- }
-
- return $response;
- }
-
/**
* Sort compare for meta headers
*
- * @internal Used to sort x-amz meta headers
* @param string $a String A
* @param string $b String B
* @return integer
+ * @internal Used to sort x-amz meta headers
*/
private function sortMetaHeadersCmp($a, $b)
{
diff --git a/src/Storage/Device/Wasabi.php b/src/Storage/Device/Wasabi.php
index 5c77ca22..07761c9f 100644
--- a/src/Storage/Device/Wasabi.php
+++ b/src/Storage/Device/Wasabi.php
@@ -2,6 +2,7 @@
namespace Utopia\Storage\Device;
+use JetBrains\PhpStorm\Pure;
use Utopia\Storage\Device\S3;
class Wasabi extends S3
@@ -34,8 +35,8 @@ class Wasabi extends S3
*/
public function __construct(string $root, string $accessKey, string $secretKey, string $bucket, string $region = self::EU_CENTRAL_1, string $acl = self::ACL_PRIVATE)
{
- parent::__construct($root, $accessKey, $secretKey, $bucket, $region, $acl);
- $this->headers['host'] = $bucket . '.'.'s3'.'.'.$region.'.'.'wasabisys'.'.'.'com';
+ $hostName = $bucket . '.'.'s3'.'.'.$region.'.'.'wasabisys'.'.'.'com';
+ parent::__construct($root, $accessKey, $secretKey, $bucket, $region, $acl, $hostName);
}
/**
@@ -53,4 +54,6 @@ public function getDescription(): string
{
return 'Wasabi Storage';
}
+
}
+
diff --git a/src/Storage/Storage.php b/src/Storage/Storage.php
index b058d017..e2b2dc94 100644
--- a/src/Storage/Storage.php
+++ b/src/Storage/Storage.php
@@ -14,9 +14,10 @@ class Storage
const DEVICE_S3 = 'S3';
const DEVICE_DO_SPACES = 'DOSpaces';
const DEVICE_WASABI = 'Wasabi';
- const DEVICE_BACKBLAZE = 'BackBlaze';
+ const DEVICE_BACKBLAZE = 'Backblaze';
const DEVICE_LINODE= 'Linode';
+
/**
* Devices.
*
@@ -24,7 +25,7 @@ class Storage
*
* @var array
*/
- public static $devices = [];
+ public static array $devices = [];
/**
* Set Device.
@@ -34,11 +35,11 @@ class Storage
* @param string $name
* @param Device $device
*
- * @throws Exception
- *
* @return void
+ *@throws Exception
+ *
*/
- public static function setDevice($name, Device $device): void
+ public static function setDevice(string $name, Device $device): void
{
self::$devices[$name] = $device;
}
@@ -54,7 +55,7 @@ public static function setDevice($name, Device $device): void
*
* @throws Exception
*/
- public static function getDevice($name)
+ public static function getDevice(string $name): Device
{
if (!\array_key_exists($name, self::$devices)) {
throw new Exception('The device "'.$name.'" is not listed');
@@ -72,7 +73,7 @@ public static function getDevice($name)
*
* @return bool
*/
- public static function exists($name)
+ public static function exists(string $name): bool
{
return (bool) \array_key_exists($name, self::$devices);
}
@@ -88,7 +89,7 @@ public static function exists($name)
*
* @return string
*/
- public static function human(int $bytes, $decimals = 2, $system = 'metric')
+ public static function human(int $bytes, int $decimals = 2, string $system = 'metric'): string
{
$mod = ($system === 'binary') ? 1024 : 1000;
diff --git a/tests/Storage/Device/BackBlazeTest.php b/tests/Storage/Device/BackblazeTest.php
similarity index 50%
rename from tests/Storage/Device/BackBlazeTest.php
rename to tests/Storage/Device/BackblazeTest.php
index b2db63eb..d084cbf9 100644
--- a/tests/Storage/Device/BackBlazeTest.php
+++ b/tests/Storage/Device/BackblazeTest.php
@@ -2,29 +2,28 @@
namespace Utopia\Tests;
-use Utopia\Storage\Device\BackBlaze;
+use Utopia\Storage\Device\Backblaze;
use Utopia\Tests\S3Base;
-class BackBlazeTest extends S3Base
+class BackblazeTest extends S3Base
{
protected function init(): void
{
- $this->root = 'root';
+
$key = $_SERVER['BACKBLAZE_ACCESS_KEY'] ?? '';
$secret = $_SERVER['BACKBLAZE_SECRET'] ?? '';
- $bucket = "backblaze-demo";
-
- $this->object = new BackBlaze($this->root, $key, $secret, $bucket, BackBlaze::US_WEST_004, BackBlaze::ACL_PRIVATE);
+ $bucket = "backblaze-demo-1";
+ $this->object = new Backblaze($this->root, $key, $secret, $bucket, BackBlaze::EU_CENTRAL_003, BackBlaze::ACL_PRIVATE);
}
protected function getAdapterName(): string
{
- return 'BackBlaze B2 Storage';
+ return 'Backblaze B2 Storage';
}
protected function getAdapterDescription(): string
{
- return 'BackBlaze B2 Storage';
+ return 'Backblaze B2 Storage';
}
}
diff --git a/tests/Storage/Device/DOSpacesTest.php b/tests/Storage/Device/DOSpacesTest.php
index d7e936af..8422d89d 100644
--- a/tests/Storage/Device/DOSpacesTest.php
+++ b/tests/Storage/Device/DOSpacesTest.php
@@ -9,7 +9,6 @@ class DOSpacesTest extends S3Base
{
protected function init(): void
{
- $this->root = '/root';
$key = $_SERVER['DO_ACCESS_KEY'] ?? '';
$secret = $_SERVER['DO_SECRET'] ?? '';
$bucket = "utopia-storage-tests";
diff --git a/tests/Storage/Device/LinodeTest.php b/tests/Storage/Device/LinodeTest.php
index e5808fe5..f3dbcd93 100644
--- a/tests/Storage/Device/LinodeTest.php
+++ b/tests/Storage/Device/LinodeTest.php
@@ -9,13 +9,11 @@ class LinodeTest extends S3Base
{
protected function init(): void
{
- $this->root = 'root';
- $key = $_SERVER['LINODE_ACCESS_KEY'] ?? '';
+ $key = $_SERVER['LINODE_ACCESS_KEY'] ?? '';
$secret = $_SERVER['LINODE_SECRET'] ?? '';
- $bucket = 'everly-test';
-
- $this->object = new Linode($this->root, $key, $secret, $bucket, Linode::EU_CENTRAL_1, Linode::ACL_PRIVATE);
+ $bucket = 'appwrite-test';
+ $this->object = new Linode($this->root, $key, $secret, $bucket, Linode::US_EAST_1, Linode::ACL_PRIVATE);
}
protected function getAdapterName(): string
diff --git a/tests/Storage/Device/S3Test.php b/tests/Storage/Device/S3Test.php
index e9e81904..94f035b1 100644
--- a/tests/Storage/Device/S3Test.php
+++ b/tests/Storage/Device/S3Test.php
@@ -10,12 +10,11 @@ class S3Test extends S3Base
protected function init(): void
{
- $this->root = '/root';
$key = $_SERVER['S3_ACCESS_KEY'] ?? '';
$secret = $_SERVER['S3_SECRET'] ?? '';
- $bucket = 'utopia-storage-tests';
+ $bucket = 'appwrite-test-bucket';
- $this->object = new S3($this->root, $key, $secret, $bucket, S3::AP_SOUTH_1, S3::ACL_PRIVATE);
+ $this->object = new S3($this->root, $key, $secret, $bucket, S3::EU_WEST_1, S3::ACL_PRIVATE);
}
/**
@@ -28,6 +27,6 @@ protected function getAdapterName() : string
protected function getAdapterDescription(): string
{
- return 'S3 Bucket Storage drive for AWS or on premise solution';
+ return 'S3 Generic Bucket Storage drive for AWS compatible solutions';
}
}
diff --git a/tests/Storage/Device/WasabiTest.php b/tests/Storage/Device/WasabiTest.php
index 0c541bd7..f7efe76b 100644
--- a/tests/Storage/Device/WasabiTest.php
+++ b/tests/Storage/Device/WasabiTest.php
@@ -5,16 +5,16 @@
use Utopia\Storage\Device\Wasabi;
use Utopia\Tests\S3Base;
+
class WasabiTest extends S3Base
{
protected function init(): void
{
- $this->root = 'root';
$key = $_SERVER['WASABI_ACCESS_KEY'] ?? '';
$secret = $_SERVER['WASABI_SECRET'] ?? '';
- $bucket = "everly-wasabi-test";
+ $bucket = "appwrite";
- $this->object = new Wasabi($this->root, $key, $secret, $bucket, Wasabi::EU_CENTRAL_1, WASABI::ACL_PRIVATE);
+ $this->object = new Wasabi($this->root, $key, $secret, $bucket, WASABI::US_EAST_1, WASABI::ACL_PRIVATE);
}
diff --git a/tests/Storage/S3Base.php b/tests/Storage/S3Base.php
index feeb8193..01dc5836 100644
--- a/tests/Storage/S3Base.php
+++ b/tests/Storage/S3Base.php
@@ -133,7 +133,7 @@ public function testDeletePath()
$this->assertEquals(true, $this->object->exists($path));
$this->assertEquals(true, $this->object->deletePath('bucket'));
$this->assertEquals(false, $this->object->exists($path));
-
+
// Test Multiple Objects
$path = $this->object->getPath('text-for-delete-path1.txt');
$path = str_ireplace($this->object->getRoot(), $this->object->getRoot() . DIRECTORY_SEPARATOR . 'bucket', $path);
@@ -148,7 +148,7 @@ public function testDeletePath()
$this->assertEquals(true, $this->object->deletePath('bucket'));
$this->assertEquals(false, $this->object->exists($path));
$this->assertEquals(false, $this->object->exists($path2));
-
+
}