From 9e0f6881db11a22ad3c8b702aa0ae6dd9bb07427 Mon Sep 17 00:00:00 2001 From: Dennis Grosch Date: Thu, 7 Jul 2022 21:36:06 +0200 Subject: [PATCH 1/3] Added MinIO Storage adapter --- src/Storage/Device/MinIO.php | 119 +++++++++++++++++++++++++++++ src/Storage/Storage.php | 2 + tests/Storage/Device/MinIOTest.php | 32 ++++++++ 3 files changed, 153 insertions(+) create mode 100644 src/Storage/Device/MinIO.php create mode 100644 tests/Storage/Device/MinIOTest.php diff --git a/src/Storage/Device/MinIO.php b/src/Storage/Device/MinIO.php new file mode 100644 index 00000000..538f1162 --- /dev/null +++ b/src/Storage/Device/MinIO.php @@ -0,0 +1,119 @@ +protocol = $protocol; + $this->headers['host'] = $host; + } + + /** + * Get list of objects in the given path. + * + * @param string $path + * + * @throws \Exception + * + * @return array + */ + public function listObjects($prefix = '', $maxKeys = 1000, $continuationToken = '') + { + $uri = '/' . $this->getRoot(); + $this->headers['content-type'] = 'text/plain'; + $this->headers['content-md5'] = \base64_encode(md5('', true)); + + $parameters = [ + 'list-type' => 2, + 'prefix' => $prefix, + 'max-keys' => $maxKeys, + ]; + if(!empty($continuationToken)) { + $parameters['continuation-token'] = $continuationToken; + } + $response = parent::call(self::METHOD_GET, $uri, '', $parameters); + return $response->body; + } + + /** + * Delete files in given path, path must be a directory. Return true on success and false on failure. + * + * @param string $path + * + * @throws \Exception + * + * @return bool + */ + public function deletePath(string $path): bool + { + $uri = '/' . $this->getRoot(); + $continuationToken = ''; + do { + $objects = $this->listObjects($path, continuationToken: $continuationToken); + $count = (int) ($objects['KeyCount'] ?? 1); + if($count < 1) { + break; + } + $continuationToken = $objects['NextContinuationToken'] ?? ''; + $body = ''; + + if($count > 1) { + foreach ($objects['Contents'] as $object) { + $body .= "{$object['Key']}"; + } + } else { + $body .= "{$objects['Contents']['Key']}"; + } + $body .= 'true'; + $body .= ''; + $this->amzHeaders['x-amz-content-sha256'] = \hash('sha256', $body); + $this->headers['content-md5'] = \base64_encode(md5($body, true)); + parent::call(self::METHOD_POST, $uri, $body, ['delete'=>'']); + } while(!empty($continuationToken)); + + return true; + } + + /** + * @return string + */ + public function getName(): string + { + return 'MinIO Object Storage'; + } + + /** + * @return string + */ + public function getDescription(): string + { + return 'MinIO Object Storage'; + } +} \ No newline at end of file diff --git a/src/Storage/Storage.php b/src/Storage/Storage.php index cabd18c8..38b2c621 100644 --- a/src/Storage/Storage.php +++ b/src/Storage/Storage.php @@ -21,6 +21,8 @@ class Storage const DEVICE_LINODE = 'linode'; + const DEVICE_MINIO= 'minio'; + /** * Devices. * diff --git a/tests/Storage/Device/MinIOTest.php b/tests/Storage/Device/MinIOTest.php new file mode 100644 index 00000000..bc5b5615 --- /dev/null +++ b/tests/Storage/Device/MinIOTest.php @@ -0,0 +1,32 @@ +root = 'minio-test-bucket'; + $key = $_SERVER['MINIO_ACCESS_KEY'] ?? ''; + $secret = $_SERVER['MINIO_SECRET'] ?? ''; + $protocol = $_SERVER['MINIO_PROTOCOL'] ?? ''; + $host = $_SERVER['MINIO_HOST'] ?? ''; + $bucket = 'minio-test-bucket'; + + $this->object = new MinIO($this->root, $key, $secret, $protocol, $host, $bucket, MinIO::EU_CENTRAL_1); + + } + + protected function getAdapterName(): string + { + return 'MinIO Object Storage'; + } + + protected function getAdapterDescription(): string + { + return 'MinIO Object Storage'; + } +} From f9800eff997306c16e0c77502a4f7f44e1563efc Mon Sep 17 00:00:00 2001 From: Dennis Grosch Date: Thu, 29 Sep 2022 14:10:01 +0200 Subject: [PATCH 2/3] change method accessibility --- src/Storage/Device/MinIO.php | 2 +- src/Storage/Device/S3.php | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Storage/Device/MinIO.php b/src/Storage/Device/MinIO.php index 538f1162..569a5320 100644 --- a/src/Storage/Device/MinIO.php +++ b/src/Storage/Device/MinIO.php @@ -44,7 +44,7 @@ public function __construct(string $root, string $accessKey, string $secretKey, * * @return array */ - public function listObjects($prefix = '', $maxKeys = 1000, $continuationToken = '') + private function listObjects($prefix = '', $maxKeys = 1000, $continuationToken = '') { $uri = '/' . $this->getRoot(); $this->headers['content-type'] = 'text/plain'; diff --git a/src/Storage/Device/S3.php b/src/Storage/Device/S3.php index 7b6161c7..323441b8 100644 --- a/src/Storage/Device/S3.php +++ b/src/Storage/Device/S3.php @@ -724,7 +724,7 @@ private function getSignatureV4(string $method, string $uri, array $parameters = * * @throws \Exception */ - private function call(string $method, string $uri, string $data = '', array $parameters = []) + protected function call(string $method, string $uri, string $data = '', array $parameters=[]) { $uri = $this->getAbsolutePath($uri); $url = 'https://'.$this->headers['host'].$uri.'?'.\http_build_query($parameters, '', '&', PHP_QUERY_RFC3986); From 7adb89cf37b595e4d07c7f93870ea0aa00f35210 Mon Sep 17 00:00:00 2001 From: Haymaker Date: Sun, 12 Mar 2023 06:17:07 -0400 Subject: [PATCH 3/3] bring in line with current main branch standard Commit amended to include fomatter/linter that somehow did not get included with inital commit. --- src/Storage/Device/MinIO.php | 67 +++++++++++++++--------------- src/Storage/Device/S3.php | 9 +++- src/Storage/Storage.php | 2 +- tests/Storage/Device/MinIOTest.php | 11 +++-- 4 files changed, 49 insertions(+), 40 deletions(-) diff --git a/src/Storage/Device/MinIO.php b/src/Storage/Device/MinIO.php index 569a5320..e54502d5 100644 --- a/src/Storage/Device/MinIO.php +++ b/src/Storage/Device/MinIO.php @@ -2,35 +2,35 @@ namespace Utopia\Storage\Device; -use Utopia\Storage\Device\S3; - class MinIO extends S3 { /** * Regions constants - * */ - const EU_CENTRAL_1='eu-central-1'; - const US_SOUTHEAST_1='us-southeast-1'; - const US_EAST_1='us-east-1'; - const AP_SOUTH_1='ap-south-1'; + const EU_CENTRAL_1 = 'eu-central-1'; + + const US_SOUTHEAST_1 = 'us-southeast-1'; + + const US_EAST_1 = 'us-east-1'; + + const AP_SOUTH_1 = 'ap-south-1'; /** * Object Storage Constructor * - * @param string $root - * @param string $accessKey - * @param string $secretKey - * @param string $protocol - * @param string $host - * @param string $bucket - * @param string $region - * @param string $acl + * @param string $root + * @param string $accessKey + * @param string $secretKey + * @param string $protocol + * @param string $host + * @param string $bucket + * @param string $region + * @param string $acl */ public function __construct(string $root, string $accessKey, string $secretKey, string $protocol, string $host, string $bucket, string $region = self::EU_CENTRAL_1, string $acl = self::ACL_PRIVATE) { parent::__construct($root, $accessKey, $secretKey, $bucket, $region, $acl); - + $this->protocol = $protocol; $this->headers['host'] = $host; } @@ -38,15 +38,14 @@ public function __construct(string $root, string $accessKey, string $secretKey, /** * Get list of objects in the given path. * - * @param string $path - * - * @throws \Exception - * + * @param string $path * @return array + * + * @throws \Exception */ private function listObjects($prefix = '', $maxKeys = 1000, $continuationToken = '') { - $uri = '/' . $this->getRoot(); + $uri = '/'.$this->getRoot(); $this->headers['content-type'] = 'text/plain'; $this->headers['content-md5'] = \base64_encode(md5('', true)); @@ -55,48 +54,48 @@ private function listObjects($prefix = '', $maxKeys = 1000, $continuationToken = 'prefix' => $prefix, 'max-keys' => $maxKeys, ]; - if(!empty($continuationToken)) { + if (! empty($continuationToken)) { $parameters['continuation-token'] = $continuationToken; } $response = parent::call(self::METHOD_GET, $uri, '', $parameters); + return $response->body; } /** * Delete files in given path, path must be a directory. Return true on success and false on failure. * - * @param string $path - * - * @throws \Exception - * + * @param string $path * @return bool + * + * @throws \Exception */ public function deletePath(string $path): bool { - $uri = '/' . $this->getRoot(); + $uri = '/'.$this->getRoot(); $continuationToken = ''; do { $objects = $this->listObjects($path, continuationToken: $continuationToken); $count = (int) ($objects['KeyCount'] ?? 1); - if($count < 1) { + if ($count < 1) { break; } $continuationToken = $objects['NextContinuationToken'] ?? ''; $body = ''; - - if($count > 1) { + + if ($count > 1) { foreach ($objects['Contents'] as $object) { $body .= "{$object['Key']}"; } } else { - $body .= "{$objects['Contents']['Key']}"; + $body .= "{$objects['Contents']['Key']}"; } $body .= 'true'; $body .= ''; $this->amzHeaders['x-amz-content-sha256'] = \hash('sha256', $body); $this->headers['content-md5'] = \base64_encode(md5($body, true)); - parent::call(self::METHOD_POST, $uri, $body, ['delete'=>'']); - } while(!empty($continuationToken)); + parent::call(self::METHOD_POST, $uri, $body, ['delete' => '']); + } while (! empty($continuationToken)); return true; } @@ -116,4 +115,4 @@ public function getDescription(): string { return 'MinIO Object Storage'; } -} \ No newline at end of file +} diff --git a/src/Storage/Device/S3.php b/src/Storage/Device/S3.php index 323441b8..0a1575e5 100644 --- a/src/Storage/Device/S3.php +++ b/src/Storage/Device/S3.php @@ -136,6 +136,11 @@ class S3 extends Device */ protected array $amzHeaders; + /** + * @var string + */ + protected string $protocol = 'https'; + /** * S3 Constructor * @@ -724,10 +729,10 @@ private function getSignatureV4(string $method, string $uri, array $parameters = * * @throws \Exception */ - protected function call(string $method, string $uri, string $data = '', array $parameters=[]) + protected function call(string $method, string $uri, string $data = '', array $parameters = []) { $uri = $this->getAbsolutePath($uri); - $url = 'https://'.$this->headers['host'].$uri.'?'.\http_build_query($parameters, '', '&', PHP_QUERY_RFC3986); + $url = $this->protocol.'://'.$this->headers['host'].$uri.'?'.\http_build_query($parameters, '', '&', PHP_QUERY_RFC3986); $response = new \stdClass; $response->body = ''; $response->headers = []; diff --git a/src/Storage/Storage.php b/src/Storage/Storage.php index 38b2c621..8dbee829 100644 --- a/src/Storage/Storage.php +++ b/src/Storage/Storage.php @@ -21,7 +21,7 @@ class Storage const DEVICE_LINODE = 'linode'; - const DEVICE_MINIO= 'minio'; + const DEVICE_MINIO = 'minio'; /** * Devices. diff --git a/tests/Storage/Device/MinIOTest.php b/tests/Storage/Device/MinIOTest.php index bc5b5615..9eaed9f0 100644 --- a/tests/Storage/Device/MinIOTest.php +++ b/tests/Storage/Device/MinIOTest.php @@ -3,7 +3,7 @@ namespace Utopia\Tests; use Utopia\Storage\Device\MinIO; -use Utopia\Tests\S3Base; +use Utopia\Tests\Storage\S3Base; class MinIOTest extends S3Base { @@ -14,10 +14,10 @@ protected function init(): void $secret = $_SERVER['MINIO_SECRET'] ?? ''; $protocol = $_SERVER['MINIO_PROTOCOL'] ?? ''; $host = $_SERVER['MINIO_HOST'] ?? ''; + $region = $_SERVER['MINIO_REGION'] ?? ''; $bucket = 'minio-test-bucket'; - $this->object = new MinIO($this->root, $key, $secret, $protocol, $host, $bucket, MinIO::EU_CENTRAL_1); - + $this->object = new MinIO($this->root, $key, $secret, $protocol, $host, $bucket, $region); } protected function getAdapterName(): string @@ -25,6 +25,11 @@ protected function getAdapterName(): string return 'MinIO Object Storage'; } + protected function getAdapterType(): string + { + return $this->object->getType(); + } + protected function getAdapterDescription(): string { return 'MinIO Object Storage';