diff --git a/composer.json b/composer.json index fb941b79..3b974e0b 100644 --- a/composer.json +++ b/composer.json @@ -26,7 +26,7 @@ "ext-curl": "*", "ext-openssl": "*", "appwrite/appwrite": "15.*", - "utopia-php/database": "4.*", + "utopia-php/database": "3.*", "utopia-php/storage": "0.18.*", "utopia-php/dsn": "0.2.*", "utopia-php/console": "0.0.*" diff --git a/composer.lock b/composer.lock index 34ab86f6..dd589852 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "9c8bb5977268bd76bf2f06af3d75f974", + "content-hash": "8b2b0894b94d976f09044c200956961a", "packages": [ { "name": "appwrite/appwrite", @@ -187,16 +187,16 @@ }, { "name": "google/protobuf", - "version": "v4.33.0", + "version": "v4.33.1", "source": { "type": "git", "url": "https://github.com/protocolbuffers/protobuf-php.git", - "reference": "b50269e23204e5ae859a326ec3d90f09efe3047d" + "reference": "0cd73ccf0cd26c3e72299cce1ea6144091a57e12" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/protocolbuffers/protobuf-php/zipball/b50269e23204e5ae859a326ec3d90f09efe3047d", - "reference": "b50269e23204e5ae859a326ec3d90f09efe3047d", + "url": "https://api.github.com/repos/protocolbuffers/protobuf-php/zipball/0cd73ccf0cd26c3e72299cce1ea6144091a57e12", + "reference": "0cd73ccf0cd26c3e72299cce1ea6144091a57e12", "shasum": "" }, "require": { @@ -225,9 +225,9 @@ "proto" ], "support": { - "source": "https://github.com/protocolbuffers/protobuf-php/tree/v4.33.0" + "source": "https://github.com/protocolbuffers/protobuf-php/tree/v4.33.1" }, - "time": "2025-10-15T20:10:28+00:00" + "time": "2025-11-12T21:58:05+00:00" }, { "name": "mongodb/mongodb", @@ -1425,16 +1425,16 @@ }, { "name": "symfony/http-client", - "version": "v7.3.4", + "version": "v7.3.6", "source": { "type": "git", "url": "https://github.com/symfony/http-client.git", - "reference": "4b62871a01c49457cf2a8e560af7ee8a94b87a62" + "reference": "3c0a55a2c8e21e30a37022801c11c7ab5a6cb2de" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/http-client/zipball/4b62871a01c49457cf2a8e560af7ee8a94b87a62", - "reference": "4b62871a01c49457cf2a8e560af7ee8a94b87a62", + "url": "https://api.github.com/repos/symfony/http-client/zipball/3c0a55a2c8e21e30a37022801c11c7ab5a6cb2de", + "reference": "3c0a55a2c8e21e30a37022801c11c7ab5a6cb2de", "shasum": "" }, "require": { @@ -1501,7 +1501,7 @@ "http" ], "support": { - "source": "https://github.com/symfony/http-client/tree/v7.3.4" + "source": "https://github.com/symfony/http-client/tree/v7.3.6" }, "funding": [ { @@ -1521,7 +1521,7 @@ "type": "tidelift" } ], - "time": "2025-09-11T10:12:26+00:00" + "time": "2025-11-05T17:41:46+00:00" }, { "name": "symfony/http-client-contracts", @@ -1928,16 +1928,16 @@ }, { "name": "symfony/service-contracts", - "version": "v3.6.0", + "version": "v3.6.1", "source": { "type": "git", "url": "https://github.com/symfony/service-contracts.git", - "reference": "f021b05a130d35510bd6b25fe9053c2a8a15d5d4" + "reference": "45112560a3ba2d715666a509a0bc9521d10b6c43" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/service-contracts/zipball/f021b05a130d35510bd6b25fe9053c2a8a15d5d4", - "reference": "f021b05a130d35510bd6b25fe9053c2a8a15d5d4", + "url": "https://api.github.com/repos/symfony/service-contracts/zipball/45112560a3ba2d715666a509a0bc9521d10b6c43", + "reference": "45112560a3ba2d715666a509a0bc9521d10b6c43", "shasum": "" }, "require": { @@ -1991,7 +1991,7 @@ "standards" ], "support": { - "source": "https://github.com/symfony/service-contracts/tree/v3.6.0" + "source": "https://github.com/symfony/service-contracts/tree/v3.6.1" }, "funding": [ { @@ -2002,12 +2002,16 @@ "url": "https://github.com/fabpot", "type": "github" }, + { + "url": "https://github.com/nicolas-grekas", + "type": "github" + }, { "url": "https://tidelift.com/funding/github/packagist/symfony/symfony", "type": "tidelift" } ], - "time": "2025-04-25T09:37:31+00:00" + "time": "2025-07-15T11:30:57+00:00" }, { "name": "tbachert/spi", @@ -2209,16 +2213,16 @@ }, { "name": "utopia-php/database", - "version": "4.0.0", + "version": "3.4.0", "source": { "type": "git", "url": "https://github.com/utopia-php/database.git", - "reference": "da9d021b2722abdf4df08a739126f1be2707cf6d" + "reference": "e10b4faa4f3a3ef30a5f6d76acdb605469924aec" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/database/zipball/da9d021b2722abdf4df08a739126f1be2707cf6d", - "reference": "da9d021b2722abdf4df08a739126f1be2707cf6d", + "url": "https://api.github.com/repos/utopia-php/database/zipball/e10b4faa4f3a3ef30a5f6d76acdb605469924aec", + "reference": "e10b4faa4f3a3ef30a5f6d76acdb605469924aec", "shasum": "" }, "require": { @@ -2261,9 +2265,9 @@ ], "support": { "issues": "https://github.com/utopia-php/database/issues", - "source": "https://github.com/utopia-php/database/tree/4.0.0" + "source": "https://github.com/utopia-php/database/tree/3.4.0" }, - "time": "2025-11-04T10:55:46+00:00" + "time": "2025-11-13T06:34:20+00:00" }, { "name": "utopia-php/dsn", @@ -3467,16 +3471,16 @@ }, { "name": "phpunit/phpunit", - "version": "11.5.43", + "version": "11.5.44", "source": { "type": "git", "url": "https://github.com/sebastianbergmann/phpunit.git", - "reference": "c6b89b6cf4324a8b4cb86e1f5dfdd6c9e0371924" + "reference": "c346885c95423eda3f65d85a194aaa24873cda82" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/phpunit/zipball/c6b89b6cf4324a8b4cb86e1f5dfdd6c9e0371924", - "reference": "c6b89b6cf4324a8b4cb86e1f5dfdd6c9e0371924", + "url": "https://api.github.com/repos/sebastianbergmann/phpunit/zipball/c346885c95423eda3f65d85a194aaa24873cda82", + "reference": "c346885c95423eda3f65d85a194aaa24873cda82", "shasum": "" }, "require": { @@ -3548,7 +3552,7 @@ "support": { "issues": "https://github.com/sebastianbergmann/phpunit/issues", "security": "https://github.com/sebastianbergmann/phpunit/security/policy", - "source": "https://github.com/sebastianbergmann/phpunit/tree/11.5.43" + "source": "https://github.com/sebastianbergmann/phpunit/tree/11.5.44" }, "funding": [ { @@ -3572,7 +3576,7 @@ "type": "tidelift" } ], - "time": "2025-10-30T08:39:39+00:00" + "time": "2025-11-13T07:17:35+00:00" }, { "name": "sebastian/cli-parser", @@ -4916,7 +4920,7 @@ ], "aliases": [], "minimum-stability": "stable", - "stability-flags": [], + "stability-flags": {}, "prefer-stable": false, "prefer-lowest": false, "platform": { @@ -4927,5 +4931,5 @@ "platform-dev": { "ext-pdo": "*" }, - "plugin-api-version": "2.2.0" + "plugin-api-version": "2.6.0" } diff --git a/src/Migration/Cache.php b/src/Migration/Cache.php index a88d1c17..0e3d1e05 100644 --- a/src/Migration/Cache.php +++ b/src/Migration/Cache.php @@ -45,6 +45,7 @@ public function resolveResourceCacheKey(Resource $resource): string case Resource::TYPE_TABLE: case Resource::TYPE_COLLECTION: /** @var Table $resource */ + $keys[] = $resource->getDatabase()->getType(); $keys[] = $resource->getDatabase()->getSequence(); break; diff --git a/src/Migration/Destinations/Appwrite.php b/src/Migration/Destinations/Appwrite.php index 523220d5..17967418 100644 --- a/src/Migration/Destinations/Appwrite.php +++ b/src/Migration/Destinations/Appwrite.php @@ -12,6 +12,7 @@ use Appwrite\Services\Storage; use Appwrite\Services\Teams; use Appwrite\Services\Users; +use Dom\Document; use Override; use Utopia\Database\Database as UtopiaDatabase; use Utopia\Database\Document as UtopiaDocument; @@ -33,6 +34,7 @@ use Utopia\Migration\Resources\Auth\Membership; use Utopia\Migration\Resources\Auth\Team; use Utopia\Migration\Resources\Auth\User; +use Utopia\Migration\Resources\Database\Attribute; use Utopia\Migration\Resources\Database\Column; use Utopia\Migration\Resources\Database\Database; use Utopia\Migration\Resources\Database\Index; @@ -57,6 +59,16 @@ class Appwrite extends Destination private Teams $teams; private Users $users; + /** + * @var callable(UtopiaDocument $database): UtopiaDatabase + */ + protected $getDatabasesDB; + + /** + * @var callable(string $databaseType):string + */ + protected $getDatabaseDSN; + /** * @var array */ @@ -66,14 +78,18 @@ class Appwrite extends Destination * @param string $project * @param string $endpoint * @param string $key - * @param UtopiaDatabase $database + * @param UtopiaDatabase $dbForProject + * @param callable(UtopiaDocument $database):UtopiaDatabase $getDatabasesDB + * @param callable(string $databaseType):string $getDatabasesDSN * @param array> $collectionStructure */ public function __construct( string $project, string $endpoint, string $key, - protected UtopiaDatabase $database, + protected UtopiaDatabase $dbForProject, + callable $getDatabasesDB, + callable $getDatabasesDSN, protected array $collectionStructure ) { $this->project = $project; @@ -89,6 +105,9 @@ public function __construct( $this->storage = new Storage($this->client); $this->teams = new Teams($this->client); $this->users = new Users($this->client); + + $this->getDatabasesDB = $getDatabasesDB; + $this->getDatabaseDSN = $getDatabasesDSN; } public static function getName(): string @@ -109,6 +128,8 @@ public static function getSupportedResources(): array // Database Resource::TYPE_DATABASE, + Resource::TYPE_DATABASE_DOCUMENTSDB, + Resource::TYPE_DATABASE_VECTORDB, Resource::TYPE_TABLE, Resource::TYPE_COLUMN, Resource::TYPE_INDEX, @@ -228,7 +249,7 @@ protected function import(array $resources, callable $callback): void $isLast = $index === $total - 1; try { - $this->database->setPreserveDates(true); + $this->dbForProject->setPreserveDates(true); $responseResource = match ($resource->getGroup()) { Transfer::GROUP_DATABASES => $this->importDatabaseResource($resource, $isLast), @@ -251,7 +272,7 @@ protected function import(array $resources, callable $callback): void $responseResource = $resource; } finally { - $this->database->setPreserveDates(false); + $this->dbForProject->setPreserveDates(false); } $this->cache->update($responseResource); @@ -269,18 +290,20 @@ public function importDatabaseResource(Resource $resource, bool $isLast): Resour { switch ($resource->getName()) { case Resource::TYPE_DATABASE: + case Resource::TYPE_DATABASE_DOCUMENTSDB: + case Resource::TYPE_DATABASE_VECTORDB: /** @var Database $resource */ $success = $this->createDatabase($resource); break; case Resource::TYPE_TABLE: case Resource::TYPE_COLLECTION: /** @var Table $resource */ - $success = $this->createTable($resource); + $success = $this->createEntity($resource); break; case Resource::TYPE_COLUMN: case Resource::TYPE_ATTRIBUTE: /** @var Column $resource */ - $success = $this->createColumn($resource); + $success = $this->createField($resource); break; case Resource::TYPE_INDEX: /** @var Index $resource */ @@ -289,7 +312,7 @@ public function importDatabaseResource(Resource $resource, bool $isLast): Resour case Resource::TYPE_ROW: case Resource::TYPE_DOCUMENT: /** @var Row $resource */ - $success = $this->createRow($resource, $isLast); + $success = $this->createRecord($resource, $isLast); break; default: $success = false; @@ -325,7 +348,7 @@ protected function createDatabase(Database $resource): bool ); } - $database = $this->database->createDocument('databases', new UtopiaDocument([ + $database = $this->dbForProject->createDocument('databases', new UtopiaDocument([ '$id' => $resource->getId(), 'name' => $resource->getDatabaseName(), 'enabled' => $resource->getEnabled(), @@ -334,6 +357,8 @@ protected function createDatabase(Database $resource): bool '$updatedAt' => $resource->getUpdatedAt(), 'originalId' => empty($resource->getOriginalId()) ? null : $resource->getOriginalId(), 'type' => empty($resource->getType()) ? 'legacy' : $resource->getType(), + // source and destination can be in different location + 'database' => ($this->getDatabaseDSN)($resource->getType()) ])); $resource->setSequence($database->getSequence()); @@ -348,7 +373,7 @@ protected function createDatabase(Database $resource): bool $this->collectionStructure['indexes'] ); - $this->database->createCollection( + $this->dbForProject->createCollection( 'database_' . $database->getSequence(), $columns, $indexes @@ -363,7 +388,7 @@ protected function createDatabase(Database $resource): bool * @throws StructureException * @throws Exception */ - protected function createTable(Table $resource): bool + protected function createEntity(Table $resource): bool { if ($resource->getId() == 'unique()') { $resource->setId(ID::unique()); @@ -380,7 +405,7 @@ protected function createTable(Table $resource): bool ); } - $database = $this->database->getDocument( + $database = $this->dbForProject->getDocument( 'databases', $resource->getDatabase()->getId() ); @@ -394,7 +419,14 @@ protected function createTable(Table $resource): bool ); } - $table = $this->database->createDocument('database_' . $database->getSequence(), new UtopiaDocument([ + $dbForDatabases = ($this->getDatabasesDB)($database); + + // passing null in creates only creates the metadata collection + if (!$dbForDatabases->exists(null, UtopiaDatabase::METADATA)) { + $dbForDatabases->create(); + } + + $table = $this->dbForProject->createDocument('database_' . $database->getSequence(), new UtopiaDocument([ '$id' => $resource->getId(), 'databaseInternalId' => $database->getSequence(), 'databaseId' => $resource->getDatabase()->getId(), @@ -409,7 +441,7 @@ protected function createTable(Table $resource): bool $resource->setSequence($table->getSequence()); - $this->database->createCollection( + $dbForDatabases->createCollection( 'database_' . $database->getSequence() . '_collection_' . $resource->getSequence(), permissions: $resource->getPermissions(), documentSecurity: $resource->getRowSecurity() @@ -423,26 +455,33 @@ protected function createTable(Table $resource): bool * @throws \Exception * @throws \Throwable */ - protected function createColumn(Column $resource): bool + protected function createField(Column|Attribute $resource): bool { + if ($resource->getTable()->getDatabase()->getType() === Resource::TYPE_DATABASE_DOCUMENTSDB) { + $resource->setStatus(Resource::STATUS_SKIPPED, 'Columns not supported for DocumentsDB'); + return false; + } + $type = match ($resource->getType()) { - Column::TYPE_DATETIME => UtopiaDatabase::VAR_DATETIME, - Column::TYPE_BOOLEAN => UtopiaDatabase::VAR_BOOLEAN, - Column::TYPE_INTEGER => UtopiaDatabase::VAR_INTEGER, - Column::TYPE_FLOAT => UtopiaDatabase::VAR_FLOAT, - Column::TYPE_RELATIONSHIP => UtopiaDatabase::VAR_RELATIONSHIP, - Column::TYPE_STRING, - Column::TYPE_IP, - Column::TYPE_EMAIL, - Column::TYPE_URL, - Column::TYPE_ENUM => UtopiaDatabase::VAR_STRING, - Column::TYPE_POINT => UtopiaDatabase::VAR_POINT, - Column::TYPE_LINE => UtopiaDatabase::VAR_LINESTRING, - Column::TYPE_POLYGON => UtopiaDatabase::VAR_POLYGON, + Column::TYPE_DATETIME, Attribute::TYPE_DATETIME => UtopiaDatabase::VAR_DATETIME, + Column::TYPE_BOOLEAN, Attribute::TYPE_BOOLEAN => UtopiaDatabase::VAR_BOOLEAN, + Column::TYPE_INTEGER, Attribute::TYPE_INTEGER => UtopiaDatabase::VAR_INTEGER, + Column::TYPE_FLOAT, Attribute::TYPE_FLOAT => UtopiaDatabase::VAR_FLOAT, + Column::TYPE_RELATIONSHIP, Attribute::TYPE_RELATIONSHIP => UtopiaDatabase::VAR_RELATIONSHIP, + Column::TYPE_STRING, Attribute::TYPE_STRING, + Column::TYPE_IP, Attribute::TYPE_IP, + Column::TYPE_EMAIL, Attribute::TYPE_EMAIL, + Column::TYPE_URL, Attribute::TYPE_URL, + Column::TYPE_ENUM, Attribute::TYPE_ENUM => UtopiaDatabase::VAR_STRING, + Column::TYPE_POINT, Attribute::TYPE_POINT => UtopiaDatabase::VAR_POINT, + Column::TYPE_LINE, Attribute::TYPE_LINE => UtopiaDatabase::VAR_LINESTRING, + Column::TYPE_POLYGON, Attribute::TYPE_POLYGON => UtopiaDatabase::VAR_POLYGON, + Column::TYPE_OBJECT, Attribute::TYPE_OBJECT => UtopiaDatabase::VAR_OBJECT, + Column::TYPE_VECTOR, Attribute::TYPE_VECTOR => UtopiaDatabase::VAR_VECTOR, default => throw new \Exception('Invalid resource type '.$resource->getType()), }; - $database = $this->database->getDocument( + $database = $this->dbForProject->getDocument( 'databases', $resource->getTable()->getDatabase()->getId(), ); @@ -456,7 +495,7 @@ protected function createColumn(Column $resource): bool ); } - $table = $this->database->getDocument( + $table = $this->dbForProject->getDocument( 'database_' . $database->getSequence(), $resource->getTable()->getId(), ); @@ -501,7 +540,7 @@ protected function createColumn(Column $resource): bool if ($type === UtopiaDatabase::VAR_RELATIONSHIP) { $resource->getOptions()['side'] = UtopiaDatabase::RELATION_SIDE_PARENT; - $relatedTable = $this->database->getDocument( + $relatedTable = $this->dbForProject->getDocument( 'database_' . $database->getSequence(), $resource->getOptions()['relatedCollection'] ); @@ -514,7 +553,7 @@ protected function createColumn(Column $resource): bool ); } } - + $dbForDatabases = ($this->getDatabasesDB)($database); try { $column = new UtopiaDocument([ '$id' => ID::custom($database->getSequence() . '_' . $table->getSequence() . '_' . $resource->getKey()), @@ -538,9 +577,9 @@ protected function createColumn(Column $resource): bool '$updatedAt' => $resource->getUpdatedAt(), ]); - $this->database->checkAttribute($table, $column); + $this->dbForProject->checkAttribute($table, $column); - $column = $this->database->createDocument('attributes', $column); + $column = $this->dbForProject->createDocument('attributes', $column); } catch (DuplicateException) { throw new Exception( resourceName: $resource->getName(), @@ -556,13 +595,13 @@ protected function createColumn(Column $resource): bool message: 'Attribute limit exceeded', ); } catch (\Throwable $e) { - $this->database->purgeCachedDocument('database_' . $database->getSequence(), $table->getId()); - $this->database->purgeCachedCollection('database_' . $database->getSequence() . '_collection_' . $table->getSequence()); + $this->dbForProject->purgeCachedDocument('database_' . $database->getSequence(), $table->getId()); + $dbForDatabases->purgeCachedCollection('database_' . $database->getSequence() . '_collection_' . $table->getSequence()); throw $e; } - $this->database->purgeCachedDocument('database_' . $database->getSequence(), $table->getId()); - $this->database->purgeCachedCollection('database_' . $database->getSequence() . '_collection_' . $table->getSequence()); + $this->dbForProject->purgeCachedDocument('database_' . $database->getSequence(), $table->getId()); + $dbForDatabases->purgeCachedCollection('database_' . $database->getSequence() . '_collection_' . $table->getSequence()); $options = $resource->getOptions(); $twoWayKey = null; @@ -596,9 +635,9 @@ protected function createColumn(Column $resource): bool '$updatedAt' => $resource->getUpdatedAt(), ]); - $this->database->createDocument('attributes', $twoWayAttribute); + $this->dbForProject->createDocument('attributes', $twoWayAttribute); } catch (DuplicateException) { - $this->database->deleteDocument('attributes', $column->getId()); + $this->dbForProject->deleteDocument('attributes', $column->getId()); throw new Exception( resourceName: $resource->getName(), @@ -607,7 +646,7 @@ protected function createColumn(Column $resource): bool message: 'Attribute already exists', ); } catch (LimitException) { - $this->database->deleteDocument('attributes', $column->getId()); + $this->dbForProject->deleteDocument('attributes', $column->getId()); throw new Exception( resourceName: $resource->getName(), @@ -616,8 +655,8 @@ protected function createColumn(Column $resource): bool message: 'Column limit exceeded', ); } catch (\Throwable $e) { - $this->database->purgeCachedDocument('database_' . $database->getSequence(), $relatedTable->getId()); - $this->database->purgeCachedCollection('database_' . $database->getSequence() . '_collection_' . $relatedTable->getSequence()); + $this->dbForProject->purgeCachedDocument('database_' . $database->getSequence(), $relatedTable->getId()); + $dbForDatabases->purgeCachedCollection('database_' . $database->getSequence() . '_collection_' . $relatedTable->getSequence()); throw $e; } } @@ -625,7 +664,7 @@ protected function createColumn(Column $resource): bool try { switch ($type) { case UtopiaDatabase::VAR_RELATIONSHIP: - if (!$this->database->createRelationship( + if (!$dbForDatabases->createRelationship( collection: 'database_' . $database->getSequence() . '_collection_' . $table->getSequence(), relatedCollection: 'database_' . $database->getSequence() . '_collection_' . $relatedTable->getSequence(), type: $options['relationType'], @@ -643,7 +682,7 @@ protected function createColumn(Column $resource): bool } break; default: - if (!$this->database->createAttribute( + if (!$dbForDatabases->createAttribute( 'database_' . $database->getSequence() . '_collection_' . $table->getSequence(), $resource->getKey(), $type, @@ -660,10 +699,10 @@ protected function createColumn(Column $resource): bool } } } catch (\Throwable) { - $this->database->deleteDocument('attributes', $column->getId()); + $this->dbForProject->deleteDocument('attributes', $column->getId()); if (isset($twoWayAttribute)) { - $this->database->deleteDocument('attributes', $twoWayAttribute->getId()); + $this->dbForProject->deleteDocument('attributes', $twoWayAttribute->getId()); } throw new Exception( @@ -675,11 +714,11 @@ protected function createColumn(Column $resource): bool } if ($type === UtopiaDatabase::VAR_RELATIONSHIP && $options['twoWay']) { - $this->database->purgeCachedDocument('database_' . $database->getSequence(), $relatedTable->getId()); + $this->dbForProject->purgeCachedDocument('database_' . $database->getSequence(), $relatedTable->getId()); } - $this->database->purgeCachedDocument('database_' . $database->getSequence(), $table->getId()); - $this->database->purgeCachedCollection('database_' . $database->getSequence() . '_collection_' . $table->getSequence()); + $this->dbForProject->purgeCachedDocument('database_' . $database->getSequence(), $table->getId()); + $dbForDatabases->purgeCachedCollection('database_' . $database->getSequence() . '_collection_' . $table->getSequence()); return true; } @@ -690,7 +729,7 @@ protected function createColumn(Column $resource): bool */ protected function createIndex(Index $resource): bool { - $database = $this->database->getDocument( + $database = $this->dbForProject->getDocument( 'databases', $resource->getTable()->getDatabase()->getId(), ); @@ -703,7 +742,7 @@ protected function createIndex(Index $resource): bool ); } - $table = $this->database->getDocument( + $table = $this->dbForProject->getDocument( 'database_' . $database->getSequence(), $resource->getTable()->getId(), ); @@ -715,13 +754,14 @@ protected function createIndex(Index $resource): bool message: 'Table not found', ); } + $dbForDatabases = ($this->getDatabasesDB)($database); - $count = $this->database->count('indexes', [ + $count = $this->dbForProject->count('indexes', [ Query::equal('collectionInternalId', [$table->getSequence()]), Query::equal('databaseInternalId', [$database->getSequence()]) - ], $this->database->getLimitForIndexes()); + ], $dbForDatabases->getLimitForIndexes()); - if ($count >= $this->database->getLimitForIndexes()) { + if ($count >= $dbForDatabases->getLimitForIndexes()) { throw new Exception( resourceName: $resource->getName(), resourceGroup: $resource->getGroup(), @@ -730,101 +770,11 @@ protected function createIndex(Index $resource): bool ); } - /** - * @var array $tableColumns - */ - $tableColumns = $table->getAttribute('attributes', []); - - /** - * @var array $tableIndexes - */ - $tableIndexes = $table->getAttribute('indexes', []); - - $oldColumns = \array_map( - fn ($attr) => $attr->getArrayCopy(), - $tableColumns - ); - - $oldColumns[] = [ - 'key' => '$id', - 'type' => UtopiaDatabase::VAR_STRING, - 'status' => 'available', - 'required' => true, - 'array' => false, - 'default' => null, - 'size' => UtopiaDatabase::LENGTH_KEY - ]; - - $oldColumns[] = [ - 'key' => '$createdAt', - 'type' => UtopiaDatabase::VAR_DATETIME, - 'status' => 'available', - 'signed' => false, - 'required' => false, - 'array' => false, - 'default' => null, - 'size' => 0 - ]; - - $oldColumns[] = [ - 'key' => '$updatedAt', - 'type' => UtopiaDatabase::VAR_DATETIME, - 'status' => 'available', - 'signed' => false, - 'required' => false, - 'array' => false, - 'default' => null, - 'size' => 0 - ]; - // Lengths hidden by default $lengths = []; - foreach ($resource->getColumns() as $i => $column) { - // find attribute metadata in collection document - $columnIndex = \array_search( - $column, - \array_column($oldColumns, 'key') - ); - - if ($columnIndex === false) { - throw new Exception( - resourceName: $resource->getName(), - resourceGroup: $resource->getGroup(), - resourceId: $resource->getId(), - message: 'Column not found in table: ' . $column, - ); - } - - $columnStatus = $oldColumns[$columnIndex]['status']; - $columnType = $oldColumns[$columnIndex]['type']; - $columnSize = $oldColumns[$columnIndex]['size']; - $columnArray = $oldColumns[$columnIndex]['array'] ?? false; - - if ($columnType === UtopiaDatabase::VAR_RELATIONSHIP) { - throw new Exception( - resourceName: $resource->getName(), - resourceGroup: $resource->getGroup(), - resourceId: $resource->getId(), - message: 'Relationship columns are not supported in indexes', - ); - } - - // Ensure attribute is available - if ($columnStatus !== 'available') { - throw new Exception( - resourceName: $resource->getName(), - resourceGroup: $resource->getGroup(), - resourceId: $resource->getId(), - message: 'Column not available: ' . $column, - ); - } - - $lengths[$i] = null; - - if ($columnArray === true) { - $lengths[$i] = UtopiaDatabase::MAX_ARRAY_INDEX_LENGTH; - } + if ($dbForDatabases->getAdapter()->getSupportForAttributes()) { + $this->validateFieldsForIndexes($resource, $table, $lengths); } $index = new UtopiaDocument([ @@ -843,20 +793,34 @@ protected function createIndex(Index $resource): bool '$updatedAt' => $resource->getUpdatedAt(), ]); + $maxIndexLength = $dbForDatabases->getAdapter()->getMaxIndexLength(); + $internalIndexesKeys = $dbForDatabases->getAdapter()->getInternalIndexesKeys(); + $supportForIndexArray = $dbForDatabases->getAdapter()->getSupportForIndexArray(); + $supportForSpatialAttributes = $dbForDatabases->getAdapter()->getSupportForSpatialAttributes(); + $supportForSpatialIndexNull = $dbForDatabases->getAdapter()->getSupportForSpatialIndexNull(); + $supportForSpatialIndexOrder = $dbForDatabases->getAdapter()->getSupportForSpatialIndexOrder(); + $supportForAttributes = $dbForDatabases->getAdapter()->getSupportForAttributes(); + $supportForMultipleFulltextIndexes = $dbForDatabases->getAdapter()->getSupportForMultipleFulltextIndexes(); + $supportForIdenticalIndexes = $dbForDatabases->getAdapter()->getSupportForIdenticalIndexes(); + $supportForVectorIndexes = $dbForDatabases->getAdapter()->getSupportForVectors(); + $supportForObjectIndexes = $dbForDatabases->getAdapter()->getSupportForObject(); + $validator = new IndexValidator( - $tableColumns, - $tableIndexes, - $this->database->getAdapter()->getMaxIndexLength(), - $this->database->getAdapter()->getInternalIndexesKeys(), - $this->database->getAdapter()->getSupportForIndexArray(), - $this->database->getAdapter()->getSupportForSpatialIndexNull(), - $this->database->getAdapter()->getSupportForSpatialIndexOrder(), - $this->database->getAdapter()->getSupportForVectors(), - $this->database->getAdapter()->getSupportForAttributes(), - $this->database->getAdapter()->getSupportForMultipleFulltextIndexes(), - $this->database->getAdapter()->getSupportForIdenticalIndexes(), + $table->getAttribute('attributes'), + $table->getAttribute('indexes', []), + $maxIndexLength, + $internalIndexesKeys, + $supportForIndexArray, + $supportForSpatialIndexNull, + $supportForSpatialIndexOrder, + $supportForVectorIndexes, + $supportForAttributes, + $supportForMultipleFulltextIndexes, + $supportForIdenticalIndexes, + $supportForObjectIndexes, ); + if (!$validator->isValid($index)) { throw new Exception( resourceName: $resource->getName(), @@ -866,10 +830,10 @@ protected function createIndex(Index $resource): bool ); } - $index = $this->database->createDocument('indexes', $index); + $index = $this->dbForProject->createDocument('indexes', $index); try { - $result = $this->database->createIndex( + $result = $dbForDatabases->createIndex( 'database_' . $database->getSequence() . '_collection_' . $table->getSequence(), $resource->getKey(), $resource->getType(), @@ -887,7 +851,7 @@ protected function createIndex(Index $resource): bool ); } } catch (\Throwable $th) { - $this->database->deleteDocument('indexes', $index->getId()); + $this->dbForProject->deleteDocument('indexes', $index->getId()); throw new Exception( resourceName: $resource->getName(), @@ -897,7 +861,7 @@ protected function createIndex(Index $resource): bool ); } - $this->database->purgeCachedDocument( + $this->dbForProject->purgeCachedDocument( 'database_' . $database->getSequence(), $table->getId() ); @@ -911,7 +875,7 @@ protected function createIndex(Index $resource): bool * @throws StructureException * @throws Exception */ - protected function createRow(Row $resource, bool $isLast): bool + protected function createRecord(Row $resource, bool $isLast): bool { if ($resource->getId() == 'unique()') { $resource->setId(ID::unique()); @@ -931,7 +895,7 @@ protected function createRow(Row $resource, bool $isLast): bool // Check if document has already been created $exists = \array_key_exists( $resource->getId(), - $this->cache->get(Resource::TYPE_ROW) + $this->cache->get($resource->getName()) ); if ($exists) { @@ -949,45 +913,46 @@ protected function createRow(Row $resource, bool $isLast): bool if ($isLast) { try { - $database = $this->database->getDocument( + $database = $this->dbForProject->getDocument( 'databases', $resource->getTable()->getDatabase()->getId(), ); - $table = $this->database->getDocument( + $table = $this->dbForProject->getDocument( 'database_' . $database->getSequence(), $resource->getTable()->getId(), ); $databaseInternalId = $database->getSequence(); $tableInternalId = $table->getSequence(); - + $dbForDatabases = ($this->getDatabasesDB)($database); /** * This is in case an attribute was deleted from Appwrite attributes collection but was not deleted from the table * When creating an archive we select * which will include orphan attribute from the schema */ - foreach ($this->rowBuffer as $row) { - foreach ($row as $key => $value) { - if (\str_starts_with($key, '$')) { - continue; - } + if ($dbForDatabases->getAdapter()->getSupportForAttributes()) { + foreach ($this->rowBuffer as $row) { + foreach ($row as $key => $value) { + if (\str_starts_with($key, '$')) { + continue; + } - /** @var \Utopia\Database\Document $attribute */ - $found = false; - foreach ($table->getAttribute('attributes', []) as $attribute) { - if ($attribute->getAttribute('key') == $key) { - $found = true; - break; + /** @var \Utopia\Database\Document $attribute */ + $found = false; + foreach ($table->getAttribute('attributes', []) as $attribute) { + if ($attribute->getAttribute('key') == $key) { + $found = true; + break; + } } - } - if (! $found) { - $row->removeAttribute($key); + if (! $found) { + $row->removeAttribute($key); + } } } } - - $this->database->skipRelationshipsExistCheck(fn () => $this->database->createDocuments( + $dbForDatabases->skipRelationshipsExistCheck(fn () => $dbForDatabases->createDocuments( 'database_' . $databaseInternalId . '_collection_' . $tableInternalId, $this->rowBuffer )); @@ -1434,4 +1399,96 @@ private function importDeployment(Deployment $deployment): Resource return $deployment; } + + private function validateFieldsForIndexes(Index $resource, UtopiaDocument $table, array &$lengths) + { + /** + * @var array $tableColumns + */ + $tableColumns = $table->getAttribute('attributes', []); + + $oldColumns = \array_map( + fn ($attr) => $attr->getArrayCopy(), + $tableColumns + ); + + $oldColumns[] = [ + 'key' => '$id', + 'type' => UtopiaDatabase::VAR_STRING, + 'status' => 'available', + 'required' => true, + 'array' => false, + 'default' => null, + 'size' => UtopiaDatabase::LENGTH_KEY + ]; + + $oldColumns[] = [ + 'key' => '$createdAt', + 'type' => UtopiaDatabase::VAR_DATETIME, + 'status' => 'available', + 'signed' => false, + 'required' => false, + 'array' => false, + 'default' => null, + 'size' => 0 + ]; + + $oldColumns[] = [ + 'key' => '$updatedAt', + 'type' => UtopiaDatabase::VAR_DATETIME, + 'status' => 'available', + 'signed' => false, + 'required' => false, + 'array' => false, + 'default' => null, + 'size' => 0 + ]; + + foreach ($resource->getColumns() as $i => $column) { + // find attribute metadata in collection document + $columnIndex = \array_search( + $column, + \array_column($oldColumns, 'key') + ); + + if ($columnIndex === false) { + throw new Exception( + resourceName: $resource->getName(), + resourceGroup: $resource->getGroup(), + resourceId: $resource->getId(), + message: 'Column not found in table: ' . $column, + ); + } + + $columnStatus = $oldColumns[$columnIndex]['status']; + $columnType = $oldColumns[$columnIndex]['type']; + $columnSize = $oldColumns[$columnIndex]['size']; + $columnArray = $oldColumns[$columnIndex]['array'] ?? false; + + if ($columnType === UtopiaDatabase::VAR_RELATIONSHIP) { + throw new Exception( + resourceName: $resource->getName(), + resourceGroup: $resource->getGroup(), + resourceId: $resource->getId(), + message: 'Relationship columns are not supported in indexes', + ); + } + + // Ensure attribute is available + if ($columnStatus !== 'available') { + throw new Exception( + resourceName: $resource->getName(), + resourceGroup: $resource->getGroup(), + resourceId: $resource->getId(), + message: 'Column not available: ' . $column, + ); + } + + $lengths[$i] = null; + + if ($columnArray === true) { + $lengths[$i] = UtopiaDatabase::MAX_ARRAY_INDEX_LENGTH; + } + } + } } diff --git a/src/Migration/Resource.php b/src/Migration/Resource.php index 9645cc6f..c5118543 100644 --- a/src/Migration/Resource.php +++ b/src/Migration/Resource.php @@ -30,6 +30,13 @@ abstract class Resource implements \JsonSerializable public const TYPE_DATABASE = 'database'; + public const TYPE_DATABASE_LEGACY = 'legacy'; + + public const TYPE_DATABASE_TABLESDB = 'tablesdb'; + + public const TYPE_DATABASE_DOCUMENTSDB = 'documentsdb'; + public const TYPE_DATABASE_VECTORDB = 'vectordb'; + public const TYPE_ROW = 'row'; public const TYPE_FILE = 'file'; @@ -70,6 +77,8 @@ abstract class Resource implements \JsonSerializable self::TYPE_BUCKET, self::TYPE_TABLE, self::TYPE_DATABASE, + self::TYPE_DATABASE_VECTORDB, + self::TYPE_DATABASE_DOCUMENTSDB, self::TYPE_ROW, self::TYPE_FILE, self::TYPE_FUNCTION, @@ -87,6 +96,39 @@ abstract class Resource implements \JsonSerializable self::TYPE_COLLECTION, ]; + // index terminology is same for all + public const DATABASE_TYPE_RESOURCE_MAP = [ + self::TYPE_DATABASE => [ + 'entity' => self::TYPE_TABLE, + 'field' => self::TYPE_COLUMN, + 'record' => self::TYPE_ROW, + ], + self::TYPE_DATABASE_DOCUMENTSDB => [ + 'entity' => self::TYPE_COLLECTION, + // HACK: not required in documentsdb but adding it for consistency in the db reader(not gonna impact) + 'field' => self::TYPE_ATTRIBUTE, + 'record' => self::TYPE_DOCUMENT, + ], + self::TYPE_DATABASE_VECTORDB => [ + 'entity' => self::TYPE_COLLECTION, + 'field' => self::TYPE_ATTRIBUTE, + 'record' => self::TYPE_DOCUMENT, + ] + ]; + + public const ENTITY_TYPE_RESOURCE_MAP = [ + self::TYPE_TABLE => [ + 'field' => self::TYPE_COLUMN, + 'record' => self::TYPE_ROW, + 'index' => self::TYPE_INDEX + ], + self::TYPE_COLLECTION => [ + 'field' => self::TYPE_ATTRIBUTE, + 'record' => self::TYPE_DOCUMENT, + 'index' => self::TYPE_INDEX + ], + ]; + protected string $id = ''; protected string $originalId = ''; diff --git a/src/Migration/Resources/Database/Attribute.php b/src/Migration/Resources/Database/Attribute.php new file mode 100644 index 00000000..244f2ba5 --- /dev/null +++ b/src/Migration/Resources/Database/Attribute.php @@ -0,0 +1,158 @@ + $formatOptions + * @param array $filters + * @param array $options + * @param string $createdAt + * @param string $updatedAt + */ + public function __construct( + protected readonly string $key, + protected readonly Table $table, + protected readonly int $size = 0, + protected readonly bool $required = false, + protected readonly mixed $default = null, + protected readonly bool $array = false, + protected readonly bool $signed = false, + protected readonly string $format = '', + protected readonly array $formatOptions = [], + protected readonly array $filters = [], + protected array $options = [], + protected string $createdAt = '', + protected string $updatedAt = '', + ) { + } + + /** + * @return array + */ + public function jsonSerialize(): array + { + return [ + 'key' => $this->key, + 'table' => $this->table, + 'type' => $this->getType(), + 'size' => $this->size, + 'required' => $this->required, + 'default' => $this->default, + 'array' => $this->array, + 'signed' => $this->signed, + 'format' => $this->format, + 'formatOptions' => $this->formatOptions, + 'filters' => $this->filters, + 'options' => $this->options, + 'createdAt' => $this->createdAt, + 'updatedAt' => $this->updatedAt, + ]; + } + + public static function getName(): string + { + return Resource::TYPE_ATTRIBUTE; + } + + abstract public function getType(): string; + + public function getGroup(): string + { + return Transfer::GROUP_DATABASES; + } + + public function getKey(): string + { + return $this->key; + } + + public function getTable(): Table + { + return $this->table; + } + + public function getSize(): int + { + return $this->size; + } + + public function isRequired(): bool + { + return $this->required; + } + + public function getDefault(): mixed + { + return $this->default; + } + + public function isArray(): bool + { + return $this->array; + } + + public function isSigned(): bool + { + return $this->signed; + } + + public function getFormat(): string + { + return $this->format; + } + + /** + * @return array + */ + public function getFormatOptions(): array + { + return $this->formatOptions; + } + + /** + * @return array + */ + public function getFilters(): array + { + return $this->filters; + } + + /** + * @return array + */ + public function &getOptions(): array + { + return $this->options; + } +} diff --git a/src/Migration/Resources/Database/Attribute/Boolean.php b/src/Migration/Resources/Database/Attribute/Boolean.php new file mode 100644 index 00000000..6bbe4b1c --- /dev/null +++ b/src/Migration/Resources/Database/Attribute/Boolean.php @@ -0,0 +1,78 @@ + + * }, + * table?: array{ + * database: array{ + * id: string, + * name: string, + * }, + * name: string, + * id: string, + * rowSecurity: bool, + * permissions: ?array + * }, + * required: bool, + * array: bool, + * default: ?bool, + * createdAt: string, + * updatedAt: string, + * } $array + * @return self + */ + public static function fromArray(array $array): self + { + return new self( + $array['key'], + Collection::fromArray($array['table'] ?? $array['collection']), + required: $array['required'], + default: $array['default'], + array: $array['array'], + createdAt: $array['createdAt'] ?? '', + updatedAt: $array['updatedAt'] ?? '', + ); + } + + public function getType(): string + { + return Attribute::TYPE_BOOLEAN; + } +} diff --git a/src/Migration/Resources/Database/Attribute/DateTime.php b/src/Migration/Resources/Database/Attribute/DateTime.php new file mode 100644 index 00000000..f81a375e --- /dev/null +++ b/src/Migration/Resources/Database/Attribute/DateTime.php @@ -0,0 +1,79 @@ + + * }, + * table?: array{ + * database: array{ + * id: string, + * name: string, + * }, + * name: string, + * id: string, + * rowSecurity: bool, + * permissions: ?array + * }, + * required: bool, + * array: bool, + * default: ?string, + * createdAt: string, + * updatedAt: string, + * } $array + * @return self + */ + public static function fromArray(array $array): self + { + return new self( + $array['key'], + Collection::fromArray($array['table'] ?? $array['collection']), + required: $array['required'], + default: $array['default'], + array: $array['array'], + createdAt: $array['createdAt'] ?? '', + updatedAt: $array['updatedAt'] ?? '', + ); + } +} diff --git a/src/Migration/Resources/Database/Attribute/Decimal.php b/src/Migration/Resources/Database/Attribute/Decimal.php new file mode 100644 index 00000000..acbcad4d --- /dev/null +++ b/src/Migration/Resources/Database/Attribute/Decimal.php @@ -0,0 +1,105 @@ + $min, + 'max' => $max, + ], + createdAt: $createdAt, + updatedAt: $updatedAt + ); + } + + /** + * @param array{ + * key: string, + * collection?: array{ + * database: array{ + * id: string, + * name: string, + * }, + * name: string, + * id: string, + * documentSecurity: bool, + * permissions: ?array + * }, + * table?: array{ + * database: array{ + * id: string, + * name: string, + * }, + * name: string, + * id: string, + * rowSecurity: bool, + * permissions: ?array + * }, + * required: bool, + * array: bool, + * default: ?float, + * formatOptions: array{ + * min: ?float, + * max: ?float + * }, + * createdAt: string, + * updatedAt: string, + * } $array + * @return self + */ + public static function fromArray(array $array): self + { + return new self( + $array['key'], + Collection::fromArray($array['table'] ?? $array['collection']), + required: $array['required'], + default: $array['default'], + array: $array['array'], + min: $array['formatOptions']['min'], + max: $array['formatOptions']['max'], + createdAt: $array['createdAt'] ?? '', + updatedAt: $array['updatedAt'] ?? '', + ); + } + + public function getType(): string + { + return Attribute::TYPE_FLOAT; + } + + public function getMin(): ?float + { + return (float)$this->formatOptions['min']; + } + + public function getMax(): ?float + { + return (float)$this->formatOptions['max']; + } +} diff --git a/src/Migration/Resources/Database/Attribute/Email.php b/src/Migration/Resources/Database/Attribute/Email.php new file mode 100644 index 00000000..2c460f37 --- /dev/null +++ b/src/Migration/Resources/Database/Attribute/Email.php @@ -0,0 +1,37 @@ + $elements + */ + public function __construct( + string $key, + Collection $collection, + array $elements, + bool $required = false, + ?string $default = null, + bool $array = false, + int $size = 256, + string $createdAt = '', + string $updatedAt = '' + ) { + parent::__construct( + $key, + $collection, + size: $size, + required: $required, + default: $default, + array: $array, + format: 'enum', + formatOptions: [ + 'elements' => $elements, + ], + createdAt: $createdAt, + updatedAt: $updatedAt + ); + } + + /** + * @param array{ + * key: string, + * collection?: array{ + * database: array{ + * id: string, + * name: string, + * }, + * name: string, + * id: string, + * documentSecurity: bool, + * permissions: ?array + * }, + * table?: array{ + * database: array{ + * id: string, + * name: string, + * }, + * name: string, + * id: string, + * rowSecurity: bool, + * permissions: ?array + * }, + * size: int, + * required: bool, + * default: ?string, + * array: bool, + * formatOptions: array{ + * elements: array + * }, + * createdAt: string, + * updatedAt: string, + * } $array + * @return self + */ + public static function fromArray(array $array): self + { + return new self( + $array['key'], + Collection::fromArray($array['table'] ?? $array['collection']), + elements: $array['formatOptions']['elements'], + required: $array['required'], + default: $array['default'], + array: $array['array'], + size: $array['size'], + createdAt: $array['createdAt'] ?? '', + updatedAt: $array['updatedAt'] ?? '', + ); + } + + public function getType(): string + { + return Attribute::TYPE_ENUM; + } + + /** + * @return array + */ + public function getElements(): array + { + return (array)$this->formatOptions['elements']; + } +} diff --git a/src/Migration/Resources/Database/Attribute/IP.php b/src/Migration/Resources/Database/Attribute/IP.php new file mode 100644 index 00000000..e2be9ade --- /dev/null +++ b/src/Migration/Resources/Database/Attribute/IP.php @@ -0,0 +1,37 @@ + 2147483647 ? 8 : 4; + + parent::__construct( + $key, + $collection, + size: $size, + required: $required, + default: $default, + array: $array, + signed: $signed, + formatOptions: [ + 'min' => $min, + 'max' => $max, + ], + createdAt: $createdAt, + updatedAt: $updatedAt + ); + } + + /** + * @param array{ + * key: string, + * collection?: array{ + * database: array{ + * id: string, + * name: string, + * }, + * name: string, + * id: string, + * documentSecurity: bool, + * permissions: ?array + * }, + * table?: array{ + * database: array{ + * id: string, + * name: string, + * }, + * name: string, + * id: string, + * rowSecurity: bool, + * permissions: ?array + * }, + * required: bool, + * array: bool, + * default: ?int, + * formatOptions: array{ + * min: ?int, + * max: ?int + * }, + * createdAt: string, + * updatedAt: string, + * } $array + * @return self + */ + public static function fromArray(array $array): self + { + return new self( + $array['key'], + Collection::fromArray($array['table'] ?? $array['collection']), + required: $array['required'], + default: $array['default'], + array: $array['array'], + min: $array['formatOptions']['min'] ?? null, + max: $array['formatOptions']['max'] ?? null, + createdAt: $array['createdAt'] ?? '', + updatedAt: $array['updatedAt'] ?? '', + ); + } + + public function getType(): string + { + return Attribute::TYPE_INTEGER; + } + + public function getMin(): ?int + { + return (int)$this->formatOptions['min']; + } + + public function getMax(): ?int + { + return (int)$this->formatOptions['max']; + } +} diff --git a/src/Migration/Resources/Database/Attribute/Line.php b/src/Migration/Resources/Database/Attribute/Line.php new file mode 100644 index 00000000..dfa360ae --- /dev/null +++ b/src/Migration/Resources/Database/Attribute/Line.php @@ -0,0 +1,74 @@ + + * }, + * table?: array{ + * database: array{ + * id: string, + * name: string, + * }, + * name: string, + * id: string, + * rowSecurity: bool, + * permissions: ?array + * }, + * required: bool, + * default: ?array, + * createdAt: string, + * updatedAt: string, + * } $array + * @return self + */ + public static function fromArray(array $array): self + { + return new self( + $array['key'], + Collection::fromArray($array['table'] ?? $array['collection']), + required: $array['required'], + default: $array['default'], + createdAt: $array['createdAt'] ?? '', + updatedAt: $array['updatedAt'] ?? '', + ); + } + + public function getType(): string + { + return Attribute::TYPE_LINE; + } +} diff --git a/src/Migration/Resources/Database/Attribute/ObjectType.php b/src/Migration/Resources/Database/Attribute/ObjectType.php new file mode 100644 index 00000000..e3ef99de --- /dev/null +++ b/src/Migration/Resources/Database/Attribute/ObjectType.php @@ -0,0 +1,74 @@ + + * }, + * table?: array{ + * database: array{ + * id: string, + * name: string, + * }, + * name: string, + * id: string, + * rowSecurity: bool, + * permissions: ?array + * }, + * required: bool, + * default: ?array, + * createdAt: string, + * updatedAt: string, + * } $array + * @return self + */ + public static function fromArray(array $array): self + { + return new self( + $array['key'], + Collection::fromArray($array['table'] ?? $array['collection']), + required: $array['required'], + default: $array['default'], + createdAt: $array['createdAt'] ?? '', + updatedAt: $array['updatedAt'] ?? '', + ); + } + + public function getType(): string + { + return Attribute::TYPE_OBJECT; + } +} diff --git a/src/Migration/Resources/Database/Attribute/Point.php b/src/Migration/Resources/Database/Attribute/Point.php new file mode 100644 index 00000000..a82d7d36 --- /dev/null +++ b/src/Migration/Resources/Database/Attribute/Point.php @@ -0,0 +1,74 @@ + + * }, + * table?: array{ + * database: array{ + * id: string, + * name: string, + * }, + * name: string, + * id: string, + * rowSecurity: bool, + * permissions: ?array + * }, + * required: bool, + * default: ?array, + * createdAt: string, + * updatedAt: string, + * } $array + * @return self + */ + public static function fromArray(array $array): self + { + return new self( + $array['key'], + Collection::fromArray($array['table'] ?? $array['collection']), + required: $array['required'], + default: $array['default'], + createdAt: $array['createdAt'] ?? '', + updatedAt: $array['updatedAt'] ?? '', + ); + } + + public function getType(): string + { + return Attribute::TYPE_POINT; + } +} diff --git a/src/Migration/Resources/Database/Attribute/Polygon.php b/src/Migration/Resources/Database/Attribute/Polygon.php new file mode 100644 index 00000000..28d2adc3 --- /dev/null +++ b/src/Migration/Resources/Database/Attribute/Polygon.php @@ -0,0 +1,74 @@ + + * }, + * table?: array{ + * database: array{ + * id: string, + * name: string, + * }, + * name: string, + * id: string, + * rowSecurity: bool, + * permissions: ?array + * }, + * required: bool, + * default: ?array, + * createdAt: string, + * updatedAt: string, + * } $array + * @return self + */ + public static function fromArray(array $array): self + { + return new self( + $array['key'], + Collection::fromArray($array['table'] ?? $array['collection']), + required: $array['required'], + default: $array['default'], + createdAt: $array['createdAt'] ?? '', + updatedAt: $array['updatedAt'] ?? '', + ); + } + + public function getType(): string + { + return Attribute::TYPE_POLYGON; + } +} diff --git a/src/Migration/Resources/Database/Attribute/Relationship.php b/src/Migration/Resources/Database/Attribute/Relationship.php new file mode 100644 index 00000000..aa252ab7 --- /dev/null +++ b/src/Migration/Resources/Database/Attribute/Relationship.php @@ -0,0 +1,125 @@ + $relatedTable, + 'relationType' => $relationType, + 'twoWay' => $twoWay, + 'twoWayKey' => $twoWayKey, + 'onDelete' => $onDelete, + 'side' => $side, + ], + createdAt: $createdAt, + updatedAt: $updatedAt + ); + } + + /** + * @param array{ + * key: string, + * collection?: array{ + * database: array{ + * id: string, + * name: string, + * }, + * name: string, + * id: string, + * documentSecurity: bool, + * permissions: ?array + * }, + * table?: array{ + * database: array{ + * id: string, + * name: string, + * }, + * name: string, + * id: string, + * rowSecurity: bool, + * permissions: ?array + * }, + * options: array{ + * relatedCollection: string, + * relationType: string, + * twoWay: bool, + * twoWayKey: ?string, + * onDelete: string, + * side: string, + * }, + * createdAt: string, + * updatedAt: string, + * } $array + * @return self + */ + public static function fromArray(array $array): self + { + return new self( + $array['key'], + Collection::fromArray($array['table'] ?? $array['collection']), + relatedTable: $array['options']['relatedTable'] ?? $array['options']['relatedCollection'], + relationType: $array['options']['relationType'], + twoWay: $array['options']['twoWay'], + twoWayKey: $array['options']['twoWayKey'], + onDelete: $array['options']['onDelete'], + side: $array['options']['side'], + createdAt: $array['createdAt'] ?? '', + updatedAt: $array['updatedAt'] ?? '', + ); + } + + public function getType(): string + { + return Attribute::TYPE_RELATIONSHIP; + } + + public function getRelatedTable(): string + { + return $this->options['relatedTable'] ?? $this->options['relatedCollection']; + } + + public function getRelationType(): string + { + return $this->options['relationType']; + } + + public function getTwoWay(): bool + { + return $this->options['twoWay']; + } + + public function getTwoWayKey(): ?string + { + return $this->options['twoWayKey']; + } + + public function getOnDelete(): string + { + return $this->options['onDelete']; + } + + public function getSide(): string + { + return $this->options['side']; + } +} diff --git a/src/Migration/Resources/Database/Attribute/Text.php b/src/Migration/Resources/Database/Attribute/Text.php new file mode 100644 index 00000000..a0c60ec8 --- /dev/null +++ b/src/Migration/Resources/Database/Attribute/Text.php @@ -0,0 +1,97 @@ + + * }, + * table?: array{ + * database: array{ + * id: string, + * name: string, + * }, + * name: string, + * id: string, + * rowSecurity: bool, + * permissions: ?array + * }, + * required: bool, + * default: ?string, + * array: bool, + * size: int, + * format: string, + * createdAt: string, + * updatedAt: string, + * } $array + * @return self + */ + public static function fromArray(array $array): self + { + return new self( + $array['key'], + Collection::fromArray($array['table'] ?? $array['collection']), + required: $array['required'], + default: $array['default'] ?? null, + array: $array['array'], + size: $array['size'], + format: $array['format'], + createdAt: $array['createdAt'] ?? '', + updatedAt: $array['updatedAt'] ?? '', + ); + } + + public function getType(): string + { + return Attribute::TYPE_STRING; + } + + public function getSize(): int + { + return $this->size; + } + + public function getFormat(): string + { + return $this->format; + } +} diff --git a/src/Migration/Resources/Database/Attribute/URL.php b/src/Migration/Resources/Database/Attribute/URL.php new file mode 100644 index 00000000..2e83a0ff --- /dev/null +++ b/src/Migration/Resources/Database/Attribute/URL.php @@ -0,0 +1,37 @@ + + * }, + * table?: array{ + * database: array{ + * id: string, + * name: string, + * }, + * name: string, + * id: string, + * rowSecurity: bool, + * permissions: ?array + * }, + * size: int, + * required: bool, + * default: ?array, + * createdAt: string, + * updatedAt: string, + * } $array + * @return self + */ + public static function fromArray(array $array): self + { + return new self( + $array['key'], + Collection::fromArray($array['table'] ?? $array['collection']), + required: $array['required'], + size:$array['size'], + default: $array['default'], + createdAt: $array['createdAt'] ?? '', + updatedAt: $array['updatedAt'] ?? '', + ); + } + + public function getType(): string + { + return Attribute::TYPE_VECTOR; + } +} diff --git a/src/Migration/Resources/Database/Collection.php b/src/Migration/Resources/Database/Collection.php new file mode 100644 index 00000000..7969b930 --- /dev/null +++ b/src/Migration/Resources/Database/Collection.php @@ -0,0 +1,50 @@ +, + * createdAt: string, + * updatedAt: string, + * enabled: bool + * } $array + */ + public static function fromArray(array $array): self + { + $database = match ($array['database']['type']) { + Resource::TYPE_DATABASE_DOCUMENTSDB => DocumentsDB::fromArray($array['database']), + Resource::TYPE_DATABASE_VECTORDB => VectorDB::fromArray($array['database']), + default => Database::fromArray($array['database']) + }; + + return new self( + $database, + name: $array['name'], + id: $array['id'], + rowSecurity: $array['rowSecurity'] ?? $array['documentSecurity'], + permissions: $array['permissions'] ?? [], + createdAt: $array['createdAt'] ?? '', + updatedAt: $array['updatedAt'] ?? '', + enabled: $array['enabled'] ?? true, + ); + } +} diff --git a/src/Migration/Resources/Database/Column.php b/src/Migration/Resources/Database/Column.php index e064a404..e80d1726 100644 --- a/src/Migration/Resources/Database/Column.php +++ b/src/Migration/Resources/Database/Column.php @@ -22,6 +22,9 @@ abstract class Column extends Resource public const TYPE_LINE = 'linestring'; public const TYPE_POLYGON = 'polygon'; + public const TYPE_OBJECT = 'object'; + public const TYPE_VECTOR = 'vector'; + /** * @param string $key * @param Table $table diff --git a/src/Migration/Resources/Database/Columns/ObjectType.php b/src/Migration/Resources/Database/Columns/ObjectType.php new file mode 100644 index 00000000..38477cd3 --- /dev/null +++ b/src/Migration/Resources/Database/Columns/ObjectType.php @@ -0,0 +1,74 @@ + + * }, + * table?: array{ + * database: array{ + * id: string, + * name: string, + * }, + * name: string, + * id: string, + * rowSecurity: bool, + * permissions: ?array + * }, + * required: bool, + * default: ?array, + * createdAt: string, + * updatedAt: string, + * } $array + * @return self + */ + public static function fromArray(array $array): self + { + return new self( + $array['key'], + Table::fromArray($array['table'] ?? $array['collection']), + required: $array['required'], + default: $array['default'], + createdAt: $array['createdAt'] ?? '', + updatedAt: $array['updatedAt'] ?? '', + ); + } + + public function getType(): string + { + return Column::TYPE_OBJECT; + } +} diff --git a/src/Migration/Resources/Database/Columns/Vector.php b/src/Migration/Resources/Database/Columns/Vector.php new file mode 100644 index 00000000..4cc898a7 --- /dev/null +++ b/src/Migration/Resources/Database/Columns/Vector.php @@ -0,0 +1,78 @@ + + * }, + * table?: array{ + * database: array{ + * id: string, + * name: string, + * }, + * name: string, + * id: string, + * rowSecurity: bool, + * permissions: ?array + * }, + * size: int, + * required: bool, + * default: ?array, + * createdAt: string, + * updatedAt: string, + * } $array + * @return self + */ + public static function fromArray(array $array): self + { + return new self( + $array['key'], + Table::fromArray($array['table'] ?? $array['collection']), + required: $array['required'], + size:$array['size'], + default: $array['default'], + createdAt: $array['createdAt'] ?? '', + updatedAt: $array['updatedAt'] ?? '', + ); + } + + public function getType(): string + { + return Column::TYPE_VECTOR; + } +} diff --git a/src/Migration/Resources/Database/Database.php b/src/Migration/Resources/Database/Database.php index 06882b21..9c904985 100644 --- a/src/Migration/Resources/Database/Database.php +++ b/src/Migration/Resources/Database/Database.php @@ -26,6 +26,7 @@ public function __construct( protected bool $enabled = true, protected string $originalId = '', protected string $type = '', + protected string $database = '' ) { $this->id = $id; } @@ -38,6 +39,7 @@ public function __construct( * updatedAt: string, * enabled: bool, * originalId: string|null, + * database: string * } $array */ public static function fromArray(array $array): self @@ -50,6 +52,7 @@ public static function fromArray(array $array): self enabled: $array['enabled'] ?? true, originalId: $array['originalId'] ?? '', type: $array['type'] ?? 'legacy', + database: $array['database'] ); } @@ -92,4 +95,9 @@ public function getType(): string { return $this->type; } + + public function getDatabase(): string + { + return $this->database; + } } diff --git a/src/Migration/Resources/Database/Document.php b/src/Migration/Resources/Database/Document.php new file mode 100644 index 00000000..07e9a5d1 --- /dev/null +++ b/src/Migration/Resources/Database/Document.php @@ -0,0 +1,51 @@ + + * }, + * table?: array{ + * database: array{ + * id: string, + * name: string, + * }, + * name: string, + * id: string, + * rowSecurity: bool, + * permissions: ?array + * }, + * data: array, + * permissions: ?array + * } $array + */ + public static function fromArray(array $array): self + { + // keeping table and collection to have backward compat + return new self( + $array['id'], + Collection::fromArray($array['table'] ?? $array['collection']), + $array['data'], + $array['permissions'] ?? [] + ); + } +} diff --git a/src/Migration/Resources/Database/DocumentsDB.php b/src/Migration/Resources/Database/DocumentsDB.php new file mode 100644 index 00000000..54091388 --- /dev/null +++ b/src/Migration/Resources/Database/DocumentsDB.php @@ -0,0 +1,38 @@ +headers['X-Appwrite-Project'] = $this->project; $this->headers['X-Appwrite-Key'] = $this->key; - switch ($this->source) { - case static::SOURCE_API: - $this->database = new APIReader(new Databases($this->client)); - break; - case static::SOURCE_DATABASE: - if (\is_null($dbForProject)) { - throw new \Exception('Database is required for database source'); - } - $this->database = new DatabaseReader($dbForProject); - break; - default: - throw new \Exception('Unknown source'); - } + $this->getDatabasesDB = $getDatabasesDB; + + $this->reader = match ($this->source) { + static::SOURCE_API => new APIReader(new Databases($this->client)), + static::SOURCE_DATABASE => new DatabaseReader($this->dbForProject, $this->getDatabasesDB), + default => throw new \Exception('Unknown source'), + }; + } public static function getName(): string @@ -131,6 +154,11 @@ public static function getSupportedResources(): array Resource::TYPE_ATTRIBUTE, Resource::TYPE_COLLECTION, + // documentsdb + Resource::TYPE_DATABASE_DOCUMENTSDB, + // vectordb + Resource::TYPE_DATABASE_VECTORDB, + // Storage Resource::TYPE_BUCKET, Resource::TYPE_FILE, @@ -270,7 +298,7 @@ private function reportAuth(array $resources, array &$report): void */ private function reportDatabases(array $resources, array &$report): void { - $this->database->report($resources, $report); + $this->reader->report($resources, $report); } /** @@ -608,32 +636,37 @@ private function exportMemberships(int $batchSize): void protected function exportGroupDatabases(int $batchSize, array $resources): void { - try { - if (\in_array(Resource::TYPE_DATABASE, $resources)) { - $this->exportDatabases($batchSize); - } - } catch (\Throwable $e) { - $this->addError( - new Exception( - Resource::TYPE_DATABASE, - Transfer::GROUP_DATABASES, - message: $e->getMessage(), - code: $e->getCode(), - previous: $e - ) - ); + $handleExportEntityScopedResources = function (string $resourceKey, callable $callback) use ($resources) { + foreach (Resource::ENTITY_TYPE_RESOURCE_MAP as $entityKey => $entityResource) { + try { + if (\in_array($entityResource[$resourceKey], $resources)) { + $callback($entityKey, $entityResource); + } + } catch (\Throwable $e) { + $this->addError( + new Exception( + $resourceKey, + Transfer::GROUP_DATABASES, + message: $e->getMessage(), + code: $e->getCode(), + previous: $e + ) + ); - return; - } + return false; + } + } + return true; + }; try { - if (Resource::isSupported(Resource::TYPE_TABLE, $resources)) { - $this->exportTables($batchSize); + if (Resource::isSupported(array_keys(Resource::DATABASE_TYPE_RESOURCE_MAP), $resources)) { + $this->exportDatabases($batchSize, $resources); } } catch (\Throwable $e) { $this->addError( new Exception( - Resource::TYPE_TABLE, + Resource::TYPE_DATABASE, Transfer::GROUP_DATABASES, message: $e->getMessage(), code: $e->getCode(), @@ -644,73 +677,55 @@ protected function exportGroupDatabases(int $batchSize, array $resources): void return; } - try { - if (Resource::isSupported(Resource::TYPE_COLUMN, $resources)) { - $this->exportColumns($batchSize); + foreach (Resource::DATABASE_TYPE_RESOURCE_MAP as $databaseKey => $databaseResource) { + try { + if (\in_array($databaseResource['entity'], $resources)) { + $this->exportEntities($databaseKey, $batchSize); + } + } catch (\Throwable $e) { + $this->addError( + new Exception( + $databaseResource['entity'], + Transfer::GROUP_DATABASES, + message: $e->getMessage(), + code: $e->getCode(), + previous: $e + ) + ); + + return; } - } catch (\Throwable $e) { - $this->addError( - new Exception( - Resource::TYPE_COLUMN, - Transfer::GROUP_DATABASES, - message: $e->getMessage(), - code: $e->getCode(), - previous: $e - ) - ); + } + // field + if (!$handleExportEntityScopedResources('field', fn ($entityKey) => $this->exportFields($entityKey, $batchSize))) { return; } - try { - if (\in_array(Resource::TYPE_INDEX, $resources)) { - $this->exportIndexes($batchSize); - } - } catch (\Throwable $e) { - $this->addError( - new Exception( - Resource::TYPE_INDEX, - Transfer::GROUP_DATABASES, - message: $e->getMessage(), - code: $e->getCode(), - previous: $e - ) - ); - + // index + if (!$handleExportEntityScopedResources('index', fn ($entityKey) => $this->exportIndexes($entityKey, $batchSize))) { return; } - try { - if (Resource::isSupported(Resource::TYPE_ROW, $resources)) { - $this->exportRows($batchSize); - } - } catch (\Throwable $e) { - $this->addError( - new Exception( - Resource::TYPE_ROW, - Transfer::GROUP_DATABASES, - message: $e->getMessage(), - code: $e->getCode(), - previous: $e - ) - ); - + // record + if (!$handleExportEntityScopedResources('record', fn ($entityKey, $entityResource) => $this->exportRecords($entityKey, $entityResource['field'], $batchSize))) { return; } } /** * @param int $batchSize + * @param array $resources * @throws Exception */ - private function exportDatabases(int $batchSize): void + private function exportDatabases(int $batchSize, array $resources = []): void { $lastDatabase = null; while (true) { - $queries = [$this->database->queryLimit($batchSize)]; + $queries = [$this->reader->queryLimit($batchSize)]; - if ($this->rootResourceId !== '' && $this->rootResourceType === Resource::TYPE_DATABASE) { + if ($this->rootResourceId !== '' && ($this->rootResourceType === Resource::TYPE_DATABASE || $this->rootResourceType === Resource::TYPE_DATABASE_DOCUMENTSDB)) { $targetDatabaseId = $this->rootResourceId; // Handle database:collection format - extract database ID @@ -721,28 +736,35 @@ private function exportDatabases(int $batchSize): void } } - $queries[] = $this->database->queryEqual('$id', [$targetDatabaseId]); - $queries[] = $this->database->queryLimit(1); + $queries[] = $this->reader->queryEqual('$id', [$targetDatabaseId]); + $queries[] = $this->reader->queryLimit(1); } $databases = []; if ($lastDatabase) { - $queries[] = $this->database->queryCursorAfter($lastDatabase); + $queries[] = $this->reader->queryCursorAfter($lastDatabase); } - $response = $this->database->listDatabases($queries); + $response = $this->reader->listDatabases($queries); foreach ($response as $database) { - $newDatabase = new Database( - $database['$id'], - $database['name'], - $database['$createdAt'], - $database['$updatedAt'], - type: $database['type'] ?? 'legacy' - ); + $databaseType = $database['type']; + if (in_array($databaseType, [Resource::TYPE_DATABASE_LEGACY,Resource::TYPE_DATABASE_TABLESDB])) { + $databaseType = Resource::TYPE_DATABASE; + } + if (Resource::isSupported($databaseType, $resources)) { + $newDatabase = self::getDatabase($databaseType, [ + 'id' => $database['$id'], + 'name' => $database['name'], + 'createdAt' => $database['$createdAt'], + 'updatedAt' => $database['$updatedAt'], + 'type' => $databaseType, + 'database' => $database['database'] + ]); + $databases[] = $newDatabase; - $databases[] = $newDatabase; + } } if (empty($databases)) { @@ -760,19 +782,19 @@ private function exportDatabases(int $batchSize): void } /** + * @param string $databaseName * @param int $batchSize * @throws Exception */ - private function exportTables(int $batchSize): void + private function exportEntities(string $databaseName, int $batchSize): void { - $databases = $this->cache->get(Database::getName()); - + $databases = $this->cache->get($databaseName); foreach ($databases as $database) { /** @var Database $database */ $lastTable = null; while (true) { - $queries = [$this->database->queryLimit($batchSize)]; + $queries = [$this->reader->queryLimit($batchSize)]; $tables = []; // Filter to specific table if rootResourceType is database with database:collection format @@ -784,32 +806,36 @@ private function exportTables(int $batchSize): void $parts = \explode(':', $this->rootResourceId, 2); if (\count($parts) === 2) { $targetTableId = $parts[1]; // table ID - $queries[] = $this->database->queryEqual('$id', [$targetTableId]); - $queries[] = $this->database->queryLimit(1); + $queries[] = $this->reader->queryEqual('$id', [$targetTableId]); + $queries[] = $this->reader->queryLimit(1); } } elseif ( $this->rootResourceId !== '' && $this->rootResourceType === Resource::TYPE_TABLE ) { $targetTableId = $this->rootResourceId; - $queries[] = $this->database->queryEqual('$id', [$targetTableId]); - $queries[] = $this->database->queryLimit(1); + $queries[] = $this->reader->queryEqual('$id', [$targetTableId]); + $queries[] = $this->reader->queryLimit(1); } elseif ($lastTable) { - $queries[] = $this->database->queryCursorAfter($lastTable); + $queries[] = $this->reader->queryCursorAfter($lastTable); } - $response = $this->database->listTables($database, $queries); - + $response = $this->reader->listTables($database, $queries); foreach ($response as $table) { - $newTable = new Table( - $database, - $table['name'], - $table['$id'], - $table['documentSecurity'], - $table['$permissions'], - $table['$createdAt'], - $table['$updatedAt'], - ); + $newTable = self::getEntity($databaseName, [ + 'id' => $table['$id'], + 'name' => $table['name'], + 'documentSecurity' => $table['documentSecurity'], + 'permissions' => $table['$permissions'], + 'createdAt' => $table['$createdAt'], + 'updatedAt' => $table['$updatedAt'], + 'database' => [ + 'id' => $database->getId(), + 'name' => $databaseName, + 'type' => $database->getType(), + 'database' => $database->getDatabase(), + ] + ]); $tables[] = $newTable; } @@ -830,26 +856,28 @@ private function exportTables(int $batchSize): void } /** + * @param string $entityType * @param int $batchSize * @throws Exception */ - private function exportColumns(int $batchSize): void + private function exportFields(string $entityType, int $batchSize): void { - $tables = $this->cache->get(Table::getName()); + $entities = $this->cache->get($entityType); + // Transfer Indexes - /** @var array $tables */ - foreach ($tables as $table) { + /** @var array $table */ + foreach ($entities as $table) { $lastColumn = null; while (true) { - $queries = [$this->database->queryLimit($batchSize)]; + $queries = [$this->reader->queryLimit($batchSize)]; $columns = []; if ($lastColumn) { - $queries[] = $this->database->queryCursorAfter($lastColumn); + $queries[] = $this->reader->queryCursorAfter($lastColumn); } - $response = $this->database->listColumns($table, $queries); + $response = $this->reader->listColumns($table, $queries); foreach ($response as $column) { if ( @@ -859,165 +887,11 @@ private function exportColumns(int $batchSize): void continue; } - switch ($column['type']) { - case Column::TYPE_STRING: - $col = match ($column['format'] ?? '') { - Column::TYPE_EMAIL => new Email( - $column['key'], - $table, - required: $column['required'], - default: $column['default'], - array: $column['array'], - size: $column['size'] ?? 254, - createdAt: $column['$createdAt'] ?? '', - updatedAt: $column['$updatedAt'] ?? '', - ), - Column::TYPE_ENUM => new Enum( - $column['key'], - $table, - elements: $column['elements'], - required: $column['required'], - default: $column['default'], - array: $column['array'], - size: $column['size'] ?? UtopiaDatabase::LENGTH_KEY, - createdAt: $column['$createdAt'] ?? '', - updatedAt: $column['$updatedAt'] ?? '', - ), - Column::TYPE_URL => new URL( - $column['key'], - $table, - required: $column['required'], - default: $column['default'], - array: $column['array'], - size: $column['size'] ?? 2000, - createdAt: $column['$createdAt'] ?? '', - updatedAt: $column['$updatedAt'] ?? '', - ), - Column::TYPE_IP => new IP( - $column['key'], - $table, - required: $column['required'], - default: $column['default'], - array: $column['array'], - size: $column['size'] ?? 39, - createdAt: $column['$createdAt'] ?? '', - updatedAt: $column['$updatedAt'] ?? '', - ), - default => new Text( - $column['key'], - $table, - required: $column['required'], - default: $column['default'], - array: $column['array'], - size: $column['size'] ?? 0, - createdAt: $column['$createdAt'] ?? '', - updatedAt: $column['$updatedAt'] ?? '', - ), - }; - - break; - case Column::TYPE_BOOLEAN: - $col = new Boolean( - $column['key'], - $table, - required: $column['required'], - default: $column['default'], - array: $column['array'], - createdAt: $column['$createdAt'] ?? '', - updatedAt: $column['$updatedAt'] ?? '', - ); - break; - case Column::TYPE_INTEGER: - $col = new Integer( - $column['key'], - $table, - required: $column['required'], - default: $column['default'], - array: $column['array'], - min: $column['min'] ?? null, - max: $column['max'] ?? null, - createdAt: $column['$createdAt'] ?? '', - updatedAt: $column['$updatedAt'] ?? '', - ); - break; - case Column::TYPE_FLOAT: - $col = new Decimal( - $column['key'], - $table, - required: $column['required'], - default: $column['default'], - array: $column['array'], - min: $column['min'] ?? null, - max: $column['max'] ?? null, - createdAt: $column['$createdAt'] ?? '', - updatedAt: $column['$updatedAt'] ?? '', - ); - break; - case Column::TYPE_RELATIONSHIP: - $col = new Relationship( - $column['key'], - $table, - relatedTable: $column['relatedTable'] ?? $column['relatedCollection'], - relationType: $column['relationType'], - twoWay: $column['twoWay'], - twoWayKey: $column['twoWayKey'], - onDelete: $column['onDelete'], - side: $column['side'], - createdAt: $column['$createdAt'] ?? '', - updatedAt: $column['$updatedAt'] ?? '', - ); - break; - case Column::TYPE_DATETIME: - $col = new DateTime( - $column['key'], - $table, - required: $column['required'], - default: $column['default'], - array: $column['array'], - createdAt: $column['$createdAt'] ?? '', - updatedAt: $column['$updatedAt'] ?? '', - ); - break; - case Column::TYPE_POINT: - $col = new Point( - $column['key'], - $table, - required: $column['required'], - default: $column['default'], - createdAt: $column['$createdAt'] ?? '', - updatedAt: $column['$updatedAt'] ?? '', - ); - break; - case Column::TYPE_LINE: - $col = new Line( - $column['key'], - $table, - required: $column['required'], - default: $column['default'], - createdAt: $column['$createdAt'] ?? '', - updatedAt: $column['$updatedAt'] ?? '', - ); - break; - case Column::TYPE_POLYGON: - $col = new Polygon( - $column['key'], - $table, - required: $column['required'], - default: $column['default'], - createdAt: $column['$createdAt'] ?? '', - updatedAt: $column['$updatedAt'] ?? '', - ); - break; - } - - if (!isset($col)) { - throw new Exception( - resourceName: Resource::TYPE_COLUMN, - resourceGroup: Transfer::GROUP_DATABASES, - resourceId: $column['$id'], - message: 'Unknown column type: ' . $column['type'] - ); - } + /** @var Table $table */ + $col = match($table->getDatabase()->getType()) { + Resource::TYPE_DATABASE_VECTORDB => self::getAttribute($table, $column), + default => self::getColumn($table, $column), + }; $columns[] = $col; } @@ -1038,27 +912,27 @@ private function exportColumns(int $batchSize): void } /** + * @param string $entityType * @param int $batchSize * @throws Exception */ - private function exportIndexes(int $batchSize): void + private function exportIndexes(string $entityType, int $batchSize): void { - $tables = $this->cache->get(Resource::TYPE_TABLE); - + $entities = $this->cache->get($entityType); // Transfer Indexes - foreach ($tables as $table) { + foreach ($entities as $table) { /** @var Table $table */ $lastIndex = null; while (true) { - $queries = [$this->database->queryLimit($batchSize)]; + $queries = [$this->reader->queryLimit($batchSize)]; $indexes = []; if ($lastIndex) { - $queries[] = $this->database->queryCursorAfter($lastIndex); + $queries[] = $this->reader->queryCursorAfter($lastIndex); } - $response = $this->database->listIndexes($table, $queries); + $response = $this->reader->listIndexes($table, $queries); foreach ($response as $index) { $indexes[] = new Index( @@ -1090,74 +964,75 @@ private function exportIndexes(int $batchSize): void } /** + * @param string $entityName + * @param string $fieldName + * @param int $batchSize * @throws Exception */ - private function exportRows(int $batchSize): void + private function exportRecords(string $entityName, string $fieldName, int $batchSize): void { - $tables = $this->cache->get(Table::getName()); - - foreach ($tables as $table) { + $entities = $this->cache->get($entityName); + foreach ($entities as $table) { /** @var Table $table */ $lastRow = null; while (true) { $queries = [ - $this->database->queryLimit($batchSize), + $this->reader->queryLimit($batchSize), ...$this->queries, ]; $rows = []; if ($lastRow) { - $queries[] = $this->database->queryCursorAfter($lastRow); + $queries[] = $this->reader->queryCursorAfter($lastRow); } $selects = ['*', '$id', '$permissions', '$updatedAt', '$createdAt']; // We want relations flat! $manyToMany = []; - $attributes = $this->cache->get(Column::getName()); - foreach ($attributes as $attribute) { - /** @var Relationship $attribute */ - if ( - $attribute->getTable()->getId() === $table->getId() && - $attribute->getType() === Column::TYPE_RELATIONSHIP && - $attribute->getSide() === 'parent' && - $attribute->getRelationType() == 'manyToMany' - ) { - /** - * Blockers: - * we should use but Does not work properly: - * $selects[] = $attribute->getKey() . '.$id'; - * when selecting for a relation we get all relations not just the one we were asking. - * when selecting for a relation like select(*, relation.$id) , all relations get resolve - */ - $manyToMany[] = $attribute->getKey(); + if ($this->reader->getSupportForAttributes()) { + $attributes = $this->cache->get($fieldName); + + foreach ($attributes as $attribute) { + /** @var Relationship $attribute */ + if ( + $attribute->getTable()->getId() === $table->getId() && + $attribute->getType() === Column::TYPE_RELATIONSHIP && + $attribute->getSide() === 'parent' && + $attribute->getRelationType() == 'manyToMany' + ) { + $manyToMany[] = $attribute->getKey(); + } } } - /** @var Column|Relationship $attribute */ - $queries[] = $this->database->querySelect($selects); + $queries[] = $this->reader->querySelect($selects); - $response = $this->database->listRows($table, $queries); + $response = $this->reader->listRows($table, $queries); foreach ($response as $row) { - // HACK: Handle many to many + // HACK: Handle many to many (only for schema-based databases) if (!empty($manyToMany)) { $stack = ['$id']; // Adding $id because we can't select only relations foreach ($manyToMany as $relation) { $stack[] = $relation . '.$id'; } - $rowItem = $this->database->getRow( + $rowItem = $this->reader->getRow( $table, $row['$id'], - [$this->database->querySelect($stack)] + [$this->reader->querySelect($stack)] ); foreach ($manyToMany as $key) { $row[$key] = []; - foreach ($rowItem[$key] as $relatedRowItem) { - $row[$key][] = $relatedRowItem['$id']; + if (isset($rowItem[$key]) && is_array($rowItem[$key])) { + foreach ($rowItem[$key] as $relatedRowItem) { + if (is_array($relatedRowItem) && isset($relatedRowItem['$id'])) { + $row[$key][] = $relatedRowItem['$id']; + } + } } } } @@ -1172,12 +1047,23 @@ private function exportRows(int $batchSize): void unset($row['$sequence']); unset($row['$collection']); - $row = new Row( - $id, - $table, - $row, - $permissions - ); + $row = self::getRecord($table->getDatabase()->getDatabaseName(), [ + 'id' => $id, + 'table' => [ + 'id' => $table->getId(), + 'name' => $table->getTableName(), + 'rowSecurity' => $table->getRowSecurity(), + 'permissions' => $table->getPermissions(), + 'database' => [ + 'id' => $table->getDatabase()->getId(), + 'name' => $table->getDatabase()->getDatabaseName(), + 'type' => $table->getDatabase()->getType(), + 'database' => $table->getDatabase()->getDatabase(), + ] + ], + 'data' => $row, + 'permissions' => $permissions + ]); $rows[] = $row; $lastRow = $row; @@ -1607,4 +1493,428 @@ private function exportDeploymentData(Func $func, array $deployment): void } } } + + /** + * @param string $databaseType + * @param array $database { + * id: string, + * name: string, + * createdAt: string, + * updatedAt: string, + * enabled: bool, + * originalId: string|null, + * database: string + * } + */ + public static function getDatabase(string $databaseType, array $database): Resource + { + switch ($databaseType) { + case Resource::TYPE_DATABASE_DOCUMENTSDB: + return DocumentsDB::fromArray($database); + case Resource::TYPE_DATABASE_VECTORDB: + return VectorDB::fromArray($database); + default: + return Database::fromArray($database); + } + } + + /** + * eg., tables,collections + * @param string $databaseType + * @param array{ + * database: array{ + * id: string, + * name: string, + * }, + * name: string, + * id: string, + * documentSecurity?: bool, + * rowSecurity?: bool, + * permissions: ?array, + * createdAt: string, + * updatedAt: string, + * enabled: bool + * } $entity + */ + public static function getEntity(string $databaseType, array $entity): Resource + { + switch ($databaseType) { + case Resource::TYPE_DATABASE_DOCUMENTSDB: + return Collection::fromArray($entity); + case Resource::TYPE_DATABASE_VECTORDB: + return Collection::fromArray($entity); + default: + return Table::fromArray($entity); + } + } + + /** + * eg.,documents/attributes + * @param string $databaseType + * @param array{ + * id: string, + * collection?: array{ + * database: array{ + * id: string, + * name: string, + * }, + * name: string, + * id: string, + * documentSecurity: bool, + * permissions: ?array + * }, + * table?: array{ + * database: array{ + * id: string, + * name: string, + * }, + * name: string, + * id: string, + * rowSecurity: bool, + * permissions: ?array + * }, + * data: array, + * permissions: ?array + * } $record + */ + public static function getRecord(string $databaseType, array $record): Resource + { + switch ($databaseType) { + case Resource::TYPE_DATABASE_DOCUMENTSDB: + return Document::fromArray($record); + case Resource::TYPE_DATABASE_VECTORDB: + return Document::fromArray($record); + default: + return Row::fromArray($record); + } + } + + public static function getColumn(Table $table, mixed $column): Column + { + return match ($column['type']) { + Column::TYPE_STRING => match ($column['format'] ?? '') { + Column::TYPE_EMAIL => new Email( + $column['key'], + $table, + required: $column['required'], + default: $column['default'], + array: $column['array'], + size: $column['size'] ?? 254, + createdAt: $column['$createdAt'] ?? '', + updatedAt: $column['$updatedAt'] ?? '', + ), + Column::TYPE_ENUM => new Enum( + $column['key'], + $table, + elements: $column['elements'], + required: $column['required'], + default: $column['default'], + array: $column['array'], + size: $column['size'] ?? UtopiaDatabase::LENGTH_KEY, + createdAt: $column['$createdAt'] ?? '', + updatedAt: $column['$updatedAt'] ?? '', + ), + Column::TYPE_URL => new URL( + $column['key'], + $table, + required: $column['required'], + default: $column['default'], + array: $column['array'], + size: $column['size'] ?? 2000, + createdAt: $column['$createdAt'] ?? '', + updatedAt: $column['$updatedAt'] ?? '', + ), + Column::TYPE_IP => new IP( + $column['key'], + $table, + required: $column['required'], + default: $column['default'], + array: $column['array'], + size: $column['size'] ?? 39, + createdAt: $column['$createdAt'] ?? '', + updatedAt: $column['$updatedAt'] ?? '', + ), + default => new Text( + $column['key'], + $table, + required: $column['required'], + default: $column['default'], + array: $column['array'], + size: $column['size'] ?? 0, + createdAt: $column['$createdAt'] ?? '', + updatedAt: $column['$updatedAt'] ?? '', + ), + }, + + Column::TYPE_BOOLEAN => new Boolean( + $column['key'], + $table, + required: $column['required'], + default: $column['default'], + array: $column['array'], + createdAt: $column['$createdAt'] ?? '', + updatedAt: $column['$updatedAt'] ?? '', + ), + + Column::TYPE_INTEGER => new Integer( + $column['key'], + $table, + required: $column['required'], + default: $column['default'], + array: $column['array'], + min: $column['min'] ?? null, + max: $column['max'] ?? null, + createdAt: $column['$createdAt'] ?? '', + updatedAt: $column['$updatedAt'] ?? '', + ), + + Column::TYPE_FLOAT => new Decimal( + $column['key'], + $table, + required: $column['required'], + default: $column['default'], + array: $column['array'], + min: $column['min'] ?? null, + max: $column['max'] ?? null, + createdAt: $column['$createdAt'] ?? '', + updatedAt: $column['$updatedAt'] ?? '', + ), + + Column::TYPE_RELATIONSHIP => new Relationship( + $column['key'], + $table, + relatedTable: $column['relatedTable'] ?? $column['relatedCollection'], + relationType: $column['relationType'], + twoWay: $column['twoWay'], + twoWayKey: $column['twoWayKey'], + onDelete: $column['onDelete'], + side: $column['side'], + createdAt: $column['$createdAt'] ?? '', + updatedAt: $column['$updatedAt'] ?? '', + ), + + Column::TYPE_DATETIME => new DateTime( + $column['key'], + $table, + required: $column['required'], + default: $column['default'], + array: $column['array'], + createdAt: $column['$createdAt'] ?? '', + updatedAt: $column['$updatedAt'] ?? '', + ), + + Column::TYPE_POINT => new Point( + $column['key'], + $table, + required: $column['required'], + default: $column['default'], + createdAt: $column['$createdAt'] ?? '', + updatedAt: $column['$updatedAt'] ?? '', + ), + + Column::TYPE_LINE => new Line( + $column['key'], + $table, + required: $column['required'], + default: $column['default'], + createdAt: $column['$createdAt'] ?? '', + updatedAt: $column['$updatedAt'] ?? '', + ), + + Column::TYPE_POLYGON => new Polygon( + $column['key'], + $table, + required: $column['required'], + default: $column['default'], + createdAt: $column['$createdAt'] ?? '', + updatedAt: $column['$updatedAt'] ?? '', + ), + + Column::TYPE_OBJECT => new ObjectType( + $column['key'], + $table, + required: $column['required'], + default: $column['default'], + createdAt: $column['$createdAt'] ?? '', + updatedAt: $column['$updatedAt'] ?? '', + ), + + Column::TYPE_VECTOR => new Vector( + $column['key'], + $table, + size: $column['size'], + required: $column['required'], + default: $column['default'], + createdAt: $column['$createdAt'] ?? '', + updatedAt: $column['$updatedAt'] ?? '', + ), + + default => throw new \InvalidArgumentException("Unsupported column type: {$column['type']}"), + }; + + } + + public static function getAttribute(Collection $collection, mixed $attribute): Attribute + { + return match ($attribute['type']) { + Attribute::TYPE_STRING => match ($attribute['format'] ?? '') { + Attribute::TYPE_EMAIL => new AttributeEmail( + $attribute['key'], + $collection, + required: $attribute['required'], + default: $attribute['default'], + array: $attribute['array'], + size: $attribute['size'] ?? 254, + createdAt: $attribute['$createdAt'] ?? '', + updatedAt: $attribute['$updatedAt'] ?? '', + ), + Attribute::TYPE_ENUM => new AttributeEnum( + $attribute['key'], + $collection, + elements: $attribute['elements'], + required: $attribute['required'], + default: $attribute['default'], + array: $attribute['array'], + size: $attribute['size'] ?? UtopiaDatabase::LENGTH_KEY, + createdAt: $attribute['$createdAt'] ?? '', + updatedAt: $attribute['$updatedAt'] ?? '', + ), + Attribute::TYPE_URL => new AttributeURL( + $attribute['key'], + $collection, + required: $attribute['required'], + default: $attribute['default'], + array: $attribute['array'], + size: $attribute['size'] ?? 2000, + createdAt: $attribute['$createdAt'] ?? '', + updatedAt: $attribute['$updatedAt'] ?? '', + ), + Attribute::TYPE_IP => new AttributeIP( + $attribute['key'], + $collection, + required: $attribute['required'], + default: $attribute['default'], + array: $attribute['array'], + size: $attribute['size'] ?? 39, + createdAt: $attribute['$createdAt'] ?? '', + updatedAt: $attribute['$updatedAt'] ?? '', + ), + default => new AttributeText( + $attribute['key'], + $collection, + required: $attribute['required'], + default: $attribute['default'], + array: $attribute['array'], + size: $attribute['size'] ?? 0, + createdAt: $attribute['$createdAt'] ?? '', + updatedAt: $attribute['$updatedAt'] ?? '', + ), + }, + + Attribute::TYPE_BOOLEAN => new AttributeBoolean( + $attribute['key'], + $collection, + required: $attribute['required'], + default: $attribute['default'], + array: $attribute['array'], + createdAt: $attribute['$createdAt'] ?? '', + updatedAt: $attribute['$updatedAt'] ?? '', + ), + + Attribute::TYPE_INTEGER => new AttributeInteger( + $attribute['key'], + $collection, + required: $attribute['required'], + default: $attribute['default'], + array: $attribute['array'], + min: $attribute['min'] ?? null, + max: $attribute['max'] ?? null, + createdAt: $attribute['$createdAt'] ?? '', + updatedAt: $attribute['$updatedAt'] ?? '', + ), + + Attribute::TYPE_FLOAT => new AttributeDecimal( + $attribute['key'], + $collection, + required: $attribute['required'], + default: $attribute['default'], + array: $attribute['array'], + min: $attribute['min'] ?? null, + max: $attribute['max'] ?? null, + createdAt: $attribute['$createdAt'] ?? '', + updatedAt: $attribute['$updatedAt'] ?? '', + ), + + Attribute::TYPE_RELATIONSHIP => new AttributeRelationship( + $attribute['key'], + $collection, + relatedTable: $attribute['relatedTable'] ?? $attribute['relatedCollection'], + relationType: $attribute['relationType'], + twoWay: $attribute['twoWay'], + twoWayKey: $attribute['twoWayKey'], + onDelete: $attribute['onDelete'], + side: $attribute['side'], + createdAt: $attribute['$createdAt'] ?? '', + updatedAt: $attribute['$updatedAt'] ?? '', + ), + + Attribute::TYPE_DATETIME => new AttributeDateTime( + $attribute['key'], + $collection, + required: $attribute['required'], + default: $attribute['default'], + array: $attribute['array'], + createdAt: $attribute['$createdAt'] ?? '', + updatedAt: $attribute['$updatedAt'] ?? '', + ), + + Attribute::TYPE_POINT => new AttributePoint( + $attribute['key'], + $collection, + required: $attribute['required'], + default: $attribute['default'], + createdAt: $attribute['$createdAt'] ?? '', + updatedAt: $attribute['$updatedAt'] ?? '', + ), + + Attribute::TYPE_LINE => new AttributeLine( + $attribute['key'], + $collection, + required: $attribute['required'], + default: $attribute['default'], + createdAt: $attribute['$createdAt'] ?? '', + updatedAt: $attribute['$updatedAt'] ?? '', + ), + + Attribute::TYPE_POLYGON => new AttributePolygon( + $attribute['key'], + $collection, + required: $attribute['required'], + default: $attribute['default'], + createdAt: $attribute['$createdAt'] ?? '', + updatedAt: $attribute['$updatedAt'] ?? '', + ), + + Attribute::TYPE_OBJECT => new AttributeObjectType( + $attribute['key'], + $collection, + required: $attribute['required'], + default: $attribute['default'], + createdAt: $attribute['$createdAt'] ?? '', + updatedAt: $attribute['$updatedAt'] ?? '', + ), + + Attribute::TYPE_VECTOR => new AttributeVector( + $attribute['key'], + $collection, + size: $attribute['size'], + required: $attribute['required'], + default: $attribute['default'], + createdAt: $attribute['$createdAt'] ?? '', + updatedAt: $attribute['$updatedAt'] ?? '', + ), + + default => throw new \InvalidArgumentException("Unsupported attribute type: {$attribute['type']}"), + }; + } } diff --git a/src/Migration/Sources/Appwrite/Reader.php b/src/Migration/Sources/Appwrite/Reader.php index b45f2bdc..04cc3bc7 100644 --- a/src/Migration/Sources/Appwrite/Reader.php +++ b/src/Migration/Sources/Appwrite/Reader.php @@ -106,4 +106,6 @@ public function queryCursorAfter(Resource|string $resource): mixed; * @return QueryType|string */ public function queryLimit(int $limit): mixed; + public function getSupportForAttributes(): bool; + } diff --git a/src/Migration/Sources/Appwrite/Reader/API.php b/src/Migration/Sources/Appwrite/Reader/API.php index da1a5022..5790a605 100644 --- a/src/Migration/Sources/Appwrite/Reader/API.php +++ b/src/Migration/Sources/Appwrite/Reader/API.php @@ -240,4 +240,9 @@ public function queryLimit(int $limit): string { return Query::limit($limit); } + + public function getSupportForAttributes(): bool + { + return true; + } } diff --git a/src/Migration/Sources/Appwrite/Reader/Database.php b/src/Migration/Sources/Appwrite/Reader/Database.php index c43e246c..edc03156 100644 --- a/src/Migration/Sources/Appwrite/Reader/Database.php +++ b/src/Migration/Sources/Appwrite/Reader/Database.php @@ -8,8 +8,10 @@ use Utopia\Database\Query; use Utopia\Migration\Exception; use Utopia\Migration\Resource; +use Utopia\Migration\Resources\Database\Collection as CollectionResource; use Utopia\Migration\Resources\Database\Column as ColumnResource; use Utopia\Migration\Resources\Database\Database as DatabaseResource; +use Utopia\Migration\Resources\Database\Document as DocumentResource; use Utopia\Migration\Resources\Database\Index as IndexResource; use Utopia\Migration\Resources\Database\Row as RowResource; use Utopia\Migration\Resources\Database\Table as TableResource; @@ -20,18 +22,46 @@ */ class Database implements Reader { - public function __construct(private readonly UtopiaDatabase $dbForProject) + /** + * @var callable(UtopiaDocument|null): UtopiaDatabase + */ + private mixed $getDatabasesDB; + + public function __construct( + private readonly UtopiaDatabase $dbForProject, + ?callable $getDatabasesDB = null + ) { + $this->getDatabasesDB = $getDatabasesDB; + } + + /** + * Get the appropriate database instance for the given database DSN + */ + private function getDatabase(?string $databaseDSN = null): UtopiaDatabase { + if ($this->getDatabasesDB !== null && $databaseDSN !== null) { + return ($this->getDatabasesDB)(new UtopiaDocument(['database' => $databaseDSN])); + } + + return $this->dbForProject; } public function report(array $resources, array &$report): mixed { $relevantResources = [ + // tablesdb Resource::TYPE_DATABASE, Resource::TYPE_TABLE, Resource::TYPE_ROW, Resource::TYPE_COLUMN, Resource::TYPE_INDEX, + // vectordb + Resource::TYPE_DATABASE_VECTORDB, + // Documentsdb + Resource::TYPE_DATABASE_DOCUMENTSDB, + Resource::TYPE_COLLECTION, + Resource::TYPE_DOCUMENT, + Resource::TYPE_ATTRIBUTE, ]; if (!Resource::isSupported($relevantResources, $resources)) { @@ -43,31 +73,35 @@ public function report(array $resources, array &$report): mixed $report[$resourceType] = 0; } } + $databases = $this->listDatabases(); - if (in_array(Resource::TYPE_DATABASE, $resources)) { - $report[Resource::TYPE_DATABASE] = $this->countResources('databases'); + foreach ($databases as $database) { + $databaseType = $database->getAttribute('type'); + if (in_array($databaseType, [Resource::TYPE_DATABASE_LEGACY,Resource::TYPE_DATABASE_TABLESDB])) { + $databaseType = Resource::TYPE_DATABASE; + } + if (Resource::isSupported($databaseType, $resources)) { + $report[$databaseType] += 1; + } } - if (count(array_intersect($resources, $relevantResources)) === 1 && - in_array(Resource::TYPE_DATABASE, $resources)) { + if ( + count(array_intersect($resources, $relevantResources)) === 1 && + Resource::isSupported(array_keys(Resource::DATABASE_TYPE_RESOURCE_MAP), $resources) + ) { return null; } $dbResources = []; - $databases = $this->listDatabases(); - - // Process each database foreach ($databases as $database) { - $databaseSequence = $database->getSequence(); - $tableId = "database_{$databaseSequence}"; - - if (Resource::isSupported(Resource::TYPE_TABLE, $resources)) { - $report[Resource::TYPE_TABLE] += $this->countResources($tableId); + $databaseType = $database->getAttribute('type'); + if (in_array($databaseType, [Resource::TYPE_DATABASE_LEGACY,Resource::TYPE_DATABASE_TABLESDB])) { + $databaseType = Resource::TYPE_DATABASE; } - if (!Resource::isSupported([Resource::TYPE_ROW, Resource::TYPE_COLUMN, Resource::TYPE_INDEX], $resources)) { - continue; - } + $databaseSpecificResources = Resource::DATABASE_TYPE_RESOURCE_MAP[$databaseType]; + + $databaseSequence = $database->getSequence(); if (!isset($dbResources[$database->getId()])) { $dbResources[$database->getId()] = new DatabaseResource( @@ -75,19 +109,29 @@ public function report(array $resources, array &$report): mixed $database->getAttribute('name'), $database->getCreatedAt(), $database->getUpdatedAt(), + $database->getAttribute('enabled', true), + $database->getAttribute('originalId', ''), + $database->getAttribute('type', ''), + $database->getAttribute('database', '') ); } $dbResource = $dbResources[$database->getId()]; $tables = $this->listTables($dbResource); + $count = count($tables); + + if (Resource::isSupported($databaseSpecificResources['entity'], $resources)) { + $report[$databaseSpecificResources['entity']] += $count; + } foreach ($tables as $table) { $tableSequence = $table->getSequence(); - if (Resource::isSupported(Resource::TYPE_ROW, $resources)) { + if (Resource::isSupported($databaseSpecificResources['record'], $resources)) { $rowTableId = "database_{$databaseSequence}_collection_{$tableSequence}"; - $report[Resource::TYPE_ROW] += $this->countResources($rowTableId); + $count = $this->countResources($rowTableId, [], $dbResource); + $report[$databaseSpecificResources['record']] += $count; } $commonQueries = [ @@ -95,8 +139,12 @@ public function report(array $resources, array &$report): mixed Query::equal('collectionInternalId', [$tableSequence]), ]; - if (Resource::isSupported(Resource::TYPE_COLUMN, $resources)) { - $report[Resource::TYPE_COLUMN] += $this->countResources('attributes', $commonQueries); + if ( + isset($databaseSpecificResources['field']) && + Resource::isSupported($databaseSpecificResources['field'], $resources) + ) { + $count = $this->countResources('attributes', $commonQueries); + $report[$databaseSpecificResources['field']] += $count; } if (in_array(Resource::TYPE_INDEX, $resources)) { @@ -295,8 +343,11 @@ public function listRows(TableResource $resource, array $queries = []): array $tableId = "database_{$database->getSequence()}_collection_{$table->getSequence()}"; + // Use the appropriate database instance for this specific database + $dbInstance = $this->getDatabase($resource->getDatabase()->getDatabase()); + try { - $rows = $this->dbForProject->find($tableId, $queries); + $rows = $dbInstance->find($tableId, $queries); } catch (DatabaseException $e) { throw new Exception( resourceName: $resource->getName(), @@ -345,7 +396,10 @@ public function getRow(TableResource $resource, string $rowId, array $queries = $tableId = "database_{$database->getSequence()}_collection_{$table->getSequence()}"; - return $this->dbForProject->getDocument( + // Use the appropriate database instance for this specific database + $dbInstance = $this->getDatabase($resource->getDatabase()->getDatabase()); + + return $dbInstance->getDocument( $tableId, $rowId, $queries @@ -385,19 +439,32 @@ public function queryCursorAfter(mixed $resource): Query switch ($resource::class) { case DatabaseResource::class: + /** @var DatabaseResource $resource */ + // Databases are always in dbForProject metadata $document = $this->dbForProject->getDocument('databases', $resource->getId()); break; case TableResource::class: + case CollectionResource::class: + /** @var TableResource|CollectionResource $resource */ + // Tables/Collections metadata is in dbForProject $database = $this->dbForProject->getDocument('databases', $resource->getDatabase()->getId()); $document = $this->dbForProject->getDocument('database_' . $database->getSequence(), $resource->getId()); break; case ColumnResource::class: + /** @var ColumnResource $resource */ + // Columns (attributes) are in dbForProject metadata $document = $this->dbForProject->getDocument('attributes', $resource->getId()); break; case IndexResource::class: + /** @var IndexResource $resource */ + // Indexes are in dbForProject metadata $document = $this->dbForProject->getDocument('indexes', $resource->getId()); break; case RowResource::class: + case DocumentResource::class: + /** @var RowResource|DocumentResource $resource */ + // Rows/Documents are in the specific database instance + // getRow() already uses getDatabase() internally $document = $this->getRow($resource->getTable(), $resource->getId()); $document = new UtopiaDocument($document); break; @@ -413,14 +480,27 @@ public function queryLimit(int $limit): Query return Query::limit($limit); } + public function getSupportForAttributes(): bool + { + return $this->dbForProject->getAdapter()->getSupportForAttributes(); + } + /** * @param string $table * @param array $queries + * @param DatabaseResource|null $databaseResource * @return int * @throws DatabaseException */ - private function countResources(string $table, array $queries = []): int + private function countResources(string $table, array $queries = [], ?DatabaseResource $databaseResource = null): int { + // Use the appropriate database instance for row data + if ($databaseResource !== null) { + $dbInstance = $this->getDatabase($databaseResource->getDatabase()); + return $dbInstance->count($table, $queries); + } + + // Use dbForProject for metadata tables return $this->dbForProject->count($table, $queries); } } diff --git a/src/Migration/Sources/CSV.php b/src/Migration/Sources/CSV.php index a6ca0c2d..c86234b7 100644 --- a/src/Migration/Sources/CSV.php +++ b/src/Migration/Sources/CSV.php @@ -5,11 +5,9 @@ use Utopia\Console; use Utopia\Database\Database as UtopiaDatabase; use Utopia\Migration\Exception; +use Utopia\Migration\Resource; use Utopia\Migration\Resource as UtopiaResource; use Utopia\Migration\Resources\Database\Column; -use Utopia\Migration\Resources\Database\Database; -use Utopia\Migration\Resources\Database\Row; -use Utopia\Migration\Resources\Database\Table; use Utopia\Migration\Resources\Storage\File; use Utopia\Migration\Source; use Utopia\Migration\Sources\Appwrite\Reader; @@ -44,12 +42,13 @@ public function __construct( string $resourceId, string $filePath, Device $device, - ?UtopiaDatabase $dbForProject + ?UtopiaDatabase $dbForProject, + ?callable $getDatabasesDB = null, ) { $this->device = $device; $this->filePath = $filePath; $this->resourceId = $resourceId; - $this->database = new DatabaseReader($dbForProject); + $this->database = new DatabaseReader($dbForProject, $getDatabasesDB); } public static function getName(): string @@ -61,6 +60,7 @@ public static function getSupportedResources(): array { return [ UtopiaResource::TYPE_ROW, + UtopiaResource::TYPE_DOCUMENT, ]; } @@ -104,7 +104,7 @@ protected function exportGroupAuth(int $batchSize, array $resources): void protected function exportGroupDatabases(int $batchSize, array $resources): void { try { - if (UtopiaResource::isSupported(UtopiaResource::TYPE_ROW, $resources)) { + if (UtopiaResource::isSupported($this->getSupportedResources(), $resources)) { $this->exportRows($batchSize); } } catch (\Throwable $e) { @@ -132,8 +132,47 @@ private function exportRows(int $batchSize): void $lastColumn = null; [$databaseId, $tableId] = \explode(':', $this->resourceId); - $database = new Database($databaseId, ''); - $table = new Table($database, '', $tableId); + + $databases = $this->database->listDatabases([ + $this->database->queryEqual('$id', [$databaseId]), + $this->database->queryLimit(1), + ]); + + if (empty($databases)) { + throw new \Exception('Database not found'); + } + + $databaseDocument = $databases[0]; + $databaseType = $databaseDocument->getAttribute('type', UtopiaResource::TYPE_DATABASE); + if (\in_array($databaseType, [UtopiaResource::TYPE_DATABASE_LEGACY, UtopiaResource::TYPE_DATABASE_TABLESDB], true)) { + $databaseType = UtopiaResource::TYPE_DATABASE; + } + + $databasePayload = [ + 'id' => $databaseDocument->getId(), + 'name' => $databaseDocument->getAttribute('name', $databaseDocument->getId()), + 'originalId' => $databaseDocument->getAttribute('originalId', ''), + 'type' => $databaseType, + 'database' => $databaseDocument->getAttribute('database', ''), + ]; + + $tablePayload = [ + 'id' => $tableId, + 'name' => $tableId, + 'documentSecurity' => false, + 'rowSecurity' => false, + 'permissions' => [], + 'createdAt' => '', + 'updatedAt' => '', + 'database' => [ + 'id' => $databasePayload['id'], + 'name' => $databasePayload['name'], + 'type' => $databasePayload['type'], + 'database' => $databasePayload['database'], + ], + ]; + + $table = Appwrite::getEntity($databaseType, $tablePayload); while (true) { $queries = [$this->database->queryLimit($batchSize)]; @@ -195,7 +234,7 @@ private function exportRows(int $batchSize): void } } - $this->withCsvStream(function ($stream, $delimiter) use ($columnTypes, $requiredColumns, $manyToManyKeys, $arrayKeys, $table, $batchSize) { + $this->withCsvStream(function ($stream, $delimiter) use ($columnTypes, $databaseType, $requiredColumns, $manyToManyKeys, $arrayKeys, $tablePayload, $batchSize) { $headers = \fgetcsv($stream, 0, $delimiter, '"', '"'); if (!\is_array($headers) || \count($headers) === 0) { @@ -300,7 +339,9 @@ private function exportRows(int $batchSize): void ), Column::TYPE_POINT, Column::TYPE_LINE, - Column::TYPE_POLYGON => \is_string($parsedValue) ? json_decode($parsedValue) : null, + Column::TYPE_POLYGON, + Column::TYPE_VECTOR, + Column::TYPE_OBJECT => \is_string($parsedValue) ? json_decode($parsedValue, true) : null, default => $parsedValue, }, }; @@ -311,12 +352,12 @@ private function exportRows(int $batchSize): void unset($parsedData['$id'], $parsedData['$permissions']); - $row = new Row( - $rowId, - $table, - $parsedData, - $permissions, - ); + $row = Appwrite::getRecord($databaseType, [ + 'id' => $rowId, + 'table' => $tablePayload, + 'data' => $parsedData, + 'permissions' => $permissions + ]); $buffer[] = $row; diff --git a/src/Migration/Transfer.php b/src/Migration/Transfer.php index 2a447eb4..2bcceef1 100644 --- a/src/Migration/Transfer.php +++ b/src/Migration/Transfer.php @@ -12,7 +12,13 @@ class Transfer public const GROUP_FUNCTIONS = 'functions'; + // separating databases and tablesdb out for easier separation in extract services + // migration can use group_databases for mentioning all resources but when mentioning specific resources go with specific type databases public const GROUP_DATABASES = 'databases'; + public const GROUP_DATABASES_TABLES_DB = 'tablesdb'; + public const GROUP_DATABASES_DOCUMENTS_DB = 'documentsdb'; + + public const GROUP_DATABASES_VECTOR_DB = 'vectordb'; public const GROUP_SETTINGS = 'settings'; @@ -34,12 +40,40 @@ class Transfer Resource::TYPE_DEPLOYMENT ]; + public const GROUP_TABLESDB_RESOURCES = [ + Resource::TYPE_DATABASE, + Resource::TYPE_TABLE, + Resource::TYPE_INDEX, + Resource::TYPE_COLUMN, + Resource::TYPE_ROW, + ]; + + public const GROUP_DOCUMENTSDB_RESOURCES = [ + Resource::TYPE_DATABASE_DOCUMENTSDB, + Resource::TYPE_COLLECTION, + Resource::TYPE_INDEX, + Resource::TYPE_DOCUMENT + ]; + + public const GROUP_VECTORDB_RESOURCES = [ + Resource::TYPE_DATABASE_VECTORDB, + Resource::TYPE_COLLECTION, + Resource::TYPE_ATTRIBUTE, + Resource::TYPE_INDEX, + Resource::TYPE_DOCUMENT + ]; + public const GROUP_DATABASES_RESOURCES = [ Resource::TYPE_DATABASE, + Resource::TYPE_DATABASE_DOCUMENTSDB, + Resource::TYPE_DATABASE_VECTORDB, Resource::TYPE_TABLE, Resource::TYPE_INDEX, Resource::TYPE_COLUMN, Resource::TYPE_ROW, + Resource::TYPE_DOCUMENT, + Resource::TYPE_COLLECTION, + Resource::TYPE_ATTRIBUTE ]; public const GROUP_SETTINGS_RESOURCES = []; @@ -68,6 +102,8 @@ class Transfer public const ROOT_RESOURCES = [ Resource::TYPE_BUCKET, Resource::TYPE_DATABASE, + Resource::TYPE_DATABASE_DOCUMENTSDB, + Resource::TYPE_DATABASE_VECTORDB, Resource::TYPE_FUNCTION, Resource::TYPE_USER, Resource::TYPE_TEAM, @@ -227,7 +263,7 @@ public function run( } if (!in_array($rootResourceType, self::ROOT_RESOURCES)) { - throw new \Exception('Resource type must be one of ' . implode(', ', self::ROOT_RESOURCES)); + throw new \Exception('Got '.$rootResourceType.' Resource type must be one of ' . implode(', ', self::ROOT_RESOURCES)); } $rootResources = \array_intersect($computedResources, self::ROOT_RESOURCES); @@ -321,6 +357,7 @@ public function getReport(string $statusLevel = ''): array public static function extractServices(array $services): array { $resources = []; + $groupDatabasesIndex = array_search(Transfer::GROUP_DATABASES, $services); foreach ($services as $service) { $resources = match ($service) { self::GROUP_FUNCTIONS => array_merge($resources, self::GROUP_FUNCTIONS_RESOURCES), @@ -328,6 +365,9 @@ public static function extractServices(array $services): array self::GROUP_GENERAL => array_merge($resources, []), self::GROUP_AUTH => array_merge($resources, self::GROUP_AUTH_RESOURCES), self::GROUP_DATABASES => array_merge($resources, self::GROUP_DATABASES_RESOURCES), + self::GROUP_DATABASES_TABLES_DB => array_merge($resources, self::GROUP_TABLESDB_RESOURCES), + self::GROUP_DATABASES_DOCUMENTS_DB => array_merge($resources, self::GROUP_DOCUMENTSDB_RESOURCES), + self::GROUP_DATABASES_VECTOR_DB => array_merge($resources, self::GROUP_VECTORDB_RESOURCES), self::GROUP_SETTINGS => array_merge($resources, self::GROUP_SETTINGS_RESOURCES), default => throw new \Exception('No service group found'), };