diff --git a/src/Migration/Sources/Appwrite.php b/src/Migration/Sources/Appwrite.php index f467de2..60c15fd 100644 --- a/src/Migration/Sources/Appwrite.php +++ b/src/Migration/Sources/Appwrite.php @@ -10,6 +10,7 @@ use Appwrite\Services\Storage; use Appwrite\Services\Teams; use Appwrite\Services\Users; +use Utopia\Console; use Utopia\Database\Database as UtopiaDatabase; use Utopia\Database\DateTime as UtopiaDateTime; use Utopia\Migration\Exception; @@ -74,6 +75,12 @@ class Appwrite extends Source public const SOURCE_API = 'api'; public const SOURCE_DATABASE = 'database'; + // Debug logging for specific projects + public static array $debugProjects = [ + '67ec0369002bd8a96885' => 'SimpMusic#Maxrave', + '6838382d0014e002589c' => 'Fastwrite#DocuTrust', + ]; + protected Client $client; private Reader $reader; @@ -120,7 +127,7 @@ public function __construct( $this->reader = match ($this->source) { static::SOURCE_API => new APIReader(new Databases($this->client)), - static::SOURCE_DATABASE => new DatabaseReader($this->dbForProject, $this->getDatabasesDB), + static::SOURCE_DATABASE => new DatabaseReader($this->dbForProject, $this->getDatabasesDB, $this->project), default => throw new \Exception('Unknown source'), }; @@ -131,6 +138,19 @@ public static function getName(): string return 'Appwrite'; } + /** + * Log migration debug info for tracked projects + */ + private function logDebugTrackedProject(string $message): void + { + $projectTag = self::$debugProjects[$this->project] ?? null; + if ($projectTag === null) { + return; + } + + Console::info("MIGRATIONS-SOURCE-$projectTag: $message"); + } + /** * @return array */ @@ -972,11 +992,22 @@ private function exportIndexes(string $entityType, int $batchSize): void private function exportRecords(string $entityName, string $fieldName, int $batchSize): void { $entities = $this->cache->get($entityName); + + $this->logDebugTrackedProject("exportRecords started | Entity: $entityName | Tables count: " . count($entities)); + foreach ($entities as $table) { /** @var Table $table */ $lastRow = null; + $iterationCount = 0; + + $this->logDebugTrackedProject("Starting table export | Table: {$table->getName()} | ID: {$table->getId()}"); while (true) { + $iterationCount++; + + $memUsage = round(memory_get_usage(true) / 1024 / 1024, 2); + $this->logDebugTrackedProject("Table: {$table->getName()} | Iteration: $iterationCount | Memory: {$memUsage}MB | LastRow: " . ($lastRow ? $lastRow->getId() : 'null')); + $queries = [ $this->reader->queryLimit($batchSize), ...$this->queries, @@ -1009,8 +1040,14 @@ private function exportRecords(string $entityName, string $fieldName, int $batch $queries[] = $this->reader->querySelect($selects); + $timestamp = microtime(true); + $this->logDebugTrackedProject("BEFORE listRows() | Table: {$table->getName()} | Batch: $batchSize | Timestamp: {$timestamp}"); + $response = $this->reader->listRows($table, $queries); + $timestamp = microtime(true); + $this->logDebugTrackedProject("AFTER listRows() | Table: {$table->getName()} | Rows: " . count($response) . " | Timestamp: $timestamp"); + foreach ($response as $row) { // HACK: Handle many to many (only for schema-based databases) if (!empty($manyToMany)) { @@ -1069,13 +1106,24 @@ private function exportRecords(string $entityName, string $fieldName, int $batch $lastRow = $row; } + $this->logDebugTrackedProject("Processed rows from response | Table: {$table->getName()} | Rows in batch: " . count($rows)); + + $this->logDebugTrackedProject("BEFORE callback() | Table: {$table->getName()} | Rows: " . count($rows)); + $this->callback($rows); + $this->logDebugTrackedProject("AFTER callback() | Table: {$table->getName()}"); + if (count($response) < $batchSize) { + $this->logDebugTrackedProject("Table export completed | Table: {$table->getName()} | Response count: " . count($response) . " < Batch size: $batchSize"); break; } } + + $this->logDebugTrackedProject("Finished table export | Table: {$table->getName()} | Total iterations: {$iterationCount}"); } + + $this->logDebugTrackedProject("exportRecords completed | Entity: {$entityName}"); } protected function exportGroupStorage(int $batchSize, array $resources): void diff --git a/src/Migration/Sources/Appwrite/Reader/Database.php b/src/Migration/Sources/Appwrite/Reader/Database.php index edc0315..84b82b4 100644 --- a/src/Migration/Sources/Appwrite/Reader/Database.php +++ b/src/Migration/Sources/Appwrite/Reader/Database.php @@ -2,6 +2,7 @@ namespace Utopia\Migration\Sources\Appwrite\Reader; +use Utopia\Console; use Utopia\Database\Database as UtopiaDatabase; use Utopia\Database\Document as UtopiaDocument; use Utopia\Database\Exception as DatabaseException; @@ -15,6 +16,7 @@ use Utopia\Migration\Resources\Database\Index as IndexResource; use Utopia\Migration\Resources\Database\Row as RowResource; use Utopia\Migration\Resources\Database\Table as TableResource; +use Utopia\Migration\Sources\Appwrite; use Utopia\Migration\Sources\Appwrite\Reader; /** @@ -29,11 +31,29 @@ class Database implements Reader public function __construct( private readonly UtopiaDatabase $dbForProject, - ?callable $getDatabasesDB = null + ?callable $getDatabasesDB = null, + private readonly ?string $projectId = null ) { $this->getDatabasesDB = $getDatabasesDB; } + /** + * Log debug message for tracked projects + */ + private function logDebug(string $message): void + { + if ($this->projectId === null) { + return; + } + + $projectTag = Appwrite::$debugProjects[$this->projectId] ?? null; + if ($projectTag === null) { + return; + } + + Console::info("MIGRATIONS-READER-$projectTag: $message"); + } + /** * Get the appropriate database instance for the given database DSN */ @@ -347,7 +367,9 @@ public function listRows(TableResource $resource, array $queries = []): array $dbInstance = $this->getDatabase($resource->getDatabase()->getDatabase()); try { + $this->logDebug("BEFORE dbInstance->find() | TableID: $tableId | Queries: " . count($queries)); $rows = $dbInstance->find($tableId, $queries); + $this->logDebug("AFTER dbInstance->find() | TableID: $tableId | Rows returned: " . count($rows)); } catch (DatabaseException $e) { throw new Exception( resourceName: $resource->getName(),