Skip to content

Add: logs #124

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
abnegate merged 3 commits into from
Nov 25, 2025
Merged

Add: logs #124

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
e2dfc95
add: logs
ItzNotABug Nov 25, 2025
e7e2e69
fix: method name.
ItzNotABug Nov 25, 2025
e8994b9
fix: import, lol.
ItzNotABug Nov 25, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
50 changes: 49 additions & 1 deletion src/Migration/Sources/Appwrite.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
use Appwrite\Services\Storage;
use Appwrite\Services\Teams;
use Appwrite\Services\Users;
use Utopia\Console;
use Utopia\Database\Database as UtopiaDatabase;
use Utopia\Database\DateTime as UtopiaDateTime;
use Utopia\Migration\Exception;
Expand Down Expand Up @@ -74,6 +75,12 @@ class Appwrite extends Source
public const SOURCE_API = 'api';
public const SOURCE_DATABASE = 'database';

// Debug logging for specific projects
public static array $debugProjects = [
'67ec0369002bd8a96885' => 'SimpMusic#Maxrave',
'6838382d0014e002589c' => 'Fastwrite#DocuTrust',
];

protected Client $client;

private Reader $reader;
Expand Down Expand Up @@ -120,7 +127,7 @@ public function __construct(

$this->reader = match ($this->source) {
static::SOURCE_API => new APIReader(new Databases($this->client)),
static::SOURCE_DATABASE => new DatabaseReader($this->dbForProject, $this->getDatabasesDB),
static::SOURCE_DATABASE => new DatabaseReader($this->dbForProject, $this->getDatabasesDB, $this->project),
default => throw new \Exception('Unknown source'),
};

Expand All @@ -131,6 +138,19 @@ public static function getName(): string
return 'Appwrite';
}

/**
* Log migration debug info for tracked projects
*/
private function logDebugTrackedProject(string $message): void
{
$projectTag = self::$debugProjects[$this->project] ?? null;
if ($projectTag === null) {
return;
}

Console::info("MIGRATIONS-SOURCE-$projectTag: $message");
}

/**
* @return array<string>
*/
Expand Down Expand Up @@ -972,11 +992,22 @@ private function exportIndexes(string $entityType, int $batchSize): void
private function exportRecords(string $entityName, string $fieldName, int $batchSize): void
{
$entities = $this->cache->get($entityName);

$this->logDebugTrackedProject("exportRecords started | Entity: $entityName | Tables count: " . count($entities));

foreach ($entities as $table) {
/** @var Table $table */
$lastRow = null;
$iterationCount = 0;

$this->logDebugTrackedProject("Starting table export | Table: {$table->getName()} | ID: {$table->getId()}");

while (true) {
$iterationCount++;

$memUsage = round(memory_get_usage(true) / 1024 / 1024, 2);
$this->logDebugTrackedProject("Table: {$table->getName()} | Iteration: $iterationCount | Memory: {$memUsage}MB | LastRow: " . ($lastRow ? $lastRow->getId() : 'null'));

$queries = [
$this->reader->queryLimit($batchSize),
...$this->queries,
Expand Down Expand Up @@ -1009,8 +1040,14 @@ private function exportRecords(string $entityName, string $fieldName, int $batch

$queries[] = $this->reader->querySelect($selects);

$timestamp = microtime(true);
$this->logDebugTrackedProject("BEFORE listRows() | Table: {$table->getName()} | Batch: $batchSize | Timestamp: {$timestamp}");

$response = $this->reader->listRows($table, $queries);

$timestamp = microtime(true);
$this->logDebugTrackedProject("AFTER listRows() | Table: {$table->getName()} | Rows: " . count($response) . " | Timestamp: $timestamp");

foreach ($response as $row) {
// HACK: Handle many to many (only for schema-based databases)
if (!empty($manyToMany)) {
Expand Down Expand Up @@ -1069,13 +1106,24 @@ private function exportRecords(string $entityName, string $fieldName, int $batch
$lastRow = $row;
}

$this->logDebugTrackedProject("Processed rows from response | Table: {$table->getName()} | Rows in batch: " . count($rows));

$this->logDebugTrackedProject("BEFORE callback() | Table: {$table->getName()} | Rows: " . count($rows));

$this->callback($rows);

$this->logDebugTrackedProject("AFTER callback() | Table: {$table->getName()}");

if (count($response) < $batchSize) {
$this->logDebugTrackedProject("Table export completed | Table: {$table->getName()} | Response count: " . count($response) . " < Batch size: $batchSize");
break;
}
}

$this->logDebugTrackedProject("Finished table export | Table: {$table->getName()} | Total iterations: {$iterationCount}");
}

$this->logDebugTrackedProject("exportRecords completed | Entity: {$entityName}");
}

protected function exportGroupStorage(int $batchSize, array $resources): void
Expand Down
24 changes: 23 additions & 1 deletion src/Migration/Sources/Appwrite/Reader/Database.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

namespace Utopia\Migration\Sources\Appwrite\Reader;

use Utopia\Console;
use Utopia\Database\Database as UtopiaDatabase;
use Utopia\Database\Document as UtopiaDocument;
use Utopia\Database\Exception as DatabaseException;
Expand All @@ -15,6 +16,7 @@
use Utopia\Migration\Resources\Database\Index as IndexResource;
use Utopia\Migration\Resources\Database\Row as RowResource;
use Utopia\Migration\Resources\Database\Table as TableResource;
use Utopia\Migration\Sources\Appwrite;
use Utopia\Migration\Sources\Appwrite\Reader;

/**
Expand All @@ -29,11 +31,29 @@ class Database implements Reader

public function __construct(
private readonly UtopiaDatabase $dbForProject,
?callable $getDatabasesDB = null
?callable $getDatabasesDB = null,
private readonly ?string $projectId = null
) {
$this->getDatabasesDB = $getDatabasesDB;
}

/**
* Log debug message for tracked projects
*/
private function logDebug(string $message): void
{
if ($this->projectId === null) {
return;
}

$projectTag = Appwrite::$debugProjects[$this->projectId] ?? null;
if ($projectTag === null) {
return;
}

Console::info("MIGRATIONS-READER-$projectTag: $message");
}

/**
* Get the appropriate database instance for the given database DSN
*/
Expand Down Expand Up @@ -347,7 +367,9 @@ public function listRows(TableResource $resource, array $queries = []): array
$dbInstance = $this->getDatabase($resource->getDatabase()->getDatabase());

try {
$this->logDebug("BEFORE dbInstance->find() | TableID: $tableId | Queries: " . count($queries));
$rows = $dbInstance->find($tableId, $queries);
$this->logDebug("AFTER dbInstance->find() | TableID: $tableId | Rows returned: " . count($rows));
} catch (DatabaseException $e) {
throw new Exception(
resourceName: $resource->getName(),
Expand Down