diff --git a/.gitignore b/.gitignore index aafb169765..86f15b1112 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,4 @@ .php-cs-fixer.cache -bin vendor composer.lock coverage.xml diff --git a/doc/loggable.md b/doc/loggable.md index 3f0b9dc55e..d3e8eaca4e 100644 --- a/doc/loggable.md +++ b/doc/loggable.md @@ -3,7 +3,7 @@ The **Loggable** behavior adds support for logging changes to and restoring prior versions of your Doctrine objects. > [!NOTE] -> The Loggable extension is NOT compatible with `doctrine/dbal` 4.0 or later +> The `data` column in the log entry table was changed from PHP's `serialize()` format to JSON in order to be compatible with `doctrine/dbal` 4.0 or later. If you are upgrading from an older version, see [Migrating existing serialized data to JSON](#migrating-existing-serialized-data-to-json) below. ## Index @@ -13,6 +13,7 @@ The **Loggable** behavior adds support for logging changes to and restoring prio - [Object Repositories](#object-repositories) - [Fetching a Model's Log Entries](#fetching-a-models-log-entries) - [Revert a Model to a Previous Version](#revert-a-model-to-a-previous-version) +- [Migrating Existing Serialized Data to JSON](#migrating-existing-serialized-data-to-json) ## Getting Started @@ -247,3 +248,77 @@ $repo = $em->getRepository(LogEntry::class); // We are now able to revert to an older version $repo->revert($article, 2); ``` + +## Migrating Existing Serialized Data to JSON + +If you have an existing `ext_log_entries` table (or custom ORM log entry tables) with data stored in the old +PHP `serialize()` format, you need to migrate it to JSON before using `doctrine/dbal` 4.0 or later. + +The project provides a Symfony console command for this migration. The command: + +1. Renames the existing `data` column to `data_serialized` (so no existing data is lost). +2. Adds a new `data` column with a JSON-compatible type. +3. Reads each row, calls `unserialize()` on the old value and `json_encode()` on the result, and writes it into the new column. +4. Optionally drops the `data_serialized` backup column automatically when `--drop-legacy` is passed. Without this flag the column is kept so you can verify the migration manually before dropping it. + +### Usage (Symfony command) + +```bash +php bin/console gedmo:loggable:migrate-data-to-json \ + [--batch-size=500] \ + [--drop-legacy] +``` + +The command uses the injected Doctrine DBAL connection and resolves the mapped ORM log entry table(s) +from Doctrine metadata, so you do not need to pass credentials or table names on the command line. + +> [!NOTE] +> Symfony does not automatically discover command services from vendor packages by the `#[AsCommand]` attribute alone. +> If you integrate this library directly, register the command as a service yourself or use a Symfony bundle/recipe that imports vendor services for you. For example: +> +> ```yaml +> services: +> Gedmo\Loggable\Command\MigrateDataToJsonCommand: ~ +> ``` + +| Option | Description | +|---|---| +| `--batch-size` | Number of rows to process per database round-trip (default: `500`). | +| `--drop-legacy` | Drop the `data_serialized` backup column automatically after a successful migration. Omit this flag if you want to keep the backup column for manual verification first. | + +> [!NOTE] +> Run this script **before** updating the entity mapping to use the `json` type and before upgrading to DBAL 4. +> Make a database backup before running the migration. + +### Manual migration approach + +If you prefer to handle the migration yourself, the steps are: + +1. **Rename** the `data` column to `data_serialized`: + ```sql + ALTER TABLE ext_log_entries RENAME COLUMN data TO data_serialized; + ``` + +2. **Add** a new `data` column (use `LONGTEXT` for MySQL/MariaDB or `TEXT` for SQLite/PostgreSQL): + ```sql + ALTER TABLE ext_log_entries ADD COLUMN data LONGTEXT DEFAULT NULL; + ``` + +3. **Convert** each row using PHP (the `unserialize()` → `json_encode()` round-trip): + + ```php + $rows = $connection->fetchAllAssociative('SELECT id, data_serialized FROM ext_log_entries WHERE data_serialized IS NOT NULL'); + foreach ($rows as $row) { + $deserialized = unserialize($row['data_serialized'], ['allowed_classes' => false]); + $json = json_encode($deserialized, JSON_THROW_ON_ERROR); + $connection->executeStatement( + 'UPDATE ext_log_entries SET data = ? WHERE id = ?', + [$json, $row['id']] + ); + } + ``` + +4. **Drop** the legacy column once you have verified the migration: + ```sql + ALTER TABLE ext_log_entries DROP COLUMN data_serialized; + ``` diff --git a/src/Loggable/Command/MigrateDataToJsonCommand.php b/src/Loggable/Command/MigrateDataToJsonCommand.php new file mode 100644 index 0000000000..82c1055f66 --- /dev/null +++ b/src/Loggable/Command/MigrateDataToJsonCommand.php @@ -0,0 +1,453 @@ + http://www.gediminasm.org + * For the full copyright and license information, please view the LICENSE + * file that was distributed with this source code. + */ + +namespace Gedmo\Loggable\Command; + +use Doctrine\DBAL\Connection; +use Doctrine\DBAL\Platforms\AbstractPlatform; +use Doctrine\Persistence\ManagerRegistry; +use Gedmo\Loggable\LogEntryInterface; +use Symfony\Component\Console\Attribute\AsCommand; +use Symfony\Component\Console\Command\Command; +use Symfony\Component\Console\Input\InputInterface; +use Symfony\Component\Console\Input\InputOption; +use Symfony\Component\Console\Output\OutputInterface; + +#[AsCommand( + name: 'gedmo:loggable:migrate-data-to-json', + description: 'Migrate Loggable data column from PHP serialized values to JSON.', +)] +final class MigrateDataToJsonCommand extends Command +{ + /** + * Matches the CREATE TABLE header and captures the table identifier as group 2. + * + * Supported identifier formats: + * - "quoted" + * - `backticked` + * - [bracketed] + * - unquoted_identifier + */ + private const SQLITE_CREATE_TABLE_HEADER_PATTERN = '/^(CREATE\\s+TABLE\\s+)((?:"[^"]*"|`[^`]*`|\\[[^\\]]*\\]|[a-zA-Z_][a-zA-Z0-9_]*))(\\s*\\()/i'; + + /** + * Matches the `data` column identifier at a column-definition boundary. + * Prevents matching identifiers like `data_type`. + */ + private const SQLITE_DATA_COLUMN_PATTERN = '/(?<=\\(|,)\\s*("data"|`data`|\\[data\\]|(?connection = $connection; + $this->managerRegistry = $managerRegistry; + } + + protected function configure(): void + { + $this + ->addOption('batch-size', null, InputOption::VALUE_REQUIRED, 'Rows per round-trip.', '500') + ->addOption('drop-legacy', null, InputOption::VALUE_NONE, 'Drop the data_serialized column after successful migration.') + ; + } + + protected function execute(InputInterface $input, OutputInterface $output): int + { + $batchSize = (int) $input->getOption('batch-size'); + $dropLegacy = (bool) $input->getOption('drop-legacy'); + + if ($batchSize < 1) { + $output->writeln('--batch-size must be greater than 0.'); + + return self::FAILURE; + } + + $tables = $this->resolveLogEntryTables($this->connection); + + if ([] === $tables) { + $output->writeln('Could not resolve any ORM log entry table from Doctrine metadata for the injected connection.'); + + return self::FAILURE; + } + + $output->writeln(sprintf( + 'Resolved %d log entry table(s): %s', + count($tables), + implode(', ', array_map(static fn (string $table): string => sprintf("'%s'", $table), $tables)) + )); + + try { + foreach ($tables as $table) { + $this->migrateTable($this->connection, $table, $batchSize, $dropLegacy, $output); + } + } catch (\Throwable $e) { + $output->writeln(sprintf('%s', $e->getMessage())); + + return self::FAILURE; + } + + return self::SUCCESS; + } + + private function migrateTable( + Connection $connection, + string $table, + int $batchSize, + bool $dropLegacy, + OutputInterface $output + ): void { + $output->writeln(''); + $output->writeln(sprintf("Migrating table '%s'...", $table)); + + $platform = $connection->getDatabasePlatform(); + $schemaManager = method_exists($connection, 'createSchemaManager') + ? $connection->createSchemaManager() + : $connection->getSchemaManager(); // DBAL 3 fallback + + $quotedTable = $platform->quoteSingleIdentifier($table); + $columns = $schemaManager->listTableColumns($table); + $columnNames = array_keys($columns); + + $hasData = in_array('data', $columnNames, true); + $hasDataSerialized = in_array('data_serialized', $columnNames, true); + + if (!$hasData && !$hasDataSerialized) { + throw new \RuntimeException(sprintf("Neither 'data' nor 'data_serialized' column found in table '%s'. Nothing to migrate.", $table)); + } + + $platformName = strtolower(get_class($platform)); + $isSqlite = str_contains($platformName, 'sqlite'); + + if ($hasData && !$hasDataSerialized) { + $output->writeln(sprintf("Step 1: Renaming column 'data' to 'data_serialized' in table '%s'...", $table)); + + if ($isSqlite) { + $this->migrateViaSqliteRecreate($connection, $platform, $table, $quotedTable, $output); + $this->finalize($connection, $platform, $table, $quotedTable, $dropLegacy, $output); + + return; + } + + $connection->executeStatement(sprintf( + 'ALTER TABLE %s RENAME COLUMN %s TO %s', + $quotedTable, + $platform->quoteSingleIdentifier('data'), + $platform->quoteSingleIdentifier('data_serialized') + )); + $output->writeln(' Done.'); + } elseif ($hasDataSerialized && !$hasData) { + $output->writeln(sprintf("Step 1: Column 'data_serialized' already exists in table '%s'; skipping rename.", $table)); + } else { + $output->writeln(sprintf("Step 1: Both 'data' and 'data_serialized' columns exist in table '%s'; assuming rename was already performed.", $table)); + } + + $columnsAfterRename = array_keys($schemaManager->listTableColumns($table)); + if (!in_array('data', $columnsAfterRename, true)) { + $output->writeln(sprintf("Step 2: Adding new 'data' column to table '%s'...", $table)); + $clobType = $platform->getClobTypeDeclarationSQL([]); + $connection->executeStatement(sprintf( + 'ALTER TABLE %s ADD COLUMN %s %s DEFAULT NULL', + $quotedTable, + $platform->quoteSingleIdentifier('data'), + $clobType + )); + $output->writeln(' Done.'); + } else { + $output->writeln(sprintf("Step 2: Column 'data' already exists in table '%s'; skipping ADD COLUMN.", $table)); + } + + $output->writeln(sprintf("Step 3: Converting serialized data to JSON in table '%s'...", $table)); + $converted = $this->convertRows($connection, $platform, $quotedTable, $batchSize, $output); + $output->writeln(sprintf(' Converted %d row(s).', $converted)); + + $this->finalize($connection, $platform, $table, $quotedTable, $dropLegacy, $output); + } + + /** + * @return list + */ + private function resolveLogEntryTables(Connection $connection): array + { + if (null === $this->managerRegistry) { + return []; + } + + $connectionIdentity = $this->getConnectionIdentity($connection); + $tablesByName = []; + + foreach ($this->managerRegistry->getManagers() as $manager) { + if (!method_exists($manager, 'getConnection')) { + continue; + } + + $managerConnection = $manager->getConnection(); + if (!$managerConnection instanceof Connection) { + continue; + } + + if (!$this->isSameConnection($managerConnection, $connection, $connectionIdentity)) { + continue; + } + + $allMetadata = $manager->getMetadataFactory()->getAllMetadata(); + + foreach ($allMetadata as $metadata) { + if (!method_exists($metadata, 'getTableName')) { + continue; + } + + $className = $metadata->getName(); + if (!is_a($className, LogEntryInterface::class, true)) { + continue; + } + + if (property_exists($metadata, 'isMappedSuperclass') && $metadata->isMappedSuperclass) { + continue; + } + + $tablesByName[$metadata->getTableName()] = true; + } + } + + return array_keys($tablesByName); + } + + /** + * @param array $connectionIdentity + */ + private function isSameConnection(Connection $first, Connection $second, array $connectionIdentity): bool + { + return $first === $second || $this->getConnectionIdentity($first) === $connectionIdentity; + } + + /** + * @return array + */ + private function getConnectionIdentity(Connection $connection): array + { + $params = $connection->getParams(); + $identity = []; + + foreach (self::CONNECTION_IDENTITY_KEYS as $key) { + if (array_key_exists($key, $params)) { + $identity[$key] = $params[$key]; + } + } + + ksort($identity); + + return $identity; + } + + private function convertRows( + Connection $connection, + AbstractPlatform $platform, + string $quotedTable, + int $batchSize, + OutputInterface $output + ): int { + $converted = 0; + $offset = 0; + + $qData = $platform->quoteSingleIdentifier('data'); + $qDataSerialized = $platform->quoteSingleIdentifier('data_serialized'); + $qId = $platform->quoteSingleIdentifier('id'); + + while (true) { + $rows = $connection->fetchAllAssociative( + sprintf( + 'SELECT %s, %s FROM %s WHERE %s IS NOT NULL AND %s IS NULL LIMIT %d OFFSET %d', + $qId, + $qDataSerialized, + $quotedTable, + $qDataSerialized, + $qData, + $batchSize, + $offset + ) + ); + + if ([] === $rows) { + break; + } + + foreach ($rows as $row) { + $serialized = $row['data_serialized']; + if (null === $serialized) { + continue; + } + + [$ok, $deserialized] = $this->safeUnserialize($serialized); + if (!$ok) { + $output->writeln(sprintf(' Warning: could not unserialize row id=%s – skipping.', $row['id'])); + continue; + } + + try { + $json = json_encode($deserialized, JSON_THROW_ON_ERROR | JSON_UNESCAPED_UNICODE); + } catch (\JsonException $e) { + $output->writeln(sprintf(' Warning: could not JSON-encode row id=%s (%s) – skipping.', $row['id'], $e->getMessage())); + continue; + } + + $connection->executeStatement( + sprintf('UPDATE %s SET %s = ? WHERE %s = ?', $quotedTable, $qData, $qId), + [$json, $row['id']] + ); + ++$converted; + } + + $offset += $batchSize; + } + + return $converted; + } + + private function migrateViaSqliteRecreate( + Connection $connection, + AbstractPlatform $platform, + string $table, + string $quotedTable, + OutputInterface $output + ): void { + $tmpTable = $table.'_migration_tmp'; + $quotedTmp = $platform->quoteSingleIdentifier($tmpTable); + + $createSql = $connection->fetchOne( + "SELECT sql FROM sqlite_master WHERE type='table' AND name=?", + [$table] + ); + + if (false === $createSql || null === $createSql) { + throw new \RuntimeException(sprintf("Could not read schema for table '%s'.", $table)); + } + + $tmpCreate = preg_replace( + self::SQLITE_CREATE_TABLE_HEADER_PATTERN, + '$1'.$quotedTmp.'$3', + $createSql, + 1 + ); + + $tmpCreate = preg_replace( + self::SQLITE_DATA_COLUMN_PATTERN, + ' "data_serialized"', + $tmpCreate, + 1 + ); + + $connection->executeStatement($tmpCreate); + + $cols = $connection->fetchAllAssociative(sprintf('PRAGMA table_info(%s)', $table)); + $colList = implode(', ', array_map( + static fn (array $c): string => $platform->quoteSingleIdentifier($c['name']), + $cols + )); + $connection->executeStatement(sprintf('INSERT INTO %s SELECT %s FROM %s', $quotedTmp, $colList, $quotedTable)); + + $connection->executeStatement(sprintf('DROP TABLE %s', $quotedTable)); + $connection->executeStatement(sprintf('ALTER TABLE %s RENAME TO %s', $quotedTmp, $quotedTable)); + + $clobType = $platform->getClobTypeDeclarationSQL([]); + $connection->executeStatement(sprintf( + 'ALTER TABLE %s ADD COLUMN %s %s DEFAULT NULL', + $quotedTable, + $platform->quoteSingleIdentifier('data'), + $clobType + )); + + $qData = $platform->quoteSingleIdentifier('data'); + $qDataSerialized = $platform->quoteSingleIdentifier('data_serialized'); + $qId = $platform->quoteSingleIdentifier('id'); + + $rows = $connection->fetchAllAssociative( + sprintf('SELECT %s, %s FROM %s WHERE %s IS NOT NULL', $qId, $qDataSerialized, $quotedTable, $qDataSerialized) + ); + $converted = 0; + foreach ($rows as $row) { + [$ok, $deserialized] = $this->safeUnserialize($row['data_serialized']); + if ($ok) { + try { + $json = json_encode($deserialized, JSON_THROW_ON_ERROR | JSON_UNESCAPED_UNICODE); + $connection->executeStatement( + sprintf('UPDATE %s SET %s = ? WHERE %s = ?', $quotedTable, $qData, $qId), + [$json, $row['id']] + ); + ++$converted; + } catch (\JsonException) { + $output->writeln(sprintf(' Warning: could not JSON-encode row id=%s – skipping.', $row['id'])); + } + } else { + $output->writeln(sprintf(' Warning: could not unserialize row id=%s – skipping.', $row['id'])); + } + } + + $output->writeln(sprintf(' SQLite migration complete (%d row(s) converted).', $converted)); + } + + private function finalize( + Connection $connection, + AbstractPlatform $platform, + string $table, + string $quotedTable, + bool $dropLegacy, + OutputInterface $output + ): void { + if ($dropLegacy) { + $output->writeln("Dropping legacy column 'data_serialized'..."); + $connection->executeStatement(sprintf( + 'ALTER TABLE %s DROP COLUMN %s', + $quotedTable, + $platform->quoteSingleIdentifier('data_serialized') + )); + $output->writeln(' Done.'); + + return; + } + + $output->writeln(''); + $output->writeln('Migration complete.'); + $output->writeln("The original data is still available in the 'data_serialized' column."); + $output->writeln('Once you have verified the migration, you can drop it:'); + $output->writeln(sprintf(' ALTER TABLE %s DROP COLUMN data_serialized;', $table)); + } + + /** + * @return array{0: bool, 1: mixed} + */ + private function safeUnserialize(string $serialized): array + { + set_error_handler(static function (): bool { + return true; + }); + + try { + $deserialized = unserialize($serialized, ['allowed_classes' => false]); + } catch (\Throwable) { + restore_error_handler(); + + return [false, null]; + } + + restore_error_handler(); + + if (false === $deserialized && 'b:0;' !== $serialized) { + return [false, null]; + } + + return [true, $deserialized]; + } +} diff --git a/src/Loggable/Entity/MappedSuperclass/AbstractLogEntry.php b/src/Loggable/Entity/MappedSuperclass/AbstractLogEntry.php index d0965547a4..730c068044 100644 --- a/src/Loggable/Entity/MappedSuperclass/AbstractLogEntry.php +++ b/src/Loggable/Entity/MappedSuperclass/AbstractLogEntry.php @@ -83,11 +83,9 @@ abstract class AbstractLogEntry implements LogEntryInterface /** * @var array|null * - * @ORM\Column(type="array", nullable=true) - * - * NOTE: The attribute uses the "array" name directly instead of the constant since it was removed in DBAL 4.0. + * @ORM\Column(type="json", nullable=true) */ - #[ORM\Column(type: 'array', nullable: true)] + #[ORM\Column(type: Types::JSON, nullable: true)] protected $data; /** diff --git a/tests/Gedmo/Loggable/LoggableEntityTest.php b/tests/Gedmo/Loggable/LoggableEntityTest.php index 21312c45a1..2b10982b91 100644 --- a/tests/Gedmo/Loggable/LoggableEntityTest.php +++ b/tests/Gedmo/Loggable/LoggableEntityTest.php @@ -11,7 +11,6 @@ namespace Gedmo\Tests\Loggable; -use Doctrine\DBAL\Types\ArrayType; use Gedmo\Loggable\Entity\LogEntry; use Gedmo\Loggable\Entity\Repository\LogEntryRepository; use Gedmo\Loggable\Loggable; @@ -40,13 +39,6 @@ abstract class LoggableEntityTest extends BaseTestCaseORM */ protected LoggableListener $listener; - public static function setUpBeforeClass(): void - { - if (!class_exists(ArrayType::class)) { - static::markTestSkipped('The loggable extension is not compatible with doctrine/dbal:>=4.0'); - } - } - public function testShouldHandleClonedEntity(): void { $art0 = new Article();