From d5dd4616ee781572716172258ccbeaabe40c22a4 Mon Sep 17 00:00:00 2001 From: Hannes Lau Date: Wed, 4 Jun 2025 10:10:19 +0200 Subject: [PATCH 1/3] chore: Migrate php linting to min version 8.1 --- .github/workflows/ci.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 6a0ff8c..f1e03ac 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -29,7 +29,6 @@ jobs: strategy: matrix: php-version: - - 8.0 - 8.1 steps: - name: Checkout @@ -49,7 +48,6 @@ jobs: strategy: matrix: php-version: - - 8.0 - 8.1 steps: - uses: actions/checkout@v2 From 9a8ded5f0af30dc26b66fd7021bf404f1b995352 Mon Sep 17 00:00:00 2001 From: Hannes Lau Date: Thu, 22 May 2025 18:22:29 +0200 Subject: [PATCH 2/3] feat: Allow to limit move/copy commands to used files (JB-1596) --- Classes/Command/AbstractCloudinaryCommand.php | 9 +++++++++ Classes/Command/CloudinaryCopyCommand.php | 1 + Classes/Command/CloudinaryMoveCommand.php | 1 + 3 files changed, 11 insertions(+) diff --git a/Classes/Command/AbstractCloudinaryCommand.php b/Classes/Command/AbstractCloudinaryCommand.php index 00948f2..a1b25fb 100644 --- a/Classes/Command/AbstractCloudinaryCommand.php +++ b/Classes/Command/AbstractCloudinaryCommand.php @@ -95,6 +95,15 @@ protected function getFiles(ResourceStorage $storage, InputInterface $input): ar } } + if ((bool)$input->getOption('used-only')) { + $query->join( + 'sys_file', + 'sys_file_reference', + 'sys_file_reference', + $query->expr()->eq('sys_file_reference.uid_local', 'sys_file.uid'), + ); + } + return $query->execute()->fetchAllAssociative(); } diff --git a/Classes/Command/CloudinaryCopyCommand.php b/Classes/Command/CloudinaryCopyCommand.php index f9b5b2e..97b6ccc 100644 --- a/Classes/Command/CloudinaryCopyCommand.php +++ b/Classes/Command/CloudinaryCopyCommand.php @@ -54,6 +54,7 @@ protected function configure(): void ->addOption('filter-file-type', '', InputArgument::OPTIONAL, 'Add a possible filter for file type as defined by FAL (e.g 1,2,3,4,5)', '') ->addOption('limit', '', InputArgument::OPTIONAL, 'Add a possible offset, limit to restrain the number of files. (eg. 0,100)', '') ->addOption('exclude', '', InputArgument::OPTIONAL, 'Exclude pattern, can contain comma separated values e.g. --exclude="/apps/%,/_temp/%"', '') + ->addOption('used-only', '', InputArgument::OPTIONAL, 'Only copy used files (with sys_file_reference)', false) ->addArgument('source', InputArgument::REQUIRED, 'Source storage identifier') ->addArgument('target', InputArgument::REQUIRED, 'Target storage identifier') ->setHelp('Usage: ./vendor/bin/typo3 cloudinary:copy 1 2'); diff --git a/Classes/Command/CloudinaryMoveCommand.php b/Classes/Command/CloudinaryMoveCommand.php index a5bb353..a44b815 100644 --- a/Classes/Command/CloudinaryMoveCommand.php +++ b/Classes/Command/CloudinaryMoveCommand.php @@ -49,6 +49,7 @@ protected function configure(): void ->addOption('filter-file-type', '', InputArgument::OPTIONAL, 'Add a possible filter for file type as defined by FAL (e.g 1,2,3,4,5)', '') ->addOption('limit', '', InputArgument::OPTIONAL, 'Add a possible offset, limit to restrain the number of files. (eg. 0,100)', '') ->addOption('exclude', '', InputArgument::OPTIONAL, 'Exclude pattern, can contain comma separated values e.g. --exclude="/apps/%,/_temp/%"', '') + ->addOption('used-only', '', InputArgument::OPTIONAL, 'Only move used files (with sys_file_reference)', false) ->addArgument('source', InputArgument::REQUIRED, 'Source storage identifier') ->addArgument('target', InputArgument::REQUIRED, 'Target storage identifier') ->setHelp('Usage: ./vendor/bin/typo3 cloudinary:move 1 2'); From c1b6bcfc8603de33ac5d8c43f550428afa99d730 Mon Sep 17 00:00:00 2001 From: Hannes Lau Date: Mon, 2 Jun 2025 15:45:50 +0200 Subject: [PATCH 3/3] feat: Allow to move files from external storage folder to cloudinary folder Instead of moving from storage to storage, allow to move files from folder to cloudinary folder. --- Classes/Command/AbstractCloudinaryCommand.php | 8 +- Classes/Command/CloudinaryCopyCommand.php | 2 +- Classes/Command/CloudinaryFixJpegCommand.php | 2 +- Classes/Command/CloudinaryMoveCommand.php | 122 +++++++++++++----- Classes/Services/FileMoveService.php | 72 +++-------- 5 files changed, 118 insertions(+), 88 deletions(-) diff --git a/Classes/Command/AbstractCloudinaryCommand.php b/Classes/Command/AbstractCloudinaryCommand.php index a1b25fb..cb181b5 100644 --- a/Classes/Command/AbstractCloudinaryCommand.php +++ b/Classes/Command/AbstractCloudinaryCommand.php @@ -9,12 +9,13 @@ * LICENSE.md file that was distributed with this source code. */ -use Doctrine\DBAL\Driver\Connection; use Symfony\Component\Console\Input\InputInterface; use Symfony\Component\Console\Style\SymfonyStyle; use Symfony\Component\Console\Command\Command; +use TYPO3\CMS\Core\Database\Connection; use TYPO3\CMS\Core\Database\ConnectionPool; use TYPO3\CMS\Core\Database\Query\QueryBuilder; +use TYPO3\CMS\Core\Resource\Folder; use TYPO3\CMS\Core\Resource\ResourceStorage; use TYPO3\CMS\Core\Utility\GeneralUtility; use Visol\Cloudinary\Driver\CloudinaryDriver; @@ -35,14 +36,15 @@ abstract class AbstractCloudinaryCommand extends Command protected string $tableName = 'sys_file'; - protected function getFiles(ResourceStorage $storage, InputInterface $input): array + protected function getFiles(Folder $folder, InputInterface $input): array { $query = $this->getQueryBuilder($this->tableName); $query ->select('*') ->from($this->tableName) ->where( - $query->expr()->eq('storage', $storage->getUid()), + $query->expr()->eq('storage', $folder->getStorage()->getUid()), + $query->expr()->like('identifier', $query->createNamedParameter($query->escapeLikeWildcards($folder->getIdentifier()) . '%')), $query->expr()->eq('missing', 0) ); diff --git a/Classes/Command/CloudinaryCopyCommand.php b/Classes/Command/CloudinaryCopyCommand.php index 97b6ccc..85a4e05 100644 --- a/Classes/Command/CloudinaryCopyCommand.php +++ b/Classes/Command/CloudinaryCopyCommand.php @@ -67,7 +67,7 @@ protected function execute(InputInterface $input, OutputInterface $output): int return Command::INVALID; } - $files = $this->getFiles($this->sourceStorage, $input); + $files = $this->getFiles($this->sourceStorage->getRootLevelFolder(), $input); if (count($files) === 0) { $this->log('No files found, no work for me!'); diff --git a/Classes/Command/CloudinaryFixJpegCommand.php b/Classes/Command/CloudinaryFixJpegCommand.php index af79fd0..95b3ffe 100644 --- a/Classes/Command/CloudinaryFixJpegCommand.php +++ b/Classes/Command/CloudinaryFixJpegCommand.php @@ -86,7 +86,7 @@ protected function execute(InputInterface $input, OutputInterface $output): int name = REPLACE(name, '.jpeg', '.jpg') WHERE storage = " . $this->targetStorage->getUid(); - $connection->query($query)->execute(); + $connection->executeStatement($query); return Command::SUCCESS; diff --git a/Classes/Command/CloudinaryMoveCommand.php b/Classes/Command/CloudinaryMoveCommand.php index a44b815..94ac515 100644 --- a/Classes/Command/CloudinaryMoveCommand.php +++ b/Classes/Command/CloudinaryMoveCommand.php @@ -10,8 +10,14 @@ */ use Exception; +use Psr\EventDispatcher\EventDispatcherInterface; use Symfony\Component\Console\Command\Command; use Symfony\Component\Console\Style\SymfonyStyle; +use TYPO3\CMS\Core\Resource\Driver\DriverInterface; +use TYPO3\CMS\Core\Resource\Event\AfterFileAddedEvent; +use TYPO3\CMS\Core\Resource\Event\AfterFileMovedEvent; +use TYPO3\CMS\Core\Resource\Folder; +use TYPO3\CMS\Core\Resource\Index\Indexer; use TYPO3\CMS\Core\Resource\ResourceStorage; use TYPO3\CMS\Core\Utility\PathUtility; use Visol\Cloudinary\Services\FileMoveService; @@ -31,13 +37,13 @@ class CloudinaryMoveCommand extends AbstractCloudinaryCommand protected array $missingFiles = []; - protected ResourceStorage $sourceStorage; - - protected ResourceStorage $targetStorage; + public function __construct( + protected ResourceFactory $resourceFactory, + protected EventDispatcherInterface $eventDispatcher, + ) { + parent::__construct(); + } - /** - * Configure the command by defining the name, options and arguments - */ protected function configure(): void { $message = 'Move bunch of images to a cloudinary storage. Consult the README.md for more info.'; @@ -60,23 +66,40 @@ protected function initialize(InputInterface $input, OutputInterface $output): v $this->io = new SymfonyStyle($input, $output); $this->isSilent = $input->getOption('silent'); - - /** @var ResourceFactory $resourceFactory */ - $resourceFactory = GeneralUtility::makeInstance(ResourceFactory::class); - - $this->sourceStorage = $resourceFactory->getStorageObject($input->getArgument('source')); - $this->targetStorage = $resourceFactory->getStorageObject($input->getArgument('target')); } protected function execute(InputInterface $input, OutputInterface $output): int { - if (!$this->checkDriverType($this->targetStorage)) { + $sourceCombinedIdentifier = $input->getArgument('source'); + if (!is_string($sourceCombinedIdentifier)) { + throw new \LogicException('source argument must be a string', 1749032224634); + } + $source = $this->resourceFactory->getFolderObjectFromCombinedIdentifier($sourceCombinedIdentifier); + $sourceStorage = $source->getStorage(); + $sourceStorageDriver = $this->getStorageDriver($sourceStorage); + + $targetCombinedIdentifier = $input->getArgument('target'); + if (!is_string($targetCombinedIdentifier)) { + throw new \LogicException('target argument must be a string', 1749032230062); + } + $target = $this->resourceFactory->getFolderObjectFromCombinedIdentifier($targetCombinedIdentifier); + $targetIndexer = GeneralUtility::makeInstance(Indexer::class, $target->getStorage()); + + $baseUrl = $input->getOption('base-url'); + if (!is_string($baseUrl)) { + throw new \LogicException('Base URL must be a string'); + } + + if (!$sourceStorage->hasHierarchicalIdentifiers()) { + $this->log('Source storage must use hierarchical identifiers'); + return Command::INVALID; + } + if (!$this->checkDriverType($target->getStorage())) { $this->log('Look out! target storage is not of type "cloudinary"'); return Command::INVALID; } - $files = $this->getFiles($this->sourceStorage, $input); - + $files = $this->getFiles($source, $input); if (count($files) === 0) { $this->log('No files found, no work for me!'); return Command::SUCCESS; @@ -84,10 +107,10 @@ protected function execute(InputInterface $input, OutputInterface $output): int $this->log('I will process %s files to be moved from storage "%s" (%s) to "%s" (%s)', [ count($files), - $this->sourceStorage->getUid(), - $this->sourceStorage->getName(), - $this->targetStorage->getUid(), - $this->targetStorage->getName(), + $source->getCombinedIdentifier(), + $sourceStorage->getName(), + $target->getCombinedIdentifier(), + $target->getStorage()->getName(), ]); // A chance to the user to confirm the action @@ -96,21 +119,18 @@ protected function execute(InputInterface $input, OutputInterface $output): int if (!$response) { $this->log('Script aborted'); - - return Command::SUCCESS; + return Command::SUCCESS; } } - /** @var ResourceFactory $resourceFactory */ - $resourceFactory = GeneralUtility::makeInstance(ResourceFactory::class); - $counter = 0; foreach ($files as $file) { $this->log(); $this->log('Starting migration with %s', [$file['identifier']]); - /** @var $fileObject */ - $fileObject = $resourceFactory->getFileObjectByStorageAndIdentifier($this->sourceStorage->getUid(), $file['identifier']); + /** @var File $fileObject */ + $fileObject = $this->resourceFactory->getFileObject($file['uid'], $file); + $sourceFileExists = $fileObject->exists(); if ($this->isFileSkipped($fileObject)) { $this->log('Skipping file ' . $fileObject->getIdentifier()); @@ -118,11 +138,17 @@ protected function execute(InputInterface $input, OutputInterface $output): int continue; } - if ($this->getFileMoveService()->fileExists($fileObject, $this->targetStorage)) { + if (! str_starts_with($fileObject->getIdentifier(), $source->getIdentifier())) { + throw new \LogicException('file is not in source folder', 1748004982814); + } + + $newIdentifier = str_replace($source->getIdentifier(), $target->getIdentifier(), $fileObject->getIdentifier()); + + if ($this->getFileMoveService()->fileExists($target->getStorage(), $newIdentifier)) { $this->log('File has already been uploaded, good for us %s', [$fileObject->getIdentifier()]); } else { // Detect if the file is existing on storage "source" (1) - if (!$fileObject->exists() && !$input->getOption('base-url')) { + if (!$sourceFileExists && empty($baseUrl)) { $this->log('Missing file %s', [$fileObject->getIdentifier()], self::WARNING); // We could log the missing files $this->missingFiles[] = $fileObject->getIdentifier(); @@ -130,11 +156,11 @@ protected function execute(InputInterface $input, OutputInterface $output): int } // Upload the file - $this->log('Uploading file from %s%s', [$input->getOption('base-url'), $fileObject->getIdentifier()]); + $this->log('Uploading file from %s%s', [$baseUrl, $fileObject->getIdentifier()]); try { $start = microtime(true); - $this->getFileMoveService()->cloudinaryUploadFile($fileObject, $this->targetStorage, $input->getOption('base-url')); + $this->getFileMoveService()->cloudinaryUploadFile($fileObject, $target, $newIdentifier, $baseUrl); $timeElapsedSeconds = microtime(true) - $start; $this->log('File uploaded, Elapsed time %.3f', [$timeElapsedSeconds]); } catch (Exception $e) { @@ -148,9 +174,29 @@ protected function execute(InputInterface $input, OutputInterface $output): int } } - // changing file storage and hard delete the file from the current storage + // Update sys_file entry + // See \TYPO3\CMS\Core\Resource\ResourceStorage::moveFile for reference $this->log('Changing storage for file %s', [$fileObject->getIdentifier()]); - $this->getFileMoveService()->changeStorage($fileObject, $this->targetStorage); + $oldIdentifier = $fileObject->getIdentifier(); + $oldFolder = $fileObject->getParentFolder(); + $fileObject->updateProperties(['storage' => $target->getStorage()->getUid(), 'identifier' => $newIdentifier]); + $newFolder = $fileObject->getParentFolder(); + if (!$newFolder instanceof Folder) { + throw new \LogicException('New folder must be a Folder', 1749032215642); + } + + // clean up processed files + $this->eventDispatcher->dispatch(new AfterFileAddedEvent($fileObject, $newFolder)); + $targetIndexer->updateIndexEntry($fileObject); + + // Delete the file from the source storage without deleting the sys_file record + if ($sourceFileExists) { + $sourceStorageDriver->deleteFile($oldIdentifier); + $sourceFileExists = false; + } + + $this->eventDispatcher->dispatch(new AfterFileMovedEvent($fileObject, $newFolder, $oldFolder)); + $counter++; } $this->log(LF); @@ -208,4 +254,16 @@ protected function getFileMoveService(): FileMoveService { return GeneralUtility::makeInstance(FileMoveService::class); } + + protected function getStorageDriver(ResourceStorage $storage): DriverInterface + { + $reflection = new \ReflectionClass($storage); + $property = $reflection->getProperty('driver'); + $property->setAccessible(true); + $driver = $property->getValue($storage); + if (!$driver instanceof DriverInterface) { + throw new \LogicException('Storage driver must implement DriverInterface', 1749032330406); + } + return $driver; + } } diff --git a/Classes/Services/FileMoveService.php b/Classes/Services/FileMoveService.php index 0302915..b513b20 100644 --- a/Classes/Services/FileMoveService.php +++ b/Classes/Services/FileMoveService.php @@ -16,6 +16,7 @@ use TYPO3\CMS\Core\Database\ConnectionPool; use TYPO3\CMS\Core\Database\Query\QueryBuilder; use TYPO3\CMS\Core\Resource\File; +use TYPO3\CMS\Core\Resource\Folder; use TYPO3\CMS\Core\Resource\ResourceStorage; use TYPO3\CMS\Core\Utility\GeneralUtility; use Visol\Cloudinary\Utility\CloudinaryApiUtility; @@ -27,22 +28,22 @@ class FileMoveService protected ?CloudinaryPathService $cloudinaryPathService = null; - public function fileExists(File $fileObject, ResourceStorage $targetStorage): bool + public function fileExists(ResourceStorage $storage, string $identifier): bool { - $this->initializeCloudinaryService($targetStorage); + $this->initializeCloudinaryService($storage); // Retrieve the Public Id based on the file identifier - $publicId = $this->getCloudinaryPathService() - ->computeCloudinaryPublicId($fileObject->getIdentifier()); + $publicId = $this->getCloudinaryPathService()->computeCloudinaryPublicId($identifier); try { - $resource = $this->getAdminApi($targetStorage)->asset($publicId); - $fileExists = !empty($resource); + $resource = (array)$this->getAdminApi($storage)->asset($publicId); + + // update resource index + (new CloudinaryResourceService($storage))->save((array)$resource); + return true; } catch (Exception $exception) { - $fileExists = false; + return false; } - - return $fileExists; } #public function forceMove(File $fileObject, ResourceStorage $targetStorage, $removeFile = true): bool @@ -74,34 +75,6 @@ public function fileExists(File $fileObject, ResourceStorage $targetStorage): bo # return $isUpdated && $isDeletedFromSourceStorage; #} - public function changeStorage(File $fileObject, ResourceStorage $targetStorage, bool $removeFile = true): bool - { - // Update the storage uid - $isMigrated = (bool)$this->updateFile( - $fileObject, - [ - 'storage' => $targetStorage->getUid(), - ] - ); - - if ($removeFile) { - // Delete the file form the local storage - $isMigrated = unlink($this->getAbsolutePath($fileObject)); - } - - return $isMigrated; - } - - protected function ensureDirectoryExistence(File $fileObject) - { - - // Make sure the directory exists - $directory = dirname($this->getAbsolutePath($fileObject)); - if (!is_dir($directory)) { - GeneralUtility::mkdir_deep($directory); - } - } - protected function getAbsolutePath(File $fileObject): string { // Compute the absolute file name of the file to move @@ -112,37 +85,34 @@ protected function getAbsolutePath(File $fileObject): string public function cloudinaryUploadFile( File $fileObject, - ResourceStorage $targetStorage, + Folder $targetFolder, + string $newIdentifier, string $baseUrl = '' ): void { - + $targetStorage = $targetFolder->getStorage(); $this->initializeCloudinaryService($targetStorage); - - $this->ensureDirectoryExistence($fileObject); - - - $fileIdentifier = $fileObject->getIdentifier(); - $publicId = $this->getCloudinaryPathService() - ->computeCloudinaryPublicId($fileIdentifier); + $publicId = $this->getCloudinaryPathService()->computeCloudinaryPublicId($newIdentifier); $options = [ 'public_id' => basename($publicId), 'folder' => $this->getCloudinaryPathService() ->computeCloudinaryFolderPath( - $fileObject->getParentFolder()->getIdentifier() + dirname($newIdentifier) ), - 'resource_type' => $this->getCloudinaryPathService()->getResourceType($fileIdentifier), + 'resource_type' => $this->getCloudinaryPathService()->getResourceType($newIdentifier), 'overwrite' => true, ]; $fileNameAndPath = $baseUrl - ? rtrim($baseUrl, DIRECTORY_SEPARATOR) . $fileIdentifier - : $this->getAbsolutePath($fileObject); + ? rtrim($baseUrl, DIRECTORY_SEPARATOR) . $fileObject->getIdentifier() + : ($fileObject->getPublicUrl() ?: $fileObject->getForLocalProcessing(false)); // Upload the file - $this->getUploadApi($targetStorage)->upload( + $response = $this->getUploadApi($targetStorage)->upload( $fileNameAndPath, $options ); + $resource = (array)$response; + (new CloudinaryResourceService($targetStorage))->save($resource); } protected function getQueryBuilder(): QueryBuilder