From bc046cab929b411984d6c887f93bd62fa7c72122 Mon Sep 17 00:00:00 2001 From: AlexisMora Date: Sat, 24 Jan 2026 21:09:00 +0100 Subject: [PATCH 1/9] Feat: feature, upload with worker pool --- .../execute-worker-pool.ts | 32 +++++++++++++++++++ .../upload-with-worker-pool/get-next-task.ts | 6 ++++ .../upload-with-worker-pool/run-worker.ts | 31 ++++++++++++++++++ .../common/upload-with-worker-pool/types.ts | 21 ++++++++++++ 4 files changed, 90 insertions(+) create mode 100644 src/backend/common/upload-with-worker-pool/execute-worker-pool.ts create mode 100644 src/backend/common/upload-with-worker-pool/get-next-task.ts create mode 100644 src/backend/common/upload-with-worker-pool/run-worker.ts create mode 100644 src/backend/common/upload-with-worker-pool/types.ts diff --git a/src/backend/common/upload-with-worker-pool/execute-worker-pool.ts b/src/backend/common/upload-with-worker-pool/execute-worker-pool.ts new file mode 100644 index 0000000000..def44b91ef --- /dev/null +++ b/src/backend/common/upload-with-worker-pool/execute-worker-pool.ts @@ -0,0 +1,32 @@ +import { DriveDesktopError } from '../../../context/shared/domain/errors/DriveDesktopError'; +import { Result } from '../../../context/shared/domain/Result'; +import { getNextTask } from './get-next-task'; +import { runWorker } from './run-worker'; +import { QueueIndex, TaskExecutor, WorkerPoolConfig, WorkerPoolResult, WorkerState } from './types'; + +export async function executeWorkerPool( + items: T[], + executor: TaskExecutor, + config: WorkerPoolConfig, +): Promise, DriveDesktopError>> { + const queue = [...items]; + const index: QueueIndex = { value: 0 }; + const state: WorkerState = { + results: { succeeded: [], failed: [] }, + fatalError: null, + }; + + const getNext = () => getNextTask(queue, index); + + const workers = Array.from({ length: config.concurrency }, () => + runWorker(getNext, executor, state, config.signal), + ); + + await Promise.all(workers); + + if (state.fatalError) { + return { error: state.fatalError }; + } + + return { data: state.results }; +} diff --git a/src/backend/common/upload-with-worker-pool/get-next-task.ts b/src/backend/common/upload-with-worker-pool/get-next-task.ts new file mode 100644 index 0000000000..ba8029e11d --- /dev/null +++ b/src/backend/common/upload-with-worker-pool/get-next-task.ts @@ -0,0 +1,6 @@ +import { QueueIndex } from './types'; + +export function getNextTask(queue: T[], index: QueueIndex): T | null { + if (index.value >= queue.length) return null; + return queue[index.value++]; +} diff --git a/src/backend/common/upload-with-worker-pool/run-worker.ts b/src/backend/common/upload-with-worker-pool/run-worker.ts new file mode 100644 index 0000000000..b44274d713 --- /dev/null +++ b/src/backend/common/upload-with-worker-pool/run-worker.ts @@ -0,0 +1,31 @@ +/* eslint-disable no-constant-condition */ +import { isFatalError } from '../../../shared/issues/SyncErrorCause'; +import { TaskExecutor, WorkerState } from './types'; + +export async function runWorker( + getNext: () => T | null, + executor: TaskExecutor, + state: WorkerState, + signal: AbortSignal, +): Promise { + while (true) { + if (signal.aborted || state.fatalError) break; + + const item = getNext(); + if (!item) break; + + // eslint-disable-next-line no-await-in-loop + const result = await executor(item, signal); + + if (result.error) { + state.results.failed.push({ item, error: result.error }); + + if (isFatalError(result.error.cause)) { + state.fatalError = result.error; + break; + } + } else { + state.results.succeeded.push(item); + } + } +} diff --git a/src/backend/common/upload-with-worker-pool/types.ts b/src/backend/common/upload-with-worker-pool/types.ts new file mode 100644 index 0000000000..69e1ca0090 --- /dev/null +++ b/src/backend/common/upload-with-worker-pool/types.ts @@ -0,0 +1,21 @@ +import { DriveDesktopError } from '../../../context/shared/domain/errors/DriveDesktopError'; +import { Result } from '../../../context/shared/domain/Result'; + +export type WorkerPoolConfig = { + concurrency: number; + signal: AbortSignal; +}; + +export type WorkerPoolResult = { + succeeded: Array; + failed: Array<{ item: T; error: DriveDesktopError }>; +}; + +export type TaskExecutor = (item: T, signal: AbortSignal) => Promise>; + +export type QueueIndex = { value: number }; + +export type WorkerState = { + results: WorkerPoolResult; + fatalError: DriveDesktopError | null; +}; From 9a33bc7ce126abd010c4c607b6a8772e7e5f04da Mon Sep 17 00:00:00 2001 From: AlexisMora Date: Sat, 24 Jan 2026 21:12:36 +0100 Subject: [PATCH 2/9] feat: comment this so that i dont forget --- .../local/localFile/application/upload/FileBatchUploader.ts | 1 + .../localFile/infrastructure/EnvironmentLocalFileUploader.ts | 2 ++ .../virtual-drive/files/application/create/SimpleFileCreator.ts | 1 + 3 files changed, 4 insertions(+) diff --git a/src/context/local/localFile/application/upload/FileBatchUploader.ts b/src/context/local/localFile/application/upload/FileBatchUploader.ts index 9d02325abe..8bd1473cd5 100644 --- a/src/context/local/localFile/application/upload/FileBatchUploader.ts +++ b/src/context/local/localFile/application/upload/FileBatchUploader.ts @@ -9,6 +9,7 @@ import { logger } from '@internxt/drive-desktop-core/build/backend'; import { backupErrorsTracker } from '../../../../../backend/features/backup'; import { deleteFileFromStorageByFileId } from '../../../../../infra/drive-server/services/files/services/delete-file-content-from-bucket'; +// TODO: THIS WHOLE FILE IS GOING TO BE REPLACED @Service() export class FileBatchUploader { constructor( diff --git a/src/context/local/localFile/infrastructure/EnvironmentLocalFileUploader.ts b/src/context/local/localFile/infrastructure/EnvironmentLocalFileUploader.ts index a7effa98ef..23bced6e79 100644 --- a/src/context/local/localFile/infrastructure/EnvironmentLocalFileUploader.ts +++ b/src/context/local/localFile/infrastructure/EnvironmentLocalFileUploader.ts @@ -9,6 +9,8 @@ import { Either, left, right } from '../../../shared/domain/Either'; import { DriveDesktopError } from '../../../shared/domain/errors/DriveDesktopError'; import { logger } from '@internxt/drive-desktop-core/build/backend'; import { MULTIPART_UPLOAD_SIZE_THRESHOLD } from '../../../shared/domain/UploadConstants'; + +// TODO: THIS WHOLE FILE IS GOING TO BE REPLACED @Service() export class EnvironmentLocalFileUploader implements LocalFileHandler { constructor( diff --git a/src/context/virtual-drive/files/application/create/SimpleFileCreator.ts b/src/context/virtual-drive/files/application/create/SimpleFileCreator.ts index 21200e0a2f..134027b4d6 100644 --- a/src/context/virtual-drive/files/application/create/SimpleFileCreator.ts +++ b/src/context/virtual-drive/files/application/create/SimpleFileCreator.ts @@ -7,6 +7,7 @@ import { FileFolderId } from '../../domain/FileFolderId'; import { File } from '../../domain/File'; import { Either, left, right } from '../../../../shared/domain/Either'; import { DriveDesktopError } from '../../../../shared/domain/errors/DriveDesktopError'; +// TODO: THIS WHOLE FILE IS GOING TO BE REPLACED @Service() export class SimpleFileCreator { constructor(private readonly remote: RemoteFileSystem) {} From 6b200d4d28ca3598f2a3b293764813401a3cc4ca Mon Sep 17 00:00:00 2001 From: AlexisMora Date: Sat, 24 Jan 2026 21:13:12 +0100 Subject: [PATCH 3/9] feat: upload files functionality with worker pool --- .../backup/upload/create-file-to-backend.ts | 138 ++++++++++++++++++ .../upload/delete-content-from-environment.ts | 13 ++ .../backup/upload/upload-backup-files.ts | 121 +++++++++++++++ .../upload/upload-content-to-environment.ts | 68 +++++++++ .../backup/upload/upload-file-with-retry.ts | 116 +++++++++++++++ .../features/backup/upload/utils/sleep.ts | 3 + 6 files changed, 459 insertions(+) create mode 100644 src/backend/features/backup/upload/create-file-to-backend.ts create mode 100644 src/backend/features/backup/upload/delete-content-from-environment.ts create mode 100644 src/backend/features/backup/upload/upload-backup-files.ts create mode 100644 src/backend/features/backup/upload/upload-content-to-environment.ts create mode 100644 src/backend/features/backup/upload/upload-file-with-retry.ts create mode 100644 src/backend/features/backup/upload/utils/sleep.ts diff --git a/src/backend/features/backup/upload/create-file-to-backend.ts b/src/backend/features/backup/upload/create-file-to-backend.ts new file mode 100644 index 0000000000..eb44adfd24 --- /dev/null +++ b/src/backend/features/backup/upload/create-file-to-backend.ts @@ -0,0 +1,138 @@ +import path from 'path'; +import { EncryptionVersion } from '@internxt/sdk/dist/drive/storage/types'; +import { DriveDesktopError } from '../../../../context/shared/domain/errors/DriveDesktopError'; +import { Result } from '../../../../context/shared/domain/Result'; +import { File } from '../../../../context/virtual-drive/files/domain/File'; +import { createFileIPC } from '../../../../infra/ipc/files-ipc'; +import crypt from '../../../../context/shared/infrastructure/crypt'; +import { CreateFileDto } from '../../../../infra/drive-server/out/dto'; + +export type CreateFileToBackendParams = { + contentsId: string; + filePath: string; + size: number; + folderId: number; + folderUuid: string; + bucket: string; +}; + +/** + * Extracts filename without extension from a path + * Replicates FilePath.name() behavior + */ +function extractName(filePath: string): string { + const base = path.posix.basename(filePath); + const { name } = path.posix.parse(base); + return name; +} + +/** + * Extracts extension without the dot from a path + * Replicates FilePath.extension() behavior + */ +function extractExtension(filePath: string): string { + const base = path.posix.basename(filePath); + const { ext } = path.posix.parse(base); + return ext.slice(1); +} + +/** + * Creates file metadata in the backend API + * + * This replicates the behavior of: + * - SimpleFileCreator.run() + * - SDKRemoteFileSystem.persist() + * + * Returns a File domain object on success, matching the original behavior exactly. + */ +export async function createFileToBackend({ + contentsId, + filePath, + size, + folderId, + folderUuid, + bucket, +}: CreateFileToBackendParams): Promise> { + const plainName = extractName(filePath); + const extension = extractExtension(filePath); + + // Encrypt name with folderId as salt (same as SDKRemoteFileSystem.persist) + const encryptedName = crypt.encryptName(plainName, folderId.toString()); + + if (!encryptedName) { + return { + error: new DriveDesktopError( + 'COULD_NOT_ENCRYPT_NAME', + `Could not encrypt the file name: ${plainName} with salt: ${folderId.toString()}`, + ), + }; + } + + // Build request body exactly as SDKRemoteFileSystem.persist does + const body: CreateFileDto = { + bucket, + fileId: undefined as string | undefined, + encryptVersion: EncryptionVersion.Aes03, + folderUuid, + size, + plainName, + type: extension, + }; + + // Only set fileId if size > 0 (same condition as SDKRemoteFileSystem.persist) + if (size > 0) { + body.fileId = contentsId; + } + + const response = await createFileIPC(body); + + if (response.data) { + // Create File domain object exactly as SimpleFileCreator does + const file = File.create({ + id: response.data.id, + uuid: response.data.uuid, + contentsId: contentsId, + folderId: folderId, + createdAt: response.data.createdAt, + modificationTime: response.data.updatedAt, + path: filePath, + size: size, + updatedAt: response.data.updatedAt, + }); + + return { data: file }; + } + + // Handle errors exactly as SDKRemoteFileSystem.persist does + if (response.error && typeof response.error === 'object' && 'cause' in response.error) { + const errorCause = (response.error as { cause: string }).cause; + + if (errorCause === 'BAD_REQUEST') { + return { + error: new DriveDesktopError('BAD_REQUEST', `Some data was not valid for ${plainName}: ${body}`), + }; + } + + if (errorCause === 'FILE_ALREADY_EXISTS') { + return { + error: new DriveDesktopError( + 'FILE_ALREADY_EXISTS', + `File with name ${plainName} on ${folderId} already exists`, + ), + }; + } + + if (errorCause === 'SERVER_ERROR') { + return { + error: new DriveDesktopError( + 'BAD_RESPONSE', + `The server could not handle the creation of ${plainName}: ${body}`, + ), + }; + } + } + + return { + error: new DriveDesktopError('UNKNOWN', `Creating file ${plainName}: ${response.error}`), + }; +} diff --git a/src/backend/features/backup/upload/delete-content-from-environment.ts b/src/backend/features/backup/upload/delete-content-from-environment.ts new file mode 100644 index 0000000000..aa50fb6f43 --- /dev/null +++ b/src/backend/features/backup/upload/delete-content-from-environment.ts @@ -0,0 +1,13 @@ +import { logger } from '@internxt/drive-desktop-core/build/backend'; +import { deleteFileFromStorageByFileId } from '../../../../infra/drive-server/services/files/services/delete-file-content-from-bucket'; + +/** + * Delete content from storage bucket (cleanup on metadata creation failure) + */ +export async function deleteContentFromEnvironment(bucket: string, contentsId: string): Promise { + try { + await deleteFileFromStorageByFileId({ bucketId: bucket, fileId: contentsId }); + } catch { + logger.error({ tag: 'BACKUPS', msg: 'Could not delete the file from the bucket', contentsId }); + } +} diff --git a/src/backend/features/backup/upload/upload-backup-files.ts b/src/backend/features/backup/upload/upload-backup-files.ts new file mode 100644 index 0000000000..3ec01974cc --- /dev/null +++ b/src/backend/features/backup/upload/upload-backup-files.ts @@ -0,0 +1,121 @@ +import { Environment } from '@internxt/inxt-js'; +import { LocalFile } from '../../../../context/local/localFile/domain/LocalFile'; +import { RemoteTree } from '../../../../context/virtual-drive/remoteTree/domain/RemoteTree'; +import { DriveDesktopError } from '../../../../context/shared/domain/errors/DriveDesktopError'; +import { Result } from '../../../../context/shared/domain/Result'; +import { relative } from '../../../../apps/backups/utils/relative'; +import { isFatalError } from '../../../../shared/issues/SyncErrorCause'; +import { backupErrorsTracker } from '..'; +import { logger } from '@internxt/drive-desktop-core/build/backend'; +import { executeWorkerPool } from '../../../common/upload-with-worker-pool/execute-worker-pool'; +import { TaskExecutor } from '../../../common/upload-with-worker-pool/types'; +import { uploadFileWithRetry } from './upload-file-with-retry'; +import { BackupProgressTracker } from '../backup-progress-tracker'; + +const DEFAULT_CONCURRENCY = 10; + +export type UploadBackupFilesParams = { + files: LocalFile[]; + localRootPath: string; + remoteTree: RemoteTree; + bucket: string; + environment: Environment; + signal: AbortSignal; + tracker: BackupProgressTracker; + concurrency?: number; +}; + +/** + * Creates a TaskExecutor for backup file uploads + * + * The executor: + * 1. Resolves parent folder from remoteTree + * 2. Calls uploadFileWithRetry (content upload + metadata creation) + * 3. Adds created file to remoteTree on success + * 4. Tracks errors in backupErrorsTracker + * 5. Returns fatal errors to stop the pool + */ +function createBackupUploadExecutor( + localRootPath: string, + remoteTree: RemoteTree, + bucket: string, + environment: Environment, + tracker: BackupProgressTracker, +): TaskExecutor { + return async (localFile: LocalFile, signal: AbortSignal): Promise> => { + const remotePath = relative(localRootPath, localFile.path); + const parent = remoteTree.getParent(remotePath); + + const result = await uploadFileWithRetry({ + path: localFile.path, + size: localFile.size, + bucket, + folderId: parent.id, + folderUuid: parent.uuid, + environment, + signal, + }); + + // Always increment progress, whether success, skip, or non-fatal error + tracker.incrementProcessed(1); + + if (result.error) { + logger.error({ tag: 'BACKUPS', msg: '[FILE UPLOAD FAILED]', error: result.error, path: localFile.path }); + + if (isFatalError(result.error.cause)) { + return { error: result.error }; + } + + backupErrorsTracker.add(parent.id, { + name: localFile.nameWithExtension(), + error: result.error.cause, + }); + + return { data: undefined }; + } + + if (result.data !== null) { + remoteTree.addFile(parent, result.data); + } + + return { data: undefined }; + }; +} + +/** + * Upload backup files using the worker pool + * + * This is the entry point that replaces FileBatchUploader.run() + * It uploads files with N concurrent workers (default 10). + * + * Behavior matches FileBatchUploader: + * - Files are uploaded to storage bucket, then metadata created in backend + * - On success, files are added to remoteTree + * - FILE_ALREADY_EXISTS is skipped silently + * - BAD_RESPONSE and other non-fatal errors are tracked in backupErrorsTracker + * - Fatal errors (NOT_ENOUGH_SPACE, etc.) stop all uploads and return error + */ +export async function uploadBackupFiles({ + files, + localRootPath, + remoteTree, + bucket, + environment, + signal, + tracker, + concurrency = DEFAULT_CONCURRENCY, +}: UploadBackupFilesParams): Promise> { + if (files.length === 0) { + return { data: undefined }; + } + + const executor = createBackupUploadExecutor(localRootPath, remoteTree, bucket, environment, tracker); + + const result = await executeWorkerPool(files, executor, { concurrency, signal }); + + if (result.error) { + return { error: result.error }; + } + + return { data: undefined }; +} diff --git a/src/backend/features/backup/upload/upload-content-to-environment.ts b/src/backend/features/backup/upload/upload-content-to-environment.ts new file mode 100644 index 0000000000..5e22e46d9d --- /dev/null +++ b/src/backend/features/backup/upload/upload-content-to-environment.ts @@ -0,0 +1,68 @@ +import { Environment } from '@internxt/inxt-js'; +import { DriveDesktopError } from '../../../../context/shared/domain/errors/DriveDesktopError'; +import { MULTIPART_UPLOAD_SIZE_THRESHOLD } from '../../../../context/shared/domain/UploadConstants'; +import { createReadStream } from 'node:fs'; +import { UploadStrategyFunction } from '@internxt/inxt-js/build/lib/core'; +import { Result } from '../../../../context/shared/domain/Result'; +import { logger } from '@internxt/drive-desktop-core/build/backend'; + +export type ContentUploadParams = { + path: string; + size: number; + bucket: string; + environment: Environment; + signal: AbortSignal; +}; + +/** + * Upload file content to storage bucket (Environment) + * Returns contentsId on success + * + */ +export function uploadContentToEnvironment({ + path, + size, + bucket, + environment, + signal, +}: ContentUploadParams): Promise> { + const uploadFn: UploadStrategyFunction = + size > MULTIPART_UPLOAD_SIZE_THRESHOLD + ? environment.uploadMultipartFile.bind(environment) + : environment.upload.bind(environment); + + const readable = createReadStream(path); + + return new Promise((resolve) => { + const state = uploadFn(bucket, { + source: readable, + fileSize: size, + finishedCallback: (err, contentsId) => { + readable.close(); + + if (err) { + logger.error({ tag: 'BACKUPS', msg: '[ENVLFU UPLOAD ERROR]', err }); + if (err.message === 'Max space used') { + return resolve({ error: new DriveDesktopError('NOT_ENOUGH_SPACE') }); + } + return resolve({ error: new DriveDesktopError('UNKNOWN') }); + } + + if (!contentsId) { + logger.error({ tag: 'BACKUPS', msg: '[ENVLFU UPLOAD ERROR] No contentsId returned' }); + return resolve({ error: new DriveDesktopError('UNKNOWN') }); + } + + resolve({ data: contentsId }); + }, + progressCallback: (progress: number) => { + logger.debug({ tag: 'SYNC-ENGINE', msg: '[UPLOAD PROGRESS]', progress }); + }, + }); + + signal.addEventListener('abort', () => { + state.stop(); + readable.destroy(); + }); + }); +} diff --git a/src/backend/features/backup/upload/upload-file-with-retry.ts b/src/backend/features/backup/upload/upload-file-with-retry.ts new file mode 100644 index 0000000000..c46a1ba343 --- /dev/null +++ b/src/backend/features/backup/upload/upload-file-with-retry.ts @@ -0,0 +1,116 @@ +/* eslint-disable no-await-in-loop */ +import { Environment } from '@internxt/inxt-js'; +import { DriveDesktopError } from '../../../../context/shared/domain/errors/DriveDesktopError'; +import { File } from '../../../../context/virtual-drive/files/domain/File'; +import { deleteContentFromEnvironment } from './delete-content-from-environment'; +import { createFileToBackend } from './create-file-to-backend'; +import { logger } from '@internxt/drive-desktop-core/build/backend'; +import { sleep } from './utils/sleep'; +import { uploadContentToEnvironment } from './upload-content-to-environment'; +import { Result } from '../../../../context/shared/domain/Result'; + +const MAX_RETRIES = 3; +const RETRY_DELAYS_MS = [1000, 2000, 4000]; + +export type UploadFileParams = { + path: string; + size: number; + bucket: string; + folderId: number; + folderUuid: string; + environment: Environment; + signal: AbortSignal; +}; + +/** + * Check if error indicates file already exists + * Same logic as FileBatchUploader error handling + */ +function isAlreadyExistsError(error: DriveDesktopError): boolean { + return error.cause === 'FILE_ALREADY_EXISTS'; +} + +/** + * Upload a file with retry logic + * + * Handles the full upload flow (replicates FileBatchUploader behavior): + * 1. Content upload (to Environment/storage) → contentsId + * 2. Metadata creation (to backend API) → File domain object + * + * Retries on failure with exponential backoff. + * Cleans up uploaded content if metadata creation fails. + * + * Special cases (matching FileBatchUploader): + * - FILE_ALREADY_EXISTS: Returns data as null (not an error) + * - BAD_RESPONSE: Treated as non-fatal, will be retried then fail + */ +export async function uploadFileWithRetry(file: UploadFileParams): Promise> { + for (let attempt = 0; attempt <= MAX_RETRIES; attempt++) { + if (file.signal.aborted) { + return { error: new DriveDesktopError('UNKNOWN', 'Upload aborted') }; + } + + try { + // Step 1: Upload content to storage bucket + const contentResult = await uploadContentToEnvironment({ + path: file.path, + size: file.size, + bucket: file.bucket, + environment: file.environment, + signal: file.signal, + }); + + if (contentResult.error) { + throw contentResult.error; + } + + const contentsId = contentResult.data; + + // Step 2: Create file metadata in backend + const metadataResult = await createFileToBackend({ + contentsId, + filePath: file.path, + size: file.size, + folderId: file.folderId, + folderUuid: file.folderUuid, + bucket: file.bucket, + }); + + if (metadataResult.error) { + // Clean up uploaded content on metadata failure (same as FileBatchUploader) + await deleteContentFromEnvironment(file.bucket, contentsId); + throw metadataResult.error; + } + + return { data: metadataResult.data }; + } catch (error) { + const driveError = error instanceof DriveDesktopError ? error : new DriveDesktopError('UNKNOWN'); + + // FILE_ALREADY_EXISTS is not an error - file is already backed up (same as FileBatchUploader) + if (isAlreadyExistsError(driveError)) { + logger.debug({ + tag: 'BACKUPS', + msg: `[FILE ALREADY EXISTS] Skipping file ${file.path} - already exists remotely`, + }); + return { data: null}; + } + + // Retry on failure + if (attempt < MAX_RETRIES) { + const delay = RETRY_DELAYS_MS[attempt]; + logger.debug({ + tag: 'BACKUPS', + msg: `Upload attempt ${attempt + 1} failed, retrying in ${delay}ms`, + path: file.path, + error: driveError.message, + }); + await sleep(delay); + continue; + } + + return { error: driveError }; + } + } + + return { error: new DriveDesktopError('UNKNOWN', 'Upload failed after max retries') }; +} diff --git a/src/backend/features/backup/upload/utils/sleep.ts b/src/backend/features/backup/upload/utils/sleep.ts new file mode 100644 index 0000000000..421bda080a --- /dev/null +++ b/src/backend/features/backup/upload/utils/sleep.ts @@ -0,0 +1,3 @@ +export function sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} From c9c4a6026e6d94bbb94e9bd034953bd764fc82b3 Mon Sep 17 00:00:00 2001 From: AlexisMora Date: Tue, 24 Mar 2026 14:35:53 +0100 Subject: [PATCH 4/9] Fix: delete unused code --- .../execute-worker-pool.ts | 32 ----- .../upload-with-worker-pool/get-next-task.ts | 6 - .../upload-with-worker-pool/run-worker.ts | 31 ----- .../common/upload-with-worker-pool/types.ts | 21 --- .../backup/upload/upload-backup-files.ts | 121 ------------------ 5 files changed, 211 deletions(-) delete mode 100644 src/backend/common/upload-with-worker-pool/execute-worker-pool.ts delete mode 100644 src/backend/common/upload-with-worker-pool/get-next-task.ts delete mode 100644 src/backend/common/upload-with-worker-pool/run-worker.ts delete mode 100644 src/backend/common/upload-with-worker-pool/types.ts delete mode 100644 src/backend/features/backup/upload/upload-backup-files.ts diff --git a/src/backend/common/upload-with-worker-pool/execute-worker-pool.ts b/src/backend/common/upload-with-worker-pool/execute-worker-pool.ts deleted file mode 100644 index def44b91ef..0000000000 --- a/src/backend/common/upload-with-worker-pool/execute-worker-pool.ts +++ /dev/null @@ -1,32 +0,0 @@ -import { DriveDesktopError } from '../../../context/shared/domain/errors/DriveDesktopError'; -import { Result } from '../../../context/shared/domain/Result'; -import { getNextTask } from './get-next-task'; -import { runWorker } from './run-worker'; -import { QueueIndex, TaskExecutor, WorkerPoolConfig, WorkerPoolResult, WorkerState } from './types'; - -export async function executeWorkerPool( - items: T[], - executor: TaskExecutor, - config: WorkerPoolConfig, -): Promise, DriveDesktopError>> { - const queue = [...items]; - const index: QueueIndex = { value: 0 }; - const state: WorkerState = { - results: { succeeded: [], failed: [] }, - fatalError: null, - }; - - const getNext = () => getNextTask(queue, index); - - const workers = Array.from({ length: config.concurrency }, () => - runWorker(getNext, executor, state, config.signal), - ); - - await Promise.all(workers); - - if (state.fatalError) { - return { error: state.fatalError }; - } - - return { data: state.results }; -} diff --git a/src/backend/common/upload-with-worker-pool/get-next-task.ts b/src/backend/common/upload-with-worker-pool/get-next-task.ts deleted file mode 100644 index ba8029e11d..0000000000 --- a/src/backend/common/upload-with-worker-pool/get-next-task.ts +++ /dev/null @@ -1,6 +0,0 @@ -import { QueueIndex } from './types'; - -export function getNextTask(queue: T[], index: QueueIndex): T | null { - if (index.value >= queue.length) return null; - return queue[index.value++]; -} diff --git a/src/backend/common/upload-with-worker-pool/run-worker.ts b/src/backend/common/upload-with-worker-pool/run-worker.ts deleted file mode 100644 index b44274d713..0000000000 --- a/src/backend/common/upload-with-worker-pool/run-worker.ts +++ /dev/null @@ -1,31 +0,0 @@ -/* eslint-disable no-constant-condition */ -import { isFatalError } from '../../../shared/issues/SyncErrorCause'; -import { TaskExecutor, WorkerState } from './types'; - -export async function runWorker( - getNext: () => T | null, - executor: TaskExecutor, - state: WorkerState, - signal: AbortSignal, -): Promise { - while (true) { - if (signal.aborted || state.fatalError) break; - - const item = getNext(); - if (!item) break; - - // eslint-disable-next-line no-await-in-loop - const result = await executor(item, signal); - - if (result.error) { - state.results.failed.push({ item, error: result.error }); - - if (isFatalError(result.error.cause)) { - state.fatalError = result.error; - break; - } - } else { - state.results.succeeded.push(item); - } - } -} diff --git a/src/backend/common/upload-with-worker-pool/types.ts b/src/backend/common/upload-with-worker-pool/types.ts deleted file mode 100644 index 69e1ca0090..0000000000 --- a/src/backend/common/upload-with-worker-pool/types.ts +++ /dev/null @@ -1,21 +0,0 @@ -import { DriveDesktopError } from '../../../context/shared/domain/errors/DriveDesktopError'; -import { Result } from '../../../context/shared/domain/Result'; - -export type WorkerPoolConfig = { - concurrency: number; - signal: AbortSignal; -}; - -export type WorkerPoolResult = { - succeeded: Array; - failed: Array<{ item: T; error: DriveDesktopError }>; -}; - -export type TaskExecutor = (item: T, signal: AbortSignal) => Promise>; - -export type QueueIndex = { value: number }; - -export type WorkerState = { - results: WorkerPoolResult; - fatalError: DriveDesktopError | null; -}; diff --git a/src/backend/features/backup/upload/upload-backup-files.ts b/src/backend/features/backup/upload/upload-backup-files.ts deleted file mode 100644 index 3ec01974cc..0000000000 --- a/src/backend/features/backup/upload/upload-backup-files.ts +++ /dev/null @@ -1,121 +0,0 @@ -import { Environment } from '@internxt/inxt-js'; -import { LocalFile } from '../../../../context/local/localFile/domain/LocalFile'; -import { RemoteTree } from '../../../../context/virtual-drive/remoteTree/domain/RemoteTree'; -import { DriveDesktopError } from '../../../../context/shared/domain/errors/DriveDesktopError'; -import { Result } from '../../../../context/shared/domain/Result'; -import { relative } from '../../../../apps/backups/utils/relative'; -import { isFatalError } from '../../../../shared/issues/SyncErrorCause'; -import { backupErrorsTracker } from '..'; -import { logger } from '@internxt/drive-desktop-core/build/backend'; -import { executeWorkerPool } from '../../../common/upload-with-worker-pool/execute-worker-pool'; -import { TaskExecutor } from '../../../common/upload-with-worker-pool/types'; -import { uploadFileWithRetry } from './upload-file-with-retry'; -import { BackupProgressTracker } from '../backup-progress-tracker'; - -const DEFAULT_CONCURRENCY = 10; - -export type UploadBackupFilesParams = { - files: LocalFile[]; - localRootPath: string; - remoteTree: RemoteTree; - bucket: string; - environment: Environment; - signal: AbortSignal; - tracker: BackupProgressTracker; - concurrency?: number; -}; - -/** - * Creates a TaskExecutor for backup file uploads - * - * The executor: - * 1. Resolves parent folder from remoteTree - * 2. Calls uploadFileWithRetry (content upload + metadata creation) - * 3. Adds created file to remoteTree on success - * 4. Tracks errors in backupErrorsTracker - * 5. Returns fatal errors to stop the pool - */ -function createBackupUploadExecutor( - localRootPath: string, - remoteTree: RemoteTree, - bucket: string, - environment: Environment, - tracker: BackupProgressTracker, -): TaskExecutor { - return async (localFile: LocalFile, signal: AbortSignal): Promise> => { - const remotePath = relative(localRootPath, localFile.path); - const parent = remoteTree.getParent(remotePath); - - const result = await uploadFileWithRetry({ - path: localFile.path, - size: localFile.size, - bucket, - folderId: parent.id, - folderUuid: parent.uuid, - environment, - signal, - }); - - // Always increment progress, whether success, skip, or non-fatal error - tracker.incrementProcessed(1); - - if (result.error) { - logger.error({ tag: 'BACKUPS', msg: '[FILE UPLOAD FAILED]', error: result.error, path: localFile.path }); - - if (isFatalError(result.error.cause)) { - return { error: result.error }; - } - - backupErrorsTracker.add(parent.id, { - name: localFile.nameWithExtension(), - error: result.error.cause, - }); - - return { data: undefined }; - } - - if (result.data !== null) { - remoteTree.addFile(parent, result.data); - } - - return { data: undefined }; - }; -} - -/** - * Upload backup files using the worker pool - * - * This is the entry point that replaces FileBatchUploader.run() - * It uploads files with N concurrent workers (default 10). - * - * Behavior matches FileBatchUploader: - * - Files are uploaded to storage bucket, then metadata created in backend - * - On success, files are added to remoteTree - * - FILE_ALREADY_EXISTS is skipped silently - * - BAD_RESPONSE and other non-fatal errors are tracked in backupErrorsTracker - * - Fatal errors (NOT_ENOUGH_SPACE, etc.) stop all uploads and return error - */ -export async function uploadBackupFiles({ - files, - localRootPath, - remoteTree, - bucket, - environment, - signal, - tracker, - concurrency = DEFAULT_CONCURRENCY, -}: UploadBackupFilesParams): Promise> { - if (files.length === 0) { - return { data: undefined }; - } - - const executor = createBackupUploadExecutor(localRootPath, remoteTree, bucket, environment, tracker); - - const result = await executeWorkerPool(files, executor, { concurrency, signal }); - - if (result.error) { - return { error: result.error }; - } - - return { data: undefined }; -} From a460bf1c9274b1bf38f1c7f9236a9cd00edaf5c7 Mon Sep 17 00:00:00 2001 From: AlexisMora Date: Tue, 24 Mar 2026 16:53:58 +0100 Subject: [PATCH 5/9] feat: upload files concurrently --- src/apps/backups/BackupService.test.ts | 122 ++++++++---------- src/apps/backups/BackupService.ts | 64 ++++----- .../BackupsDependencyContainerFactory.ts | 15 ++- .../local/registerLocalFileServices.ts | 7 +- .../async-queue/execute-async-queue.test.ts | 102 +++++++++++++++ .../common/async-queue/execute-async-queue.ts | 27 ++++ src/backend/common/async-queue/types.ts | 8 ++ .../features/backup/upload/constants.ts | 3 + .../create-backup-update-executor.test.ts | 89 +++++++++++++ .../upload/create-backup-update-executor.ts | 52 ++++++++ .../create-backup-upload-executor.test.ts | 105 +++++++++++++++ .../upload/create-backup-upload-executor.ts | 58 +++++++++ .../upload/create-file-to-backend.test.ts | 91 +++++++++++++ .../backup/upload/create-file-to-backend.ts | 79 ++---------- .../upload/update-file-with-retry.test.ts | 94 ++++++++++++++ .../backup/upload/update-file-with-retry.ts | 70 ++++++++++ .../backup/upload/upload-file-with-retry.ts | 33 +---- 17 files changed, 816 insertions(+), 203 deletions(-) create mode 100644 src/backend/common/async-queue/execute-async-queue.test.ts create mode 100644 src/backend/common/async-queue/execute-async-queue.ts create mode 100644 src/backend/common/async-queue/types.ts create mode 100644 src/backend/features/backup/upload/constants.ts create mode 100644 src/backend/features/backup/upload/create-backup-update-executor.test.ts create mode 100644 src/backend/features/backup/upload/create-backup-update-executor.ts create mode 100644 src/backend/features/backup/upload/create-backup-upload-executor.test.ts create mode 100644 src/backend/features/backup/upload/create-backup-upload-executor.ts create mode 100644 src/backend/features/backup/upload/create-file-to-backend.test.ts create mode 100644 src/backend/features/backup/upload/update-file-with-retry.test.ts create mode 100644 src/backend/features/backup/upload/update-file-with-retry.ts diff --git a/src/apps/backups/BackupService.test.ts b/src/apps/backups/BackupService.test.ts index 960f41eee3..2039942452 100644 --- a/src/apps/backups/BackupService.test.ts +++ b/src/apps/backups/BackupService.test.ts @@ -1,10 +1,8 @@ -import { vi, Mock } from 'vitest'; +import { Mock } from 'vitest'; import { mockDeep } from 'vitest-mock-extended'; import { BackupService } from './BackupService'; import LocalTreeBuilder from '../../context/local/localTree/application/LocalTreeBuilder'; import { RemoteTreeBuilder } from '../../context/virtual-drive/remoteTree/application/RemoteTreeBuilder'; -import { FileBatchUploader } from '../../context/local/localFile/application/upload/FileBatchUploader'; -import { FileBatchUpdater } from '../../context/local/localFile/application/update/FileBatchUpdater'; import { SimpleFolderCreator } from '../../context/virtual-drive/folders/application/create/SimpleFolderCreator'; import { BackupInfo } from './BackupInfo'; import { DriveDesktopError } from '../../context/shared/domain/errors/DriveDesktopError'; @@ -15,66 +13,59 @@ import { RemoteTree } from '../../context/virtual-drive/remoteTree/domain/Remote import { UsageModule } from '../../backend/features/usage/usage.module'; import { FolderMother } from '../../context/virtual-drive/folders/domain/__test-helpers__/FolderMother'; import { BackupProgressTracker } from '../../backend/features/backup/backup-progress-tracker'; +import * as executeAsyncQueueModule from '../../backend/common/async-queue/execute-async-queue'; +import * as addFileToTrashModule from '../../infra/drive-server/services/files/services/add-file-to-trash'; +import { partialSpyOn } from '../../../tests/vitest/utils.helper'; -// Mock the UsageModule -vi.mock('../../backend/features/usage/usage.module', () => ({ - UsageModule: { - validateSpace: vi.fn(), - }, -})); - -// Mock the Environment module -vi.mock('@internxt/inxt-js', () => ({ - Environment: { - get: vi.fn(), - }, -})); +vi.mock(import('../../backend/features/usage/usage.module')); +vi.mock(import('@internxt/inxt-js')); describe('BackupService', () => { + const executeAsyncQueueMock = partialSpyOn(executeAsyncQueueModule, 'executeAsyncQueue'); + const addFileToTrashMock = partialSpyOn(addFileToTrashModule, 'addFileToTrash'); + let backupService: BackupService; let localTreeBuilder: LocalTreeBuilder; let remoteTreeBuilder: RemoteTreeBuilder; - let fileBatchUploader: FileBatchUploader; - let fileBatchUpdater: FileBatchUpdater; let simpleFolderCreator: SimpleFolderCreator; let mockValidateSpace: Mock; let abortController: AbortController; let tracker: BackupProgressTracker; + const info: BackupInfo = { + pathname: '/path/to/backup', + folderId: 123, + folderUuid: 'uuid', + tmpPath: '/tmp/path', + backupsBucket: 'backups-bucket', + name: 'backup-name', + }; + beforeEach(() => { localTreeBuilder = mockDeep(); remoteTreeBuilder = mockDeep(); - fileBatchUploader = mockDeep(); - fileBatchUpdater = mockDeep(); simpleFolderCreator = mockDeep(); tracker = mockDeep(); - mockValidateSpace = UsageModule.validateSpace as Mock; + mockValidateSpace = vi.mocked(UsageModule.validateSpace); abortController = new AbortController(); - // Setup default mock implementations vi.mocked(simpleFolderCreator.run).mockResolvedValue(FolderMother.any()); + executeAsyncQueueMock.mockResolvedValue({ data: undefined }); + addFileToTrashMock.mockResolvedValue({ data: true }); backupService = new BackupService( localTreeBuilder, remoteTreeBuilder, - fileBatchUploader, - fileBatchUpdater, simpleFolderCreator, + {} as any, + 'backups-bucket', ); mockValidateSpace.mockClear(); }); it('should successfully run the backup process', async () => { - const info: BackupInfo = { - pathname: '/path/to/backup', - folderId: 123, - folderUuid: 'uuid', - tmpPath: '/tmp/path', - backupsBucket: 'backups-bucket', - name: 'backup-name', - }; const localTree = LocalTreeMother.oneLevel(10); const remoteTree = RemoteTreeMother.oneLevel(10); @@ -92,14 +83,6 @@ describe('BackupService', () => { }); it('should return an error if local tree generation fails', async () => { - const info: BackupInfo = { - pathname: '/path/to/backup', - folderId: 123, - folderUuid: 'uuid', - tmpPath: '/tmp/path', - backupsBucket: 'backups-bucket', - name: 'backup-name', - }; const error = new DriveDesktopError('NOT_EXISTS', 'Failed to generate local tree'); vi.mocked(localTreeBuilder.run).mockResolvedValueOnce(left(error)); @@ -111,17 +94,8 @@ describe('BackupService', () => { }); it('should return an error if remote tree generation fails', async () => { - const info: BackupInfo = { - pathname: '/path/to/backup', - folderId: 123, - folderUuid: 'uuid', - tmpPath: '/tmp/path', - backupsBucket: 'backups-bucket', - name: 'backup-name', - }; const error = new DriveDesktopError('NOT_EXISTS', 'Failed to generate remote tree'); - // Mock the behavior of dependencies vi.mocked(localTreeBuilder.run).mockResolvedValueOnce(right(LocalTreeMother.oneLevel(10))); vi.mocked(remoteTreeBuilder.run).mockResolvedValueOnce(left(error) as unknown as RemoteTree); @@ -132,17 +106,8 @@ describe('BackupService', () => { }); it('should return an error if there is not enough space', async () => { - const info: BackupInfo = { - pathname: '/path/to/backup', - folderId: 123, - folderUuid: 'uuid', - tmpPath: '/tmp/path', - backupsBucket: 'backups-bucket', - name: 'backup-name', - }; - - (localTreeBuilder.run as Mock).mockResolvedValueOnce(right(LocalTreeMother.oneLevel(10))); - (remoteTreeBuilder.run as Mock).mockResolvedValueOnce(RemoteTreeMother.oneLevel(10)); + vi.mocked(localTreeBuilder.run).mockResolvedValueOnce(right(LocalTreeMother.oneLevel(10))); + vi.mocked(remoteTreeBuilder.run).mockResolvedValueOnce(RemoteTreeMother.oneLevel(10)); mockValidateSpace.mockResolvedValueOnce({ data: { hasSpace: false } }); const result = await backupService.run(info, abortController.signal, tracker); @@ -151,15 +116,6 @@ describe('BackupService', () => { }); it('should return an unknown error for unexpected issues', async () => { - const info: BackupInfo = { - pathname: '/path/to/backup', - folderId: 123, - folderUuid: 'uuid', - tmpPath: '/tmp/path', - backupsBucket: 'backups-bucket', - name: 'backup-name', - }; - vi.mocked(localTreeBuilder.run).mockImplementationOnce(() => { throw new Error('Unexpected error'); }); @@ -169,4 +125,32 @@ describe('BackupService', () => { expect(result).toBeInstanceOf(DriveDesktopError); expect(result?.message).toBe('An unknown error occurred'); }); + + it('should propagate fatal error from uploadAndCreate', async () => { + const fatalError = new DriveDesktopError('NOT_ENOUGH_SPACE', 'No space left'); + + vi.mocked(localTreeBuilder.run).mockResolvedValueOnce(right(LocalTreeMother.oneLevel(10))); + vi.mocked(remoteTreeBuilder.run).mockResolvedValueOnce(RemoteTreeMother.oneLevel(10)); + mockValidateSpace.mockResolvedValueOnce({ data: { hasSpace: true } }); + executeAsyncQueueMock.mockResolvedValueOnce({ error: fatalError }); + + const result = await backupService.run(info, abortController.signal, tracker); + + expect(result).toBe(fatalError); + }); + + it('should propagate fatal error from uploadAndUpdate', async () => { + const fatalError = new DriveDesktopError('NOT_ENOUGH_SPACE', 'No space left'); + + vi.mocked(localTreeBuilder.run).mockResolvedValueOnce(right(LocalTreeMother.oneLevel(10))); + vi.mocked(remoteTreeBuilder.run).mockResolvedValueOnce(RemoteTreeMother.oneLevel(10)); + mockValidateSpace.mockResolvedValueOnce({ data: { hasSpace: true } }); + executeAsyncQueueMock + .mockResolvedValueOnce({ data: undefined }) + .mockResolvedValueOnce({ error: fatalError }); + + const result = await backupService.run(info, abortController.signal, tracker); + + expect(result).toBe(fatalError); + }); }); diff --git a/src/apps/backups/BackupService.ts b/src/apps/backups/BackupService.ts index a89520767e..baa2dd679b 100644 --- a/src/apps/backups/BackupService.ts +++ b/src/apps/backups/BackupService.ts @@ -1,6 +1,8 @@ +import { Environment } from '@internxt/inxt-js'; import { Service } from 'diod'; -import { FileBatchUpdater } from '../../context/local/localFile/application/update/FileBatchUpdater'; -import { FileBatchUploader } from '../../context/local/localFile/application/upload/FileBatchUploader'; +import { executeAsyncQueue } from '../../backend/common/async-queue/execute-async-queue'; +import { createBackupUpdateExecutor, ModifiedFilePair } from '../../backend/features/backup/upload/create-backup-update-executor'; +import { DEFAULT_CONCURRENCY } from '../../backend/features/backup/upload/constants'; import { LocalFile } from '../../context/local/localFile/domain/LocalFile'; import { AbsolutePath } from '../../context/local/localFile/infrastructure/AbsolutePath'; import LocalTreeBuilder from '../../context/local/localTree/application/LocalTreeBuilder'; @@ -10,8 +12,6 @@ import { SimpleFolderCreator } from '../../context/virtual-drive/folders/applica import { RemoteTreeBuilder } from '../../context/virtual-drive/remoteTree/application/RemoteTreeBuilder'; import { RemoteTree } from '../../context/virtual-drive/remoteTree/domain/RemoteTree'; import { BackupInfo } from './BackupInfo'; -import { AddedFilesBatchCreator } from './batches/AddedFilesBatchCreator'; -import { ModifiedFilesBatchCreator } from './batches/ModifiedFilesBatchCreator'; import { DiffFilesCalculatorService, FilesDiff } from './diff/DiffFilesCalculatorService'; import { FoldersDiff, FoldersDiffCalculator } from './diff/FoldersDiffCalculator'; import { relative } from './utils/relative'; @@ -24,15 +24,16 @@ import { BackupProgressTracker } from '../../backend/features/backup/backup-prog import { RetryError } from '../shared/retry/RetryError'; import { Either, left, right } from '../../context/shared/domain/Either'; import { addFileToTrash } from '../../infra/drive-server/services/files/services/add-file-to-trash'; +import { createBackupUploadExecutor } from '../../backend/features/backup/upload/create-backup-upload-executor'; @Service() export class BackupService { constructor( private readonly localTreeBuilder: LocalTreeBuilder, private readonly remoteTreeBuilder: RemoteTreeBuilder, - private readonly fileBatchUploader: FileBatchUploader, - private readonly fileBatchUpdater: FileBatchUpdater, private readonly simpleFolderCreator: SimpleFolderCreator, + private readonly environment: Environment, + private readonly bucket: string, ) {} // TODO: PB-3897 - Change Signature of this method for a better error handling @@ -188,52 +189,51 @@ export class BackupService { const { added, modified, deleted } = filesDiff; - logger.debug({ tag: 'BACKUPS', msg: 'Files added', count: added.length }); - await this.uploadAndCreate(local.root.path, added, remote, signal, tracker); + if (added.length > 0) { + logger.debug({ tag: 'BACKUPS', msg: 'Files added', count: added.length }); + await this.uploadAndCreate(local.root.path, added, remote, signal, tracker); + } - logger.debug({ tag: 'BACKUPS', msg: 'Files modified', count: modified.size }); - await this.uploadAndUpdate(modified, local, remote, signal, tracker); + if (modified.size > 0) { + logger.debug({ tag: 'BACKUPS', msg: 'Files modified', count: modified.size }); + await this.uploadAndUpdate(modified, signal, tracker); + } - logger.debug({ tag: 'BACKUPS', msg: 'Files deleted', count: deleted.length }); - await this.deleteRemoteFiles(deleted, signal, tracker); + if (deleted.length > 0) { + logger.debug({ tag: 'BACKUPS', msg: 'Files deleted', count: deleted.length }); + await this.deleteRemoteFiles(deleted, signal, tracker); + } } private async uploadAndCreate( localRootPath: string, added: Array, - tree: RemoteTree, + remoteTree: RemoteTree, signal: AbortSignal, tracker: BackupProgressTracker, ): Promise { - const batches = AddedFilesBatchCreator.run(added); + const executor = createBackupUploadExecutor(localRootPath, remoteTree, this.bucket, this.environment, tracker); - for (const batch of batches) { - if (signal.aborted) { - return; - } - // eslint-disable-next-line no-await-in-loop - await this.fileBatchUploader.run(localRootPath, tree, batch, signal); - tracker.incrementProcessed(batch.length); + const result = await executeAsyncQueue(added, executor, { concurrency: DEFAULT_CONCURRENCY, signal }); + + if (result.error) { + throw result.error; } } private async uploadAndUpdate( modified: Map, - localTree: LocalTree, - remoteTree: RemoteTree, signal: AbortSignal, tracker: BackupProgressTracker, ): Promise { - const batches = ModifiedFilesBatchCreator.run(modified); + const items: ModifiedFilePair[] = Array.from(modified.entries()); - for (const batch of batches) { - logger.debug({ tag: 'BACKUPS', msg: 'Signal aborted', aborted: signal.aborted }); - if (signal.aborted) { - return; - } - // eslint-disable-next-line no-await-in-loop - await this.fileBatchUpdater.run(localTree.root, remoteTree, Array.from(batch.keys()), signal); - tracker.incrementProcessed(batch.size); + const executor = createBackupUpdateExecutor(this.bucket, this.environment, tracker); + + const result = await executeAsyncQueue(items, executor, { concurrency: DEFAULT_CONCURRENCY, signal }); + + if (result.error) { + throw result.error; } } diff --git a/src/apps/backups/dependency-injection/BackupsDependencyContainerFactory.ts b/src/apps/backups/dependency-injection/BackupsDependencyContainerFactory.ts index bdd0eb27f6..e2b8ddf051 100644 --- a/src/apps/backups/dependency-injection/BackupsDependencyContainerFactory.ts +++ b/src/apps/backups/dependency-injection/BackupsDependencyContainerFactory.ts @@ -1,4 +1,5 @@ import { Container } from 'diod'; +import { Environment } from '@internxt/inxt-js'; import { backgroundProcessSharedInfraBuilder } from '../../shared/dependency-injection/background/backgroundProcessSharedInfraBuilder'; import { registerFilesServices } from './virtual-drive/registerFilesServices'; import { registerFolderServices } from './virtual-drive/registerFolderServices'; @@ -9,7 +10,9 @@ import { registerRemoteTreeServices } from './virtual-drive/registerRemoteTreeSe import { DependencyInjectionUserProvider } from '../../shared/dependency-injection/DependencyInjectionUserProvider'; import { DownloaderHandlerFactory } from '../../../context/storage/StorageFiles/domain/download/DownloaderHandlerFactory'; import { EnvironmentFileDownloaderHandlerFactory } from '../../../context/storage/StorageFiles/infrastructure/download/EnvironmentRemoteFileContentsManagersFactory'; -import { Environment } from '@internxt/inxt-js'; +import LocalTreeBuilder from '../../../context/local/localTree/application/LocalTreeBuilder'; +import { RemoteTreeBuilder } from '../../../context/virtual-drive/remoteTree/application/RemoteTreeBuilder'; +import { SimpleFolderCreator } from '../../../context/virtual-drive/folders/application/create/SimpleFolderCreator'; export class BackupsDependencyContainerFactory { private static container: Container | null = null; @@ -33,7 +36,15 @@ export class BackupsDependencyContainerFactory { .register(DownloaderHandlerFactory) .useFactory((c) => new EnvironmentFileDownloaderHandlerFactory(c.get(Environment), user.backupsBucket)); - builder.registerAndUse(BackupService); + builder.register(BackupService).useFactory((c) => { + return new BackupService( + c.get(LocalTreeBuilder), + c.get(RemoteTreeBuilder), + c.get(SimpleFolderCreator), + c.get(Environment), + user.backupsBucket, + ); + }); this.container = builder.build(); diff --git a/src/apps/backups/dependency-injection/local/registerLocalFileServices.ts b/src/apps/backups/dependency-injection/local/registerLocalFileServices.ts index 276dbc29b4..f15bd23392 100644 --- a/src/apps/backups/dependency-injection/local/registerLocalFileServices.ts +++ b/src/apps/backups/dependency-injection/local/registerLocalFileServices.ts @@ -2,8 +2,6 @@ import { INTERNXT_CLIENT, INTERNXT_VERSION } from './../../../../core/utils/util import { ContainerBuilder } from 'diod'; import { FileBatchUpdater } from '../../../../context/local/localFile/application/update/FileBatchUpdater'; import { LocalFileHandler } from '../../../../context/local/localFile/domain/LocalFileUploader'; -import { FileBatchUploader } from '../../../../context/local/localFile/application/upload/FileBatchUploader'; -import { SimpleFileCreator } from '../../../../context/virtual-drive/files/application/create/SimpleFileCreator'; import { EnvironmentLocalFileUploader } from '../../../../context/local/localFile/infrastructure/EnvironmentLocalFileUploader'; import { DependencyInjectionUserProvider } from '../../../shared/dependency-injection/DependencyInjectionUserProvider'; import { Environment } from '@internxt/inxt-js'; @@ -26,7 +24,7 @@ export function registerLocalFileServices(builder: ContainerBuilder) { }, }); - builder.register(Environment).useInstance(environment).private(); + builder.register(Environment).useInstance(environment); builder .register(LocalFileHandler) @@ -38,7 +36,4 @@ export function registerLocalFileServices(builder: ContainerBuilder) { // Services builder.registerAndUse(FileBatchUpdater); - builder.register(FileBatchUploader).useFactory((c) => { - return new FileBatchUploader(c.get(LocalFileHandler), c.get(SimpleFileCreator), user.backupsBucket); - }); } diff --git a/src/backend/common/async-queue/execute-async-queue.test.ts b/src/backend/common/async-queue/execute-async-queue.test.ts new file mode 100644 index 0000000000..1d1d82aefe --- /dev/null +++ b/src/backend/common/async-queue/execute-async-queue.test.ts @@ -0,0 +1,102 @@ +import { DriveDesktopError } from '../../../context/shared/domain/errors/DriveDesktopError'; +import { executeAsyncQueue } from './execute-async-queue'; +import { TaskExecutor } from './types'; + +describe('executeAsyncQueue', () => { + let abortController: AbortController; + + beforeEach(() => { + abortController = new AbortController(); + }); + + it('should process all items successfully', async () => { + const processed: number[] = []; + const executor: TaskExecutor = async (item) => { + processed.push(item); + return { data: undefined }; + }; + + const result = await executeAsyncQueue([1, 2, 3], executor, { + concurrency: 10, + signal: abortController.signal, + }); + + expect(result.data).toBeUndefined(); + expect(result.error).toBeUndefined(); + expect(processed).toHaveLength(3); + }); + + it('should stop processing when executor returns an error', async () => { + const processed: number[] = []; + const fatalError = new DriveDesktopError('NOT_ENOUGH_SPACE', 'No space'); + const executor: TaskExecutor = async (item) => { + processed.push(item); + if (item === 2) { + return { error: fatalError }; + } + return { data: undefined }; + }; + + const result = await executeAsyncQueue([1, 2, 3, 4, 5], executor, { + concurrency: 1, + signal: abortController.signal, + }); + + expect(result.error).toBe(fatalError); + expect(processed).toStrictEqual([1, 2]); + }); + + it('should respect concurrency limit', async () => { + let maxConcurrent = 0; + let running = 0; + + const executor: TaskExecutor = async () => { + running++; + if (running > maxConcurrent) maxConcurrent = running; + await new Promise((resolve) => setTimeout(resolve, 10)); + running--; + return { data: undefined }; + }; + + await executeAsyncQueue([1, 2, 3, 4, 5, 6], executor, { + concurrency: 3, + signal: abortController.signal, + }); + + expect(maxConcurrent).toBe(3); + }); + + it('should skip items when signal is aborted', async () => { + const processed: number[] = []; + const executor: TaskExecutor = async (item) => { + processed.push(item); + if (item === 2) { + abortController.abort(); + } + return { data: undefined }; + }; + + const result = await executeAsyncQueue([1, 2, 3, 4, 5], executor, { + concurrency: 1, + signal: abortController.signal, + }); + + expect(result.data).toBeUndefined(); + expect(processed).toStrictEqual([1, 2]); + }); + + it('should pass the signal to the executor', async () => { + let receivedSignal: AbortSignal | undefined; + const executor: TaskExecutor = async (_item, signal) => { + receivedSignal = signal; + return { data: undefined }; + }; + + await executeAsyncQueue([1], executor, { + concurrency: 1, + signal: abortController.signal, + }); + + expect(receivedSignal).toBe(abortController.signal); + }); +}); diff --git a/src/backend/common/async-queue/execute-async-queue.ts b/src/backend/common/async-queue/execute-async-queue.ts new file mode 100644 index 0000000000..dda8abf0c7 --- /dev/null +++ b/src/backend/common/async-queue/execute-async-queue.ts @@ -0,0 +1,27 @@ +import { queue } from 'async'; +import { DriveDesktopError } from '../../../context/shared/domain/errors/DriveDesktopError'; +import { Result } from '../../../context/shared/domain/Result'; +import { AsyncQueueConfig, TaskExecutor } from './types'; + +export async function executeAsyncQueue( + items: T[], + executor: TaskExecutor, + config: AsyncQueueConfig, +): Promise> { + return new Promise((resolve) => { + const taskQueue = queue(async (item: T) => { + if (config.signal.aborted) return; + + const result = await executor(item, config.signal); + + if (result.error) { + taskQueue.kill(); + resolve({ error: result.error }); + } + }, config.concurrency); + + taskQueue.drain(() => resolve({ data: undefined })); + + taskQueue.push(items); + }); +} diff --git a/src/backend/common/async-queue/types.ts b/src/backend/common/async-queue/types.ts new file mode 100644 index 0000000000..94c9e738a9 --- /dev/null +++ b/src/backend/common/async-queue/types.ts @@ -0,0 +1,8 @@ +import { DriveDesktopError } from '../../../context/shared/domain/errors/DriveDesktopError'; +import { Result } from '../../../context/shared/domain/Result'; + +export type TaskExecutor = (item: T, signal: AbortSignal) => Promise>; +export type AsyncQueueConfig = { + concurrency: number; + signal: AbortSignal; +}; diff --git a/src/backend/features/backup/upload/constants.ts b/src/backend/features/backup/upload/constants.ts new file mode 100644 index 0000000000..64f38e6fca --- /dev/null +++ b/src/backend/features/backup/upload/constants.ts @@ -0,0 +1,3 @@ +export const DEFAULT_CONCURRENCY = 10; +export const MAX_RETRIES = 3; +export const RETRY_DELAYS_MS = [1000, 2000, 4000]; diff --git a/src/backend/features/backup/upload/create-backup-update-executor.test.ts b/src/backend/features/backup/upload/create-backup-update-executor.test.ts new file mode 100644 index 0000000000..735294909a --- /dev/null +++ b/src/backend/features/backup/upload/create-backup-update-executor.test.ts @@ -0,0 +1,89 @@ +import { partialSpyOn } from '../../../../../tests/vitest/utils.helper'; +import { DriveDesktopError } from '../../../../context/shared/domain/errors/DriveDesktopError'; +import { FileMother } from '../../../../context/virtual-drive/files/domain/__test-helpers__/FileMother'; +import { LocalFileMother } from '../../../../context/local/localFile/domain/__test-helpers__/LocalFileMother'; +import { BackupProgressTracker } from '../backup-progress-tracker'; +import { mockDeep } from 'vitest-mock-extended'; +import { createBackupUpdateExecutor, ModifiedFilePair } from './create-backup-update-executor'; +import * as updateFileWithRetryModule from './update-file-with-retry'; +import * as backupErrorsTrackerModule from '..'; + +describe('createBackupUpdateExecutor', () => { + const updateFileWithRetryMock = partialSpyOn(updateFileWithRetryModule, 'updateFileWithRetry'); + const backupErrorsTrackerAddMock = partialSpyOn(backupErrorsTrackerModule.backupErrorsTracker, 'add'); + + let tracker: BackupProgressTracker; + let abortController: AbortController; + + beforeEach(() => { + tracker = mockDeep(); + abortController = new AbortController(); + }); + + function createExecutor() { + return createBackupUpdateExecutor('bucket', {} as any, tracker); + } + + function createPair(): ModifiedFilePair { + return [LocalFileMother.any(), FileMother.any()]; + } + + it('should update a file successfully', async () => { + updateFileWithRetryMock.mockResolvedValue({ data: undefined }); + const executor = createExecutor(); + const pair = createPair(); + + const result = await executor(pair, abortController.signal); + + expect(result.data).toBeUndefined(); + expect(result.error).toBeUndefined(); + expect(tracker.incrementProcessed).toHaveBeenCalledWith(1); + }); + + it('should return fatal error without tracking it', async () => { + const fatalError = new DriveDesktopError('NOT_ENOUGH_SPACE', 'No space'); + updateFileWithRetryMock.mockResolvedValue({ error: fatalError }); + const executor = createExecutor(); + const pair = createPair(); + + const result = await executor(pair, abortController.signal); + + expect(result.error).toBe(fatalError); + expect(backupErrorsTrackerAddMock).not.toHaveBeenCalled(); + expect(tracker.incrementProcessed).toHaveBeenCalledWith(1); + }); + + it('should track non-fatal error and return success', async () => { + const nonFatalError = new DriveDesktopError('BAD_RESPONSE', 'Network error'); + updateFileWithRetryMock.mockResolvedValue({ error: nonFatalError }); + const executor = createExecutor(); + const [localFile, remoteFile] = createPair(); + + const result = await executor([localFile, remoteFile], abortController.signal); + + expect(result.data).toBeUndefined(); + expect(result.error).toBeUndefined(); + expect(backupErrorsTrackerAddMock).toHaveBeenCalledWith(remoteFile.folderId, { + name: localFile.nameWithExtension(), + error: nonFatalError.cause, + }); + expect(tracker.incrementProcessed).toHaveBeenCalledWith(1); + }); + + it('should call updateFileWithRetry with correct params', async () => { + updateFileWithRetryMock.mockResolvedValue({ data: undefined }); + const executor = createExecutor(); + const [localFile, remoteFile] = createPair(); + + await executor([localFile, remoteFile], abortController.signal); + + expect(updateFileWithRetryMock).toHaveBeenCalledWith({ + path: localFile.path, + size: localFile.size, + bucket: 'bucket', + fileUuid: remoteFile.uuid, + environment: {}, + signal: abortController.signal, + }); + }); +}); diff --git a/src/backend/features/backup/upload/create-backup-update-executor.ts b/src/backend/features/backup/upload/create-backup-update-executor.ts new file mode 100644 index 0000000000..00e32060a5 --- /dev/null +++ b/src/backend/features/backup/upload/create-backup-update-executor.ts @@ -0,0 +1,52 @@ +import { Environment } from '@internxt/inxt-js'; +import { BackupProgressTracker } from '../backup-progress-tracker'; +import { LocalFile } from '../../../../context/local/localFile/domain/LocalFile'; +import { File } from '../../../../context/virtual-drive/files/domain/File'; +import { Result } from '../../../../context/shared/domain/Result'; +import { DriveDesktopError } from '../../../../context/shared/domain/errors/DriveDesktopError'; +import { updateFileWithRetry } from './update-file-with-retry'; +import { logger } from '@internxt/drive-desktop-core/build/backend'; +import { isFatalError } from '../../../../shared/issues/SyncErrorCause'; +import { TaskExecutor } from '../../../common/async-queue/types'; +import { backupErrorsTracker } from '..'; + +export type ModifiedFilePair = [LocalFile, File]; + +export function createBackupUpdateExecutor( + bucket: string, + environment: Environment, + tracker: BackupProgressTracker, +): TaskExecutor { + return async ( + [localFile, remoteFile]: ModifiedFilePair, + signal: AbortSignal, + ): Promise> => { + const result = await updateFileWithRetry({ + path: localFile.path, + size: localFile.size, + bucket, + fileUuid: remoteFile.uuid, + environment, + signal, + }); + + tracker.incrementProcessed(1); + + if (result.error) { + logger.error({ tag: 'BACKUPS', msg: '[FILE UPDATE FAILED]', error: result.error, path: localFile.path }); + + if (isFatalError(result.error.cause)) { + return { error: result.error }; + } + + backupErrorsTracker.add(remoteFile.folderId, { + name: localFile.nameWithExtension(), + error: result.error.cause, + }); + + return { data: undefined }; + } + + return { data: undefined }; + }; +} diff --git a/src/backend/features/backup/upload/create-backup-upload-executor.test.ts b/src/backend/features/backup/upload/create-backup-upload-executor.test.ts new file mode 100644 index 0000000000..f3d49770b1 --- /dev/null +++ b/src/backend/features/backup/upload/create-backup-upload-executor.test.ts @@ -0,0 +1,105 @@ +import path from 'path'; +import { partialSpyOn } from '../../../../../tests/vitest/utils.helper'; +import { DriveDesktopError } from '../../../../context/shared/domain/errors/DriveDesktopError'; +import { FileMother } from '../../../../context/virtual-drive/files/domain/__test-helpers__/FileMother'; +import { LocalFileMother } from '../../../../context/local/localFile/domain/__test-helpers__/LocalFileMother'; +import { RemoteTreeMother } from '../../../../context/virtual-drive/remoteTree/domain/__test-helpers__/RemoteTreeMother'; +import { BackupProgressTracker } from '../backup-progress-tracker'; +import { mockDeep } from 'vitest-mock-extended'; +import { createBackupUploadExecutor } from './create-backup-upload-executor'; +import * as uploadFileWithRetryModule from './upload-file-with-retry'; +import * as backupErrorsTrackerModule from '..'; +import { AbsolutePath } from '../../../../context/local/localFile/infrastructure/AbsolutePath'; +import { Environment } from '@internxt/inxt-js'; + +describe('createBackupUploadExecutor', () => { + const uploadFileWithRetryMock = partialSpyOn(uploadFileWithRetryModule, 'uploadFileWithRetry'); + const backupErrorsTrackerAddMock = partialSpyOn(backupErrorsTrackerModule.backupErrorsTracker, 'add'); + + let tracker: BackupProgressTracker; + let abortController: AbortController; + + beforeEach(() => { + tracker = mockDeep(); + abortController = new AbortController(); + }); + + function setup() { + const remoteTree = RemoteTreeMother.onlyRoot(); + // file placed directly under root so getParent always returns root + const localFile = LocalFileMother.fromPartial({ + path: path.join(remoteTree.root.path, 'test-file.txt') as AbsolutePath, + }); + const executor = createBackupUploadExecutor(remoteTree.root.path, remoteTree, 'bucket', {} as Environment, tracker); + return { remoteTree, localFile, executor }; + } + + it('should upload a file successfully and add it to the remote tree', async () => { + const { remoteTree, localFile, executor } = setup(); + const createdFile = FileMother.any(); + uploadFileWithRetryMock.mockResolvedValue({ data: createdFile }); + const addFileMock = partialSpyOn(remoteTree, 'addFile'); + + const result = await executor(localFile, abortController.signal); + + expect(result.data).toBeUndefined(); + expect(result.error).toBeUndefined(); + expect(addFileMock).toHaveBeenCalled(); + expect(tracker.incrementProcessed).toHaveBeenCalledWith(1); + }); + + it('should skip adding to remote tree when file already exists (data is null)', async () => { + const { remoteTree, localFile, executor } = setup(); + uploadFileWithRetryMock.mockResolvedValue({ data: null }); + const addFileMock = partialSpyOn(remoteTree, 'addFile'); + + const result = await executor(localFile, abortController.signal); + + expect(result.data).toBeUndefined(); + expect(result.error).toBeUndefined(); + expect(addFileMock).not.toHaveBeenCalled(); + expect(tracker.incrementProcessed).toHaveBeenCalledWith(1); + }); + + it('should return fatal error without tracking it', async () => { + const { localFile, executor } = setup(); + const fatalError = new DriveDesktopError('NOT_ENOUGH_SPACE', 'No space'); + uploadFileWithRetryMock.mockResolvedValue({ error: fatalError }); + + const result = await executor(localFile, abortController.signal); + + expect(result.error).toBe(fatalError); + expect(backupErrorsTrackerAddMock).not.toHaveBeenCalled(); + expect(tracker.incrementProcessed).toHaveBeenCalledWith(1); + }); + + it('should track non-fatal error and return success', async () => { + const { localFile, executor } = setup(); + const nonFatalError = new DriveDesktopError('BAD_RESPONSE', 'Network error'); + uploadFileWithRetryMock.mockResolvedValue({ error: nonFatalError }); + + const result = await executor(localFile, abortController.signal); + + expect(result.data).toBeUndefined(); + expect(result.error).toBeUndefined(); + expect(backupErrorsTrackerAddMock).toHaveBeenCalled(); + expect(tracker.incrementProcessed).toHaveBeenCalledWith(1); + }); + + it('should call uploadFileWithRetry with correct params', async () => { + const { remoteTree, localFile, executor } = setup(); + uploadFileWithRetryMock.mockResolvedValue({ data: FileMother.any() }); + + await executor(localFile, abortController.signal); + + expect(uploadFileWithRetryMock).toHaveBeenCalledWith({ + path: localFile.path, + size: localFile.size, + bucket: 'bucket', + folderId: remoteTree.root.id, + folderUuid: remoteTree.root.uuid, + environment: {} as Environment, + signal: abortController.signal, + }); + }); +}); diff --git a/src/backend/features/backup/upload/create-backup-upload-executor.ts b/src/backend/features/backup/upload/create-backup-upload-executor.ts new file mode 100644 index 0000000000..9ed238104f --- /dev/null +++ b/src/backend/features/backup/upload/create-backup-upload-executor.ts @@ -0,0 +1,58 @@ +import { Environment } from '@internxt/inxt-js'; +import { RemoteTree } from '../../../../context/virtual-drive/remoteTree/domain/RemoteTree'; +import { BackupProgressTracker } from '../backup-progress-tracker'; +import { LocalFile } from '../../../../context/local/localFile/domain/LocalFile'; +import { Result } from '../../../../context/shared/domain/Result'; +import { DriveDesktopError } from '../../../../context/shared/domain/errors/DriveDesktopError'; +import { relative } from 'node:path'; +import { uploadFileWithRetry } from './upload-file-with-retry'; +import { logger } from '@internxt/drive-desktop-core/build/backend'; +import { isFatalError } from '../../../../shared/issues/SyncErrorCause'; +import { TaskExecutor } from '../../../common/async-queue/types'; +import { backupErrorsTracker } from '..'; + +export function createBackupUploadExecutor( + localRootPath: string, + remoteTree: RemoteTree, + bucket: string, + environment: Environment, + tracker: BackupProgressTracker, +): TaskExecutor { + return async (localFile: LocalFile, signal: AbortSignal): Promise> => { + const remotePath = relative(localRootPath, localFile.path); + const parent = remoteTree.getParent(remotePath); + + const result = await uploadFileWithRetry({ + path: localFile.path, + size: localFile.size, + bucket, + folderId: parent.id, + folderUuid: parent.uuid, + environment, + signal, + }); + + tracker.incrementProcessed(1); + + if (result.error) { + logger.error({ tag: 'BACKUPS', msg: '[FILE UPLOAD FAILED]', error: result.error, path: localFile.path }); + + if (isFatalError(result.error.cause)) { + return { error: result.error }; + } + + backupErrorsTracker.add(parent.id, { + name: localFile.nameWithExtension(), + error: result.error.cause, + }); + + return { data: undefined }; + } + + if (result.data !== null) { + remoteTree.addFile(parent, result.data); + } + + return { data: undefined }; + }; +} diff --git a/src/backend/features/backup/upload/create-file-to-backend.test.ts b/src/backend/features/backup/upload/create-file-to-backend.test.ts new file mode 100644 index 0000000000..4fca5ceb4c --- /dev/null +++ b/src/backend/features/backup/upload/create-file-to-backend.test.ts @@ -0,0 +1,91 @@ +import { partialSpyOn } from '../../../../../tests/vitest/utils.helper'; +import { DriveDesktopError } from '../../../../context/shared/domain/errors/DriveDesktopError'; +import { DriveServerError } from '../../../../infra/drive-server/drive-server.error'; +import { createFileToBackend } from './create-file-to-backend'; +import * as createFileModule from '../../../../infra/drive-server/services/files/services/create-file'; +import { UuidMother } from '../../../../context/shared/domain/__test-helpers__/UuidMother'; +import { BucketEntryIdMother } from '../../../../context/virtual-drive/shared/domain/__test-helpers__/BucketEntryIdMother'; + +describe('createFileToBackend', () => { + const createFileMock = partialSpyOn(createFileModule, 'createFile'); + + const baseParams = { + contentsId: BucketEntryIdMother.primitive(), + filePath: '/backup/my-file.txt', + size: 1024, + folderId: 123, + folderUuid: UuidMother.primitive(), + bucket: 'bucket', + }; + + const fileResponse = { + id: 1, + uuid: UuidMother.primitive(), + createdAt: new Date().toISOString(), + updatedAt: new Date().toISOString(), + }; + + it('should return a File domain object on success', async () => { + createFileMock.mockResolvedValue({ data: fileResponse }); + + const result = await createFileToBackend(baseParams); + + expect(result.error).toBeUndefined(); + expect(result.data?.path).toBe(baseParams.filePath); + expect(result.data?.size).toBe(baseParams.size); + }); + + it('should not set fileId when size is 0', async () => { + createFileMock.mockResolvedValue({ data: fileResponse }); + + await createFileToBackend({ ...baseParams, size: 0 }); + + const calledBody = createFileMock.mock.calls[0][0]; + expect(calledBody.fileId).toBeUndefined(); + }); + + it('should set fileId when size is greater than 0', async () => { + createFileMock.mockResolvedValue({ data: fileResponse }); + + await createFileToBackend(baseParams); + + const calledBody = createFileMock.mock.calls[0][0]; + expect(calledBody.fileId).toBe(baseParams.contentsId); + }); + + it('should return FILE_ALREADY_EXISTS error on CONFLICT', async () => { + createFileMock.mockResolvedValue({ error: new DriveServerError('CONFLICT', 409) }); + + const result = await createFileToBackend(baseParams); + + expect(result.error).toBeInstanceOf(DriveDesktopError); + expect(result.error?.cause).toBe('FILE_ALREADY_EXISTS'); + }); + + it('should return BAD_REQUEST error on BAD_REQUEST', async () => { + createFileMock.mockResolvedValue({ error: new DriveServerError('BAD_REQUEST', 400) }); + + const result = await createFileToBackend(baseParams); + + expect(result.error).toBeInstanceOf(DriveDesktopError); + expect(result.error?.cause).toBe('BAD_REQUEST'); + }); + + it('should return BAD_RESPONSE error on SERVER_ERROR', async () => { + createFileMock.mockResolvedValue({ error: new DriveServerError('SERVER_ERROR', 500) }); + + const result = await createFileToBackend(baseParams); + + expect(result.error).toBeInstanceOf(DriveDesktopError); + expect(result.error?.cause).toBe('BAD_RESPONSE'); + }); + + it('should return UNKNOWN error for unmapped causes', async () => { + createFileMock.mockResolvedValue({ error: new DriveServerError('NETWORK_ERROR') }); + + const result = await createFileToBackend(baseParams); + + expect(result.error).toBeInstanceOf(DriveDesktopError); + expect(result.error?.cause).toBe('UNKNOWN'); + }); +}); diff --git a/src/backend/features/backup/upload/create-file-to-backend.ts b/src/backend/features/backup/upload/create-file-to-backend.ts index eb44adfd24..8c972f34bc 100644 --- a/src/backend/features/backup/upload/create-file-to-backend.ts +++ b/src/backend/features/backup/upload/create-file-to-backend.ts @@ -1,11 +1,11 @@ import path from 'path'; import { EncryptionVersion } from '@internxt/sdk/dist/drive/storage/types'; import { DriveDesktopError } from '../../../../context/shared/domain/errors/DriveDesktopError'; +import { SyncError } from '../../../../shared/issues/SyncErrorCause'; import { Result } from '../../../../context/shared/domain/Result'; import { File } from '../../../../context/virtual-drive/files/domain/File'; -import { createFileIPC } from '../../../../infra/ipc/files-ipc'; -import crypt from '../../../../context/shared/infrastructure/crypt'; import { CreateFileDto } from '../../../../infra/drive-server/out/dto'; +import { createFile } from '../../../../infra/drive-server/services/files/services/create-file'; export type CreateFileToBackendParams = { contentsId: string; @@ -16,35 +16,18 @@ export type CreateFileToBackendParams = { bucket: string; }; -/** - * Extracts filename without extension from a path - * Replicates FilePath.name() behavior - */ function extractName(filePath: string): string { const base = path.posix.basename(filePath); const { name } = path.posix.parse(base); return name; } -/** - * Extracts extension without the dot from a path - * Replicates FilePath.extension() behavior - */ function extractExtension(filePath: string): string { const base = path.posix.basename(filePath); const { ext } = path.posix.parse(base); return ext.slice(1); } -/** - * Creates file metadata in the backend API - * - * This replicates the behavior of: - * - SimpleFileCreator.run() - * - SDKRemoteFileSystem.persist() - * - * Returns a File domain object on success, matching the original behavior exactly. - */ export async function createFileToBackend({ contentsId, filePath, @@ -56,19 +39,6 @@ export async function createFileToBackend({ const plainName = extractName(filePath); const extension = extractExtension(filePath); - // Encrypt name with folderId as salt (same as SDKRemoteFileSystem.persist) - const encryptedName = crypt.encryptName(plainName, folderId.toString()); - - if (!encryptedName) { - return { - error: new DriveDesktopError( - 'COULD_NOT_ENCRYPT_NAME', - `Could not encrypt the file name: ${plainName} with salt: ${folderId.toString()}`, - ), - }; - } - - // Build request body exactly as SDKRemoteFileSystem.persist does const body: CreateFileDto = { bucket, fileId: undefined as string | undefined, @@ -79,60 +49,37 @@ export async function createFileToBackend({ type: extension, }; - // Only set fileId if size > 0 (same condition as SDKRemoteFileSystem.persist) if (size > 0) { body.fileId = contentsId; } - const response = await createFileIPC(body); + const response = await createFile(body); if (response.data) { - // Create File domain object exactly as SimpleFileCreator does const file = File.create({ id: response.data.id, uuid: response.data.uuid, - contentsId: contentsId, - folderId: folderId, + contentsId, + folderId, createdAt: response.data.createdAt, modificationTime: response.data.updatedAt, path: filePath, - size: size, + size, updatedAt: response.data.updatedAt, }); return { data: file }; } - // Handle errors exactly as SDKRemoteFileSystem.persist does - if (response.error && typeof response.error === 'object' && 'cause' in response.error) { - const errorCause = (response.error as { cause: string }).cause; - - if (errorCause === 'BAD_REQUEST') { - return { - error: new DriveDesktopError('BAD_REQUEST', `Some data was not valid for ${plainName}: ${body}`), - }; - } - - if (errorCause === 'FILE_ALREADY_EXISTS') { - return { - error: new DriveDesktopError( - 'FILE_ALREADY_EXISTS', - `File with name ${plainName} on ${folderId} already exists`, - ), - }; - } + const causeMap: Record = { + BAD_REQUEST: 'BAD_REQUEST', + CONFLICT: 'FILE_ALREADY_EXISTS', + SERVER_ERROR: 'BAD_RESPONSE', + }; - if (errorCause === 'SERVER_ERROR') { - return { - error: new DriveDesktopError( - 'BAD_RESPONSE', - `The server could not handle the creation of ${plainName}: ${body}`, - ), - }; - } - } + const cause = causeMap[response.error.cause] ?? 'UNKNOWN'; return { - error: new DriveDesktopError('UNKNOWN', `Creating file ${plainName}: ${response.error}`), + error: new DriveDesktopError(cause, response.error.message ?? `Creating file ${plainName} failed`), }; } diff --git a/src/backend/features/backup/upload/update-file-with-retry.test.ts b/src/backend/features/backup/upload/update-file-with-retry.test.ts new file mode 100644 index 0000000000..5aa76f0e26 --- /dev/null +++ b/src/backend/features/backup/upload/update-file-with-retry.test.ts @@ -0,0 +1,94 @@ +import { partialSpyOn } from '../../../../../tests/vitest/utils.helper'; +import { DriveDesktopError } from '../../../../context/shared/domain/errors/DriveDesktopError'; +import { DriveServerError } from '../../../../infra/drive-server/drive-server.error'; +import { updateFileWithRetry } from './update-file-with-retry'; +import * as uploadContentToEnvironmentModule from './upload-content-to-environment'; +import * as overrideFileModule from '../../../../infra/drive-server/services/files/services/override-file'; +import * as sleepModule from './utils/sleep'; +import { BucketEntryIdMother } from '../../../../context/virtual-drive/shared/domain/__test-helpers__/BucketEntryIdMother'; +import { UuidMother } from '../../../../context/shared/domain/__test-helpers__/UuidMother'; + +describe('updateFileWithRetry', () => { + const uploadContentMock = partialSpyOn(uploadContentToEnvironmentModule, 'uploadContentToEnvironment'); + const overrideFileMock = partialSpyOn(overrideFileModule, 'overrideFile'); + const sleepMock = partialSpyOn(sleepModule, 'sleep'); + + beforeEach(() => { + sleepMock.mockResolvedValue(undefined); + }); + + const baseParams = { + path: '/backup/file.txt', + size: 1024, + bucket: 'bucket', + fileUuid: UuidMother.primitive(), + environment: {} as any, + signal: new AbortController().signal, + }; + + it('should update file successfully', async () => { + uploadContentMock.mockResolvedValue({ data: BucketEntryIdMother.primitive() }); + overrideFileMock.mockResolvedValue({ data: true }); + + const result = await updateFileWithRetry(baseParams); + + expect(result.data).toBeUndefined(); + expect(result.error).toBeUndefined(); + }); + + it('should return error when signal is aborted', async () => { + const abortController = new AbortController(); + abortController.abort(); + + const result = await updateFileWithRetry({ ...baseParams, signal: abortController.signal }); + + expect(result.error).toBeInstanceOf(DriveDesktopError); + expect(uploadContentMock).not.toHaveBeenCalled(); + }); + + it('should return error after max retries when content upload fails', async () => { + const uploadError = new DriveDesktopError('BAD_RESPONSE', 'Upload failed'); + uploadContentMock.mockResolvedValue({ error: uploadError }); + + const result = await updateFileWithRetry(baseParams); + + expect(result.error).toBeInstanceOf(DriveDesktopError); + expect(uploadContentMock).toHaveBeenCalledTimes(4); + expect(sleepMock).toHaveBeenCalledTimes(3); + }); + + it('should return error after max retries when override fails', async () => { + uploadContentMock.mockResolvedValue({ data: BucketEntryIdMother.primitive() }); + overrideFileMock.mockResolvedValue({ error: new DriveServerError('SERVER_ERROR', 500) }); + + const result = await updateFileWithRetry(baseParams); + + expect(result.error).toBeInstanceOf(DriveDesktopError); + expect(result.error?.cause).toBe('BAD_RESPONSE'); + expect(overrideFileMock).toHaveBeenCalledTimes(4); + }); + + it('should retry with correct delays', async () => { + uploadContentMock.mockResolvedValue({ error: new DriveDesktopError('BAD_RESPONSE', 'fail') }); + + await updateFileWithRetry(baseParams); + + expect(sleepMock).toHaveBeenNthCalledWith(1, 1000); + expect(sleepMock).toHaveBeenNthCalledWith(2, 2000); + expect(sleepMock).toHaveBeenNthCalledWith(3, 4000); + }); + + it('should succeed on retry after initial failure', async () => { + const contentsId = BucketEntryIdMother.primitive(); + uploadContentMock + .mockResolvedValueOnce({ error: new DriveDesktopError('BAD_RESPONSE', 'fail') }) + .mockResolvedValue({ data: contentsId }); + overrideFileMock.mockResolvedValue({ data: true }); + + const result = await updateFileWithRetry(baseParams); + + expect(result.data).toBeUndefined(); + expect(result.error).toBeUndefined(); + expect(uploadContentMock).toHaveBeenCalledTimes(2); + }); +}); diff --git a/src/backend/features/backup/upload/update-file-with-retry.ts b/src/backend/features/backup/upload/update-file-with-retry.ts new file mode 100644 index 0000000000..5789a359aa --- /dev/null +++ b/src/backend/features/backup/upload/update-file-with-retry.ts @@ -0,0 +1,70 @@ +/* eslint-disable no-await-in-loop */ +import { Environment } from '@internxt/inxt-js'; +import { DriveDesktopError } from '../../../../context/shared/domain/errors/DriveDesktopError'; +import { Result } from '../../../../context/shared/domain/Result'; +import { overrideFile } from '../../../../infra/drive-server/services/files/services/override-file'; +import { logger } from '@internxt/drive-desktop-core/build/backend'; +import { sleep } from './utils/sleep'; +import { uploadContentToEnvironment } from './upload-content-to-environment'; +import { MAX_RETRIES, RETRY_DELAYS_MS } from './constants'; + +export type UpdateFileParams = { + path: string; + size: number; + bucket: string; + fileUuid: string; + environment: Environment; + signal: AbortSignal; +}; + +export async function updateFileWithRetry(file: UpdateFileParams): Promise> { + for (let attempt = 0; attempt <= MAX_RETRIES; attempt++) { + if (file.signal.aborted) { + return { error: new DriveDesktopError('UNKNOWN', 'Upload aborted') }; + } + + try { + const { data: contentsId, error } = await uploadContentToEnvironment({ + path: file.path, + size: file.size, + bucket: file.bucket, + environment: file.environment, + signal: file.signal, + }); + + if (error) { + throw error; + } + + const overrideResult = await overrideFile({ + fileUuid: file.fileUuid, + fileContentsId: contentsId, + fileSize: file.size, + }); + + if (overrideResult.error) { + throw new DriveDesktopError('BAD_RESPONSE', `Failed to override file ${file.path}`); + } + + return { data: undefined }; + } catch (error) { + const driveError = error instanceof DriveDesktopError ? error : new DriveDesktopError('UNKNOWN'); + + if (attempt < MAX_RETRIES) { + const delay = RETRY_DELAYS_MS[attempt]; + logger.debug({ + tag: 'BACKUPS', + msg: `Update attempt ${attempt + 1} failed, retrying in ${delay}ms`, + path: file.path, + error: driveError.message, + }); + await sleep(delay); + continue; + } + + return { error: driveError }; + } + } + + return { error: new DriveDesktopError('UNKNOWN', 'Update failed after max retries') }; +} diff --git a/src/backend/features/backup/upload/upload-file-with-retry.ts b/src/backend/features/backup/upload/upload-file-with-retry.ts index c46a1ba343..dc365c22f2 100644 --- a/src/backend/features/backup/upload/upload-file-with-retry.ts +++ b/src/backend/features/backup/upload/upload-file-with-retry.ts @@ -8,9 +8,8 @@ import { logger } from '@internxt/drive-desktop-core/build/backend'; import { sleep } from './utils/sleep'; import { uploadContentToEnvironment } from './upload-content-to-environment'; import { Result } from '../../../../context/shared/domain/Result'; +import { MAX_RETRIES, RETRY_DELAYS_MS } from './constants'; -const MAX_RETRIES = 3; -const RETRY_DELAYS_MS = [1000, 2000, 4000]; export type UploadFileParams = { path: string; @@ -29,21 +28,6 @@ export type UploadFileParams = { function isAlreadyExistsError(error: DriveDesktopError): boolean { return error.cause === 'FILE_ALREADY_EXISTS'; } - -/** - * Upload a file with retry logic - * - * Handles the full upload flow (replicates FileBatchUploader behavior): - * 1. Content upload (to Environment/storage) → contentsId - * 2. Metadata creation (to backend API) → File domain object - * - * Retries on failure with exponential backoff. - * Cleans up uploaded content if metadata creation fails. - * - * Special cases (matching FileBatchUploader): - * - FILE_ALREADY_EXISTS: Returns data as null (not an error) - * - BAD_RESPONSE: Treated as non-fatal, will be retried then fail - */ export async function uploadFileWithRetry(file: UploadFileParams): Promise> { for (let attempt = 0; attempt <= MAX_RETRIES; attempt++) { if (file.signal.aborted) { @@ -51,8 +35,7 @@ export async function uploadFileWithRetry(file: UploadFileParams): Promise Date: Tue, 24 Mar 2026 17:03:22 +0100 Subject: [PATCH 6/9] delete: FileBatchUpdater --- .../local/registerLocalFileServices.ts | 4 -- .../virtual-drive/registerFilesServices.ts | 2 - .../__mocks__/LocalFileUploaderMock.ts | 14 ----- .../update/FileBatchUpdater.test.ts | 61 ------------------- .../application/update/FileBatchUpdater.ts | 48 --------------- .../local/localFile/domain/LocalFile.ts | 13 ---- .../__test-helpers__/LocalFileMother.ts | 13 ---- .../local/localFolder/domain/LocalFolder.ts | 4 -- .../override/SimpleFileOverrider.ts | 18 ------ .../domain/__test-helpers__/FileMother.ts | 7 --- .../remoteTree/domain/RemoteTree.ts | 8 --- 11 files changed, 192 deletions(-) delete mode 100644 src/context/local/localFile/application/__mocks__/LocalFileUploaderMock.ts delete mode 100644 src/context/local/localFile/application/update/FileBatchUpdater.test.ts delete mode 100644 src/context/local/localFile/application/update/FileBatchUpdater.ts delete mode 100644 src/context/virtual-drive/files/application/override/SimpleFileOverrider.ts diff --git a/src/apps/backups/dependency-injection/local/registerLocalFileServices.ts b/src/apps/backups/dependency-injection/local/registerLocalFileServices.ts index f15bd23392..abdfeffb4d 100644 --- a/src/apps/backups/dependency-injection/local/registerLocalFileServices.ts +++ b/src/apps/backups/dependency-injection/local/registerLocalFileServices.ts @@ -1,6 +1,5 @@ import { INTERNXT_CLIENT, INTERNXT_VERSION } from './../../../../core/utils/utils'; import { ContainerBuilder } from 'diod'; -import { FileBatchUpdater } from '../../../../context/local/localFile/application/update/FileBatchUpdater'; import { LocalFileHandler } from '../../../../context/local/localFile/domain/LocalFileUploader'; import { EnvironmentLocalFileUploader } from '../../../../context/local/localFile/infrastructure/EnvironmentLocalFileUploader'; import { DependencyInjectionUserProvider } from '../../../shared/dependency-injection/DependencyInjectionUserProvider'; @@ -33,7 +32,4 @@ export function registerLocalFileServices(builder: ContainerBuilder) { return new EnvironmentLocalFileUploader(env, user.backupsBucket); }) .private(); - - // Services - builder.registerAndUse(FileBatchUpdater); } diff --git a/src/apps/backups/dependency-injection/virtual-drive/registerFilesServices.ts b/src/apps/backups/dependency-injection/virtual-drive/registerFilesServices.ts index ef43cbcd4a..beba2affdb 100644 --- a/src/apps/backups/dependency-injection/virtual-drive/registerFilesServices.ts +++ b/src/apps/backups/dependency-injection/virtual-drive/registerFilesServices.ts @@ -1,6 +1,5 @@ import { ContainerBuilder } from 'diod'; import { SimpleFileCreator } from '../../../../context/virtual-drive/files/application/create/SimpleFileCreator'; -import { SimpleFileOverrider } from '../../../../context/virtual-drive/files/application/override/SimpleFileOverrider'; import { RemoteFileSystem } from '../../../../context/virtual-drive/files/domain/file-systems/RemoteFileSystem'; import { SDKRemoteFileSystem } from '../../../../context/virtual-drive/files/infrastructure/SDKRemoteFileSystem'; import { DependencyInjectionUserProvider } from '../../../shared/dependency-injection/DependencyInjectionUserProvider'; @@ -15,5 +14,4 @@ export function registerFilesServices(builder: ContainerBuilder) { .private(); builder.registerAndUse(SimpleFileCreator); - builder.registerAndUse(SimpleFileOverrider); } diff --git a/src/context/local/localFile/application/__mocks__/LocalFileUploaderMock.ts b/src/context/local/localFile/application/__mocks__/LocalFileUploaderMock.ts deleted file mode 100644 index 175203d76b..0000000000 --- a/src/context/local/localFile/application/__mocks__/LocalFileUploaderMock.ts +++ /dev/null @@ -1,14 +0,0 @@ -import { LocalFileHandler } from '../../domain/LocalFileUploader'; -import { AbsolutePath } from '../../infrastructure/AbsolutePath'; -import { Either } from '../../../../shared/domain/Either'; -import { DriveDesktopError } from '../../../../shared/domain/errors/DriveDesktopError'; -import { vi } from 'vitest'; - -export class LocalFileUploaderMock implements LocalFileHandler { - private readonly uploadMock = vi.fn(); - private readonly deleteMock = vi.fn(); - - upload(path: AbsolutePath, size: number, abortSignal: AbortSignal): Promise> { - return this.uploadMock(path, size, abortSignal); - } -} diff --git a/src/context/local/localFile/application/update/FileBatchUpdater.test.ts b/src/context/local/localFile/application/update/FileBatchUpdater.test.ts deleted file mode 100644 index 896ddf0ee9..0000000000 --- a/src/context/local/localFile/application/update/FileBatchUpdater.test.ts +++ /dev/null @@ -1,61 +0,0 @@ -import { FileBatchUpdater } from './FileBatchUpdater'; -import { AbsolutePath } from '../../infrastructure/AbsolutePath'; -import { right } from '../../../../shared/domain/Either'; -import { SimpleFileOverrider } from '../../../../virtual-drive/files/application/override/SimpleFileOverrider'; -import { RemoteFileSystem } from '../../../../virtual-drive/files/domain/file-systems/RemoteFileSystem'; -import { FileMother } from '../../../../virtual-drive/files/domain/__test-helpers__/FileMother'; -import { RemoteTreeMother } from '../../../../virtual-drive/remoteTree/domain/__test-helpers__/RemoteTreeMother'; -import { LocalFolderMother } from '../../../localFolder/domain/__test-helpers__/LocalFolderMother'; -import { LocalFileMother } from '../../domain/__test-helpers__/LocalFileMother'; -import { LocalFileUploaderMock } from '../__mocks__/LocalFileUploaderMock'; -import { vi, MockInstance } from 'vitest'; - -describe('File Batch Updater', () => { - let SUT: FileBatchUpdater; - - let uploader: LocalFileUploaderMock; - let simpleFileOverrider: SimpleFileOverrider; - let simpleFileOverriderSpy: MockInstance; - - let abortController: AbortController; - const localRoot = LocalFolderMother.fromPartial({ - path: '/home/user/Documents' as AbsolutePath, - }); - - beforeAll(() => { - uploader = new LocalFileUploaderMock(); - simpleFileOverrider = new SimpleFileOverrider({} as RemoteFileSystem); - - simpleFileOverriderSpy = vi.spyOn(simpleFileOverrider, 'run'); - - SUT = new FileBatchUpdater(uploader, simpleFileOverrider); - }); - - beforeEach(() => { - vi.resetAllMocks(); - abortController = new AbortController(); - }); - - it('resolves when all updates are completed', async () => { - const remoteFiles = FileMother.array(); - const numberOfFilesToUpdate = remoteFiles.length; - - const localFiles = LocalFileMother.array(numberOfFilesToUpdate, (i) => ({ - path: (localRoot.path + remoteFiles[i].path) as AbsolutePath, - })); - - const tree = RemoteTreeMother.onlyRoot(); - - remoteFiles.forEach((file) => { - tree.addFile(tree.root, file); - }); - - const mockContentsId = 'mock-contents-id'; - vi.spyOn(uploader, 'upload').mockReturnValue(Promise.resolve(right(mockContentsId))); - simpleFileOverriderSpy.mockReturnValue(right(Promise.resolve())); - - await SUT.run(localRoot, tree, localFiles, abortController.signal); - - expect(simpleFileOverriderSpy).toBeCalledTimes(numberOfFilesToUpdate); - }); -}); diff --git a/src/context/local/localFile/application/update/FileBatchUpdater.ts b/src/context/local/localFile/application/update/FileBatchUpdater.ts deleted file mode 100644 index 89b7f8f9a8..0000000000 --- a/src/context/local/localFile/application/update/FileBatchUpdater.ts +++ /dev/null @@ -1,48 +0,0 @@ -import { Service } from 'diod'; -import { LocalFile } from '../../domain/LocalFile'; -import { LocalFileHandler } from '../../domain/LocalFileUploader'; -import { RemoteTree } from '../../../../virtual-drive/remoteTree/domain/RemoteTree'; -import { SimpleFileOverrider } from '../../../../virtual-drive/files/application/override/SimpleFileOverrider'; -import { LocalFolder } from '../../../localFolder/domain/LocalFolder'; -import { relative } from '../../../../../apps/backups/utils/relative'; - -@Service() -export class FileBatchUpdater { - constructor( - private readonly uploader: LocalFileHandler, - private readonly simpleFileOverrider: SimpleFileOverrider, - ) {} - - async run( - localRoot: LocalFolder, - remoteTree: RemoteTree, - batch: Array, - signal: AbortSignal, - ): Promise { - for (const localFile of batch) { - if (signal.aborted) { - return; - } - - // eslint-disable-next-line no-await-in-loop - const upload = await this.uploader.upload(localFile.path, localFile.size, signal); - - if (upload.isLeft()) { - throw upload.getLeft(); - } - - const contentsId = upload.getRight(); - - const remotePath = relative(localRoot.path, localFile.path); - - const file = remoteTree.get(remotePath); - - if (file.isFolder()) { - throw new Error(`Expected file, found folder on ${file.path}`); - } - - // eslint-disable-next-line no-await-in-loop - await this.simpleFileOverrider.run(file, contentsId, localFile.size); - } - } -} diff --git a/src/context/local/localFile/domain/LocalFile.ts b/src/context/local/localFile/domain/LocalFile.ts index 1f1dab4294..b9a84b425c 100644 --- a/src/context/local/localFile/domain/LocalFile.ts +++ b/src/context/local/localFile/domain/LocalFile.ts @@ -30,10 +30,6 @@ export class LocalFile extends AggregateRoot { return this._size.value; } - holdsSubpath(otherPath: string): boolean { - return this._path.endsWith(otherPath); - } - isEmpty(): boolean { return this._size.isEmpty(); } @@ -50,15 +46,6 @@ export class LocalFile extends AggregateRoot { return this._size.isBig(); } - basedir(): string { - const dirname = path.posix.dirname(this._path); - if (dirname === '.') { - return path.posix.sep; - } - - return dirname; - } - nameWithExtension() { const basename = path.posix.basename(this._path); const { base } = path.posix.parse(basename); diff --git a/src/context/local/localFile/domain/__test-helpers__/LocalFileMother.ts b/src/context/local/localFile/domain/__test-helpers__/LocalFileMother.ts index a2e6d7ae98..d96078ccf0 100644 --- a/src/context/local/localFile/domain/__test-helpers__/LocalFileMother.ts +++ b/src/context/local/localFile/domain/__test-helpers__/LocalFileMother.ts @@ -18,17 +18,4 @@ export class LocalFileMother { ...partial, }); } - - static array( - numberOfElements: number, - generator?: (position: number) => Partial, - ): Array { - const array = []; - - for (let i = 0; i < numberOfElements; i++) { - array.push(LocalFileMother.fromPartial(generator ? generator(i) : {})); - } - - return array; - } } diff --git a/src/context/local/localFolder/domain/LocalFolder.ts b/src/context/local/localFolder/domain/LocalFolder.ts index 12e9801d68..645fa06315 100644 --- a/src/context/local/localFolder/domain/LocalFolder.ts +++ b/src/context/local/localFolder/domain/LocalFolder.ts @@ -19,10 +19,6 @@ export class LocalFolder extends AggregateRoot { return this._path; } - get modificationTime(): number { - return this._modificationTime; - } - basedir(): string { const dirname = path.posix.dirname(this._path); if (dirname === '.') { diff --git a/src/context/virtual-drive/files/application/override/SimpleFileOverrider.ts b/src/context/virtual-drive/files/application/override/SimpleFileOverrider.ts deleted file mode 100644 index 7f2d7a0222..0000000000 --- a/src/context/virtual-drive/files/application/override/SimpleFileOverrider.ts +++ /dev/null @@ -1,18 +0,0 @@ -import { Service } from 'diod'; -import { FileSize } from '../../domain/FileSize'; -import { File } from '../../domain/File'; -import { FileContentsId } from '../../domain/FileContentsId'; -import { overrideFile } from '../../../../../infra/drive-server/services/files/services/override-file'; - -@Service() -export class SimpleFileOverrider { - async run(file: File, contentsId: string, size: number): Promise { - file.changeContents(new FileContentsId(contentsId), new FileSize(size)); - - await overrideFile({ - fileUuid: file.uuid, - fileContentsId: file.contentsId, - fileSize: file.size, - }); - } -} diff --git a/src/context/virtual-drive/files/domain/__test-helpers__/FileMother.ts b/src/context/virtual-drive/files/domain/__test-helpers__/FileMother.ts index 7678e7fe14..70876c3f30 100644 --- a/src/context/virtual-drive/files/domain/__test-helpers__/FileMother.ts +++ b/src/context/virtual-drive/files/domain/__test-helpers__/FileMother.ts @@ -44,13 +44,6 @@ export class FileMother { }); } - static thumbnable() { - return File.from({ - ...FileMother.any().attributes(), - path: FilePathMother.thumbnable(2).value, - }); - } - static noThumbnable() { return File.from({ ...FileMother.any().attributes(), diff --git a/src/context/virtual-drive/remoteTree/domain/RemoteTree.ts b/src/context/virtual-drive/remoteTree/domain/RemoteTree.ts index 82c66ad100..a7dfa74b1e 100644 --- a/src/context/virtual-drive/remoteTree/domain/RemoteTree.ts +++ b/src/context/virtual-drive/remoteTree/domain/RemoteTree.ts @@ -115,14 +115,6 @@ export class RemoteTree { return node.folder; } - - hasParent(id: string): boolean { - const dirname = path.posix.dirname(id); - const parentId = dirname === '.' ? path.posix.sep : dirname; - - return this.has(parentId); - } - getParent(id: string): Folder { const dirname = path.posix.dirname(id); const parentId = dirname === '.' ? path.posix.sep : dirname; From 2362739b0b447c1b849ade8b57b7b48579320d97 Mon Sep 17 00:00:00 2001 From: AlexisMora Date: Tue, 24 Mar 2026 17:21:13 +0100 Subject: [PATCH 7/9] delete: FileBatchUploader --- .../local/registerLocalFileServices.ts | 10 - .../virtual-drive/registerFilesServices.ts | 3 - .../thumbnails/domain/ThumbnableExtension.ts | 15 - src/apps/shared/FileTypes/FileTypes.ts | 190 ----- .../__mocks__/LocalFileRepositoryMock.ts | 41 -- .../application/upload/FileBatchUploader.ts | 89 --- .../localFile/domain/LocalFileRepository.ts | 7 - .../localFile/domain/LocalFileUploader.ts | 11 - .../EnvironmentLocalFileUploader.ts | 71 -- .../FsLocalFileRepository.test.ts | 66 -- .../infrastructure/FsLocalFileRepository.ts | 41 -- .../__test-helpers__/folderStructure.ts | 29 - .../application/create/SimpleFileCreator.ts | 55 -- .../virtual-drive/files/domain/File.ts | 16 - .../virtual-drive/files/domain/FilePath.ts | 23 - .../files/domain/MimeTypesMap.ts | 651 ------------------ 16 files changed, 1318 deletions(-) delete mode 100644 src/apps/main/thumbnails/domain/ThumbnableExtension.ts delete mode 100644 src/apps/shared/FileTypes/FileTypes.ts delete mode 100644 src/context/local/localFile/__mocks__/LocalFileRepositoryMock.ts delete mode 100644 src/context/local/localFile/application/upload/FileBatchUploader.ts delete mode 100644 src/context/local/localFile/domain/LocalFileRepository.ts delete mode 100644 src/context/local/localFile/domain/LocalFileUploader.ts delete mode 100644 src/context/local/localFile/infrastructure/EnvironmentLocalFileUploader.ts delete mode 100644 src/context/local/localFile/infrastructure/FsLocalFileRepository.test.ts delete mode 100644 src/context/local/localFile/infrastructure/FsLocalFileRepository.ts delete mode 100644 src/context/local/localFile/infrastructure/__test-helpers__/folderStructure.ts delete mode 100644 src/context/virtual-drive/files/application/create/SimpleFileCreator.ts delete mode 100644 src/context/virtual-drive/files/domain/MimeTypesMap.ts diff --git a/src/apps/backups/dependency-injection/local/registerLocalFileServices.ts b/src/apps/backups/dependency-injection/local/registerLocalFileServices.ts index abdfeffb4d..7c780a1bce 100644 --- a/src/apps/backups/dependency-injection/local/registerLocalFileServices.ts +++ b/src/apps/backups/dependency-injection/local/registerLocalFileServices.ts @@ -1,7 +1,5 @@ import { INTERNXT_CLIENT, INTERNXT_VERSION } from './../../../../core/utils/utils'; import { ContainerBuilder } from 'diod'; -import { LocalFileHandler } from '../../../../context/local/localFile/domain/LocalFileUploader'; -import { EnvironmentLocalFileUploader } from '../../../../context/local/localFile/infrastructure/EnvironmentLocalFileUploader'; import { DependencyInjectionUserProvider } from '../../../shared/dependency-injection/DependencyInjectionUserProvider'; import { Environment } from '@internxt/inxt-js'; import { getCredentials } from '../../../main/auth/get-credentials'; @@ -24,12 +22,4 @@ export function registerLocalFileServices(builder: ContainerBuilder) { }); builder.register(Environment).useInstance(environment); - - builder - .register(LocalFileHandler) - .useFactory((c) => { - const env = c.get(Environment); - return new EnvironmentLocalFileUploader(env, user.backupsBucket); - }) - .private(); } diff --git a/src/apps/backups/dependency-injection/virtual-drive/registerFilesServices.ts b/src/apps/backups/dependency-injection/virtual-drive/registerFilesServices.ts index beba2affdb..f8e6327829 100644 --- a/src/apps/backups/dependency-injection/virtual-drive/registerFilesServices.ts +++ b/src/apps/backups/dependency-injection/virtual-drive/registerFilesServices.ts @@ -1,5 +1,4 @@ import { ContainerBuilder } from 'diod'; -import { SimpleFileCreator } from '../../../../context/virtual-drive/files/application/create/SimpleFileCreator'; import { RemoteFileSystem } from '../../../../context/virtual-drive/files/domain/file-systems/RemoteFileSystem'; import { SDKRemoteFileSystem } from '../../../../context/virtual-drive/files/infrastructure/SDKRemoteFileSystem'; import { DependencyInjectionUserProvider } from '../../../shared/dependency-injection/DependencyInjectionUserProvider'; @@ -12,6 +11,4 @@ export function registerFilesServices(builder: ContainerBuilder) { .register(RemoteFileSystem) .useFactory(() => new SDKRemoteFileSystem(user.backupsBucket)) .private(); - - builder.registerAndUse(SimpleFileCreator); } diff --git a/src/apps/main/thumbnails/domain/ThumbnableExtension.ts b/src/apps/main/thumbnails/domain/ThumbnableExtension.ts deleted file mode 100644 index 6621892dc0..0000000000 --- a/src/apps/main/thumbnails/domain/ThumbnableExtension.ts +++ /dev/null @@ -1,15 +0,0 @@ -import fileExtensionGroups, { FileExtensionGroup } from '../../../shared/FileTypes/FileTypes'; - -const imageExtensions = fileExtensionGroups[FileExtensionGroup.Image]; -const pdfExtensions = fileExtensionGroups[FileExtensionGroup.Pdf]; - -const thumbnableImageExtension = [ - ...imageExtensions.jpg, - ...imageExtensions.png, - ...imageExtensions.bmp, - ...imageExtensions.gif, -] as const; - -const thumbnablePdfExtension = [...pdfExtensions.pdf] as const; - -export const thumbnableExtensions = [...thumbnableImageExtension, ...thumbnablePdfExtension] as const; diff --git a/src/apps/shared/FileTypes/FileTypes.ts b/src/apps/shared/FileTypes/FileTypes.ts deleted file mode 100644 index f500b6a64c..0000000000 --- a/src/apps/shared/FileTypes/FileTypes.ts +++ /dev/null @@ -1,190 +0,0 @@ -/*TODO: DELETE DEAD CODE */ -type FileExtensionMap = Record; - -const audioExtensions: FileExtensionMap = { - '3gp': ['3gp'], - aa: ['aa'], - aac: ['aac'], - aax: ['aax'], - act: ['act'], - aiff: ['aiff'], - alac: ['alac'], - amr: ['amr'], - ape: ['ape'], - au: ['au'], - awd: ['awd'], - dss: ['dss'], - dvf: ['dvf'], - flac: ['flac'], - gsm: ['gsm'], - iklax: ['iklax'], - ivs: ['ivs'], - m4a: ['m4a'], - m4b: ['m4b'], - m4p: ['m4p'], - mmf: ['mmf'], - mp3: ['mp3'], - mpc: ['mpc'], - msv: ['msv'], - nmf: ['nmf'], - ogg: ['ogg', 'oga', 'mogg'], - opus: ['opus'], - ra: ['ra', 'rm'], - rf64: ['rf64'], - sln: ['sln'], - tta: ['tta'], - voc: ['voc'], - vox: ['vox'], - wav: ['wav'], - wma: ['wma'], - wv: ['wv'], - webm: ['webm'], - '8svx': ['8svx'], - cda: ['cda'], -}; - -const codeExtensions: FileExtensionMap = { - c: ['c', 'h'], - 'c++': ['cpp', 'c++', 'cc', 'cxx', 'hpp', 'h++', 'hh', 'hxx'], - cobol: ['cob', 'cpy'], - 'c#': ['cs'], - cmake: ['cmake'], - coffee: ['coffee'], - css: ['css'], - less: ['less'], - sass: ['sass'], - scss: ['scss'], - fortran: ['f', 'for', 'f77', 'f90'], - 'asp.net': ['aspx'], - html: ['html', 'hmn'], - java: ['java'], - jsp: ['jsp'], - javascript: ['js'], - typescript: ['ts'], - json: ['json'], - jsx: ['jsx'], - kotlin: ['kt'], - mathematica: ['m', 'nb'], - php: ['php', 'php3', 'php4', 'php5', 'phtml'], - python: ['BUILD', 'bzl', 'py', 'pyw'], - ruby: ['rb'], - sql: ['sql'], - vue: ['vue'], - yaml: ['yaml', 'yml'], -}; - -const figmaExtensions: FileExtensionMap = { - fig: ['fig'], -}; - -const imageExtensions: FileExtensionMap = { - tiff: ['tif', 'tiff'], - bmp: ['bmp'], - heic: ['heic'], - jpg: ['jpg', 'jpeg'], - gif: ['gif'], - png: ['png'], - eps: ['eps'], - raw: ['raw', 'cr2', 'nef', 'orf', 'sr2'], -}; - -const pdfExtensions: FileExtensionMap = { - pdf: ['pdf'], -}; - -const pptExtensions: FileExtensionMap = { - ppt: ['ppt', 'pptx', 'pptm'], -}; - -const txtExtensions: FileExtensionMap = { - txt: ['txt', 'text', 'conf', 'def', 'list', 'log', 'md', 'lock'], -}; - -const videoExtensions: FileExtensionMap = { - webm: ['webm'], - mkv: ['mkv'], - vob: ['vob'], - ogg: ['ogv', 'ogg'], - drc: ['drc'], - avi: ['avi'], - mts: ['mts', 'm2ts'], - quicktime: ['mov', 'qt'], - 'windows-media-video': ['wmv'], - raw: ['yuv'], - 'real-media': ['rm', 'rmvb'], - 'vivo-active': ['viv'], - asf: ['asf'], - amv: ['amv'], - 'mpeg-4': ['mp4', 'm4p', 'm4v'], - 'mpeg-1': ['mpg', 'mp2', 'mpeg', 'mpe', 'mpv'], - 'mpeg-2': ['mpg', 'mpeg', 'm2v'], - m4v: ['m4v'], - svi: ['svi'], - '3gpp': ['3gp'], - '3gpp2': ['3g2'], - mxf: ['mxf'], - roq: ['roq'], - nsv: ['nsv'], - flv: ['flv', 'f4v', 'f4p', 'f4a', 'f4b'], -}; - -const WordExtensions: FileExtensionMap = { - doc: ['doc', 'docx'], -}; - -const xlsExtensions: FileExtensionMap = { - xls: ['xls', 'xlsx'], -}; - -const xmlExtensions: FileExtensionMap = { - xml: ['xml', 'xsl', 'xsd'], - svg: ['svg'], -}; - -const csvExtensions: FileExtensionMap = { - csv: ['csv'], -}; - -const zipExtensions: FileExtensionMap = { - zip: ['zip', 'zipx'], -}; - -const defaultExtensions: FileExtensionMap = {}; - -export enum FileExtensionGroup { - Audio, - Code, - Figma, - Image, - Pdf, - Ppt, - Txt, - Video, - Word, - Xls, - Xml, - Csv, - Zip, - Default, -} - -type FileExtensionsDictionary = Record; - -const fileExtensionGroups: FileExtensionsDictionary = { - [FileExtensionGroup.Audio]: audioExtensions, - [FileExtensionGroup.Code]: codeExtensions, - [FileExtensionGroup.Figma]: figmaExtensions, - [FileExtensionGroup.Image]: imageExtensions, - [FileExtensionGroup.Pdf]: pdfExtensions, - [FileExtensionGroup.Ppt]: pptExtensions, - [FileExtensionGroup.Txt]: txtExtensions, - [FileExtensionGroup.Video]: videoExtensions, - [FileExtensionGroup.Word]: WordExtensions, - [FileExtensionGroup.Xls]: xlsExtensions, - [FileExtensionGroup.Xml]: xmlExtensions, - [FileExtensionGroup.Csv]: csvExtensions, - [FileExtensionGroup.Zip]: zipExtensions, - [FileExtensionGroup.Default]: defaultExtensions, -}; - -export default fileExtensionGroups; diff --git a/src/context/local/localFile/__mocks__/LocalFileRepositoryMock.ts b/src/context/local/localFile/__mocks__/LocalFileRepositoryMock.ts deleted file mode 100644 index 957165617f..0000000000 --- a/src/context/local/localFile/__mocks__/LocalFileRepositoryMock.ts +++ /dev/null @@ -1,41 +0,0 @@ -import { LocalFile } from '../domain/LocalFile'; -import { LocalFileRepository } from '../domain/LocalFileRepository'; -import { AbsolutePath } from '../infrastructure/AbsolutePath'; - -export class LocalFileRepositoryMock implements LocalFileRepository { - private readonly filesMock = vi.fn(); - private readonly foldersMock = vi.fn(); - - files(absolutePath: AbsolutePath): Promise { - return this.filesMock(absolutePath); - } - - returnsFiles(files: LocalFile | Array) { - if (Array.isArray(files)) { - this.filesMock.mockResolvedValueOnce(files); - return; - } - - this.filesMock.mockResolvedValueOnce([files]); - } - - assertFilesHasBeenCalledWith(folder: AbsolutePath) { - expect(this.filesMock).toHaveBeenCalledWith(folder); - } - - folders(absolutePath: AbsolutePath): Promise { - return this.foldersMock(absolutePath); - } - - returnsFolders(paths: Array) { - this.foldersMock.mockResolvedValueOnce(paths); - } - - withOutFolders() { - this.foldersMock.mockResolvedValue([]); - } - - assertFoldersHasBeenCalledWith(folder: AbsolutePath) { - expect(this.foldersMock).toHaveBeenCalledWith(folder); - } -} diff --git a/src/context/local/localFile/application/upload/FileBatchUploader.ts b/src/context/local/localFile/application/upload/FileBatchUploader.ts deleted file mode 100644 index 8bd1473cd5..0000000000 --- a/src/context/local/localFile/application/upload/FileBatchUploader.ts +++ /dev/null @@ -1,89 +0,0 @@ -import { Service } from 'diod'; -import { LocalFile } from '../../domain/LocalFile'; -import { LocalFileHandler } from '../../domain/LocalFileUploader'; -import { SimpleFileCreator } from '../../../../virtual-drive/files/application/create/SimpleFileCreator'; -import { RemoteTree } from '../../../../virtual-drive/remoteTree/domain/RemoteTree'; -import { relative } from '../../../../../apps/backups/utils/relative'; -import { isFatalError } from '../../../../../shared/issues/SyncErrorCause'; -import { logger } from '@internxt/drive-desktop-core/build/backend'; -import { backupErrorsTracker } from '../../../../../backend/features/backup'; -import { deleteFileFromStorageByFileId } from '../../../../../infra/drive-server/services/files/services/delete-file-content-from-bucket'; - -// TODO: THIS WHOLE FILE IS GOING TO BE REPLACED -@Service() -export class FileBatchUploader { - constructor( - private readonly localHandler: LocalFileHandler, - private readonly creator: SimpleFileCreator, - private readonly bucket: string, - ) {} - - async run( - localRootPath: string, - remoteTree: RemoteTree, - batch: Array, - signal: AbortSignal, - ): Promise { - for (const localFile of batch) { - if (signal.aborted) { - return; - } - - const remotePath = relative(localRootPath, localFile.path); - const parent = remoteTree.getParent(remotePath); - - let uploadEither; - try { - // eslint-disable-next-line no-await-in-loop - uploadEither = await this.localHandler.upload(localFile.path, localFile.size, signal); - } catch (error) { - logger.error({ msg: '[UPLOAD ERROR]', error }); - continue; - } - - if (uploadEither.isLeft()) { - const error = uploadEither.getLeft(); - logger.error({ msg: '[FILE UPLOAD FAILED]', error }); - - if (isFatalError(error.cause)) { - throw error; - } - backupErrorsTracker.add(parent.id, { name: localFile.nameWithExtension(), error: error.cause }); - continue; - } - - const contentsId = uploadEither.getRight(); - - // eslint-disable-next-line no-await-in-loop - const either = await this.creator.run(contentsId, localFile.path, localFile.size, parent.id, parent.uuid); - - if (either.isLeft()) { - logger.debug({ msg: '[FILE CREATION FAILED]', error: either.getLeft() }); - // eslint-disable-next-line no-await-in-loop - await deleteFileFromStorageByFileId({ - bucketId: this.bucket, - fileId: contentsId, - }); - const error = either.getLeft(); - - if (error.cause === 'FILE_ALREADY_EXISTS') { - logger.debug({ - msg: `[FILE ALREADY EXISTS] Skipping file ${localFile.path} - already exists remotely`, - }); - continue; - } - - if (error.cause === 'BAD_RESPONSE') { - backupErrorsTracker.add(parent.id, { name: localFile.nameWithExtension(), error: error.cause }); - continue; - } - - throw error; - } - - const file = either.getRight(); - - remoteTree.addFile(parent, file); - } - } -} diff --git a/src/context/local/localFile/domain/LocalFileRepository.ts b/src/context/local/localFile/domain/LocalFileRepository.ts deleted file mode 100644 index 72ef7c2883..0000000000 --- a/src/context/local/localFile/domain/LocalFileRepository.ts +++ /dev/null @@ -1,7 +0,0 @@ -import { AbsolutePath } from '../infrastructure/AbsolutePath'; -import { LocalFile } from './LocalFile'; - -export abstract class LocalFileRepository { - abstract files(absolutePath: AbsolutePath): Promise>; - abstract folders(absolutePath: AbsolutePath): Promise>; -} diff --git a/src/context/local/localFile/domain/LocalFileUploader.ts b/src/context/local/localFile/domain/LocalFileUploader.ts deleted file mode 100644 index 22aebfd086..0000000000 --- a/src/context/local/localFile/domain/LocalFileUploader.ts +++ /dev/null @@ -1,11 +0,0 @@ -import { Either } from '../../../shared/domain/Either'; -import { DriveDesktopError } from '../../../shared/domain/errors/DriveDesktopError'; -import { AbsolutePath } from '../infrastructure/AbsolutePath'; - -export abstract class LocalFileHandler { - abstract upload( - path: AbsolutePath, - size: number, - abortSignal: AbortSignal, - ): Promise>; -} diff --git a/src/context/local/localFile/infrastructure/EnvironmentLocalFileUploader.ts b/src/context/local/localFile/infrastructure/EnvironmentLocalFileUploader.ts deleted file mode 100644 index 23bced6e79..0000000000 --- a/src/context/local/localFile/infrastructure/EnvironmentLocalFileUploader.ts +++ /dev/null @@ -1,71 +0,0 @@ -import { UploadStrategyFunction } from '@internxt/inxt-js/build/lib/core'; -import { Service } from 'diod'; -import { createReadStream } from 'fs'; -import { Stopwatch } from '../../../../apps/shared/types/Stopwatch'; -import { AbsolutePath } from './AbsolutePath'; -import { LocalFileHandler } from '../domain/LocalFileUploader'; -import { Environment } from '@internxt/inxt-js'; -import { Either, left, right } from '../../../shared/domain/Either'; -import { DriveDesktopError } from '../../../shared/domain/errors/DriveDesktopError'; -import { logger } from '@internxt/drive-desktop-core/build/backend'; -import { MULTIPART_UPLOAD_SIZE_THRESHOLD } from '../../../shared/domain/UploadConstants'; - -// TODO: THIS WHOLE FILE IS GOING TO BE REPLACED -@Service() -export class EnvironmentLocalFileUploader implements LocalFileHandler { - constructor( - private readonly environment: Environment, - private readonly bucket: string, - ) {} - - upload(path: AbsolutePath, size: number, abortSignal: AbortSignal): Promise> { - const fn: UploadStrategyFunction = - size > MULTIPART_UPLOAD_SIZE_THRESHOLD - ? this.environment.uploadMultipartFile.bind(this.environment) - : this.environment.upload.bind(this.environment); - - const readable = createReadStream(path); - - const stopwatch = new Stopwatch(); - - stopwatch.start(); - - return new Promise>((resolve) => { - const state = fn(this.bucket, { - source: readable, - fileSize: size, - finishedCallback: (err, contentsId) => { - readable.close(); - stopwatch.finish(); - - if (err) { - logger.error({ tag: 'SYNC-ENGINE', msg: '[ENVLFU UPLOAD ERROR]', err }); - if (err.message === 'Max space used') { - return resolve(left(new DriveDesktopError('NOT_ENOUGH_SPACE'))); - } - return resolve(left(new DriveDesktopError('UNKNOWN'))); - } - - if (!contentsId) { - logger.error({ tag: 'SYNC-ENGINE', msg: '[ENVLFU UPLOAD ERROR] No contentsId returned' }); - return resolve(left(new DriveDesktopError('UNKNOWN'))); - } - - resolve(right(contentsId)); - }, - progressCallback: (progress: number) => { - logger.debug({ tag: 'SYNC-ENGINE', msg: '[UPLOAD PROGRESS]', progress }); - }, - }); - - abortSignal.addEventListener( - 'abort', - () => { - state.stop(); - readable.destroy(); - }, - { once: true }, - ); - }); - } -} diff --git a/src/context/local/localFile/infrastructure/FsLocalFileRepository.test.ts b/src/context/local/localFile/infrastructure/FsLocalFileRepository.test.ts deleted file mode 100644 index 2edb5b2278..0000000000 --- a/src/context/local/localFile/infrastructure/FsLocalFileRepository.test.ts +++ /dev/null @@ -1,66 +0,0 @@ -import path from 'node:path'; -import { FsLocalFileRepository } from './FsLocalFileRepository'; -import { AbsolutePath } from './AbsolutePath'; -import { createFolderStructure, removeFolderStructure } from './__test-helpers__/folderStructure'; - -const TEST_FOLDER = path.join(__dirname, 'FsLocalFileRepositoryTestFolder') as AbsolutePath; -/** I need to skip this test as is giving this error: - * FAIL main src/context/local/localFile/infrastructure/FsLocalFileRepository.test.ts > FsLocalFileRepository - * Error: EEXIST: file already exists, - * mkdir 'drive-desktop-linux/src/context/local/localFile/infrastructure/FsLocalFileRepositoryTestFolder' - */ -describe.skip('FsLocalFileRepository', () => { - let SUT: FsLocalFileRepository; - - beforeAll(async () => { - await createFolderStructure(TEST_FOLDER); - - SUT = new FsLocalFileRepository(); - }); - - afterAll(async () => { - await removeFolderStructure(TEST_FOLDER); - }); - - describe('folders', () => { - it('obtains all the folders', async () => { - const folders = await SUT.folders(path.join(TEST_FOLDER, 'folder') as AbsolutePath); - - expect(folders).toStrictEqual( - expect.arrayContaining([ - path.join(TEST_FOLDER, 'folder', 'subfolder'), - path.join(TEST_FOLDER, 'folder', 'empty_folder'), - ]), - ); - }); - - it('returns an empty array if there are no folders', async () => { - const folders = await SUT.folders(path.join(TEST_FOLDER, 'folder', 'empty_folder') as AbsolutePath); - - expect(folders).toEqual([]); - }); - }); - - describe('files', () => { - it('obtains all files in the folder', async () => { - const files = await SUT.files(TEST_FOLDER); - - expect(files).toStrictEqual( - expect.arrayContaining([ - expect.objectContaining({ - _path: path.join(TEST_FOLDER, '.hidden'), - }), - expect.objectContaining({ - _path: path.join(TEST_FOLDER, 'hello_world.txt'), - }), - ]), - ); - }); - - it('returns an empty array if there are no files', async () => { - const files = await SUT.files(path.join(TEST_FOLDER, 'folder', 'empty_folder') as AbsolutePath); - - expect(files).toEqual([]); - }); - }); -}); diff --git a/src/context/local/localFile/infrastructure/FsLocalFileRepository.ts b/src/context/local/localFile/infrastructure/FsLocalFileRepository.ts deleted file mode 100644 index 9a18961464..0000000000 --- a/src/context/local/localFile/infrastructure/FsLocalFileRepository.ts +++ /dev/null @@ -1,41 +0,0 @@ -import { Service } from 'diod'; -import { LocalFileRepository } from '../domain/LocalFileRepository'; -import { LocalFile } from '../domain/LocalFile'; -import { AbsolutePath } from './AbsolutePath'; -import fs from 'fs/promises'; -import path from 'path'; - -@Service() -export class FsLocalFileRepository implements LocalFileRepository { - async files(absolutePath: AbsolutePath): Promise { - const dirents = await fs.readdir(absolutePath, { - withFileTypes: true, - }); - - const conversion = dirents - .filter((dirent) => dirent.isFile()) - .map(async (file) => { - const fileAbsolutePath = path.join(absolutePath, file.name) as AbsolutePath; - - const { mtime, size } = await fs.stat(fileAbsolutePath); - - return LocalFile.from({ - size: size, - path: fileAbsolutePath, - modificationTime: mtime.getTime(), - }); - }); - - const files = await Promise.all(conversion); - - return files; - } - - async folders(absolutePath: AbsolutePath): Promise { - const dirents = await fs.readdir(absolutePath, { withFileTypes: true }); - - return dirents - .filter((dirent) => dirent.isDirectory()) - .map((folder) => path.join(absolutePath, folder.name) as AbsolutePath); - } -} diff --git a/src/context/local/localFile/infrastructure/__test-helpers__/folderStructure.ts b/src/context/local/localFile/infrastructure/__test-helpers__/folderStructure.ts deleted file mode 100644 index 3673ce2608..0000000000 --- a/src/context/local/localFile/infrastructure/__test-helpers__/folderStructure.ts +++ /dev/null @@ -1,29 +0,0 @@ -import path from 'node:path'; -import { AbsolutePath } from '../AbsolutePath'; -import fs from 'fs/promises'; - -export const tree = { - folder: ['hello_world.txt', '.hidden', 'without_extension'], - 'folder/subfolder': ['myfile.txt'], - 'folder/empty_folder': [], -}; - -export async function createFolderStructure(basePath: AbsolutePath): Promise { - const foldersFilesTuple = Object.entries(tree); - - await fs.mkdir(basePath); - - const treeCreation = foldersFilesTuple.map(async ([folder, files]) => { - await fs.mkdir(path.join(basePath, folder), { recursive: true }); - - const write = files.map((file) => fs.writeFile(path.join(basePath, file), 'test file content')); - - await Promise.all(write); - }); - - await Promise.all(treeCreation); -} - -export async function removeFolderStructure(basePath: AbsolutePath) { - fs.rmdir(basePath, { recursive: true }); -} diff --git a/src/context/virtual-drive/files/application/create/SimpleFileCreator.ts b/src/context/virtual-drive/files/application/create/SimpleFileCreator.ts deleted file mode 100644 index 134027b4d6..0000000000 --- a/src/context/virtual-drive/files/application/create/SimpleFileCreator.ts +++ /dev/null @@ -1,55 +0,0 @@ -import { Service } from 'diod'; -import { RemoteFileSystem } from '../../domain/file-systems/RemoteFileSystem'; -import { FilePath } from '../../domain/FilePath'; -import { FileSize } from '../../domain/FileSize'; -import { FileContentsId } from '../../domain/FileContentsId'; -import { FileFolderId } from '../../domain/FileFolderId'; -import { File } from '../../domain/File'; -import { Either, left, right } from '../../../../shared/domain/Either'; -import { DriveDesktopError } from '../../../../shared/domain/errors/DriveDesktopError'; -// TODO: THIS WHOLE FILE IS GOING TO BE REPLACED -@Service() -export class SimpleFileCreator { - constructor(private readonly remote: RemoteFileSystem) {} - - async run( - contentsId: string, - path: string, - size: number, - folderId: number, - folderUuid: string, - ): Promise> { - const fileSize = new FileSize(size); - const fileContentsId = new FileContentsId(contentsId); - const filePath = new FilePath(path); - const fileFolderId = new FileFolderId(folderId); - - const either = await this.remote.persist({ - contentsId: fileContentsId, - path: filePath, - size: fileSize, - folderId: fileFolderId, - folderUuid, - }); - - if (either.isLeft()) { - return left(either.getLeft()); - } - - const dto = either.getRight(); - - const file = File.create({ - id: dto.id, - uuid: dto.uuid, - contentsId: fileContentsId.value, - folderId: fileFolderId.value, - createdAt: dto.createdAt, - modificationTime: dto.modificationTime, - path: filePath.value, - size: fileSize.value, - updatedAt: dto.modificationTime, - }); - - return right(file); - } -} diff --git a/src/context/virtual-drive/files/domain/File.ts b/src/context/virtual-drive/files/domain/File.ts index 94cbedfc83..090926d469 100644 --- a/src/context/virtual-drive/files/domain/File.ts +++ b/src/context/virtual-drive/files/domain/File.ts @@ -227,10 +227,6 @@ export class File extends AggregateRoot { } } - hasParent(id: number): boolean { - return this.folderId === id; - } - isFolder(): this is Folder { return false; } @@ -239,18 +235,6 @@ export class File extends AggregateRoot { return true; } - isThumbnable(): boolean { - return this._path.isThumbnable(); - } - - mimeType() { - return this._path.mimeType(); - } - - hasStatus(status: FileStatuses): boolean { - return this._status.is(status); - } - attributes(): FileAttributes { return { id: this.id, diff --git a/src/context/virtual-drive/files/domain/FilePath.ts b/src/context/virtual-drive/files/domain/FilePath.ts index b676c6c2a4..7f063173ae 100644 --- a/src/context/virtual-drive/files/domain/FilePath.ts +++ b/src/context/virtual-drive/files/domain/FilePath.ts @@ -1,8 +1,5 @@ import path from 'path'; import { Path } from '../../../shared/domain/value-objects/Path'; -import { thumbnableExtensions } from '../../../../apps/main/thumbnails/domain/ThumbnableExtension'; -import MimeTypesMap, { MimeType } from './MimeTypesMap'; - export class FilePath extends Path { constructor(value: string) { super(value); @@ -57,24 +54,4 @@ export class FilePath extends Path { updateName(name: string): FilePath { return FilePath.fromParts([this.dirname(), name]); } - - isThumbnable(): boolean { - if (!this.hasExtension()) { - return false; - } - - return thumbnableExtensions.includes(this.extension()); - } - - mimeType(): MimeType { - const extension = `.${this.extension()}`; - - const mimeType = MimeTypesMap[extension]; - - if (!mimeType) { - return 'application/octet-stream'; - } - - return mimeType; - } } diff --git a/src/context/virtual-drive/files/domain/MimeTypesMap.ts b/src/context/virtual-drive/files/domain/MimeTypesMap.ts deleted file mode 100644 index d79b91dee2..0000000000 --- a/src/context/virtual-drive/files/domain/MimeTypesMap.ts +++ /dev/null @@ -1,651 +0,0 @@ -const map = { - '.323': 'text/h323', - '.3g2': 'video/3gpp2', - '.3gp': 'video/3gpp', - '.3gp2': 'video/3gpp2', - '.3gpp': 'video/3gpp', - '.7z': 'application/x-7z-compressed', - '.aa': 'audio/audible', - '.AAC': 'audio/aac', - '.aaf': 'application/octet-stream', - '.aax': 'audio/vnd.audible.aax', - '.ac3': 'audio/ac3', - '.aca': 'application/octet-stream', - '.accda': 'application/msaccess.addin', - '.accdb': 'application/msaccess', - '.accdc': 'application/msaccess.cab', - '.accde': 'application/msaccess', - '.accdr': 'application/msaccess.runtime', - '.accdt': 'application/msaccess', - '.accdw': 'application/msaccess.webapplication', - '.accft': 'application/msaccess.ftemplate', - '.acx': 'application/internet-property-stream', - '.AddIn': 'text/xml', - '.ade': 'application/msaccess', - '.adobebridge': 'application/x-bridge-url', - '.adp': 'application/msaccess', - '.ADT': 'audio/vnd.dlna.adts', - '.ADTS': 'audio/aac', - '.afm': 'application/octet-stream', - '.ai': 'application/postscript', - '.aif': 'audio/aiff', - '.aifc': 'audio/aiff', - '.aiff': 'audio/aiff', - '.air': 'application/vnd.adobe.air-application-installer-package+zip', - '.amc': 'application/mpeg', - '.anx': 'application/annodex', - '.apk': 'application/vnd.android.package-archive', - '.apng': 'image/apng', - '.application': 'application/x-ms-application', - '.art': 'image/x-jg', - '.asa': 'application/xml', - '.asax': 'application/xml', - '.ascx': 'application/xml', - '.asd': 'application/octet-stream', - '.asf': 'video/x-ms-asf', - '.ashx': 'application/xml', - '.asi': 'application/octet-stream', - '.asm': 'text/plain', - '.asmx': 'application/xml', - '.aspx': 'application/xml', - '.asr': 'video/x-ms-asf', - '.asx': 'video/x-ms-asf', - '.atom': 'application/atom+xml', - '.au': 'audio/basic', - '.avci': 'image/avci', - '.avcs': 'image/avcs', - '.avi': 'video/x-msvideo', - '.avif': 'image/avif', - '.avifs': 'image/avif-sequence', - '.axa': 'audio/annodex', - '.axs': 'application/olescript', - '.axv': 'video/annodex', - '.bas': 'text/plain', - '.bcpio': 'application/x-bcpio', - '.bin': 'application/octet-stream', - '.bmp': 'image/bmp', - '.c': 'text/plain', - '.cab': 'application/octet-stream', - '.caf': 'audio/x-caf', - '.calx': 'application/vnd.ms-office.calx', - '.cat': 'application/vnd.ms-pki.seccat', - '.cc': 'text/plain', - '.cd': 'text/plain', - '.cdda': 'audio/aiff', - '.cdf': 'application/x-cdf', - '.cer': 'application/x-x509-ca-cert', - '.cfg': 'text/plain', - '.chm': 'application/octet-stream', - '.class': 'application/x-java-applet', - '.clp': 'application/x-msclip', - '.cmd': 'text/plain', - '.cmx': 'image/x-cmx', - '.cnf': 'text/plain', - '.cod': 'image/cis-cod', - '.config': 'application/xml', - '.contact': 'text/x-ms-contact', - '.coverage': 'application/xml', - '.cpio': 'application/x-cpio', - '.cpp': 'text/plain', - '.crd': 'application/x-mscardfile', - '.crl': 'application/pkix-crl', - '.crt': 'application/x-x509-ca-cert', - '.cs': 'text/plain', - '.csdproj': 'text/plain', - '.csh': 'application/x-csh', - '.csproj': 'text/plain', - '.css': 'text/css', - '.csv': 'text/csv', - '.cur': 'application/octet-stream', - '.czx': 'application/x-czx', - '.cxx': 'text/plain', - '.dat': 'application/octet-stream', - '.datasource': 'application/xml', - '.dbproj': 'text/plain', - '.dcr': 'application/x-director', - '.def': 'text/plain', - '.deploy': 'application/octet-stream', - '.der': 'application/x-x509-ca-cert', - '.dgml': 'application/xml', - '.dib': 'image/bmp', - '.dif': 'video/x-dv', - '.dir': 'application/x-director', - '.disco': 'text/xml', - '.divx': 'video/divx', - '.dll': 'application/x-msdownload', - '.dll.config': 'text/xml', - '.dlm': 'text/dlm', - '.doc': 'application/msword', - '.docm': 'application/vnd.ms-word.document.macroEnabled.12', - '.docx': 'application/vnd.openxmlformats-officedocument.wordprocessingml.document', - '.dot': 'application/msword', - '.dotm': 'application/vnd.ms-word.template.macroEnabled.12', - '.dotx': 'application/vnd.openxmlformats-officedocument.wordprocessingml.template', - '.dsp': 'application/octet-stream', - '.dsw': 'text/plain', - '.dtd': 'text/xml', - '.dtsConfig': 'text/xml', - '.dv': 'video/x-dv', - '.dvi': 'application/x-dvi', - '.dwf': 'drawing/x-dwf', - '.dwg': 'application/acad', - '.dwp': 'application/octet-stream', - '.dxf': 'application/x-dxf', - '.dxr': 'application/x-director', - '.eml': 'message/rfc822', - '.emf': 'image/emf', - '.emz': 'application/octet-stream', - '.eot': 'application/vnd.ms-fontobject', - '.eps': 'application/postscript', - '.es': 'application/ecmascript', - '.etl': 'application/etl', - '.etx': 'text/x-setext', - '.evy': 'application/envoy', - '.exe': 'application/vnd.microsoft.portable-executable', - '.exe.config': 'text/xml', - '.f4v': 'video/mp4', - '.fdf': 'application/vnd.fdf', - '.fif': 'application/fractals', - '.filters': 'application/xml', - '.fla': 'application/octet-stream', - '.flac': 'audio/flac', - '.flr': 'x-world/x-vrml', - '.flv': 'video/x-flv', - '.fsscript': 'application/fsharp-script', - '.fsx': 'application/fsharp-script', - '.generictest': 'application/xml', - '.geojson': 'application/geo+json', - '.gif': 'image/gif', - '.gpx': 'application/gpx+xml', - '.group': 'text/x-ms-group', - '.gsm': 'audio/x-gsm', - '.gtar': 'application/x-gtar', - '.gz': 'application/x-gzip', - '.h': 'text/plain', - '.hdf': 'application/x-hdf', - '.hdml': 'text/x-hdml', - '.heic': 'image/heic', - '.heics': 'image/heic-sequence', - '.heif': 'image/heif', - '.heifs': 'image/heif-sequence', - '.hhc': 'application/x-oleobject', - '.hhk': 'application/octet-stream', - '.hhp': 'application/octet-stream', - '.hlp': 'application/winhlp', - '.hpp': 'text/plain', - '.hqx': 'application/mac-binhex40', - '.hta': 'application/hta', - '.htc': 'text/x-component', - '.htm': 'text/html', - '.html': 'text/html', - '.htt': 'text/webviewhtml', - '.hxa': 'application/xml', - '.hxc': 'application/xml', - '.hxd': 'application/octet-stream', - '.hxe': 'application/xml', - '.hxf': 'application/xml', - '.hxh': 'application/octet-stream', - '.hxi': 'application/octet-stream', - '.hxk': 'application/xml', - '.hxq': 'application/octet-stream', - '.hxr': 'application/octet-stream', - '.hxs': 'application/octet-stream', - '.hxt': 'text/html', - '.hxv': 'application/xml', - '.hxw': 'application/octet-stream', - '.hxx': 'text/plain', - '.i': 'text/plain', - '.ical': 'text/calendar', - '.icalendar': 'text/calendar', - '.ico': 'image/x-icon', - '.ics': 'text/calendar', - '.idl': 'text/plain', - '.ief': 'image/ief', - '.ifb': 'text/calendar', - '.iii': 'application/x-iphone', - '.inc': 'text/plain', - '.inf': 'application/octet-stream', - '.ini': 'text/plain', - '.inl': 'text/plain', - '.ins': 'application/x-internet-signup', - '.ipa': 'application/x-itunes-ipa', - '.ipg': 'application/x-itunes-ipg', - '.ipproj': 'text/plain', - '.ipsw': 'application/x-itunes-ipsw', - '.iqy': 'text/x-ms-iqy', - '.isp': 'application/x-internet-signup', - '.isma': 'application/octet-stream', - '.ismv': 'application/octet-stream', - '.ite': 'application/x-itunes-ite', - '.itlp': 'application/x-itunes-itlp', - '.itms': 'application/x-itunes-itms', - '.itpc': 'application/x-itunes-itpc', - '.IVF': 'video/x-ivf', - '.jar': 'application/java-archive', - '.java': 'application/octet-stream', - '.jck': 'application/liquidmotion', - '.jcz': 'application/liquidmotion', - '.jfif': 'image/pjpeg', - '.jnlp': 'application/x-java-jnlp-file', - '.jpb': 'application/octet-stream', - '.jpe': 'image/jpeg', - '.jpeg': 'image/jpeg', - '.jpg': 'image/jpeg', - '.js': 'application/javascript', - '.json': 'application/json', - '.jsx': 'text/jscript', - '.jsxbin': 'text/plain', - '.key': 'application/vnd.apple.keynote', - '.latex': 'application/x-latex', - '.library-ms': 'application/windows-library+xml', - '.lit': 'application/x-ms-reader', - '.loadtest': 'application/xml', - '.lpk': 'application/octet-stream', - '.lsf': 'video/x-la-asf', - '.lst': 'text/plain', - '.lsx': 'video/x-la-asf', - '.lzh': 'application/octet-stream', - '.m13': 'application/x-msmediaview', - '.m14': 'application/x-msmediaview', - '.m1v': 'video/mpeg', - '.m2t': 'video/vnd.dlna.mpeg-tts', - '.m2ts': 'video/vnd.dlna.mpeg-tts', - '.m2v': 'video/mpeg', - '.m3u': 'audio/x-mpegurl', - '.m3u8': 'audio/x-mpegurl', - '.m4a': 'audio/m4a', - '.m4b': 'audio/m4b', - '.m4p': 'audio/m4p', - '.m4r': 'audio/x-m4r', - '.m4v': 'video/x-m4v', - '.mac': 'image/x-macpaint', - '.mak': 'text/plain', - '.man': 'application/x-troff-man', - '.manifest': 'application/x-ms-manifest', - '.map': 'text/plain', - '.master': 'application/xml', - '.mbox': 'application/mbox', - '.mda': 'application/msaccess', - '.mdb': 'application/x-msaccess', - '.mde': 'application/msaccess', - '.mdp': 'application/octet-stream', - '.me': 'application/x-troff-me', - '.mfp': 'application/x-shockwave-flash', - '.mht': 'message/rfc822', - '.mhtml': 'message/rfc822', - '.mid': 'audio/mid', - '.midi': 'audio/mid', - '.mix': 'application/octet-stream', - '.mk': 'text/plain', - '.mk3d': 'video/x-matroska-3d', - '.mka': 'audio/x-matroska', - '.mkv': 'video/x-matroska', - '.mmf': 'application/x-smaf', - '.mno': 'text/xml', - '.mny': 'application/x-msmoney', - '.mod': 'video/mpeg', - '.mov': 'video/quicktime', - '.movie': 'video/x-sgi-movie', - '.mp2': 'video/mpeg', - '.mp2v': 'video/mpeg', - '.mp3': 'audio/mpeg', - '.mp4': 'video/mp4', - '.mp4v': 'video/mp4', - '.mpa': 'video/mpeg', - '.mpe': 'video/mpeg', - '.mpeg': 'video/mpeg', - '.mpf': 'application/vnd.ms-mediapackage', - '.mpg': 'video/mpeg', - '.mpp': 'application/vnd.ms-project', - '.mpv2': 'video/mpeg', - '.mqv': 'video/quicktime', - '.ms': 'application/x-troff-ms', - '.msg': 'application/vnd.ms-outlook', - '.msi': 'application/octet-stream', - '.mso': 'application/octet-stream', - '.mts': 'video/vnd.dlna.mpeg-tts', - '.mtx': 'application/xml', - '.mvb': 'application/x-msmediaview', - '.mvc': 'application/x-miva-compiled', - '.mxf': 'application/mxf', - '.mxp': 'application/x-mmxp', - '.nc': 'application/x-netcdf', - '.nsc': 'video/x-ms-asf', - '.numbers': 'application/vnd.apple.numbers', - '.nws': 'message/rfc822', - '.ocx': 'application/octet-stream', - '.oda': 'application/oda', - '.odb': 'application/vnd.oasis.opendocument.database', - '.odc': 'application/vnd.oasis.opendocument.chart', - '.odf': 'application/vnd.oasis.opendocument.formula', - '.odg': 'application/vnd.oasis.opendocument.graphics', - '.odh': 'text/plain', - '.odi': 'application/vnd.oasis.opendocument.image', - '.odl': 'text/plain', - '.odm': 'application/vnd.oasis.opendocument.text-master', - '.odp': 'application/vnd.oasis.opendocument.presentation', - '.ods': 'application/vnd.oasis.opendocument.spreadsheet', - '.odt': 'application/vnd.oasis.opendocument.text', - '.oga': 'audio/ogg', - '.ogg': 'audio/ogg', - '.ogv': 'video/ogg', - '.ogx': 'application/ogg', - '.one': 'application/onenote', - '.onea': 'application/onenote', - '.onepkg': 'application/onenote', - '.onetmp': 'application/onenote', - '.onetoc': 'application/onenote', - '.onetoc2': 'application/onenote', - '.opus': 'audio/ogg', - '.orderedtest': 'application/xml', - '.osdx': 'application/opensearchdescription+xml', - '.otf': 'application/font-sfnt', - '.otg': 'application/vnd.oasis.opendocument.graphics-template', - '.oth': 'application/vnd.oasis.opendocument.text-web', - '.otp': 'application/vnd.oasis.opendocument.presentation-template', - '.ots': 'application/vnd.oasis.opendocument.spreadsheet-template', - '.ott': 'application/vnd.oasis.opendocument.text-template', - '.oxps': 'application/oxps', - '.oxt': 'application/vnd.openofficeorg.extension', - '.p10': 'application/pkcs10', - '.p12': 'application/x-pkcs12', - '.p7b': 'application/x-pkcs7-certificates', - '.p7c': 'application/pkcs7-mime', - '.p7m': 'application/pkcs7-mime', - '.p7r': 'application/x-pkcs7-certreqresp', - '.p7s': 'application/pkcs7-signature', - '.pages': 'application/vnd.apple.pages', - '.pbm': 'image/x-portable-bitmap', - '.pcast': 'application/x-podcast', - '.pct': 'image/pict', - '.pcx': 'application/octet-stream', - '.pcz': 'application/octet-stream', - '.pdf': 'application/pdf', - '.pfb': 'application/octet-stream', - '.pfm': 'application/octet-stream', - '.pfx': 'application/x-pkcs12', - '.pgm': 'image/x-portable-graymap', - '.pic': 'image/pict', - '.pict': 'image/pict', - '.pkgdef': 'text/plain', - '.pkgundef': 'text/plain', - '.pko': 'application/vnd.ms-pki.pko', - '.pls': 'audio/scpls', - '.pma': 'application/x-perfmon', - '.pmc': 'application/x-perfmon', - '.pml': 'application/x-perfmon', - '.pmr': 'application/x-perfmon', - '.pmw': 'application/x-perfmon', - '.png': 'image/png', - '.pnm': 'image/x-portable-anymap', - '.pnt': 'image/x-macpaint', - '.pntg': 'image/x-macpaint', - '.pnz': 'image/png', - '.pot': 'application/vnd.ms-powerpoint', - '.potm': 'application/vnd.ms-powerpoint.template.macroEnabled.12', - '.potx': 'application/vnd.openxmlformats-officedocument.presentationml.template', - '.ppa': 'application/vnd.ms-powerpoint', - '.ppam': 'application/vnd.ms-powerpoint.addin.macroEnabled.12', - '.ppm': 'image/x-portable-pixmap', - '.pps': 'application/vnd.ms-powerpoint', - '.ppsm': 'application/vnd.ms-powerpoint.slideshow.macroEnabled.12', - '.ppsx': 'application/vnd.openxmlformats-officedocument.presentationml.slideshow', - '.ppt': 'application/vnd.ms-powerpoint', - '.pptm': 'application/vnd.ms-powerpoint.presentation.macroEnabled.12', - '.pptx': 'application/vnd.openxmlformats-officedocument.presentationml.presentation', - '.prf': 'application/pics-rules', - '.prm': 'application/octet-stream', - '.prx': 'application/octet-stream', - '.ps': 'application/postscript', - '.psc1': 'application/PowerShell', - '.psd': 'application/octet-stream', - '.psess': 'application/xml', - '.psm': 'application/octet-stream', - '.psp': 'application/octet-stream', - '.pst': 'application/vnd.ms-outlook', - '.pub': 'application/x-mspublisher', - '.pwz': 'application/vnd.ms-powerpoint', - '.qht': 'text/x-html-insertion', - '.qhtm': 'text/x-html-insertion', - '.qt': 'video/quicktime', - '.qti': 'image/x-quicktime', - '.qtif': 'image/x-quicktime', - '.qtl': 'application/x-quicktimeplayer', - '.qxd': 'application/octet-stream', - '.ra': 'audio/x-pn-realaudio', - '.ram': 'audio/x-pn-realaudio', - '.rar': 'application/x-rar-compressed', - '.ras': 'image/x-cmu-raster', - '.rat': 'application/rat-file', - '.rc': 'text/plain', - '.rc2': 'text/plain', - '.rct': 'text/plain', - '.rdlc': 'application/xml', - '.reg': 'text/plain', - '.resx': 'application/xml', - '.rf': 'image/vnd.rn-realflash', - '.rgb': 'image/x-rgb', - '.rgs': 'text/plain', - '.rm': 'application/vnd.rn-realmedia', - '.rmi': 'audio/mid', - '.rmp': 'application/vnd.rn-rn_music_package', - '.rmvb': 'application/vnd.rn-realmedia-vbr', - '.roff': 'application/x-troff', - '.rpm': 'audio/x-pn-realaudio-plugin', - '.rqy': 'text/x-ms-rqy', - '.rtf': 'application/rtf', - '.rtx': 'text/richtext', - '.rvt': 'application/octet-stream', - '.ruleset': 'application/xml', - '.s': 'text/plain', - '.safariextz': 'application/x-safari-safariextz', - '.scd': 'application/x-msschedule', - '.scr': 'text/plain', - '.sct': 'text/scriptlet', - '.sd2': 'audio/x-sd2', - '.sdp': 'application/sdp', - '.sea': 'application/octet-stream', - '.searchConnector-ms': 'application/windows-search-connector+xml', - '.setpay': 'application/set-payment-initiation', - '.setreg': 'application/set-registration-initiation', - '.settings': 'application/xml', - '.sgimb': 'application/x-sgimb', - '.sgml': 'text/sgml', - '.sh': 'application/x-sh', - '.shar': 'application/x-shar', - '.shtml': 'text/html', - '.sit': 'application/x-stuffit', - '.sitemap': 'application/xml', - '.skin': 'application/xml', - '.skp': 'application/x-koan', - '.sldm': 'application/vnd.ms-powerpoint.slide.macroEnabled.12', - '.sldx': 'application/vnd.openxmlformats-officedocument.presentationml.slide', - '.slk': 'application/vnd.ms-excel', - '.sln': 'text/plain', - '.slupkg-ms': 'application/x-ms-license', - '.smd': 'audio/x-smd', - '.smi': 'application/octet-stream', - '.smx': 'audio/x-smd', - '.smz': 'audio/x-smd', - '.snd': 'audio/basic', - '.snippet': 'application/xml', - '.snp': 'application/octet-stream', - '.sql': 'application/sql', - '.sol': 'text/plain', - '.sor': 'text/plain', - '.spc': 'application/x-pkcs7-certificates', - '.spl': 'application/futuresplash', - '.spx': 'audio/ogg', - '.src': 'application/x-wais-source', - '.srf': 'text/plain', - '.SSISDeploymentManifest': 'text/xml', - '.ssm': 'application/streamingmedia', - '.sst': 'application/vnd.ms-pki.certstore', - '.stl': 'application/vnd.ms-pki.stl', - '.sv4cpio': 'application/x-sv4cpio', - '.sv4crc': 'application/x-sv4crc', - '.svc': 'application/xml', - '.svg': 'image/svg+xml', - '.swf': 'application/x-shockwave-flash', - '.step': 'application/step', - '.stp': 'application/step', - '.t': 'application/x-troff', - '.tar': 'application/x-tar', - '.tcl': 'application/x-tcl', - '.testrunconfig': 'application/xml', - '.testsettings': 'application/xml', - '.tex': 'application/x-tex', - '.texi': 'application/x-texinfo', - '.texinfo': 'application/x-texinfo', - '.tgz': 'application/x-compressed', - '.thmx': 'application/vnd.ms-officetheme', - '.thn': 'application/octet-stream', - '.tif': 'image/tiff', - '.tiff': 'image/tiff', - '.tlh': 'text/plain', - '.tli': 'text/plain', - '.toc': 'application/octet-stream', - '.tr': 'application/x-troff', - '.trm': 'application/x-msterminal', - '.trx': 'application/xml', - '.ts': 'video/vnd.dlna.mpeg-tts', - '.tsv': 'text/tab-separated-values', - '.ttf': 'application/font-sfnt', - '.tts': 'video/vnd.dlna.mpeg-tts', - '.txt': 'text/plain', - '.u32': 'application/octet-stream', - '.uls': 'text/iuls', - '.user': 'text/plain', - '.ustar': 'application/x-ustar', - '.vb': 'text/plain', - '.vbdproj': 'text/plain', - '.vbk': 'video/mpeg', - '.vbproj': 'text/plain', - '.vbs': 'text/vbscript', - '.vcf': 'text/x-vcard', - '.vcproj': 'application/xml', - '.vcs': 'text/plain', - '.vcxproj': 'application/xml', - '.vddproj': 'text/plain', - '.vdp': 'text/plain', - '.vdproj': 'text/plain', - '.vdx': 'application/vnd.ms-visio.viewer', - '.vml': 'text/xml', - '.vscontent': 'application/xml', - '.vsct': 'text/xml', - '.vsd': 'application/vnd.visio', - '.vsi': 'application/ms-vsi', - '.vsix': 'application/vsix', - '.vsixlangpack': 'text/xml', - '.vsixmanifest': 'text/xml', - '.vsmdi': 'application/xml', - '.vspscc': 'text/plain', - '.vss': 'application/vnd.visio', - '.vsscc': 'text/plain', - '.vssettings': 'text/xml', - '.vssscc': 'text/plain', - '.vst': 'application/vnd.visio', - '.vstemplate': 'text/xml', - '.vsto': 'application/x-ms-vsto', - '.vsw': 'application/vnd.visio', - '.vsx': 'application/vnd.visio', - '.vtt': 'text/vtt', - '.vtx': 'application/vnd.visio', - '.wasm': 'application/wasm', - '.wav': 'audio/wav', - '.wave': 'audio/wav', - '.wax': 'audio/x-ms-wax', - '.wbk': 'application/msword', - '.wbmp': 'image/vnd.wap.wbmp', - '.wcm': 'application/vnd.ms-works', - '.wdb': 'application/vnd.ms-works', - '.wdp': 'image/vnd.ms-photo', - '.webarchive': 'application/x-safari-webarchive', - '.webm': 'video/webm', - '.webp': 'image/webp', - '.webtest': 'application/xml', - '.wiq': 'application/xml', - '.wiz': 'application/msword', - '.wks': 'application/vnd.ms-works', - '.WLMP': 'application/wlmoviemaker', - '.wlpginstall': 'application/x-wlpg-detect', - '.wlpginstall3': 'application/x-wlpg3-detect', - '.wm': 'video/x-ms-wm', - '.wma': 'audio/x-ms-wma', - '.wmd': 'application/x-ms-wmd', - '.wmf': 'application/x-msmetafile', - '.wml': 'text/vnd.wap.wml', - '.wmlc': 'application/vnd.wap.wmlc', - '.wmls': 'text/vnd.wap.wmlscript', - '.wmlsc': 'application/vnd.wap.wmlscriptc', - '.wmp': 'video/x-ms-wmp', - '.wmv': 'video/x-ms-wmv', - '.wmx': 'video/x-ms-wmx', - '.wmz': 'application/x-ms-wmz', - '.woff': 'application/font-woff', - '.woff2': 'application/font-woff2', - '.wpl': 'application/vnd.ms-wpl', - '.wps': 'application/vnd.ms-works', - '.wri': 'application/x-mswrite', - '.wrl': 'x-world/x-vrml', - '.wrz': 'x-world/x-vrml', - '.wsc': 'text/scriptlet', - '.wsdl': 'text/xml', - '.wvx': 'video/x-ms-wvx', - '.x': 'application/directx', - '.xaf': 'x-world/x-vrml', - '.xaml': 'application/xaml+xml', - '.xap': 'application/x-silverlight-app', - '.xbap': 'application/x-ms-xbap', - '.xbm': 'image/x-xbitmap', - '.xdr': 'text/plain', - '.xht': 'application/xhtml+xml', - '.xhtml': 'application/xhtml+xml', - '.xla': 'application/vnd.ms-excel', - '.xlam': 'application/vnd.ms-excel.addin.macroEnabled.12', - '.xlc': 'application/vnd.ms-excel', - '.xld': 'application/vnd.ms-excel', - '.xlk': 'application/vnd.ms-excel', - '.xll': 'application/vnd.ms-excel', - '.xlm': 'application/vnd.ms-excel', - '.xls': 'application/vnd.ms-excel', - '.xlsb': 'application/vnd.ms-excel.sheet.binary.macroEnabled.12', - '.xlsm': 'application/vnd.ms-excel.sheet.macroEnabled.12', - '.xlsx': 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', - '.xlt': 'application/vnd.ms-excel', - '.xltm': 'application/vnd.ms-excel.template.macroEnabled.12', - '.xltx': 'application/vnd.openxmlformats-officedocument.spreadsheetml.template', - '.xlw': 'application/vnd.ms-excel', - '.xml': 'text/xml', - '.xmp': 'application/octet-stream', - '.xmta': 'application/xml', - '.xof': 'x-world/x-vrml', - '.XOML': 'text/plain', - '.xpm': 'image/x-xpixmap', - '.xps': 'application/vnd.ms-xpsdocument', - '.xrm-ms': 'text/xml', - '.xsc': 'application/xml', - '.xsd': 'text/xml', - '.xsf': 'text/xml', - '.xsl': 'text/xml', - '.xslt': 'text/xml', - '.xsn': 'application/octet-stream', - '.xss': 'application/xml', - '.xspf': 'application/xspf+xml', - '.xtp': 'application/octet-stream', - '.xwd': 'image/x-xwindowdump', - '.z': 'application/x-compress', - '.zip': 'application/zip', -} as const; - -const keys = Object.keys(map); - -const types = Object.values(map); - -export type MimeType = (typeof types)[number]; - -type MimeTypesMap = { - [key: (typeof keys)[number]]: MimeType; -}; - -export default map as MimeTypesMap; From 393abceab1807a14d81755a82fb3b7cab0a80588 Mon Sep 17 00:00:00 2001 From: AlexisMora Date: Tue, 24 Mar 2026 17:26:04 +0100 Subject: [PATCH 8/9] delete: ModifiedBatchCreator --- .../batches/AddedFilesBatchCreator.test.ts | 31 ---------- .../backups/batches/AddedFilesBatchCreator.ts | 18 ------ .../backups/batches/GroupFilesBySize.test.ts | 59 ------------------- src/apps/backups/batches/GroupFilesBySize.ts | 19 ------ .../batches/GroupFilesInChunksBySize.test.ts | 41 ------------- .../batches/GroupFilesInChunksBySize.ts | 32 ---------- .../batches/ModifiedFilesBatchCreator.test.ts | 45 -------------- .../batches/ModifiedFilesBatchCreator.ts | 28 --------- 8 files changed, 273 deletions(-) delete mode 100644 src/apps/backups/batches/AddedFilesBatchCreator.test.ts delete mode 100644 src/apps/backups/batches/AddedFilesBatchCreator.ts delete mode 100644 src/apps/backups/batches/GroupFilesBySize.test.ts delete mode 100644 src/apps/backups/batches/GroupFilesBySize.ts delete mode 100644 src/apps/backups/batches/GroupFilesInChunksBySize.test.ts delete mode 100644 src/apps/backups/batches/GroupFilesInChunksBySize.ts delete mode 100644 src/apps/backups/batches/ModifiedFilesBatchCreator.test.ts delete mode 100644 src/apps/backups/batches/ModifiedFilesBatchCreator.ts diff --git a/src/apps/backups/batches/AddedFilesBatchCreator.test.ts b/src/apps/backups/batches/AddedFilesBatchCreator.test.ts deleted file mode 100644 index be23ef343e..0000000000 --- a/src/apps/backups/batches/AddedFilesBatchCreator.test.ts +++ /dev/null @@ -1,31 +0,0 @@ -import { AddedFilesBatchCreator } from './AddedFilesBatchCreator'; -import { LocalFileSize } from '../../../context/local/localFile/domain/LocalFileSize'; -import { LocalFileMother } from '../../../context/local/localFile/domain/__test-helpers__/LocalFileMother'; - -describe('AddedFilesBatchCreator', () => { - it('should create batches of added files grouped by size', () => { - const localFileSmall = LocalFileMother.fromPartial({ - size: LocalFileSize.MAX_SMALL_FILE_SIZE - 1, - }); - const localFileMedium = LocalFileMother.fromPartial({ - size: LocalFileSize.MAX_SMALL_FILE_SIZE + 1, - }); - const localFileBig = LocalFileMother.fromPartial({ - size: LocalFileSize.MAX_MEDIUM_FILE_SIZE + 1, - }); - - const files = [localFileSmall, localFileMedium, localFileBig]; - - const batches = AddedFilesBatchCreator.run(files); - - expect(batches.length).toBe(3); - - expect(batches[0]).toContain(localFileSmall); - expect(batches[1]).toContain(localFileMedium); - expect(batches[2]).toContain(localFileBig); - - expect(batches[0].length).toBe(1); - expect(batches[1].length).toBe(1); - expect(batches[2].length).toBe(1); - }); -}); diff --git a/src/apps/backups/batches/AddedFilesBatchCreator.ts b/src/apps/backups/batches/AddedFilesBatchCreator.ts deleted file mode 100644 index cbd91c7b3a..0000000000 --- a/src/apps/backups/batches/AddedFilesBatchCreator.ts +++ /dev/null @@ -1,18 +0,0 @@ -import { GroupFilesBySize } from './GroupFilesBySize'; -import { GroupFilesInChunksBySize } from './GroupFilesInChunksBySize'; -import { LocalFile } from '../../../context/local/localFile/domain/LocalFile'; - -export class AddedFilesBatchCreator { - private static readonly sizes = ['empty', 'small', 'medium', 'big'] as const; - - static run(files: Array): Array> { - const nonEmptyFiles = files.filter((f) => f.size > 0); - - const batches = AddedFilesBatchCreator.sizes.flatMap((size) => { - const groupedBySize = GroupFilesBySize[size](nonEmptyFiles); - return GroupFilesInChunksBySize[size](groupedBySize); - }); - - return batches; - } -} diff --git a/src/apps/backups/batches/GroupFilesBySize.test.ts b/src/apps/backups/batches/GroupFilesBySize.test.ts deleted file mode 100644 index 9e4fc5cf72..0000000000 --- a/src/apps/backups/batches/GroupFilesBySize.test.ts +++ /dev/null @@ -1,59 +0,0 @@ -import { GroupFilesBySize } from './GroupFilesBySize'; -import { LocalFileMother } from '../../../context/local/localFile/domain/__test-helpers__/LocalFileMother'; -import { LocalFileSize } from '../../../context/local/localFile/domain/LocalFileSize'; - -describe('GroupFilesBySize', () => { - it('should return an empty array when no files are provided', () => { - const groupedFiles = GroupFilesBySize.small([]); // Test for small files - expect(groupedFiles).toEqual([]); - }); - - it('should group only empty files', () => { - const emptyFile1 = LocalFileMother.fromPartial({ size: 0 }); - const emptyFile2 = LocalFileMother.fromPartial({ size: 0 }); - - const groupedFiles = GroupFilesBySize.empty([emptyFile1, emptyFile2]); - expect(groupedFiles).toEqual([emptyFile1, emptyFile2]); - }); - - // TODO: This test is skipped because LocalFileSize.isSmall() includes empty files (size 0) - // in the "small" category. The logic should be: isSmall() should return true only for - // files with size > 0 AND size <= MAX_SMALL_FILE_SIZE. Empty files should be a separate - // category. This needs to be fixed in LocalFileSize.ts line 24. - it.skip('should group mixed file sizes correctly', () => { - const emptyFile = LocalFileMother.fromPartial({ size: 0 }); - const smallFile = LocalFileMother.fromPartial({ - size: LocalFileSize.MAX_SMALL_FILE_SIZE - 1, - }); - const mediumFile = LocalFileMother.fromPartial({ - size: LocalFileSize.MAX_SMALL_FILE_SIZE + 1, - }); - const bigFile = LocalFileMother.fromPartial({ - size: LocalFileSize.MAX_MEDIUM_FILE_SIZE + 1, - }); - - const files = [emptyFile, smallFile, mediumFile, bigFile]; - - const groupedSmallFiles = GroupFilesBySize.small(files); - const groupedMediumFiles = GroupFilesBySize.medium(files); - const groupedBigFiles = GroupFilesBySize.big(files); - const groupedEmptyFiles = GroupFilesBySize.empty(files); - - expect(groupedSmallFiles).toEqual([smallFile]); - expect(groupedMediumFiles).toEqual([mediumFile]); - expect(groupedBigFiles).toEqual([bigFile]); - expect(groupedEmptyFiles).toEqual([emptyFile]); - }); - - it('should group all files of the same size correctly', () => { - const smallFile1 = LocalFileMother.fromPartial({ - size: LocalFileSize.MAX_SMALL_FILE_SIZE - 1, - }); - const smallFile2 = LocalFileMother.fromPartial({ - size: LocalFileSize.MAX_SMALL_FILE_SIZE - 1, - }); - - const groupedFiles = GroupFilesBySize.small([smallFile1, smallFile2]); - expect(groupedFiles).toEqual([smallFile1, smallFile2]); - }); -}); diff --git a/src/apps/backups/batches/GroupFilesBySize.ts b/src/apps/backups/batches/GroupFilesBySize.ts deleted file mode 100644 index 225e4526c8..0000000000 --- a/src/apps/backups/batches/GroupFilesBySize.ts +++ /dev/null @@ -1,19 +0,0 @@ -import { LocalFile } from '../../../context/local/localFile/domain/LocalFile'; - -export class GroupFilesBySize { - static empty(files: Array) { - return files.filter((file) => file.isEmpty()); - } - - static small(files: Array) { - return files.filter((file) => file.isSmall()); - } - - static medium(files: Array) { - return files.filter((file) => file.isMedium()); - } - - static big(files: Array) { - return files.filter((file) => file.isBig()); - } -} diff --git a/src/apps/backups/batches/GroupFilesInChunksBySize.test.ts b/src/apps/backups/batches/GroupFilesInChunksBySize.test.ts deleted file mode 100644 index 915c849c85..0000000000 --- a/src/apps/backups/batches/GroupFilesInChunksBySize.test.ts +++ /dev/null @@ -1,41 +0,0 @@ -import { GroupFilesInChunksBySize } from './GroupFilesInChunksBySize'; -import { LocalFile } from '../../../context/local/localFile/domain/LocalFile'; -import { LocalFileMother } from '../../../context/local/localFile/domain/__test-helpers__/LocalFileMother'; - -describe('GroupFilesInChunksBySize', () => { - const generateFiles = (count: number): Array => { - return Array.from({ length: count }, (_, i) => LocalFileMother.fromPartial({ size: i })); - }; - - test('should group small files into 16 chunks', () => { - const files = generateFiles(32); - const chunks = GroupFilesInChunksBySize.small(files); - expect(chunks.length).toBe(2); - expect(chunks[0].length).toBe(16); - expect(chunks[1].length).toBe(16); - }); - - test('should group medium files into 6 chunks', () => { - const files = generateFiles(18); - const chunks = GroupFilesInChunksBySize.medium(files); - expect(chunks.length).toBe(3); - expect(chunks[0].length).toBe(6); - expect(chunks[1].length).toBe(6); - expect(chunks[2].length).toBe(6); - }); - - test('should group big files into 2 chunks', () => { - const files = generateFiles(4); - const chunks = GroupFilesInChunksBySize.big(files); - expect(chunks.length).toBe(2); - expect(chunks[0].length).toBe(2); - expect(chunks[1].length).toBe(2); - }); - - test('should group files into a single chunk when empty is called', () => { - const files = generateFiles(5); - const chunks = GroupFilesInChunksBySize.empty(files); - expect(chunks.length).toBe(1); - expect(chunks[0].length).toBe(5); - }); -}); diff --git a/src/apps/backups/batches/GroupFilesInChunksBySize.ts b/src/apps/backups/batches/GroupFilesInChunksBySize.ts deleted file mode 100644 index 1e75486244..0000000000 --- a/src/apps/backups/batches/GroupFilesInChunksBySize.ts +++ /dev/null @@ -1,32 +0,0 @@ -import _ from 'lodash'; -import { LocalFile } from '../../../context/local/localFile/domain/LocalFile'; - -export type Chucks = Array>; - -const NUMBER_OF_PARALLEL_QUEUES_FOR_SMALL_FILES = 16; - -const NUMBER_OF_PARALLEL_QUEUES_FOR_MEDIUM_FILES = 6; - -const NUMBER_OF_PARALLEL_QUEUES_FOR_BIG_FILES = 2; - -export class GroupFilesInChunksBySize { - static small(all: Array): Chucks { - return GroupFilesInChunksBySize.chunk(all, NUMBER_OF_PARALLEL_QUEUES_FOR_SMALL_FILES); - } - - static medium(all: Array): Chucks { - return GroupFilesInChunksBySize.chunk(all, NUMBER_OF_PARALLEL_QUEUES_FOR_MEDIUM_FILES); - } - - static big(all: Array): Chucks { - return GroupFilesInChunksBySize.chunk(all, NUMBER_OF_PARALLEL_QUEUES_FOR_BIG_FILES); - } - - static empty(all: Array): Chucks { - return GroupFilesInChunksBySize.chunk(all, all.length); - } - - private static chunk(files: Array, size: number) { - return _.chunk(files, size); - } -} diff --git a/src/apps/backups/batches/ModifiedFilesBatchCreator.test.ts b/src/apps/backups/batches/ModifiedFilesBatchCreator.test.ts deleted file mode 100644 index 93affefb6b..0000000000 --- a/src/apps/backups/batches/ModifiedFilesBatchCreator.test.ts +++ /dev/null @@ -1,45 +0,0 @@ -import { ModifiedFilesBatchCreator } from './ModifiedFilesBatchCreator'; -import { LocalFile } from '../../../context/local/localFile/domain/LocalFile'; -import { LocalFileSize } from '../../../context/local/localFile/domain/LocalFileSize'; -import { File } from '../../../context/virtual-drive/files/domain/File'; -import { LocalFileMother } from '../../../context/local/localFile/domain/__test-helpers__/LocalFileMother'; -import { FileMother } from '../../../context/virtual-drive/files/domain/__test-helpers__/FileMother'; -describe('ModifiedFilesBatchCreator', () => { - it('should create batches of modified files grouped by size', () => { - const localFileSmall = LocalFileMother.fromPartial({ - size: LocalFileSize.MAX_SMALL_FILE_SIZE - 1, - }); - const localFileMedium = LocalFileMother.fromPartial({ - size: LocalFileSize.MAX_SMALL_FILE_SIZE + 1, - }); - const localFileBig = LocalFileMother.fromPartial({ - size: LocalFileSize.MAX_MEDIUM_FILE_SIZE + 1, - }); - - const smallFile = FileMother.fromPartial({}); - const mediumFile = FileMother.fromPartial({}); - const bigFile = FileMother.fromPartial({}); - - const files = new Map([ - [localFileSmall, smallFile], - [localFileMedium, mediumFile], - [localFileBig, bigFile], - ]); - - const batches = ModifiedFilesBatchCreator.run(files); - - expect(batches.length).toBe(3); - - expect(batches[0].get(localFileSmall)).toBeDefined(); - expect(batches[1].get(localFileMedium)).toBeDefined(); - expect(batches[2].get(localFileBig)).toBeDefined(); - - expect(batches[0].size).toBe(1); - expect(batches[1].size).toBe(1); - expect(batches[2].size).toBe(1); - - expect(batches[0].get(localFileSmall)).toBe(smallFile); - expect(batches[1].get(localFileMedium)).toBe(mediumFile); - expect(batches[2].get(localFileBig)).toBe(bigFile); - }); -}); diff --git a/src/apps/backups/batches/ModifiedFilesBatchCreator.ts b/src/apps/backups/batches/ModifiedFilesBatchCreator.ts deleted file mode 100644 index 0e40ead338..0000000000 --- a/src/apps/backups/batches/ModifiedFilesBatchCreator.ts +++ /dev/null @@ -1,28 +0,0 @@ -import { GroupFilesBySize } from './GroupFilesBySize'; -import { GroupFilesInChunksBySize } from './GroupFilesInChunksBySize'; -import { LocalFile } from '../../../context/local/localFile/domain/LocalFile'; -import { File } from '../../../context/virtual-drive/files/domain/File'; - -export class ModifiedFilesBatchCreator { - private static readonly sizes = ['small', 'medium', 'big'] as const; - - static run(files: Map): Array> { - const batches = ModifiedFilesBatchCreator.sizes.flatMap((size) => { - const localFiles = Array.from(files.keys()); - - const groupedBySize = GroupFilesBySize[size](localFiles); - - return GroupFilesInChunksBySize[size](groupedBySize); - }); - - return batches.map((batch) => - batch.reduce((map, local) => { - const file = files.get(local); - - map.set(local, file); - - return map; - }, new Map()), - ); - } -} From 09d3cfa0d9baa1b79e71ed200dc7214e7d36baa2 Mon Sep 17 00:00:00 2001 From: AlexisMora Date: Tue, 24 Mar 2026 17:33:42 +0100 Subject: [PATCH 9/9] format:fix --- src/apps/backups/BackupService.test.ts | 4 +--- src/apps/backups/BackupService.ts | 5 ++++- src/backend/features/backup/upload/upload-file-with-retry.ts | 1 - 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/apps/backups/BackupService.test.ts b/src/apps/backups/BackupService.test.ts index 2039942452..dfed5b5a26 100644 --- a/src/apps/backups/BackupService.test.ts +++ b/src/apps/backups/BackupService.test.ts @@ -145,9 +145,7 @@ describe('BackupService', () => { vi.mocked(localTreeBuilder.run).mockResolvedValueOnce(right(LocalTreeMother.oneLevel(10))); vi.mocked(remoteTreeBuilder.run).mockResolvedValueOnce(RemoteTreeMother.oneLevel(10)); mockValidateSpace.mockResolvedValueOnce({ data: { hasSpace: true } }); - executeAsyncQueueMock - .mockResolvedValueOnce({ data: undefined }) - .mockResolvedValueOnce({ error: fatalError }); + executeAsyncQueueMock.mockResolvedValueOnce({ data: undefined }).mockResolvedValueOnce({ error: fatalError }); const result = await backupService.run(info, abortController.signal, tracker); diff --git a/src/apps/backups/BackupService.ts b/src/apps/backups/BackupService.ts index baa2dd679b..f3286cc7cf 100644 --- a/src/apps/backups/BackupService.ts +++ b/src/apps/backups/BackupService.ts @@ -1,7 +1,10 @@ import { Environment } from '@internxt/inxt-js'; import { Service } from 'diod'; import { executeAsyncQueue } from '../../backend/common/async-queue/execute-async-queue'; -import { createBackupUpdateExecutor, ModifiedFilePair } from '../../backend/features/backup/upload/create-backup-update-executor'; +import { + createBackupUpdateExecutor, + ModifiedFilePair, +} from '../../backend/features/backup/upload/create-backup-update-executor'; import { DEFAULT_CONCURRENCY } from '../../backend/features/backup/upload/constants'; import { LocalFile } from '../../context/local/localFile/domain/LocalFile'; import { AbsolutePath } from '../../context/local/localFile/infrastructure/AbsolutePath'; diff --git a/src/backend/features/backup/upload/upload-file-with-retry.ts b/src/backend/features/backup/upload/upload-file-with-retry.ts index dc365c22f2..4abb1c38e1 100644 --- a/src/backend/features/backup/upload/upload-file-with-retry.ts +++ b/src/backend/features/backup/upload/upload-file-with-retry.ts @@ -10,7 +10,6 @@ import { uploadContentToEnvironment } from './upload-content-to-environment'; import { Result } from '../../../../context/shared/domain/Result'; import { MAX_RETRIES, RETRY_DELAYS_MS } from './constants'; - export type UploadFileParams = { path: string; size: number;