Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 5 additions & 1 deletion .talismanrc
Original file line number Diff line number Diff line change
Expand Up @@ -202,5 +202,9 @@ fileignoreconfig:
- filename: packages/contentstack-import/test/unit/utils/logger.test.ts
checksum: 794e06e657a7337c8f094d6042fb04c779683f97b860efae14e075098d2af024
- filename: packages/contentstack-import-setup/src/import/modules/taxonomies.ts
checksum: 49dd8e754a0d3635585a74e943ab097593f061089a7cddc22683ec6caddbb3c5
checksum: c1bccc885b3f41f187f150c739b4bbd1608b01f09b0d9be0ad9214127cac071d
- filename: packages/contentstack-import-setup/src/commands/cm/stacks/import-setup.ts
checksum: 06035980b36802260f190af6e63632efe167f5b336693163f59268f3e788fba1
- filename: packages/contentstack-import-setup/src/utils/constants.ts
checksum: fcfabb4c53ee822e05903db77595413842d656b55e2869bae97bb6c0e0e209c3
version: '1.0'
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,8 @@ import {
ContentstackClient,
pathValidator,
formatError,
CLIProgressManager,
configHandler,
} from '@contentstack/cli-utilities';

import { ImportConfig } from '../../../types';
Expand Down Expand Up @@ -71,9 +73,27 @@ export default class ImportSetupCommand extends Command {
importSetupConfig.host = this.cmaHost;
importSetupConfig.region = this.region;
importSetupConfig.developerHubBaseUrl = this.developerHubUrl;

// Prepare the context object
const context = this.createImportContext(importSetupConfig.apiKey, importSetupConfig.authenticationMethod);
importSetupConfig.context = { ...context };

if (flags.branch) {
CLIProgressManager.initializeGlobalSummary(
`IMPORT-SETUP-${flags.branch}`,
flags.branch,
`Setting up import for "${flags.branch}" branch...`,
);
} else {
CLIProgressManager.initializeGlobalSummary(`IMPORT-SETUP`, flags.branch, 'Setting up import...');
}

const managementAPIClient: ContentstackClient = await managementSDKClient(importSetupConfig);
const importSetup = new ImportSetup(importSetupConfig, managementAPIClient);
await importSetup.start();

CLIProgressManager.printGlobalSummary();

log(
importSetupConfig,
`Backup folder and mapper files have been successfully created for the stack using the API key ${importSetupConfig.apiKey}.`,
Expand All @@ -85,11 +105,24 @@ export default class ImportSetupCommand extends Command {
'success',
);
} catch (error) {
CLIProgressManager.printGlobalSummary();
log(
{ data: '' } as ImportConfig,
`Failed to create the backup folder and mapper files: ${formatError(error)}`,
'error',
);
}
}

private createImportContext(apiKey: string, authenticationMethod?: string): any {
return {
command: this.context?.info?.command || 'cm:stacks:import-setup',
module: '',
userId: configHandler.get('userUid') || '',
sessionId: this.context?.sessionId,
apiKey: apiKey || '',
orgId: configHandler.get('oauthOrgUid') || '',
authenticationMethod: authenticationMethod || 'Basic Auth',
};
}
}
47 changes: 47 additions & 0 deletions packages/contentstack-import-setup/src/import/modules/assets.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import { AssetRecord, ImportConfig, ModuleClassParams } from '../../types';
import { isEmpty, orderBy, values } from 'lodash';
import { formatError, FsUtility, sanitizePath } from '@contentstack/cli-utilities';
import BaseImportSetup from './base-setup';
import { MODULE_NAMES, MODULE_CONTEXTS, PROCESS_NAMES, PROCESS_STATUS } from '../../utils';

export default class AssetImportSetup extends BaseImportSetup {
private assetsFilePath: string;
Expand All @@ -20,6 +21,7 @@ export default class AssetImportSetup extends BaseImportSetup {

constructor({ config, stackAPIClient, dependencies }: ModuleClassParams) {
super({ config, stackAPIClient, dependencies });
this.currentModuleName = MODULE_NAMES[MODULE_CONTEXTS.ASSETS];
this.assetsFolderPath = join(sanitizePath(this.config.contentDir), 'assets');
this.assetsFilePath = join(sanitizePath(this.config.contentDir), 'assets', 'assets.json');
this.assetsConfig = config.modules.assets;
Expand All @@ -39,10 +41,51 @@ export default class AssetImportSetup extends BaseImportSetup {
*/
async start() {
try {
const progress = this.createNestedProgress(this.currentModuleName);

// Analyze to get chunk count
const indexerCount = await this.withLoadingSpinner('ASSETS: Analyzing import data...', async () => {
const basePath = this.assetsFolderPath;
const fs = new FsUtility({ basePath, indexFileName: 'assets.json' });
const indexer = fs.indexFileContent;
return values(indexer).length;
});

if (indexerCount === 0) {
log(this.config, 'No assets found in the content folder.', 'info');
return;
}

// Add processes - use a large number for total assets since we don't know exact count
// The progress will update as we process each asset
progress.addProcess(PROCESS_NAMES.ASSETS_MAPPER_GENERATION, 1);
progress.addProcess(PROCESS_NAMES.ASSETS_FETCH_AND_MAP, indexerCount * 10); // Estimate: ~10 assets per chunk

// Create mapper directory
progress
.startProcess(PROCESS_NAMES.ASSETS_MAPPER_GENERATION)
.updateStatus(
PROCESS_STATUS.ASSETS_MAPPER_GENERATION.GENERATING,
PROCESS_NAMES.ASSETS_MAPPER_GENERATION,
);
fsUtil.makeDirectory(this.mapperDirPath);
this.progressManager?.tick(true, 'mapper directory created', null, PROCESS_NAMES.ASSETS_MAPPER_GENERATION);
progress.completeProcess(PROCESS_NAMES.ASSETS_MAPPER_GENERATION, true);

// Fetch and map assets
progress
.startProcess(PROCESS_NAMES.ASSETS_FETCH_AND_MAP)
.updateStatus(
PROCESS_STATUS.ASSETS_FETCH_AND_MAP.FETCHING,
PROCESS_NAMES.ASSETS_FETCH_AND_MAP,
);
await this.fetchAndMapAssets();
progress.completeProcess(PROCESS_NAMES.ASSETS_FETCH_AND_MAP, true);

this.completeProgress(true);
log(this.config, `The required setup files for the asset have been generated successfully.`, 'success');
} catch (error) {
this.completeProgress(false, error?.message || 'Assets mapper generation failed');
log(this.config, `Error occurred while generating the asset mapper: ${formatError(error)}.`, 'error');
}
}
Expand All @@ -67,17 +110,21 @@ export default class AssetImportSetup extends BaseImportSetup {
if (items.length === 1) {
this.assetUidMapper[uid] = items[0].uid;
this.assetUrlMapper[url] = items[0].url;
this.progressManager?.tick(true, `asset: ${title}`, null, PROCESS_NAMES.ASSETS_FETCH_AND_MAP);
log(this.config, `Mapped asset successfully: '${title}'`, 'info');
} else if (items.length > 1) {
this.duplicateAssets[uid] = items.map((asset: any) => {
return { uid: asset.uid, title: asset.title, url: asset.url };
});
this.progressManager?.tick(true, `asset: ${title} (duplicate)`, null, PROCESS_NAMES.ASSETS_FETCH_AND_MAP);
log(this.config, `Multiple assets found with the title '${title}'.`, 'info');
} else {
this.progressManager?.tick(false, `asset: ${title}`, 'Not found in stack', PROCESS_NAMES.ASSETS_FETCH_AND_MAP);
log(this.config, `Asset with title '${title}' not found in the stack!`, 'info');
}
};
const onReject = ({ error, apiData: { title } = undefined }: any) => {
this.progressManager?.tick(false, `asset: ${title}`, formatError(error), PROCESS_NAMES.ASSETS_FETCH_AND_MAP);
log(this.config, `Failed to map the asset '${title}'.`, 'error');
log(this.config, formatError(error), 'error');
};
Expand Down
Original file line number Diff line number Diff line change
@@ -1,11 +1,14 @@
import { log, fsUtil } from '../../utils';
import { ApiOptions, CustomPromiseHandler, EnvType, ImportConfig, ModuleClassParams } from '../../types';
import { chunk, entries, isEmpty, isEqual, last } from 'lodash';
import { CLIProgressManager, configHandler } from '@contentstack/cli-utilities';

export default class BaseImportSetup {
public config: ImportConfig;
public stackAPIClient: ModuleClassParams['stackAPIClient'];
public dependencies: ModuleClassParams['dependencies'];
protected progressManager: CLIProgressManager | null = null;
protected currentModuleName: string = '';

constructor({ config, stackAPIClient, dependencies }: ModuleClassParams) {
this.config = config;
Expand Down Expand Up @@ -205,4 +208,48 @@ export default class BaseImportSetup {
return Promise.resolve();
}
}

/**
* Create simple progress manager
*/
protected createSimpleProgress(moduleName: string, total?: number): CLIProgressManager {
this.currentModuleName = moduleName;
const logConfig = configHandler.get('log') || {};
const showConsoleLogs = logConfig.showConsoleLogs ?? false;
this.progressManager = CLIProgressManager.createSimple(moduleName, total, showConsoleLogs);
return this.progressManager;
}

/**
* Create nested progress manager
*/
protected createNestedProgress(moduleName: string): CLIProgressManager {
this.currentModuleName = moduleName;
const logConfig = configHandler.get('log') || {};
const showConsoleLogs = logConfig.showConsoleLogs ?? false;
this.progressManager = CLIProgressManager.createNested(moduleName, showConsoleLogs);
return this.progressManager;
}

/**
* Complete progress manager
*/
protected completeProgress(success: boolean = true, error?: string): void {
this.progressManager?.complete(success, error);
this.progressManager = null;
}

/**
* Show a loading spinner before initializing progress
*/
protected async withLoadingSpinner<T>(message: string, action: () => Promise<T>): Promise<T> {
const logConfig = configHandler.get('log') || {};
const showConsoleLogs = logConfig.showConsoleLogs ?? false;

if (showConsoleLogs) {
// If console logs are enabled, don't show spinner, just execute the action
return await action();
}
return await CLIProgressManager.withLoadingSpinner(message, action);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -4,17 +4,52 @@ import { join } from 'path';
import { ImportConfig, ModuleClassParams } from '../../types';
import ExtensionImportSetup from './extensions';
import BaseImportSetup from './base-setup';
import { MODULE_NAMES, MODULE_CONTEXTS, PROCESS_NAMES, PROCESS_STATUS } from '../../utils';

export default class ContentTypesImportSetup extends BaseImportSetup {
constructor(options: ModuleClassParams) {
super(options);
this.currentModuleName = MODULE_NAMES[MODULE_CONTEXTS.CONTENT_TYPES];
}

async start() {
try {
await this.setupDependencies();
const progress = this.createNestedProgress(this.currentModuleName);

// Add processes
progress.addProcess(PROCESS_NAMES.CONTENT_TYPES_DEPENDENCY_SETUP, this.dependencies?.length || 0);
progress.addProcess(PROCESS_NAMES.CONTENT_TYPES_MAPPER_GENERATION, 1);

// Setup dependencies
if (this.dependencies && this.dependencies.length > 0) {
progress
.startProcess(PROCESS_NAMES.CONTENT_TYPES_DEPENDENCY_SETUP)
.updateStatus(
PROCESS_STATUS.CONTENT_TYPES_DEPENDENCY_SETUP.SETTING_UP,
PROCESS_NAMES.CONTENT_TYPES_DEPENDENCY_SETUP,
);

await this.setupDependencies();

this.progressManager?.tick(true, 'dependencies setup', null, PROCESS_NAMES.CONTENT_TYPES_DEPENDENCY_SETUP);
progress.completeProcess(PROCESS_NAMES.CONTENT_TYPES_DEPENDENCY_SETUP, true);
}

// Mapper generation
progress
.startProcess(PROCESS_NAMES.CONTENT_TYPES_MAPPER_GENERATION)
.updateStatus(
PROCESS_STATUS.CONTENT_TYPES_MAPPER_GENERATION.GENERATING,
PROCESS_NAMES.CONTENT_TYPES_MAPPER_GENERATION,
);

this.progressManager?.tick(true, 'mapper generation', null, PROCESS_NAMES.CONTENT_TYPES_MAPPER_GENERATION);
progress.completeProcess(PROCESS_NAMES.CONTENT_TYPES_MAPPER_GENERATION, true);

this.completeProgress(true);
log(this.config, `The required setup files for content types have been generated successfully.`, 'success');
} catch (error) {
this.completeProgress(false, error?.message || 'Content types mapper generation failed');
log(this.config, `Error occurred while generating the content type mapper: ${error.message}.`, 'error');
}
}
Expand Down
Original file line number Diff line number Diff line change
@@ -1,17 +1,27 @@
import { log } from '../../utils';
import { ModuleClassParams } from '../../types';
import BaseImportSetup from './base-setup';
import { MODULE_NAMES, MODULE_CONTEXTS, PROCESS_NAMES, PROCESS_STATUS } from '../../utils';

export default class EntriesImportSetup extends BaseImportSetup {
constructor(options: ModuleClassParams) {
super(options);
this.currentModuleName = MODULE_NAMES[MODULE_CONTEXTS.ENTRIES];
}

async start() {
try {
const progress = this.createSimpleProgress(this.currentModuleName, 1);

progress.updateStatus('Setting up dependencies...');
await this.setupDependencies();

this.progressManager?.tick(true, 'entries mapper setup', null);
this.completeProgress(true);

log(this.config, `The required setup files for entries have been generated successfully.`, 'success');
} catch (error) {
this.completeProgress(false, error?.message || 'Entries mapper generation failed');
log(this.config, `Error occurred while generating the entry mapper: ${error.message}.`, 'error');
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,21 +4,19 @@ import { join } from 'path';
import { ImportConfig, ModuleClassParams } from '../../types';
import { isEmpty } from 'lodash';
import { formatError, sanitizePath } from '@contentstack/cli-utilities';
import BaseImportSetup from './base-setup';
import { MODULE_NAMES, MODULE_CONTEXTS, PROCESS_NAMES, PROCESS_STATUS } from '../../utils';

export default class ExtensionImportSetup {
private config: ImportConfig;
export default class ExtensionImportSetup extends BaseImportSetup {
private extensionsFilePath: string;
private extensionMapper: Record<string, string>;
private stackAPIClient: ModuleClassParams['stackAPIClient'];
private dependencies: ModuleClassParams['dependencies'];
private extensionsConfig: ImportConfig['modules']['extensions'];
private mapperDirPath: string;
private extensionsFolderPath: string;
private extUidMapperPath: string;

constructor({ config, stackAPIClient }: ModuleClassParams) {
this.config = config;
this.stackAPIClient = stackAPIClient;
super({ config, stackAPIClient, dependencies: [] });
this.currentModuleName = MODULE_NAMES[MODULE_CONTEXTS.EXTENSIONS];
this.extensionsFilePath = join(sanitizePath(this.config.contentDir), 'extensions', 'extensions.json');
this.extensionsConfig = config.modules.extensions;
this.extUidMapperPath = join(sanitizePath(this.config.backupDir), 'mapper', 'extensions', 'uid-mapping.json');
Expand All @@ -32,28 +30,53 @@ export default class ExtensionImportSetup {
*/
async start() {
try {
const extensions: any = await fsUtil.readFile(this.extensionsFilePath);
const extensions: any = await this.withLoadingSpinner('EXTENSIONS: Analyzing import data...', async () => {
return await fsUtil.readFile(this.extensionsFilePath);
});

if (!isEmpty(extensions)) {
// 2. Create mapper directory
const extensionsArray = Object.values(extensions) as any[];
const progress = this.createNestedProgress(this.currentModuleName);

// Add process
progress.addProcess(PROCESS_NAMES.EXTENSIONS_MAPPER_GENERATION, extensionsArray.length);

// Create mapper directory
const mapperFilePath = join(sanitizePath(this.config.backupDir), 'mapper', 'extensions');
fsUtil.makeDirectory(mapperFilePath); // Use fsUtil
fsUtil.makeDirectory(mapperFilePath);

progress
.startProcess(PROCESS_NAMES.EXTENSIONS_MAPPER_GENERATION)
.updateStatus(
PROCESS_STATUS.EXTENSIONS_MAPPER_GENERATION.GENERATING,
PROCESS_NAMES.EXTENSIONS_MAPPER_GENERATION,
);

for (const extension of Object.values(extensions) as any) {
const targetExtension: any = await this.getExtension(extension);
if (!targetExtension) {
log(this.config, `Extension with the title '${extension.title}' not found in the stack.`, 'info');
continue;
for (const extension of extensionsArray) {
try {
const targetExtension: any = await this.getExtension(extension);
if (!targetExtension) {
log(this.config, `Extension with the title '${extension.title}' not found in the stack.`, 'info');
this.progressManager?.tick(false, `extension: ${extension.title}`, 'Not found in stack', PROCESS_NAMES.EXTENSIONS_MAPPER_GENERATION);
continue;
}
this.extensionMapper[extension.uid] = targetExtension.uid;
this.progressManager?.tick(true, `extension: ${extension.title}`, null, PROCESS_NAMES.EXTENSIONS_MAPPER_GENERATION);
} catch (error) {
this.progressManager?.tick(false, `extension: ${extension.title}`, formatError(error), PROCESS_NAMES.EXTENSIONS_MAPPER_GENERATION);
}
this.extensionMapper[extension.uid] = targetExtension.uid;
}

await fsUtil.writeFile(this.extUidMapperPath, this.extensionMapper);
progress.completeProcess(PROCESS_NAMES.EXTENSIONS_MAPPER_GENERATION, true);
this.completeProgress(true);

log(this.config, `The required setup files for extensions have been generated successfully.`, 'success');
} else {
log(this.config, 'No extensions found in the content folder.', 'info');
}
} catch (error) {
this.completeProgress(false, error?.message || 'Extensions mapper generation failed');
log(this.config, `Error occurred while generating the extension mapper: ${formatError(error)}.`, 'error');
}
}
Expand Down
Loading
Loading