From 7d8c879f630c4e26277be701ac635e4bab18b1b4 Mon Sep 17 00:00:00 2001 From: Sajith Subramanian <31767159+sajith-subramanian@users.noreply.github.com> Date: Thu, 12 Dec 2024 10:18:22 +0530 Subject: [PATCH 1/3] add updated download and upload methods and deprecate old methods with async suffix --- oss/source/custom-code/ossClient.ts | 96 +++++++++++++++++++++++++++++ oss/source/package.json | 2 +- 2 files changed, 97 insertions(+), 1 deletion(-) diff --git a/oss/source/custom-code/ossClient.ts b/oss/source/custom-code/ossClient.ts index 5c2a5e53..5da12d88 100644 --- a/oss/source/custom-code/ossClient.ts +++ b/oss/source/custom-code/ossClient.ts @@ -5,6 +5,7 @@ import { BucketsApi, ObjectsApi } from "../api"; import { FileTransferConfigurations } from "./fileTransferConfigurations"; import { promises as fs } from "fs"; import { Stream } from "stream"; +import { deprecate } from "util"; export class OssClient extends BaseClient { @@ -23,6 +24,7 @@ export class OssClient extends BaseClient { this.ossFileTransfer = new OSSFileTransfer(new FileTransferConfigurations(3), optionalArgs.sdkManager, optionalArgs.authenticationProvider); } + /** *Downloads a file by transparently handling operations like obtaining signed download URLs and chunking large files for optimal transfer. * @param {string} bucketKey URL-encoded bucket key @@ -35,6 +37,9 @@ export class OssClient extends BaseClient { * @throws {RequiredError} * @memberof OSSApiInterface */ + /** + * @deprecated Use the `downloadObject` method instead. + */ public async downloadObjectAsync(bucketKey: string, objectKey: string, filePath: string, optionalArgs?: { cancellationToken?: AbortController, requestIdPrefix?: string, accessToken?: string, onProgress?: (percentCompleted: number) => void }): Promise; /** *Downloads a file by transparently handling operations like obtaining signed download URLs and chunking large files for optimal transfer. @@ -49,7 +54,13 @@ export class OssClient extends BaseClient { * @memberof OSSApiInterface * */ + /** + * @deprecated Use the `downloadObject` method instead. + */ public async downloadObjectAsync(bucketKey: string, objectKey: string, optionalArgs?: { cancellationToken?: AbortController, requestIdPrefix?: string, accessToken?: string, onProgress?: (percentCompleted: number) => void }): Promise; + /** + * @deprecated Use the `downloadObject` method instead. + */ public async downloadObjectAsync(bucketKey: string, objectKey: string, filePathOrOptionalArgs: string | { cancellationToken?: AbortController, requestIdPrefix?: string, accessToken?: string, onProgress?: (percentCompleted: number) => void }, optionalArgs?: { cancellationToken?: AbortController, requestIdPrefix?: string, accessToken?: string, onProgress?: (percentCompleted: number) => void }): Promise { let filePath: string | undefined; if (typeof filePathOrOptionalArgs === 'string') { @@ -71,6 +82,55 @@ export class OssClient extends BaseClient { } } + /** + *Downloads a file by transparently handling operations like obtaining signed download URLs and chunking large files for optimal transfer. + * @param {string} bucketKey URL-encoded bucket key + * @param {string} objectKey URL-encoded object name + * @param {string} filePath The Path of the file where should be downloaded + * @param accessToken bearer access token + * @param {AbortController} cancellationToken + * @param {string} requestIdPrefix + * @param {*} [options] Override http request option. + * @throws {RequiredError} + * @memberof OSSApiInterface + */ + public async downloadObject(bucketKey: string, objectKey: string, filePath: string, optionalArgs?: { cancellationToken?: AbortController, requestIdPrefix?: string, accessToken?: string, onProgress?: (percentCompleted: number) => void }): Promise; + /** + *Downloads a file by transparently handling operations like obtaining signed download URLs and chunking large files for optimal transfer. + * @param {string} bucketKey URL-encoded bucket key + * @param {string} objectKey URL-encoded object name + * @param accessToken bearer access token + * @param {AbortController} cancellationToken + * @param {string} requestIdPrefix + * @param {*} [options] Override http request option. + * @returns {Stream} + * @throws {RequiredError} + * @memberof OSSApiInterface + * + */ + public async downloadObject(bucketKey: string, objectKey: string, optionalArgs?: { cancellationToken?: AbortController, requestIdPrefix?: string, accessToken?: string, onProgress?: (percentCompleted: number) => void }): Promise; + public async downloadObject(bucketKey: string, objectKey: string, filePathOrOptionalArgs: string | { cancellationToken?: AbortController, requestIdPrefix?: string, accessToken?: string, onProgress?: (percentCompleted: number) => void }, optionalArgs?: { cancellationToken?: AbortController, requestIdPrefix?: string, accessToken?: string, onProgress?: (percentCompleted: number) => void }): Promise { + let filePath: string | undefined; + if (typeof filePathOrOptionalArgs === 'string') { + filePath = filePathOrOptionalArgs; + } else { + optionalArgs = filePathOrOptionalArgs; + } + + if (!optionalArgs?.accessToken && !this.authenticationProvider) { + throw new Error("Please provide a valid access token or an authentication provider"); + } else if (!optionalArgs?.accessToken) { + (optionalArgs ??= {}).accessToken = await this.authenticationProvider.getAccessToken(); + } + + if (filePath) { + await this.ossFileTransfer.download(bucketKey, objectKey, optionalArgs?.accessToken, filePath, optionalArgs?.cancellationToken || new AbortController(), optionalArgs?.requestIdPrefix, optionalArgs?.onProgress); + } else { + return this.ossFileTransfer.download(bucketKey, objectKey, optionalArgs?.accessToken, null, optionalArgs?.cancellationToken || new AbortController(), optionalArgs?.requestIdPrefix, optionalArgs?.onProgress); + } + } + + /** * Instructs OSS to complete the object creation process for numerous objects after their bytes have been uploaded directly to S3. An object will not be accessible until you complete the object creation process, either with this endpoint or the single Complete Upload endpoint. This endpoint accepts batch sizes of up to 25. Any larger and the request will fail. * @param {string} bucketKey URL-encoded bucket key @@ -83,6 +143,9 @@ export class OssClient extends BaseClient { * @throws {RequiredError} * @memberof OSSApiInterface */ + /** + * @deprecated Use the `uploadObject` method instead. + */ public async uploadObjectAsync(bucketKey: string, objectKey: string, sourceToUpload: Buffer | string, optionalArgs?: { cancellationToken?: AbortController, requestIdPrefix?: string, accessToken?: string, onProgress?: (percentCompleted: number) => void }): Promise { if (!optionalArgs?.accessToken && !this.authenticationProvider) { throw new Error("Please provide a valid access token or an authentication provider"); @@ -100,6 +163,39 @@ export class OssClient extends BaseClient { } return response.content; } + + + /** + * Instructs OSS to complete the object creation process for numerous objects after their bytes have been uploaded directly to S3. An object will not be accessible until you complete the object creation process, either with this endpoint or the single Complete Upload endpoint. This endpoint accepts batch sizes of up to 25. Any larger and the request will fail. + * @param {string} bucketKey URL-encoded bucket key + * @param {string} objectKey URL-encoded object name + * @param {Buffer|string} sourceToUpload The Path of the file to be uploaded or the buffer of the file . + * @param accessToken bearer access token + * @param {AbortController} cancellationToken + * @param {string} requestIdPrefix + * @param {*} [options] Override http request option. + * @throws {RequiredError} + * @memberof OSSApiInterface + */ + public async uploadObject(bucketKey: string, objectKey: string, sourceToUpload: Buffer | string, optionalArgs?: { cancellationToken?: AbortController, requestIdPrefix?: string, accessToken?: string, onProgress?: (percentCompleted: number) => void }): Promise { + if (!optionalArgs?.accessToken && !this.authenticationProvider) { + throw new Error("Please provide a valid access token or an authentication provider"); + } + else if (!optionalArgs?.accessToken) { + (optionalArgs ??= {}).accessToken = await this.authenticationProvider.getAccessToken(); + } + var response; + if (typeof sourceToUpload === 'string') { + var buffer = await fs.readFile(sourceToUpload); + response = await this.ossFileTransfer.upload(bucketKey, objectKey, buffer, optionalArgs?.accessToken, optionalArgs?.cancellationToken || new AbortController, optionalArgs?.requestIdPrefix, optionalArgs?.onProgress); + } + else { + response = await this.ossFileTransfer.upload(bucketKey, objectKey, sourceToUpload, optionalArgs?.accessToken, optionalArgs?.cancellationToken || new AbortController, optionalArgs?.requestIdPrefix, optionalArgs?.onProgress); + } + return response.content; + } + + /** * Instructs OSS to complete the object creation process for numerous objects after their bytes have been uploaded directly to S3. An object will not be accessible until you complete the object creation process, either with this endpoint or the single Complete Upload endpoint. This endpoint accepts batch sizes of up to 25. Any larger and the request will fail. * @param {string} bucketKey URL-encoded bucket key diff --git a/oss/source/package.json b/oss/source/package.json index 01a0a9be..9403a6ac 100644 --- a/oss/source/package.json +++ b/oss/source/package.json @@ -1,6 +1,6 @@ { "name": "@aps_sdk/oss", - "version": "1.0.0", + "version": "1.0.1", "description": "Client sdk for Data Management Oss API", "license": "Apache-2.0", "author": "Autodesk Platform Services", From 5d7b2d4fb560e671cff9e20e9d4ae112b29d95db Mon Sep 17 00:00:00 2001 From: Sajith Subramanian <31767159+sajith-subramanian@users.noreply.github.com> Date: Thu, 12 Dec 2024 10:20:28 +0530 Subject: [PATCH 2/3] remove unnecessary import --- oss/source/custom-code/ossClient.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/oss/source/custom-code/ossClient.ts b/oss/source/custom-code/ossClient.ts index 5da12d88..778be92d 100644 --- a/oss/source/custom-code/ossClient.ts +++ b/oss/source/custom-code/ossClient.ts @@ -5,7 +5,7 @@ import { BucketsApi, ObjectsApi } from "../api"; import { FileTransferConfigurations } from "./fileTransferConfigurations"; import { promises as fs } from "fs"; import { Stream } from "stream"; -import { deprecate } from "util"; + export class OssClient extends BaseClient { From b2996b8516438c782cd21e6cd355e7fb8821a063 Mon Sep 17 00:00:00 2001 From: srivastavarahull <139200493+srivastavarahull@users.noreply.github.com> Date: Thu, 12 Dec 2024 16:41:08 +0530 Subject: [PATCH 3/3] Fixed upload and download method naming --- oss/source/custom-code/ossFileTransfer.ts | 1 + samples/oss.ts | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/oss/source/custom-code/ossFileTransfer.ts b/oss/source/custom-code/ossFileTransfer.ts index e443956b..df913add 100644 --- a/oss/source/custom-code/ossFileTransfer.ts +++ b/oss/source/custom-code/ossFileTransfer.ts @@ -149,6 +149,7 @@ export class OSSFileTransfer implements IOSSFileTransfer { let partsDownloaded: number = 0; let start: number = 0; let outStream = new PassThrough(); + outStream.setMaxListeners(numberOfChunks+1); let fileStream: WriteStream; if(filePath){ fileStream= createWriteStream(filePath, { flags: 'a' }); diff --git a/samples/oss.ts b/samples/oss.ts index e902d697..bc402ca1 100644 --- a/samples/oss.ts +++ b/samples/oss.ts @@ -26,17 +26,17 @@ const ossClient = new OssClient({ authenticationProvider: staticAuthenticationPr async function upload() { //sourceToUpload can be either file path or stream of the object - const response = await ossClient.uploadObjectAsync(bucketKey, objectKey, sourceToUpload); + const response = await ossClient.uploadObject(bucketKey, objectKey, sourceToUpload); console.log(response); } //Downloaing a file can be directly to the file or get the stream of file. async function download() { - await ossClient.downloadObjectAsync(bucketKey, objectKey, filePath); + await ossClient.downloadObject(bucketKey, objectKey, filePath); } async function downloadAsStream() { let fileStream = new Stream(); - fileStream = await ossClient.downloadObjectAsync(bucketKey, objectKey); + fileStream = await ossClient.downloadObject(bucketKey, objectKey); } /** * This function will return the details about the specified bucket.