diff --git a/CHANGELOG.md b/CHANGELOG.md index f071e8978..d3b8cde4b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -56,6 +56,11 @@ and this project adheres to `--api-base-url` so it is shown in the help output. Any other value for `--api-base-url` will cause an error to be thrown if `--development` is set to `"true"`. +- Improved type definitions to allow for expressing the type of the + `IntegrationExecutionConfig` that is returned from `loadExecutionConfig` and + passed to `executionHandler` functions. +- Added `IntegrationStepExecutionContext.stepMetadata` to allow + `executionHandler` functions to readily access their `StepMetadata`. ## 8.24.0 - 2022-09-15 @@ -162,25 +167,25 @@ Options: specified, logging to the job event log is disabled. Here is an example of usage: -```typescript -{ - ['fetch-prs']: { - disabled: false - }, - ['fetch-issues']: { - disabled: !scopes.repoIssues, - disabledReason: DisabledStepReason.PERMISSION - } -} -``` + ```typescript + const stepStartStates: StepStartStates = { + ['fetch-prs']: { + disabled: false, + }, + ['fetch-issues']: { + disabled: !scopes.repoIssues, + disabledReason: DisabledStepReason.PERMISSION, + }, + }; + ``` -Sample text output: + Sample text output: -``` -Skipped step "Fetch Issues". The required permission was not provided to perform this step. -Skipped step "Fetch Issues". This step is disabled via configuration. Please contact support to enabled. -Skipped step "Fetch Issues". Beta feature, please contact support to enable. -``` + ``` + Skipped step "Fetch Issues". The required permission was not provided to perform this step. + Skipped step "Fetch Issues". This step is disabled via configuration. Please contact support to enabled. + Skipped step "Fetch Issues". Beta feature, please contact support to enable. + ``` ## [8.17.0] - 2022-06-29 @@ -230,7 +235,7 @@ Skipped step "Fetch Issues". Beta feature, please contact support to enable. ### Changed -- Allow an integration job id to be passed in when initializing syncronization. +- Allow an integration job id to be passed in when initializing synchronization. ## [8.13.11] - 2022-05-27 @@ -249,14 +254,14 @@ Skipped step "Fetch Issues". Beta feature, please contact support to enable. ### Fixed - Fixed issue when unzipping gzipped polly recording entries. Now removes the - content.encoding value once content is decoded. + `content.encoding` value once content is decoded. - Fixes issue introduced in 8.13.4 ## [8.13.8] - 2022-05-19 ### Changed -- Moved `shrinkBatchRawData` to its own module for readablity and easy mocking +- Moved `shrinkBatchRawData` to its own module for readability and easy mocking in test - Increased threshold by which we continue to shrink rawData from 6 million bytes to 5.5 million bytes @@ -341,8 +346,8 @@ Skipped step "Fetch Issues". Beta feature, please contact support to enable. ### Added - Additional error type `IntegrationProviderRetriesExceededError` to be used - when integration has exhausted all of the retries. This error type won't be - sent in as an alert to the operators. + when integration has exhausted all the retries. This error type won't be sent + in as an alert to the operators. ## [8.10.1] - 2022-04-08 @@ -663,13 +668,16 @@ of the support.jupiterone.io site. ```ts import { fromTemporaryCredentials } from '@aws-sdk/credential-providers'; + type IntegrationConfig = { + roleArn: string; + externalId: string; + }; + /** * The AWS integration uses shared `fromTemporaryCredentials` across all of * its clients. */ - export function loadExecutionConfig({ - config: { roleArn: string, externalId: string }, - }) { + export function loadExecutionConfig({ config: IntegrationConfig }) { return { credentials: fromTemporaryCredentials({ params: { @@ -740,15 +748,17 @@ of the support.jupiterone.io site. const virtualMachineId = await jobState.findEntity( nic.virtualMachine?.id as string, ); + ``` + ```ts // by allowing `undefined`, we can more safely use these methods without type assertions const virtualMachineId = await jobState.findEntity(nic.virtualMachine?.id); ``` ### Fixed -- Fixed the way that symlinks are created on windows machines. Directories are - still created as simlinks, but files are now hardlinks to prevent the +- Fixed the way that symlinks are created on Windows machines. Directories are + still created as symlinks, but files are now hardlinks to prevent the requirement that `yarn start` be run with admin credentials. ## [7.4.0] - 2021-11-03 @@ -896,7 +906,7 @@ of the support.jupiterone.io site. - Fix `j1-integration document --output-file` to reflect that it is a path relative to `--project-path` -- Fixed the way that symlinks are created on windows machines, which previously +- Fixed the way that symlinks are created on Windows machines, which previously threw `EPERM: operation not permitted, symlink` ## [6.15.0] - 2021-08-19 @@ -941,8 +951,8 @@ of the support.jupiterone.io site. ### Added - a `dependencyGraphOrder` property to the InvocationConfig and a - `dependencyGraphId` property to the StepMetadata which togeather can be used - to create multiple ordered dependency graphs per execution. + `dependencyGraphId` property to the StepMetadata which together can be used to + create multiple ordered dependency graphs per execution. ## 6.10.0 - 2021-07-09 @@ -972,7 +982,7 @@ of the support.jupiterone.io site. ### Added -- Added `j1-integration diff` command to ouptut colorized diffs of old/new +- Added `j1-integration diff` command to output colorized diffs of old/new integrations. - Allow overriding integration instance properties when running integrations locally. @@ -1201,7 +1211,7 @@ getData: (key: string) => Promise; Usage in an integration step: ```typescript - { + const integrationMetadata = { id: 'my-step', name: 'My step', entities: [ @@ -1217,9 +1227,9 @@ getData: (key: string) => Promise; ], relationships: [], async exeutionHandler() { - ... - } - } + // work here + }, + }; ``` See PR [#404](https://github.com/JupiterOne/sdk/pull/404) @@ -1584,16 +1594,14 @@ Example: ```typescript const entity = await jobState.addEntity(convertToEntity(data)); const entity2 = await jobState.addEntity(convertToOtherEntity(entity2)); -await jobState.addRelationship( - convertToRelationship(entity, entity2) -); +await jobState.addRelationship(convertToRelationship(entity, entity2)); // Or this: await jobState.addRelationship( convertToRelationship( - await jobState.addEntity(convertToEntity(data)) - await jobState.addEntity(convertToOtherEntity(entity2)) - ) + await jobState.addEntity(convertToEntity(data)), + await jobState.addEntity(convertToOtherEntity(entity2)), + ), ); ``` @@ -1641,8 +1649,9 @@ expect(context.jobState.collectedEntities).toMatchGraphObjectSchema({ _rawData: { type: 'array', items: { type: 'object' }, - } - } + }, + }, + }, }); ``` @@ -1748,7 +1757,7 @@ export const invocationConfig: IntegrationInvocationConfig = convert properties that are named with common suffixes (on, at, time, date) to a UNIX timestamp (number). - Added `publishMetric` function to `IntegrationLogger` that now causes a - `metric` event to be emit. + `metric` event to be emitted. ## 1.1.1 - 2020-06-08 @@ -1767,7 +1776,7 @@ export const invocationConfig: IntegrationInvocationConfig = ### Fixed - `createIntegrationRelationship` made `_key` optional for relationship - mappings, a fine thing to do because specifying the `_key` for those insn't + mappings, a fine thing to do because specifying the `_key` for them isn't necessary. However, the function was changed at the same time to stop generating a `_key`, which is required to ensure the collected relationships are unique. This fixes things so the `_key` remains an optional argument, and @@ -1781,8 +1790,7 @@ export const invocationConfig: IntegrationInvocationConfig = `_type` for relationship mappings, overriding the generated value or values provided in `properties` option. - Removed `@types/vis` from dependencies to devDependencies because having the - type forces typescript consumers to have `DOM` in the their `lib` compiler - option. + type forces typescript consumers to have `DOM` in their `lib` compiler option. ## 1.0.1 - 2020-06-03 @@ -1832,4 +1840,4 @@ into the following packages: - `@jupiterone/integration-sdk-cli` To view the changes that went into `@jupiterone/integration-sdk`, please see -[LEGACY_SDK_CHANGELOG.md](./LEGACY_SDK_CHANGELOG.md). +[LEGACY_SDK_CHANGELOG.md](packages/integration-sdk/LEGACY_SDK_CHANGELOG.md). diff --git a/packages/integration-sdk-cli/src/__tests__/cli.test.ts b/packages/integration-sdk-cli/src/__tests__/cli.test.ts index 1e1a9c796..09d8997a3 100644 --- a/packages/integration-sdk-cli/src/__tests__/cli.test.ts +++ b/packages/integration-sdk-cli/src/__tests__/cli.test.ts @@ -509,6 +509,11 @@ describe('collect/visualize integration', () => { }); }); +test('custom context and config types', async () => { + loadProjectStructure('typeScriptCustomConfigsProject'); + await createCli().parseAsync(['node', 'j1-integration', 'collect']); +}); + describe('document', () => { test('loads the integration with entities and relationships and writes documentation results', async () => { await documentCommandSnapshotTest('docsInstanceWithRelationships'); diff --git a/packages/integration-sdk-core/src/types/config.ts b/packages/integration-sdk-core/src/types/config.ts index 46f19cd6a..fb39a3704 100644 --- a/packages/integration-sdk-core/src/types/config.ts +++ b/packages/integration-sdk-core/src/types/config.ts @@ -3,10 +3,10 @@ import { GetStepStartStatesFunction, Step } from './step'; import { InvocationValidationFunction } from './validation'; import { ExecutionContext, + IntegrationExecutionConfig, IntegrationExecutionContext, - StepExecutionContext, IntegrationStepExecutionContext, - IntegrationExecutionConfig, + StepExecutionContext, } from './context'; import { Entity } from './entity'; import { Relationship } from './relationship'; @@ -61,12 +61,13 @@ export interface InvocationConfig< } export interface IntegrationInvocationConfig< - TConfig extends IntegrationInstanceConfig = IntegrationInstanceConfig, + TInstanceConfig extends IntegrationInstanceConfig = IntegrationInstanceConfig, + TExecutionConfig extends IntegrationExecutionConfig = IntegrationExecutionConfig, > extends InvocationConfig< - IntegrationExecutionContext, - IntegrationStepExecutionContext + IntegrationExecutionContext, + IntegrationStepExecutionContext > { - instanceConfigFields?: IntegrationInstanceConfigFieldMap; + instanceConfigFields?: IntegrationInstanceConfigFieldMap; } export interface IntegrationInstanceConfigField { @@ -76,5 +77,5 @@ export interface IntegrationInstanceConfigField { } export type IntegrationInstanceConfigFieldMap< - TConfig extends IntegrationInstanceConfig = IntegrationInstanceConfig, -> = Record; + TInstanceConfig extends IntegrationInstanceConfig = IntegrationInstanceConfig, +> = Record; diff --git a/packages/integration-sdk-core/src/types/context.ts b/packages/integration-sdk-core/src/types/context.ts index 5c3b0c19b..3cf9b0db4 100644 --- a/packages/integration-sdk-core/src/types/context.ts +++ b/packages/integration-sdk-core/src/types/context.ts @@ -1,6 +1,7 @@ import { IntegrationInstance, IntegrationInstanceConfig } from './instance'; import { JobState } from './jobState'; import { IntegrationLogger } from './logger'; +import { StepMetadata } from './step'; export type Execution = { startedOn: number; @@ -24,43 +25,44 @@ export interface ExecutionContext { * `IntegrationInstanceConfig`, containing dynamic values perhaps calculated * based on the instance config. */ -export type IntegrationExecutionConfig = object; +export type IntegrationExecutionConfig = Record; /** - * @param TConfig the integration specific type of the `instance.config` + * @param TInstanceConfig the integration specific type of the `instance.config` * property */ export type IntegrationLoadExecutionConfigContext< - TConfig extends IntegrationInstanceConfig, + TInstanceConfig extends IntegrationInstanceConfig, > = ExecutionContext & { - instance: IntegrationInstance; + instance: IntegrationInstance; }; /** - * @param TConfig the integration specific type of the `instance.config` + * @param TInstanceConfig the integration specific type of the `instance.config` * property * @param TExecutionConfig the configuration type produced by the * integration's optional `loadExecutionConfig` function */ export type IntegrationExecutionContext< - TConfig extends IntegrationInstanceConfig = IntegrationInstanceConfig, + TInstanceConfig extends IntegrationInstanceConfig = IntegrationInstanceConfig, TExecutionConfig extends IntegrationExecutionConfig = IntegrationExecutionConfig, -> = IntegrationLoadExecutionConfigContext & { +> = IntegrationLoadExecutionConfigContext & { executionConfig: TExecutionConfig; }; export type StepExecutionContext = ExecutionContext & { jobState: JobState; + stepMetadata: StepMetadata; }; /** - * @param TConfig the integration specific type of the `instance.config` + * @param TInstanceConfig the integration specific type of the `instance.config` * property * @param TExecutionConfig the configuration type produced by the * integration's optional `loadExecutionConfig` function */ export interface IntegrationStepExecutionContext< - TConfig extends IntegrationInstanceConfig = IntegrationInstanceConfig, + TInstanceConfig extends IntegrationInstanceConfig = IntegrationInstanceConfig, TExecutionConfig extends IntegrationExecutionConfig = IntegrationExecutionConfig, -> extends IntegrationExecutionContext, +> extends IntegrationExecutionContext, StepExecutionContext {} diff --git a/packages/integration-sdk-core/src/types/instance.ts b/packages/integration-sdk-core/src/types/instance.ts index 69fab3b82..b792d1b7c 100644 --- a/packages/integration-sdk-core/src/types/instance.ts +++ b/packages/integration-sdk-core/src/types/instance.ts @@ -4,10 +4,10 @@ export type IntegrationInstanceConfig = Record; * A stored user configuration for executing the integration defined by * the associated `integrationDefinitionId`. * - * @param TConfig the integration specific type of the `config` property + * @param TInstanceConfig the integration specific type of the `config` property */ export interface IntegrationInstance< - TConfig extends IntegrationInstanceConfig = IntegrationInstanceConfig, + TInstanceConfig extends IntegrationInstanceConfig = IntegrationInstanceConfig, > { /** * Unique identifier for the activated integration instance. @@ -44,5 +44,5 @@ export interface IntegrationInstance< * configuration upon invocation and provide useful configuration error * messages. */ - config: TConfig; + config: TInstanceConfig; } diff --git a/packages/integration-sdk-core/src/types/spec.ts b/packages/integration-sdk-core/src/types/spec.ts index 09126bdb5..d2206bffb 100644 --- a/packages/integration-sdk-core/src/types/spec.ts +++ b/packages/integration-sdk-core/src/types/spec.ts @@ -1,18 +1,24 @@ import { IntegrationInvocationConfig } from './config'; -import { IntegrationStepExecutionContext } from './context'; +import { + IntegrationExecutionConfig, + IntegrationStepExecutionContext, +} from './context'; import { IntegrationInstanceConfig } from './instance'; import { Step } from './step'; -export interface StepSpec - extends Omit< - Step>, +export interface StepSpec< + TInstanceConfig extends IntegrationInstanceConfig, + TExecutionConfig extends IntegrationExecutionConfig, +> extends Omit< + Step>, 'executionHandler' > { implemented: boolean; } export interface IntegrationSpecConfig< - TConfig extends IntegrationInstanceConfig = IntegrationInstanceConfig, + TInstanceConfig extends IntegrationInstanceConfig = IntegrationInstanceConfig, + TExecutionConfig extends IntegrationExecutionConfig = IntegrationExecutionConfig, > extends Omit { - integrationSteps: StepSpec[]; + integrationSteps: StepSpec[]; } diff --git a/packages/integration-sdk-core/src/types/step.ts b/packages/integration-sdk-core/src/types/step.ts index 8c86e4631..ea848342a 100644 --- a/packages/integration-sdk-core/src/types/step.ts +++ b/packages/integration-sdk-core/src/types/step.ts @@ -1,10 +1,11 @@ import { - RelationshipClass, IntegrationEntitySchema, + RelationshipClass, } from '@jupiterone/data-model'; import { ExecutionContext, + IntegrationExecutionConfig, IntegrationStepExecutionContext, StepExecutionContext, } from './context'; @@ -48,10 +49,6 @@ export type ExecutionHandlerFunction = ( context: T, ) => Promise | void; -export type StepExecutionHandlerFunction< - TConfig extends IntegrationInstanceConfig = IntegrationInstanceConfig, -> = ExecutionHandlerFunction>; - export enum StepResultStatus { SUCCESS = 'success', CACHED = 'cached', @@ -63,7 +60,7 @@ export enum StepResultStatus { export type Step = StepMetadata & { /** - * Function that runs to perform the stpe that + * Function that runs to perform the work of a `Step`. */ executionHandler: ExecutionHandlerFunction; }; @@ -92,13 +89,15 @@ export type IntegrationStepResult = Omit< }; export type IntegrationStep< - TConfig extends IntegrationInstanceConfig = IntegrationInstanceConfig, -> = StepMetadata & Step>; + TInstanceConfig extends IntegrationInstanceConfig = IntegrationInstanceConfig, + TExecutionConfig extends IntegrationExecutionConfig = IntegrationExecutionConfig, +> = StepMetadata & + Step>; export interface GraphObjectIndexMetadata { /** * Whether the index of the graph object store is enabled or not. For example, - * in the case leveraging the `FileSystemGraphObjectStore`, this value + * in the case of leveraging the `FileSystemGraphObjectStore`, this value * determines whether we need to write the specific graph object to disk. */ enabled: boolean; @@ -142,7 +141,7 @@ export interface StepGraphObjectMetadata { partial?: boolean; /** - * Contains metadadata that can be leveraged inside of the graph object store + * Contains metadata that can be leveraged inside the graph object store. */ indexMetadata?: GraphObjectIndexMetadata; } diff --git a/packages/integration-sdk-core/src/types/validation.ts b/packages/integration-sdk-core/src/types/validation.ts index ce0d479ae..52330b507 100644 --- a/packages/integration-sdk-core/src/types/validation.ts +++ b/packages/integration-sdk-core/src/types/validation.ts @@ -1,4 +1,8 @@ -import { ExecutionContext, IntegrationExecutionContext } from './context'; +import { + ExecutionContext, + IntegrationExecutionConfig, + IntegrationExecutionContext, +} from './context'; import { IntegrationInstanceConfig } from './instance'; export type InvocationValidationFunction = ( @@ -6,5 +10,8 @@ export type InvocationValidationFunction = ( ) => Promise | void; export type IntegrationInvocationValidationFunction< - TConfig extends IntegrationInstanceConfig = IntegrationInstanceConfig, -> = InvocationValidationFunction>; + TInstanceConfig extends IntegrationInstanceConfig = IntegrationInstanceConfig, + TExecutionConfig extends IntegrationExecutionConfig = IntegrationExecutionConfig, +> = InvocationValidationFunction< + IntegrationExecutionContext +>; diff --git a/packages/integration-sdk-private-test-utils/__fixtures__/typeScriptCustomConfigsProject/.env b/packages/integration-sdk-private-test-utils/__fixtures__/typeScriptCustomConfigsProject/.env new file mode 100644 index 000000000..bb535cb6c --- /dev/null +++ b/packages/integration-sdk-private-test-utils/__fixtures__/typeScriptCustomConfigsProject/.env @@ -0,0 +1 @@ +MY_INSTANCE_CONFIG_FIELD=myInstanceConfigFieldValue diff --git a/packages/integration-sdk-private-test-utils/__fixtures__/typeScriptCustomConfigsProject/.gitignore b/packages/integration-sdk-private-test-utils/__fixtures__/typeScriptCustomConfigsProject/.gitignore new file mode 100644 index 000000000..0188fb5d2 --- /dev/null +++ b/packages/integration-sdk-private-test-utils/__fixtures__/typeScriptCustomConfigsProject/.gitignore @@ -0,0 +1 @@ +.j1-integration \ No newline at end of file diff --git a/packages/integration-sdk-private-test-utils/__fixtures__/typeScriptCustomConfigsProject/src/config.ts b/packages/integration-sdk-private-test-utils/__fixtures__/typeScriptCustomConfigsProject/src/config.ts new file mode 100644 index 000000000..54323edae --- /dev/null +++ b/packages/integration-sdk-private-test-utils/__fixtures__/typeScriptCustomConfigsProject/src/config.ts @@ -0,0 +1,45 @@ +import { + IntegrationExecutionConfig, + IntegrationExecutionContext, + IntegrationInstanceConfig, + IntegrationStepExecutionContext, +} from '@jupiterone/integration-sdk-core'; + +export interface CustomInstanceConfig extends IntegrationInstanceConfig { + myInstanceConfigField: string; +} + +export interface CustomExecutionConfig extends IntegrationExecutionConfig { + myExecutionConfigField: string; +} + +export type CustomExecutionContext = IntegrationExecutionContext< + CustomInstanceConfig, + CustomExecutionConfig +>; + +export interface CustomStepExecutionContext + extends IntegrationStepExecutionContext< + CustomInstanceConfig, + CustomExecutionConfig + > { + myStepExecutionContextField: string; +} + +export function validateInvocation(context: CustomExecutionContext) { + if ( + context.instance.config.myInstanceConfigField !== + 'myInstanceConfigFieldValue' + ) + throw new Error('Invalid instance config'); +} + +export function loadExecutionConfig(options: { + config: CustomInstanceConfig; +}): CustomExecutionConfig { + if (options.config.myInstanceConfigField !== 'myInstanceConfigFieldValue') + throw new Error('Invalid instance config'); + return { + myExecutionConfigField: 'myExecutionConfigFieldValue', + }; +} diff --git a/packages/integration-sdk-private-test-utils/__fixtures__/typeScriptCustomConfigsProject/src/index.ts b/packages/integration-sdk-private-test-utils/__fixtures__/typeScriptCustomConfigsProject/src/index.ts new file mode 100644 index 000000000..0fbbc4631 --- /dev/null +++ b/packages/integration-sdk-private-test-utils/__fixtures__/typeScriptCustomConfigsProject/src/index.ts @@ -0,0 +1,23 @@ +import { IntegrationInvocationConfig } from '@jupiterone/integration-sdk-core'; +import { + CustomExecutionConfig, + CustomInstanceConfig, + loadExecutionConfig, + validateInvocation, +} from './config'; +import step from './steps'; + +export const invocationConfig: IntegrationInvocationConfig< + CustomInstanceConfig, + CustomExecutionConfig +> = { + instanceConfigFields: { + myInstanceConfigField: { + mask: true, + type: 'string', + }, + }, + integrationSteps: [step], + validateInvocation, + loadExecutionConfig, +}; diff --git a/packages/integration-sdk-private-test-utils/__fixtures__/typeScriptCustomConfigsProject/src/steps.ts b/packages/integration-sdk-private-test-utils/__fixtures__/typeScriptCustomConfigsProject/src/steps.ts new file mode 100644 index 000000000..121044824 --- /dev/null +++ b/packages/integration-sdk-private-test-utils/__fixtures__/typeScriptCustomConfigsProject/src/steps.ts @@ -0,0 +1,70 @@ +import { + createIntegrationEntity, + IntegrationStep, +} from '@jupiterone/integration-sdk-core'; +import { + CustomExecutionConfig, + CustomInstanceConfig, + CustomStepExecutionContext, +} from './config'; + +const stepId = 'step-id'; +const stepName = 'Step Name'; +const entities = [ + { + resourceName: 'The Account', + _type: 'my_account', + _class: 'Account', + }, +]; +const relationships = []; + +const step: IntegrationStep = { + id: stepId, + name: stepName, + entities, + relationships, + + executionHandler: async (context: CustomStepExecutionContext) => { + const { instance, jobState, executionConfig, stepMetadata } = context; + + if ( + context.myStepExecutionContextField != 'myStepExecutionContextFieldValue' + ) + throw new Error('Invalid step execution context'); + + if ( + executionConfig.myExecutionConfigField !== 'myExecutionConfigFieldValue' + ) + throw new Error('Invalid execution config'); + + if (instance.config.myInstanceConfigField !== 'myInstanceConfigFieldValue') + throw new Error('Invalid instance config'); + + if (stepMetadata.id != stepId) throw new Error('Invalid step metadata'); + if (stepMetadata.name != stepName) throw new Error('Invalid step metadata'); + if (stepMetadata.entities != entities) + throw new Error('Invalid step metadata'); + if (stepMetadata.relationships != relationships) + throw new Error('Invalid step metadata'); + + await jobState.addEntities([ + createIntegrationEntity({ + entityData: { + source: { + id: '1234', + name: instance.config.myInstanceConfigField, + description: executionConfig.myExecutionConfigField, + }, + assign: { + _key: 'account:1234', + _type: 'my_account', + _class: 'Account', + }, + }, + }), + ]); + }, +}; + +export default step; diff --git a/packages/integration-sdk-private-test-utils/__fixtures__/typeScriptIntegrationProject/src/index.ts b/packages/integration-sdk-private-test-utils/__fixtures__/typeScriptIntegrationProject/src/index.ts index 899ae617b..35f212dec 100644 --- a/packages/integration-sdk-private-test-utils/__fixtures__/typeScriptIntegrationProject/src/index.ts +++ b/packages/integration-sdk-private-test-utils/__fixtures__/typeScriptIntegrationProject/src/index.ts @@ -3,8 +3,9 @@ import fetchUsers from './steps/fetchUsers'; import validateInvocation from './validateInvocation'; import getStepStartStates from './getStepStartStates'; +import { IntegrationInvocationConfig } from '@jupiterone/integration-sdk-core'; -export const invocationConfig = { +export const invocationConfig: IntegrationInvocationConfig = { instanceConfigFields: { myConfig: { mask: true, diff --git a/packages/integration-sdk-private-test-utils/__fixtures__/typeScriptIntegrationProject/src/steps/fetchUsers.ts b/packages/integration-sdk-private-test-utils/__fixtures__/typeScriptIntegrationProject/src/steps/fetchUsers.ts index 80e6e7526..972fcef01 100644 --- a/packages/integration-sdk-private-test-utils/__fixtures__/typeScriptIntegrationProject/src/steps/fetchUsers.ts +++ b/packages/integration-sdk-private-test-utils/__fixtures__/typeScriptIntegrationProject/src/steps/fetchUsers.ts @@ -1,7 +1,7 @@ import { - IntegrationStepExecutionContext, - createIntegrationEntity, createDirectRelationship, + createIntegrationEntity, + IntegrationStepExecutionContext, Step, StepExecutionContext, } from '@jupiterone/integration-sdk-core'; @@ -25,9 +25,7 @@ const fetchUsersStep: Step = { targetType: 'my_user', }, ], - executionHandler: async ({ - jobState, - }: IntegrationStepExecutionContext<{}>) => { + executionHandler: async ({ jobState }: IntegrationStepExecutionContext) => { await jobState.addEntities([ createIntegrationEntity({ entityData: { diff --git a/packages/integration-sdk-private-test-utils/__fixtures__/typeScriptIntegrationProject/src/validateInvocation.ts b/packages/integration-sdk-private-test-utils/__fixtures__/typeScriptIntegrationProject/src/validateInvocation.ts index 87e6f00e7..adeb4bbbc 100644 --- a/packages/integration-sdk-private-test-utils/__fixtures__/typeScriptIntegrationProject/src/validateInvocation.ts +++ b/packages/integration-sdk-private-test-utils/__fixtures__/typeScriptIntegrationProject/src/validateInvocation.ts @@ -1,3 +1,7 @@ -export default function validateInvocation() { - return 'loaded'; +import { IntegrationExecutionContext } from '@jupiterone/integration-sdk-core'; + +export default function validateInvocation( + _context: IntegrationExecutionContext, +) { + // no problem-o } diff --git a/packages/integration-sdk-runtime/src/execution/__tests__/executeIntegration.test.ts b/packages/integration-sdk-runtime/src/execution/__tests__/executeIntegration.test.ts index 5838e9f7c..817254bd6 100644 --- a/packages/integration-sdk-runtime/src/execution/__tests__/executeIntegration.test.ts +++ b/packages/integration-sdk-runtime/src/execution/__tests__/executeIntegration.test.ts @@ -7,12 +7,14 @@ import * as zlib from 'zlib'; import { createDirectRelationship, Entity, + IntegrationExecutionConfig, IntegrationExecutionContext, IntegrationInstance, IntegrationInstanceConfig, IntegrationInvocationConfig, IntegrationInvocationValidationFunction, IntegrationLogger, + IntegrationStepExecutionContext, IntegrationValidationError, Relationship, RelationshipClass, @@ -72,11 +74,18 @@ async function getSortedLocalGraphData(): Promise< } export interface InstanceConfigurationData< - TIntegrationConfig extends IntegrationInstanceConfig = IntegrationInstanceConfig, + TInstanceConfig extends IntegrationInstanceConfig = IntegrationInstanceConfig, + TExecutionConfig extends IntegrationExecutionConfig = IntegrationExecutionConfig, > { - validateInvocation: IntegrationInvocationValidationFunction; - instance: IntegrationInstance; - invocationConfig: IntegrationInvocationConfig; + validateInvocation: IntegrationInvocationValidationFunction< + TInstanceConfig, + TExecutionConfig + >; + instance: IntegrationInstance; + invocationConfig: IntegrationInvocationConfig< + TInstanceConfig, + TExecutionConfig + >; logger: IntegrationLogger; } @@ -89,12 +98,13 @@ describe('executeIntegrationInstance', () => { const executionStartedOn = Date.now(); async function executeIntegrationInstanceWithConfig< - TIntegrationConfig extends IntegrationInstanceConfig = IntegrationInstanceConfig, + TInstanceConfig extends IntegrationInstanceConfig = IntegrationInstanceConfig, + TExecutionConfig extends IntegrationExecutionConfig = IntegrationExecutionConfig, >( - config: InstanceConfigurationData, + config: InstanceConfigurationData, options: ExecuteIntegrationOptions = {}, ) { - return executeIntegrationInstance( + return executeIntegrationInstance( config.logger, config.instance, config.invocationConfig, @@ -114,6 +124,35 @@ describe('executeIntegrationInstance', () => { .mockResolvedValue(Promise.resolve(1000)); }); + test('provides stepMetadata to executionHandler context', async () => { + let stepContext: IntegrationStepExecutionContext | undefined; + const executionHandler = jest + .fn() + .mockImplementation((context: IntegrationStepExecutionContext) => { + stepContext = context; + }); + + const config = createInstanceConfiguration({ + invocationConfig: { + integrationSteps: [ + { + id: 'step', + name: 'Step', + entities: [], + relationships: [], + executionHandler, + }, + ], + }, + }); + + await executeIntegrationInstanceWithConfig(config); + + expect(stepContext).toBeDefined(); + expect(stepContext?.stepMetadata).toBeDefined(); + expect(stepContext?.stepMetadata.id).toEqual('step'); + }); + test('executes validateInvocation function if provided in config', async () => { const config = createInstanceConfiguration(); await executeIntegrationInstanceWithConfig(config); diff --git a/packages/integration-sdk-runtime/src/execution/__tests__/utils/createIntegrationConfig.ts b/packages/integration-sdk-runtime/src/execution/__tests__/utils/createIntegrationConfig.ts index 9daa0bc4c..a71c02698 100644 --- a/packages/integration-sdk-runtime/src/execution/__tests__/utils/createIntegrationConfig.ts +++ b/packages/integration-sdk-runtime/src/execution/__tests__/utils/createIntegrationConfig.ts @@ -1,22 +1,31 @@ import { - IntegrationInvocationValidationFunction, - IntegrationInvocationConfig, - IntegrationInstanceConfig, + IntegrationExecutionConfig, IntegrationInstance, + IntegrationInstanceConfig, + IntegrationInvocationConfig, + IntegrationInvocationValidationFunction, } from '@jupiterone/integration-sdk-core'; import { LOCAL_INTEGRATION_INSTANCE } from '../..'; import { createIntegrationLogger } from '../../..'; import { InstanceConfigurationData } from '../executeIntegration.test'; export function createInstanceConfiguration< - TIntegrationConfig extends IntegrationInstanceConfig = IntegrationInstanceConfig, + TInstanceConfig extends IntegrationInstanceConfig = IntegrationInstanceConfig, + TExecutionConfig extends IntegrationExecutionConfig = IntegrationExecutionConfig, >( - options?: Partial>, -): InstanceConfigurationData { - const validateInvocation: IntegrationInvocationValidationFunction = - options?.validateInvocation || jest.fn(); + options?: Partial< + InstanceConfigurationData + >, +): InstanceConfigurationData { + const validateInvocation: IntegrationInvocationValidationFunction< + TInstanceConfig, + TExecutionConfig + > = options?.validateInvocation || jest.fn(); - const invocationConfig: IntegrationInvocationConfig = { + const invocationConfig: IntegrationInvocationConfig< + TInstanceConfig, + TExecutionConfig + > = { validateInvocation, integrationSteps: [], ...options?.invocationConfig, @@ -26,7 +35,7 @@ export function createInstanceConfiguration< validateInvocation, invocationConfig, instance: - LOCAL_INTEGRATION_INSTANCE as IntegrationInstance, + LOCAL_INTEGRATION_INSTANCE as IntegrationInstance, logger: createIntegrationLogger({ name: 'integration-name', invocationConfig, diff --git a/packages/integration-sdk-runtime/src/execution/config.ts b/packages/integration-sdk-runtime/src/execution/config.ts index 012e1b22d..83a4082d9 100644 --- a/packages/integration-sdk-runtime/src/execution/config.ts +++ b/packages/integration-sdk-runtime/src/execution/config.ts @@ -2,11 +2,11 @@ import dotenv from 'dotenv'; import snakeCase from 'lodash/snakeCase'; import { - IntegrationLocalConfigFieldMissingError, - IntegrationLocalConfigFieldTypeMismatchError, IntegrationInstanceConfig, IntegrationInstanceConfigField, IntegrationInstanceConfigFieldMap, + IntegrationLocalConfigFieldMissingError, + IntegrationLocalConfigFieldTypeMismatchError, } from '@jupiterone/integration-sdk-core'; const dotenvExpand = require('dotenv-expand'); @@ -15,8 +15,10 @@ const dotenvExpand = require('dotenv-expand'); * Reads integration configuration from environment variables */ export function loadConfigFromEnvironmentVariables< - TConfig extends IntegrationInstanceConfig, ->(configMap: IntegrationInstanceConfigFieldMap): TConfig { + TInstanceConfig extends IntegrationInstanceConfig, +>( + configMap: IntegrationInstanceConfigFieldMap, +): TInstanceConfig { // pull in environment variables from .env file if available dotenvExpand(dotenv.config()); @@ -46,7 +48,7 @@ export function loadConfigFromEnvironmentVariables< acc[field] = value; } return acc; - }, {}) as TConfig; + }, {}) as TInstanceConfig; } function convertEnvironmentVariableValueForField( diff --git a/packages/integration-sdk-runtime/src/execution/dependencyGraph.ts b/packages/integration-sdk-runtime/src/execution/dependencyGraph.ts index 85a885393..aaf4efbe5 100644 --- a/packages/integration-sdk-runtime/src/execution/dependencyGraph.ts +++ b/packages/integration-sdk-runtime/src/execution/dependencyGraph.ts @@ -11,6 +11,7 @@ import { Relationship, Step, StepExecutionContext, + StepMetadata, StepResultStatus, StepStartStates, } from '@jupiterone/integration-sdk-core'; @@ -306,7 +307,7 @@ export function executeStepDependencyGraph< typeTracker, graphObjectStore, dataStore, - stepId, + stepMetadata: step, beforeAddEntity, beforeAddRelationship, uploader, @@ -437,7 +438,7 @@ function buildStepContext< TExecutionContext extends ExecutionContext, TStepExecutionContext extends StepExecutionContext, >({ - stepId, + stepMetadata, context, duplicateKeyTracker, typeTracker, @@ -447,7 +448,7 @@ function buildStepContext< beforeAddEntity, beforeAddRelationship, }: { - stepId: string; + stepMetadata: StepMetadata; context: TExecutionContext; duplicateKeyTracker: DuplicateKeyTracker; typeTracker: TypeTracker; @@ -475,7 +476,7 @@ function buildStepContext< : undefined; const jobState = createStepJobState({ - stepId, + stepId: stepMetadata.id, duplicateKeyTracker, typeTracker, graphObjectStore, @@ -488,9 +489,10 @@ function buildStepContext< const stepExecutionContext: StepExecutionContext = { ...context, logger: context.logger.child({ - stepId, + stepId: stepMetadata.id, }), jobState, + stepMetadata, }; return stepExecutionContext as TStepExecutionContext; diff --git a/packages/integration-sdk-runtime/src/execution/executeIntegration.ts b/packages/integration-sdk-runtime/src/execution/executeIntegration.ts index 35bd6b719..28c6f441d 100644 --- a/packages/integration-sdk-runtime/src/execution/executeIntegration.ts +++ b/packages/integration-sdk-runtime/src/execution/executeIntegration.ts @@ -1,6 +1,7 @@ import { ExecutionContext, ExecutionHistory, + IntegrationExecutionConfig, IntegrationInstance, IntegrationInstanceConfig, IntegrationInvocationConfig, @@ -95,11 +96,12 @@ export async function executeIntegrationLocally( * Starts execution of an integration instance. */ export async function executeIntegrationInstance< - TIntegrationConfig extends IntegrationInstanceConfig = IntegrationInstanceConfig, + TInstanceConfig extends IntegrationInstanceConfig = IntegrationInstanceConfig, + TExecutionConfig extends IntegrationExecutionConfig = IntegrationExecutionConfig, >( logger: IntegrationLogger, - instance: IntegrationInstance, - config: IntegrationInvocationConfig, + instance: IntegrationInstance, + config: IntegrationInvocationConfig, executionHistory: ExecutionHistory, options: ExecuteIntegrationOptions = {}, ): Promise { @@ -120,8 +122,9 @@ export async function executeIntegrationInstance< instance: instanceWithTrimmedConfig, logger, executionHistory, - executionConfig: - config.loadExecutionConfig?.(instanceWithTrimmedConfig) || {}, + executionConfig: (config.loadExecutionConfig?.( + instanceWithTrimmedConfig, + ) || {}) as TExecutionConfig, }, config, options, diff --git a/packages/integration-sdk-runtime/src/execution/utils/getMaskedFields.ts b/packages/integration-sdk-runtime/src/execution/utils/getMaskedFields.ts index 1838a5442..8f83675d0 100644 --- a/packages/integration-sdk-runtime/src/execution/utils/getMaskedFields.ts +++ b/packages/integration-sdk-runtime/src/execution/utils/getMaskedFields.ts @@ -1,11 +1,13 @@ import { + IntegrationExecutionConfig, IntegrationInstanceConfig, IntegrationInvocationConfig, } from '@jupiterone/integration-sdk-core'; import { keys, pickBy } from 'lodash'; export function getMaskedFields< - TIntegrationConfig extends IntegrationInstanceConfig = IntegrationInstanceConfig, ->(config: IntegrationInvocationConfig) { + TInstanceConfig extends IntegrationInstanceConfig = IntegrationInstanceConfig, + TExecutionConfig extends IntegrationExecutionConfig = IntegrationExecutionConfig, +>(config: IntegrationInvocationConfig) { return keys(pickBy(config.instanceConfigFields, (val) => val.mask)); } diff --git a/packages/integration-sdk-runtime/src/logger/index.ts b/packages/integration-sdk-runtime/src/logger/index.ts index 8cf4495eb..66afeec55 100644 --- a/packages/integration-sdk-runtime/src/logger/index.ts +++ b/packages/integration-sdk-runtime/src/logger/index.ts @@ -3,10 +3,14 @@ import { EventEmitter } from 'events'; import { v4 as uuid } from 'uuid'; import { + DisabledStepReason, ExecutionContext, IntegrationError, + IntegrationEvent, + IntegrationExecutionConfig, IntegrationExecutionContext, IntegrationInstance, + IntegrationInstanceConfig, IntegrationInstanceConfigFieldMap, IntegrationInvocationConfig, IntegrationLogger as IntegrationLoggerType, @@ -14,27 +18,24 @@ import { InvocationConfig, isProviderAuthError, Metric, + PublishErrorEventInput, + PublishEventInput, + PublishEventLevel, + PublishInfoEventInput, + PublishWarnEventInput, shouldReportErrorToOperator, StepExecutionContext, StepMetadata, SynchronizationJob, UNEXPECTED_ERROR_CODE, UNEXPECTED_ERROR_REASON, - IntegrationInstanceConfig, - PublishEventInput, - PublishWarnEventInput, - PublishEventLevel, - IntegrationEvent, - PublishInfoEventInput, - PublishErrorEventInput, - DisabledStepReason, } from '@jupiterone/integration-sdk-core'; export * from './registerEventHandlers'; export const PROVIDER_AUTH_ERROR_HELP = ' Failed to access provider resource.' + - ' This integration is likely misconfigured or has insufficient permissions required to access the resource.' + + ' This integration is likely mis-configured or has insufficient permissions required to access the resource.' + " Please ensure your integration's configuration settings are set up correctly."; // eslint-disable-next-line @@ -52,12 +53,16 @@ interface CreateLoggerInput< } interface CreateIntegrationLoggerInput< - TIntegrationConfig extends IntegrationInstanceConfig = IntegrationInstanceConfig, + TInstanceConfig extends IntegrationInstanceConfig = IntegrationInstanceConfig, + TExecutionConfig extends IntegrationExecutionConfig = IntegrationExecutionConfig, > extends CreateLoggerInput< - IntegrationExecutionContext, - IntegrationStepExecutionContext + IntegrationExecutionContext, + IntegrationStepExecutionContext > { - invocationConfig?: IntegrationInvocationConfig; + invocationConfig?: IntegrationInvocationConfig< + TInstanceConfig, + TExecutionConfig + >; } interface PublishMetricOptions { @@ -113,14 +118,18 @@ export function createLogger< * serializers common to all integrations. */ export function createIntegrationLogger< - TIntegrationConfig extends IntegrationInstanceConfig = IntegrationInstanceConfig, + TInstanceConfig extends IntegrationInstanceConfig = IntegrationInstanceConfig, + TExecutionConfig extends IntegrationExecutionConfig = IntegrationExecutionConfig, >({ name, invocationConfig, pretty, serializers, onFailure, -}: CreateIntegrationLoggerInput): IntegrationLogger { +}: CreateIntegrationLoggerInput< + TInstanceConfig, + TExecutionConfig +>): IntegrationLogger { const serializeInstanceConfig = createInstanceConfigSerializer( invocationConfig?.instanceConfigFields, ); @@ -144,8 +153,8 @@ export function createIntegrationLogger< } function createInstanceConfigSerializer< - TConfig extends IntegrationInstanceConfig = IntegrationInvocationConfig, ->(fields?: IntegrationInstanceConfigFieldMap) { + TInstanceConfig extends IntegrationInstanceConfig = IntegrationInvocationConfig, +>(fields?: IntegrationInstanceConfigFieldMap) { return (config: any) => { if (!config) { return config; @@ -185,8 +194,8 @@ export class IntegrationLogger extends EventEmitter implements IntegrationLoggerType { - private _logger: Logger; - private _errorSet: Set; + private readonly _logger: Logger; + private readonly _errorSet: Set; readonly onFailure: OnFailureFunction; constructor(input: IntegrationLoggerInput) { @@ -199,7 +208,7 @@ export class IntegrationLogger } /** - * Answers `true` when the err has been reported to the logger instance + * Answers `true` when err has been reported to the logger instance * through these functions: * * * warn(err, ...) diff --git a/packages/integration-sdk-testing/src/__tests__/context.test.ts b/packages/integration-sdk-testing/src/__tests__/context.test.ts index c5074dab7..2863a0f48 100644 --- a/packages/integration-sdk-testing/src/__tests__/context.test.ts +++ b/packages/integration-sdk-testing/src/__tests__/context.test.ts @@ -1,13 +1,16 @@ import { Entity, - Relationship, - IntegrationStep, + IntegrationExecutionConfig, + IntegrationInstanceConfig, IntegrationInstanceConfigFieldMap, + IntegrationStep, + Relationship, + StepMetadata, } from '@jupiterone/integration-sdk-core'; import { - LOCAL_INTEGRATION_INSTANCE, IntegrationLogger, + LOCAL_INTEGRATION_INSTANCE, } from '@jupiterone/integration-sdk-runtime'; import { @@ -23,20 +26,32 @@ import { } from '../context'; import { v4 as uuid } from 'uuid'; -interface Config { +interface InstanceConfig extends IntegrationInstanceConfig { myBooleanConfig: boolean; myStringConfig: string; myTypelessConfig: string; } -const instanceConfigFields: IntegrationInstanceConfigFieldMap = { - myBooleanConfig: { - type: 'boolean', - }, - myStringConfig: { - type: 'string', - }, - myTypelessConfig: {}, +interface ExecutionConfig extends IntegrationExecutionConfig { + myExecutionConfig: string; +} + +const instanceConfigFields: IntegrationInstanceConfigFieldMap = + { + myBooleanConfig: { + type: 'boolean', + }, + myStringConfig: { + type: 'string', + }, + myTypelessConfig: {}, + }; + +const stepMetadata: StepMetadata = { + id: 'my-step', + name: 'My Step', + entities: [], + relationships: [], }; /** @@ -82,7 +97,7 @@ const instanceConfigFields: IntegrationInstanceConfigFieldMap = { }); }); - test('accepts an instanceConfig for prepopulating configuration values', () => { + test('accepts an instanceConfig for pre-populating configuration values', () => { const config = { test: true }; const { instance } = createContext({ instanceConfig: config }); expect(instance).toEqual({ ...LOCAL_INTEGRATION_INSTANCE, config }); @@ -125,7 +140,7 @@ const instanceConfigFields: IntegrationInstanceConfigFieldMap = { describe('createMockExecutionContext', () => { test('accepts generic for typed instance config', () => { - const { instance } = createMockExecutionContext({ + const { instance } = createMockExecutionContext({ instanceConfigFields, }); @@ -136,9 +151,13 @@ describe('createMockExecutionContext', () => { }); describe('createMockStepExecutionContext', () => { - test('accepts generic for typed instance config', () => { - const { instance } = createMockStepExecutionContext({ + test('accepts generic for typed configs', () => { + const { instance } = createMockStepExecutionContext< + InstanceConfig, + ExecutionConfig + >({ instanceConfigFields, + stepMetadata, }); expect(instance.config.myTypelessConfig).toBeDefined(); @@ -167,6 +186,7 @@ describe('createMockStepExecutionContext', () => { const { jobState } = createMockStepExecutionContext({ instanceConfigFields, + stepMetadata, }); await jobState.addEntities(entities); @@ -191,6 +211,7 @@ describe('createMockStepExecutionContext', () => { ]; const { jobState } = createMockStepExecutionContext({ instanceConfigFields, + stepMetadata, }); await expect(jobState.addEntities(entities)).rejects.toThrow( @@ -211,13 +232,17 @@ describe('createMockStepExecutionContext', () => { }, }; - const context = createMockStepExecutionContext({ instanceConfigFields }); + const context = createMockStepExecutionContext({ + instanceConfigFields, + stepMetadata, + }); await step.executionHandler(context); }); test('tracks encounteredTypes', async () => { const { jobState } = createMockStepExecutionContext({ instanceConfigFields, + stepMetadata, }); await jobState.addEntity({ diff --git a/packages/integration-sdk-testing/src/context.ts b/packages/integration-sdk-testing/src/context.ts index 892e3bce5..c70f7a5b8 100644 --- a/packages/integration-sdk-testing/src/context.ts +++ b/packages/integration-sdk-testing/src/context.ts @@ -1,11 +1,13 @@ import { ExecutionHistory, + IntegrationExecutionConfig, IntegrationExecutionContext, IntegrationInstance, IntegrationInstanceConfig, IntegrationInstanceConfigField, IntegrationInstanceConfigFieldMap, IntegrationStepExecutionContext, + StepMetadata, } from '@jupiterone/integration-sdk-core'; import { loadConfigFromEnvironmentVariables, @@ -20,45 +22,60 @@ import { import { createMockIntegrationLogger } from './logger'; export type CreateMockExecutionContextOptions< - TConfig extends IntegrationInstanceConfig = IntegrationInstanceConfig, + TInstanceConfig extends IntegrationInstanceConfig = IntegrationInstanceConfig, + TExecutionConfig extends IntegrationExecutionConfig = IntegrationExecutionConfig, > = - | CreateMockExecutionContextOptionsWithInstanceConfig - | CreateMockExecutionContextOptionsWithInstanceConfigFields; + | CreateMockExecutionContextOptionsWithConfigs< + TInstanceConfig, + TExecutionConfig + > + | CreateMockExecutionContextOptionsWithInstanceConfigFields< + TInstanceConfig, + TExecutionConfig + >; -interface CreateMockExecutionContextOptionsWithInstanceConfig< - TConfig extends IntegrationInstanceConfig = IntegrationInstanceConfig, +interface CreateMockExecutionContextOptionsWithConfigs< + TInstanceConfig extends IntegrationInstanceConfig = IntegrationInstanceConfig, + TExecutionConfig extends IntegrationExecutionConfig = IntegrationExecutionConfig, > { - instanceConfig: TConfig; + instanceConfig: TInstanceConfig; + executionConfig: TExecutionConfig; executionHistory?: ExecutionHistory; } interface CreateMockExecutionContextOptionsWithInstanceConfigFields< - TConfig extends IntegrationInstanceConfig = IntegrationInstanceConfig, + TInstanceConfig extends IntegrationInstanceConfig = IntegrationInstanceConfig, + TExecutionConfig extends IntegrationExecutionConfig = IntegrationExecutionConfig, > { - instanceConfigFields: IntegrationInstanceConfigFieldMap; + instanceConfigFields: IntegrationInstanceConfigFieldMap; executionHistory?: ExecutionHistory; } export function createMockExecutionContext< - TConfig extends IntegrationInstanceConfig = IntegrationInstanceConfig, + TInstanceConfig extends IntegrationInstanceConfig = IntegrationInstanceConfig, + TExecutionConfig extends IntegrationExecutionConfig = IntegrationExecutionConfig, >( - options: CreateMockExecutionContextOptions = { - instanceConfigFields: {} as IntegrationInstanceConfigFieldMap, + options: CreateMockExecutionContextOptions< + TInstanceConfig, + TExecutionConfig + > = { + instanceConfigFields: + {} as IntegrationInstanceConfigFieldMap, }, -): IntegrationExecutionContext { +): IntegrationExecutionContext { const logger = createMockIntegrationLogger(); const accountId = process.env.JUPITERONE_LOCAL_INTEGRATION_INSTANCE_ACCOUNT_ID || LOCAL_INTEGRATION_INSTANCE.accountId; // copy local instance properties so that tests cannot - // mutate the original object and cause unpredicable behavior + // mutate the original object and cause unpredictable behavior const instance = { ...LOCAL_INTEGRATION_INSTANCE, accountId, - } as IntegrationInstance; + } as IntegrationInstance; - if (isOptionsWithInstanceConfig(options)) { + if (isOptionsWithInstanceConfig(options)) { instance.config = options.instanceConfig; } else { const { instanceConfigFields } = options; @@ -75,14 +92,15 @@ export function createMockExecutionContext< // this would generally only happen when a developer does not // have an .env file configured or when an integration's test suite // runs in CI - instance.config = generateInstanceConfig(instanceConfigFields); + instance.config = + generateInstanceConfig(instanceConfigFields); } } return { logger, instance, - executionConfig: {}, + executionConfig: {} as TExecutionConfig, executionHistory: options.executionHistory || { current: { startedOn: Date.now(), @@ -92,32 +110,45 @@ export function createMockExecutionContext< } function isOptionsWithInstanceConfig< - TConfig extends IntegrationInstanceConfig = IntegrationInstanceConfig, + TInstanceConfig extends IntegrationInstanceConfig = IntegrationInstanceConfig, >( - options: CreateMockExecutionContextOptions, -): options is CreateMockExecutionContextOptionsWithInstanceConfig { + options: CreateMockExecutionContextOptions, +): options is CreateMockExecutionContextOptionsWithConfigs { return !!(options as any).instanceConfig; } export type CreateMockStepExecutionContextOptions< - TConfig extends IntegrationInstanceConfig = IntegrationInstanceConfig, -> = CreateMockExecutionContextOptions & CreateMockJobStateOptions; + TInstanceConfig extends IntegrationInstanceConfig = IntegrationInstanceConfig, + TExecutionConfig extends IntegrationExecutionConfig = IntegrationExecutionConfig, +> = CreateMockExecutionContextOptions & + CreateMockJobStateOptions & { + stepMetadata: StepMetadata; + }; export interface MockIntegrationStepExecutionContext< - TConfig extends IntegrationInstanceConfig = IntegrationInstanceConfig, -> extends IntegrationStepExecutionContext { + TInstanceConfig extends IntegrationInstanceConfig = IntegrationInstanceConfig, + TExecutionConfig extends IntegrationExecutionConfig = IntegrationExecutionConfig, +> extends IntegrationStepExecutionContext { jobState: MockJobState; + stepMetadata: StepMetadata; } export function createMockStepExecutionContext< - TConfig extends IntegrationInstanceConfig = IntegrationInstanceConfig, + TInstanceConfig extends IntegrationInstanceConfig = IntegrationInstanceConfig, + TExecutionConfig extends IntegrationExecutionConfig = IntegrationExecutionConfig, >( - options: CreateMockStepExecutionContextOptions = { - instanceConfigFields: {} as IntegrationInstanceConfigFieldMap, + options: CreateMockStepExecutionContextOptions< + TInstanceConfig, + TExecutionConfig + > = { + instanceConfigFields: + {} as IntegrationInstanceConfigFieldMap, + stepMetadata: {} as StepMetadata, }, -): MockIntegrationStepExecutionContext { +): MockIntegrationStepExecutionContext { return { - ...createMockExecutionContext(options), + ...createMockExecutionContext(options), + stepMetadata: options.stepMetadata, jobState: createMockJobState(options), }; } diff --git a/packages/integration-sdk-testing/src/executeStepWithDependencies.ts b/packages/integration-sdk-testing/src/executeStepWithDependencies.ts index f74221416..fbab918e6 100644 --- a/packages/integration-sdk-testing/src/executeStepWithDependencies.ts +++ b/packages/integration-sdk-testing/src/executeStepWithDependencies.ts @@ -1,4 +1,7 @@ -import { IntegrationExecutionConfig } from '@jupiterone/integration-sdk-core'; +import { + IntegrationExecutionConfig, + StepMetadata, +} from '@jupiterone/integration-sdk-core'; import { buildStepDependencyGraph } from '@jupiterone/integration-sdk-runtime'; import { StepTestConfig } from './config'; import { @@ -28,13 +31,25 @@ export async function executeStepWithDependencies(params: StepTestConfig) { const preContext: MockIntegrationStepExecutionContext & { executionConfig: IntegrationExecutionConfig; } = { - ...createMockStepExecutionContext({ instanceConfig }), + ...createMockStepExecutionContext({ + instanceConfig, + executionConfig, + stepMetadata: {} as StepMetadata, + }), executionConfig, }; for (const dependencyStepId of dependencyStepIds) { const dependencyStep = stepDependencyGraph.getNodeData(dependencyStepId); - await dependencyStep.executionHandler(preContext); + await dependencyStep.executionHandler({ + ...preContext, + stepMetadata: { + id: dependencyStep.id, + name: dependencyStep.name, + entities: dependencyStep.entities, + relationships: dependencyStep.relationships, + }, + }); } const context: MockIntegrationStepExecutionContext & { @@ -42,6 +57,13 @@ export async function executeStepWithDependencies(params: StepTestConfig) { } = { ...createMockStepExecutionContext({ instanceConfig, + executionConfig, + stepMetadata: { + id: step.id, + name: step.name, + entities: step.entities, + relationships: step.relationships, + }, entities: preContext.jobState.collectedEntities, relationships: preContext.jobState.collectedRelationships, setData: preContext.jobState.collectedData,