CLI: support credential maps (#1191)

* cli: added credential map utility

* integrate credential map into cli

* project: map project_credential_id to configuration

* delete test

* add test

* tweak error messages

* tidy

* changeset

* integration test

* fix test

* support yaml credential map

* skip flaky engine test

Author: josephjclark

.changeset/orange-pigs-cross.md

@@ -0,0 +1,5 @@
+---
+'@openfn/cli': minor
+---
+
+Allow credential map, as json or yaml, to be passed via --credentials

.changeset/public-dragons-study.md

@@ -0,0 +1,5 @@
+---
+'@openfn/project': patch
+---
+
+Map project_credential_id to configuration

integration-tests/cli/test/execute-workflow.test.ts

@@ -147,6 +147,18 @@ test.serial(
   }
 );
 
+test.serial(
+  `openfn ${jobsPath}/wf-creds.json --credentials ${jobsPath}/creds.json`,
+  async (t) => {
+    const { err, stdout, stderr } = await run(t.title);
+    console.log({ stdout, stderr });
+    t.falsy(err);
+
+    const out = getJSON();
+    t.is(out.value, 'admin:admin');
+  }
+);
+
 test.serial(
   `openfn ${jobsPath}/wf-errors.json -S "{ \\"data\\": { \\"number\\": 2 } }"`,
   async (t) => {

integration-tests/cli/test/fixtures/creds.js

@@ -0,0 +1,4 @@
+fn((s) => {
+  s.value = `${s.configuration.user}:${s.configuration.password}`;
+  return s;
+});

integration-tests/cli/test/fixtures/creds.json

@@ -0,0 +1,6 @@
+{
+  "08089249-0890-4a73-8799-e2ec2b9e5d77": {
+    "user": "admin",
+    "password": "admin"
+  }
+}

integration-tests/cli/test/fixtures/wf-creds.json

@@ -0,0 +1,11 @@
+{
+  "workflow": {
+    "steps": [
+      {
+        "adaptor": "common",
+        "configuration": "08089249-0890-4a73-8799-e2ec2b9e5d77",
+        "expression": "creds.js"
+      }
+    ]
+  }
+}

packages/cli/src/execute/apply-credential-map.ts

@@ -0,0 +1,55 @@
+/**
+ * utility to take a workflow and a credential map
+ * and apply credentials to each step
+ */
+
+import { ExecutionPlan } from '@openfn/lexicon';
+import { Logger } from '../util';
+
+type JobId = string;
+
+export type CredentialMap = Record<JobId, any>;
+
+const applyCredentialMap = (
+  plan: ExecutionPlan,
+  map: CredentialMap = {},
+  logger?: Logger
+) => {
+  const stepsWithCredentialIds = plan.workflow.steps.filter(
+    (step: any) =>
+      typeof step.configuration === 'string' &&
+      !step.configuration.endsWith('.json')
+  ) as { configuration: string; name?: string; id: string }[];
+
+  const unmapped: Record<string, true> = {};
+
+  for (const step of stepsWithCredentialIds) {
+    if (map[step.configuration]) {
+      logger?.debug(
+        `Applying credential ${step.configuration} to "${step.name ?? step.id}"`
+      );
+      step.configuration = map[step.configuration];
+    } else {
+      unmapped[step.configuration] = true;
+      // @ts-ignore
+      delete step.configuration;
+    }
+  }
+
+  if (Object.keys(unmapped).length) {
+    logger?.warn(
+      `WARNING: credential IDs were found in the workflow, but values have not been provided:`
+    );
+    logger?.warn('  ', Object.keys(unmapped).join(','));
+    if (map) {
+      logger?.warn(
+        'If the workflow fails, add these credentials to the credential map'
+      );
+    } else {
+      // TODO if running from project file this might be bad advice
+      logger?.warn('Pass a credential map with --credentials');
+    }
+  }
+};
+
+export default applyCredentialMap;

packages/cli/src/execute/command.ts

@@ -13,6 +13,7 @@ export type ExecuteOptions = Required<
     | 'cacheSteps'
     | 'command'
     | 'compile'
+    | 'credentials'
     | 'expandAdaptors'
     | 'end'
     | 'immutable'
@@ -46,6 +47,7 @@ const options = [
   o.autoinstall,
   o.cacheSteps,
   o.compile,
+  o.credentials,
   o.end,
   o.ignoreImports,
   o.immutable,

packages/cli/src/execute/handler.ts

@@ -1,9 +1,13 @@
 import type { ExecutionPlan } from '@openfn/lexicon';
+import { yamlToJson } from '@openfn/project';
+import { readFile } from 'node:fs/promises';
+import path from 'node:path';
 
 import type { ExecuteOptions } from './command';
 import execute from './execute';
 import serializeOutput from './serialize-output';
 import getAutoinstallTargets from './get-autoinstall-targets';
+import applyCredentialMap from './apply-credential-map';
 
 import { install } from '../repo/handler';
 import compile from '../compile/compile';
@@ -44,14 +48,43 @@ const matchStep = (
   return '';
 };
 
+const loadAndApplyCredentialMap = async (
+  plan: ExecutionPlan,
+  options: ExecuteOptions,
+  logger: Logger
+) => {
+  let creds = {};
+  if (options.credentials) {
+    try {
+      const credsRaw = await readFile(
+        path.resolve(options.credentials),
+        'utf8'
+      );
+      if (options.credentials.endsWith('.json')) {
+        creds = JSON.parse(credsRaw);
+      } else {
+        creds = yamlToJson(credsRaw);
+      }
+    } catch (e) {
+      logger.error('Error processing credential map:');
+      logger.error(e);
+      // probably want to exist if the credential map is invalid
+      process.exitCode = 1;
+      return;
+    }
+    logger.info('Credential map loaded ');
+  }
+  return applyCredentialMap(plan, creds, logger);
+};
+
 const executeHandler = async (options: ExecuteOptions, logger: Logger) => {
   const start = new Date().getTime();
   assertPath(options.path);
   await validateAdaptors(options, logger);
 
   let plan = await loadPlan(options, logger);
   validatePlan(plan, logger);
-
+  await loadAndApplyCredentialMap(plan, options, logger);
   if (options.cacheSteps) {
     await clearCache(plan, options, logger);
   }

packages/cli/src/options.ts

@@ -31,6 +31,7 @@ export type Opts = {
   compile?: boolean;
   configPath?: string;
   confirm?: boolean;
+  credentials?: string;
   describe?: string;
   end?: string; // workflow end node
   env?: string;
@@ -245,6 +246,14 @@ export const configPath: CLIOption = {
   },
 };
 
+export const credentials: CLIOption = {
+  name: 'credentials',
+  yargs: {
+    alias: ['creds'],
+    description: 'A path which points to a credential map',
+  },
+};
+
 export const describe: CLIOption = {
   name: 'describe',
   yargs: {