feat(testing-sdk): support running cloud tests asynchronously (#92)

*Issue #, if available:*
DAR-SDK-305

*Description of changes:*

Adding support for running cloud tests with `InvocationType.Event`
(async invoke).

The implementation uses a HistoryPoller class which will poll the
execution history to get all events. Customers will be able to specify
if they want to use async or sync executions using the `invocationType`
parameter. By default, the runner will use sync executions only and
`waitFor` will throw an error unless they use `InvocationType.Event`.

Additionally:
- Enabling wait-for-callback.test.ts to run on both cloud and local
runners
- Adjust integration-test.js script to allow running specific tests

TODO:
- Stop the execution and polling if the test fails. Currently it will
run forever if the test fails.
- Add a test that has many events (this will take time to run since it
needs to generate a lot of events -> maybe run this only on push?)

By submitting this pull request, I confirm that you can use, modify,
copy, and redistribute this contribution, under the terms of your
choice.

Author: anthonyting

.github/workflows/scripts/integration-test/integration-test.js

@@ -219,7 +219,7 @@ class IntegrationTestRunner {
   }
 
   // Run Jest integration tests
-  async runJestTests() {
+  async runJestTests(/** @type {string | undefined} */ testPattern) {
     log.info("Running Jest integration tests...");
 
     const examplesDir = CONFIG.EXAMPLES_PACKAGE_PATH;
@@ -235,7 +235,14 @@ class IntegrationTestRunner {
     console.log(JSON.stringify(this.getFunctionNameMap(), null, 2));
     log.info(`Lambda Endpoint: ${CONFIG.LAMBDA_ENDPOINT}`);
 
-    this.execCommand("npm run test:integration", {
+    // Build test command with optional pattern
+    let testCommand = "npm run test:integration";
+    if (testPattern) {
+      testCommand += ` -- ${testPattern}`;
+      log.info(`Running tests with pattern: ${testPattern}`);
+    }
+
+    this.execCommand(testCommand, {
       cwd: examplesDir,
       env,
     });
@@ -273,12 +280,14 @@ class IntegrationTestRunner {
    * @param {boolean} [options.deployOnly]
    * @param {boolean} [options.testOnly]
    * @param {boolean} [options.cleanupOnly]
+   * @param {string} [options.testPattern]
    */
   async run(options = {}) {
     const {
       deployOnly = false,
       testOnly = false,
       cleanupOnly = false,
+      testPattern,
     } = options;
 
     log.info("Starting integration test...");
@@ -297,7 +306,7 @@ class IntegrationTestRunner {
     }
 
     if (!deployOnly) {
-      await this.runJestTests();
+      await this.runJestTests(testPattern);
     }
 
     log.success("Integration test completed successfully!");
@@ -312,16 +321,23 @@ class IntegrationTestRunner {
 
 // CLI interface
 function showUsage() {
-  console.log("Usage: node integration-test.js [OPTIONS]");
+  console.log("Usage: node integration-test.js [OPTIONS] [TEST_PATTERN]");
   console.log("");
   console.log("Options:");
   console.log("  --deploy-only   Only deploy functions, don't run tests");
   console.log(
-    "  --test-only     Only run tests (assumes functions are already deployed)"
+    "  --test-only [pattern]  Only run tests (assumes functions are already deployed)"
+  );
+  console.log(
+    "                         Optional test pattern to filter specific tests"
   );
   console.log("  --cleanup-only  Only cleanup existing functions");
   console.log("  --help          Show this help message");
   console.log("");
+  console.log("Examples:");
+  console.log("  node integration-test.js --test-only my-test-name");
+  console.log("  node integration-test.js --test-only");
+  console.log("");
   console.log("Environment Variables:");
   console.log("  AWS_REGION      AWS region (default: us-west-2)");
   console.log("  LAMBDA_ENDPOINT Custom Lambda endpoint URL");
@@ -332,14 +348,18 @@ async function main() {
   const args = process.argv.slice(2);
 
   // Parse command line arguments
+  /** @type {{ cleanupOnExit: boolean, deployOnly: boolean, testOnly: boolean, cleanupOnly: boolean, testPattern?: string }} */
   const options = {
     cleanupOnExit: true,
     deployOnly: false,
     testOnly: false,
     cleanupOnly: false,
+    testPattern: undefined,
   };
 
-  for (const arg of args) {
+  for (let i = 0; i < args.length; i++) {
+    const arg = args[i];
+
     switch (arg) {
       case "--deploy-only":
         options.deployOnly = true;
@@ -348,6 +368,11 @@ async function main() {
       case "--test-only":
         options.testOnly = true;
         options.cleanupOnExit = false;
+        // Check if the next argument is a test pattern (not another flag)
+        if (i + 1 < args.length && !args[i + 1].startsWith("--")) {
+          options.testPattern = args[i + 1];
+          i++; // Skip the next argument since we consumed it as test pattern
+        }
         break;
       case "--cleanup-only":
         options.cleanupOnly = true;

packages/aws-durable-execution-sdk-js-examples/examples-catalog.json

@@ -188,9 +188,10 @@
       "handler": "wait-for-callback.handler",
       "durableConfig": {
         "RetentionPeriodInDays": 7,
-        "ExecutionTimeout": 300
+        "ExecutionTimeout": 60
       },
-      "path": "./src/examples/wait-for-callback.ts"
+      "path": "./src/examples/wait-for-callback.ts",
+      "integration": true
     },
     {
       "name": "Named Wait",

packages/aws-durable-execution-sdk-js-examples/src/examples/__tests__/shared/test-helper.ts

@@ -4,6 +4,7 @@ import {
   DurableTestRunner,
   DurableOperation,
   LocalTestRunnerHandlerFunction,
+  InvocationType,
 } from "@aws/durable-execution-sdk-js-testing";
 
 export interface TestDefinition<ResultType> {
@@ -13,6 +14,7 @@ export interface TestDefinition<ResultType> {
   tests: (
     runner: DurableTestRunner<DurableOperation<unknown>, ResultType>
   ) => void;
+  invocationType?: InvocationType;
 }
 
 /**
@@ -36,9 +38,18 @@ export function createTests<ResultType>(testDef: TestDefinition<ResultType>) {
 
     const runner = new CloudDurableTestRunner<ResultType>({
       functionName,
-      config: {
+      clientConfig: {
         endpoint: process.env.LAMBDA_ENDPOINT,
       },
+      config: {
+        invocationType: testDef.invocationType,
+      },
+    });
+
+    beforeEach(() => {
+      // TODO: fix the testing library to allow the same runner to run multiple times on the same handler
+      // instead of needing to reset the operation storage
+      runner.reset();
     });
 
     describe(`${testDef.name} (cloud)`, () => {

packages/aws-durable-execution-sdk-js-examples/src/examples/__tests__/wait-for-callback.test.ts

@@ -1,44 +1,41 @@
-import {LocalDurableTestRunner, WaitingOperationStatus} from "@aws/durable-execution-sdk-js-testing";
-import {handler} from '../wait-for-callback';
-
-beforeAll(() => LocalDurableTestRunner.setupTestEnvironment());
-afterAll(() => LocalDurableTestRunner.teardownTestEnvironment());
-
-let durableTestRunner: LocalDurableTestRunner<void>;
-
-beforeEach(async () => {
-    durableTestRunner = new LocalDurableTestRunner({
-        handlerFunction: handler,
-        skipTime: true,
-    });
-})
-
-describe("wait-for-callback test", () => {
+import {
+  InvocationType,
+  WaitingOperationStatus,
+} from "@aws/durable-execution-sdk-js-testing";
+import { handler } from "../wait-for-callback";
+import { createTests } from "./shared/test-helper";
+
+createTests({
+  name: "wait-for-callback test",
+  functionName: "wait-for-callback",
+  handler,
+  invocationType: InvocationType.Event,
+  tests: (runner) => {
     it("function completes when callback succeeds - happy case", async () => {
-        const executionPromise = durableTestRunner.run();
+      const executionPromise = runner.run();
 
-        const waitForCallbackOp = durableTestRunner.getOperationByIndex(0);
-        await waitForCallbackOp.waitForData(WaitingOperationStatus.STARTED)
-        await waitForCallbackOp.sendCallbackSuccess("succeeded");
+      const waitForCallbackOp = runner.getOperationByIndex(0);
+      await waitForCallbackOp.waitForData(WaitingOperationStatus.STARTED);
+      await waitForCallbackOp.sendCallbackSuccess("succeeded");
 
-        const execution = await executionPromise;
+      const execution = await executionPromise;
 
-        expect(execution.getResult()).toBeDefined()
+      expect(execution.getResult()).toEqual("succeeded");
     });
 
     it("function completes when callback fails - happy case", async () => {
-        const executionPromise = durableTestRunner.run();
+      const executionPromise = runner.run();
 
-        const waitForCallbackOp = durableTestRunner.getOperationByIndex(0);
-        await waitForCallbackOp.waitForData(WaitingOperationStatus.STARTED)
-        await waitForCallbackOp.sendCallbackFailure({ErrorMessage: "ERROR"});
+      const waitForCallbackOp = runner.getOperationByIndex(0);
+      await waitForCallbackOp.waitForData(WaitingOperationStatus.STARTED);
+      await waitForCallbackOp.sendCallbackFailure({ ErrorMessage: "ERROR" });
 
-        const execution = await executionPromise;
+      const execution = await executionPromise;
 
-        expect(execution.getError()).toBeDefined()
+      expect(execution.getError()).toBeDefined();
     });
 
     it("function times out when callback is not called - failure case", async () => {
-        // TODO: add test when callback timeout is handled in TS SDK: https://tiny.amazon.com/1f5679y8l
     });
-});
+  },
+});

packages/aws-durable-execution-sdk-js-examples/src/examples/wait-for-callback.ts

@@ -6,8 +6,10 @@ const mySubmitterFunction = async (callbackId: string): Promise<void> => {
 
 export const handler = withDurableFunctions(async (event: any, context: DurableContext) => {
     console.log("Hello world before callback!");
-    await context.waitForCallback("my callback function", mySubmitterFunction, {
+    const result = await context.waitForCallback("my callback function", mySubmitterFunction, {
         timeout: 10000,
     });
     console.log("Hello world after callback!");
+
+    return result
 });