From 3ebf31469e0792209256dc1da83b0629c348f7c5 Mon Sep 17 00:00:00 2001 From: Kieran Osgood Date: Mon, 29 Jun 2026 16:20:22 +0100 Subject: [PATCH 1/3] Add E2E matrix launch smoke foundation Assisted-By: devx/6c1e3ad5-96c8-4972-b087-da7ff7b195c3 --- e2e/README.md | 100 ++++++++++++++++--- e2e/config/matrix.yml | 19 ++++ e2e/lib/e2e_matrix.rb | 150 +++++++++++++++++++++++++++++ e2e/scripts/e2e_matrix | 51 ++++++++++ e2e/tests/shared/launch-smoke.yaml | 7 ++ 5 files changed, 315 insertions(+), 12 deletions(-) create mode 100644 e2e/config/matrix.yml create mode 100644 e2e/lib/e2e_matrix.rb create mode 100755 e2e/scripts/e2e_matrix create mode 100644 e2e/tests/shared/launch-smoke.yaml diff --git a/e2e/README.md b/e2e/README.md index 45b8ad3e..29ee55bf 100644 --- a/e2e/README.md +++ b/e2e/README.md @@ -1,19 +1,22 @@ # Checkout Kit End-to-End Tests -This directory contains Maestro end-to-end smoke flows for Checkout Kit sample -apps. +This directory contains Maestro end-to-end flows and configuration for Checkout +Kit sample apps. Two complementary setups live here: -The current runnable suite starts with React Native. It verifies a full guest -checkout from a seeded cart in the sample app, through Shopify checkout, and -back to the app after completion. +- A **local** guest-checkout smoke flow, run with `dev rn e2e`, that exercises a + full checkout from a seeded cart through Shopify checkout and back to the app. +- A **CI matrix** that expands applications, OS tracks, and suites into + BrowserStack Maestro run rows, starting with a shared launch smoke. ## Run locally -Run the matching command from the repo root. - Run `dev up` first to provision the local toolchain. Install Maestro separately and make sure `maestro --version` succeeds before running these flows. +### Full guest checkout (`dev rn e2e`) + +Run the matching command from the repo root. + React Native iOS: ```bash @@ -31,17 +34,90 @@ sample app, then run Maestro. They require the standard storefront `.env` setup, but the E2E flow seeds its own cart through the bootstrap deep link; no manual sample cart setup is required. +### Shared launch smoke + +The launch smoke launches a sample app and waits for the shared ready marker +exposed by that app, using the same environment contract used by CI. + +React Native iOS: + +```bash +E2E_APP_ID=com.shopify.checkoutkit.reactnativedemo \ +E2E_READY_MARKER=catalog-tab \ +maestro --platform ios test e2e/tests/shared/launch-smoke.yaml +``` + +React Native Android: + +```bash +E2E_APP_ID=com.shopify.checkoutkit.reactnativedemo \ +E2E_READY_MARKER=catalog-tab \ +maestro --platform android test e2e/tests/shared/launch-smoke.yaml +``` + +React Native E2E runs should use the released native SDK artifacts declared by +the React Native sample configuration, not local in-repo native SDK overrides. + +## Matrix + +CI runs are described by `config/matrix.yml`. The matrix expands applications, OS +tracks, and suites into BrowserStack Maestro run rows. Because Bitrise has no +built-in matrix support, `e2e/lib/e2e_matrix.rb` performs the expansion and the +pipeline parallelizes over the resulting rows. + +Current applications: + +- React Native iOS sample app +- React Native Android sample app + +Current OS tracks: + +- `latest` + +Current suites: + +- `tests/shared/launch-smoke.yaml` + +Validate the matrix: + +```bash +ruby e2e/scripts/e2e_matrix validate +``` + +Expand all run rows: + +```bash +ruby e2e/scripts/e2e_matrix expand +``` + +Expand a single run row by index: + +```bash +ruby e2e/scripts/e2e_matrix expand --index 0 +``` + ## Files - `config.yaml` configures Maestro for shared platform behavior. - `flows/` contains reusable Maestro subflows for app setup and checkout steps. - `tests/react-native/full-guest-checkout.yaml` composes the React Native guest checkout smoke test from those subflows. +- `config/matrix.yml`, `lib/e2e_matrix.rb`, and `scripts/` drive the CI matrix. +- `tests/shared/launch-smoke.yaml` is the shared launch smoke suite. + +## Shared app contract + +Shared flows rely on stable cross-app identifiers. The launch smoke requires each +target app to expose this ready marker: + +- `catalog-tab` + +Future shared flows should add identifiers here before they are used across +React Native, Swift, and Android sample apps. ## Scope -This smoke flow is intended to catch regressions in the React Native sample app -integration surface: cart bootstrap, checkout presentation, checkout -completion, and return to the sample app. It is not a replacement for -checkout-web's browser-based coverage or for future native Swift and Android -sample-app E2E coverage. +These flows catch regressions in the React Native sample app integration +surface: cart bootstrap, checkout presentation, checkout completion, and return +to the sample app. They are not a replacement for checkout-web's browser-based +coverage or for future native Swift and Android sample-app E2E coverage. diff --git a/e2e/config/matrix.yml b/e2e/config/matrix.yml new file mode 100644 index 00000000..f6cc9f60 --- /dev/null +++ b/e2e/config/matrix.yml @@ -0,0 +1,19 @@ +version: 1 +applications: + - id: react-native-ios + target: react-native + platform: ios + app_id: com.shopify.checkoutkit.reactnativedemo + artifact_env: E2E_REACT_NATIVE_IOS_APP_PATH + ready_marker: catalog-tab + - id: react-native-android + target: react-native + platform: android + app_id: com.shopify.checkoutkit.reactnativedemo + artifact_env: E2E_REACT_NATIVE_ANDROID_APP_PATH + ready_marker: catalog-tab +os_tracks: + - latest +suites: + - id: launch-smoke + execute: tests/shared/launch-smoke.yaml diff --git a/e2e/lib/e2e_matrix.rb b/e2e/lib/e2e_matrix.rb new file mode 100644 index 00000000..cd80fba2 --- /dev/null +++ b/e2e/lib/e2e_matrix.rb @@ -0,0 +1,150 @@ +# frozen_string_literal: true + +require "yaml" + +class E2EMatrix + attr_reader :config_path + + def self.load(config_path) + new(config_path, YAML.safe_load_file(config_path)) + end + + def initialize(config_path, config) + @config_path = config_path + @config = config || {} + end + + def expand + applications.flat_map do |application| + os_tracks.flat_map do |os_track| + suites.map do |suite| + build_run(application, os_track, suite) + end + end + end + end + + def run_at(index) + runs = expand + runs.fetch(index) + end + + def validation_errors + errors = [] + errors << "version must be 1" unless @config.fetch("version", nil) == 1 + validate_collection(errors, "applications", applications) + validate_collection(errors, "os_tracks", os_tracks) + validate_collection(errors, "suites", suites) + validate_applications(errors) + validate_os_tracks(errors) + validate_suites(errors) + errors + end + + private + + def build_run(application, os_track, suite) + platform = application.fetch("platform") + os_track_id = os_track_id(os_track) + suite_id = suite.fetch("id") + application_id = application.fetch("id") + + { + "id" => "#{application_id}-#{os_track_id}-#{suite_id}", + "application_id" => application_id, + "target" => application.fetch("target"), + "platform" => platform, + "os_track" => os_track_id, + "device_selector" => device_selector(platform, os_track), + "app_id" => application.fetch("app_id"), + "artifact_env" => application.fetch("artifact_env"), + "execute" => suite.fetch("execute"), + "ready_marker" => application.fetch("ready_marker"), + "status_context" => "checkout-kit/e2e/#{application_id}/#{os_track_id}/#{suite_id}" + } + end + + def device_selector(platform, os_track) + return os_track.fetch("device_selector") if os_track.is_a?(Hash) && os_track.key?("device_selector") + + "#{platform}:phone:#{os_track_id(os_track)}" + end + + def os_track_id(os_track) + os_track.is_a?(Hash) ? os_track.fetch("id") : os_track + end + + def applications + @config.fetch("applications", []) || [] + end + + def os_tracks + @config.fetch("os_tracks", []) || [] + end + + def suites + @config.fetch("suites", []) || [] + end + + def validate_collection(errors, name, collection) + errors << "#{name} must be a non-empty array" unless collection.is_a?(Array) && !collection.empty? + end + + def validate_applications(errors) + return unless applications.is_a?(Array) + + validate_unique_ids(errors, "application", applications) + applications.each do |application| + id = application.fetch("id", "") + required_application_keys.each do |key| + errors << "application #{id} missing #{key}" if application.fetch(key, "").to_s.empty? + end + platform = application.fetch("platform", nil) + errors << "application #{id} platform must be ios or android" unless ["ios", "android"].include?(platform) + end + end + + def validate_os_tracks(errors) + return unless os_tracks.is_a?(Array) + + ids = os_tracks.map { |os_track| safe_os_track_id(os_track) } + errors << "os_track ids must be unique" unless ids.compact.uniq.length == ids.compact.length + os_tracks.each do |os_track| + id = safe_os_track_id(os_track) + errors << "os_track missing id" if id.to_s.empty? + end + end + + def safe_os_track_id(os_track) + os_track_id(os_track) + rescue KeyError + nil + end + + def validate_suites(errors) + return unless suites.is_a?(Array) + + validate_unique_ids(errors, "suite", suites) + suites.each do |suite| + id = suite.fetch("id", "") + execute = suite.fetch("execute", "") + errors << "suite #{id} missing execute" if execute.empty? + next if execute.empty? + + errors << "suite #{id} execute path does not exist: #{execute}" unless File.exist?(File.join(e2e_root, execute)) + end + end + + def validate_unique_ids(errors, label, collection) + ids = collection.map { |item| item.fetch("id", nil) } + errors << "#{label} ids must be unique" unless ids.compact.uniq.length == ids.compact.length + end + + def required_application_keys + ["id", "target", "platform", "app_id", "artifact_env", "ready_marker"] + end + + def e2e_root + File.expand_path("..", File.dirname(config_path)) + end +end diff --git a/e2e/scripts/e2e_matrix b/e2e/scripts/e2e_matrix new file mode 100755 index 00000000..5f521888 --- /dev/null +++ b/e2e/scripts/e2e_matrix @@ -0,0 +1,51 @@ +#!/usr/bin/env ruby +# frozen_string_literal: true + +require "json" +require "optparse" +require_relative "../lib/e2e_matrix" + +options = { + config: "e2e/config/matrix.yml" +} + +parser = OptionParser.new do |opts| + opts.banner = "Usage: e2e/scripts/e2e_matrix validate|expand [options]" + opts.on("--config PATH") { |path| options[:config] = path } + opts.on("--index INDEX", Integer) { |index| options[:index] = index } +end + +command = ARGV.shift +parser.parse!(ARGV) + +unless ["validate", "expand"].include?(command) + warn parser + exit 1 +end + +matrix = E2EMatrix.load(options.fetch(:config)) + +case command +when "validate" + errors = matrix.validation_errors + if errors.empty? + puts "E2E matrix is valid" + else + warn errors.join("\n") + exit 1 + end +when "expand" + errors = matrix.validation_errors + unless errors.empty? + warn errors.join("\n") + exit 1 + end + + output = if options.key?(:index) + matrix.run_at(options.fetch(:index)) + else + matrix.expand + end + + puts JSON.pretty_generate(output) +end diff --git a/e2e/tests/shared/launch-smoke.yaml b/e2e/tests/shared/launch-smoke.yaml new file mode 100644 index 00000000..5f534a86 --- /dev/null +++ b/e2e/tests/shared/launch-smoke.yaml @@ -0,0 +1,7 @@ +appId: ${E2E_APP_ID} +--- +- launchApp +- extendedWaitUntil: + visible: + id: ${E2E_READY_MARKER} + timeout: 60000 From b733667295e80e45bcd64c48ec06808c134970f4 Mon Sep 17 00:00:00 2001 From: Kieran Osgood Date: Mon, 6 Jul 2026 09:43:32 +0100 Subject: [PATCH 2/3] Document E2EMatrix expansion purpose Assisted-By: devx/96a590ff-685a-49ce-8199-0810f0ac2379 --- e2e/README.md | 6 ++-- e2e/config/matrix.yml | 2 +- e2e/lib/e2e_matrix.rb | 65 ++++++++++++++++++++++++++----------------- 3 files changed, 43 insertions(+), 30 deletions(-) diff --git a/e2e/README.md b/e2e/README.md index 29ee55bf..425c608f 100644 --- a/e2e/README.md +++ b/e2e/README.md @@ -5,7 +5,7 @@ Kit sample apps. Two complementary setups live here: - A **local** guest-checkout smoke flow, run with `dev rn e2e`, that exercises a full checkout from a seeded cart through Shopify checkout and back to the app. -- A **CI matrix** that expands applications, OS tracks, and suites into +- A **CI matrix** that expands applications, OS version tags, and suites into BrowserStack Maestro run rows, starting with a shared launch smoke. ## Run locally @@ -61,7 +61,7 @@ the React Native sample configuration, not local in-repo native SDK overrides. ## Matrix CI runs are described by `config/matrix.yml`. The matrix expands applications, OS -tracks, and suites into BrowserStack Maestro run rows. Because Bitrise has no +version tags, and suites into BrowserStack Maestro run rows. Because Bitrise has no built-in matrix support, `e2e/lib/e2e_matrix.rb` performs the expansion and the pipeline parallelizes over the resulting rows. @@ -70,7 +70,7 @@ Current applications: - React Native iOS sample app - React Native Android sample app -Current OS tracks: +Current OS version tags: - `latest` diff --git a/e2e/config/matrix.yml b/e2e/config/matrix.yml index f6cc9f60..bddfb8cb 100644 --- a/e2e/config/matrix.yml +++ b/e2e/config/matrix.yml @@ -12,7 +12,7 @@ applications: app_id: com.shopify.checkoutkit.reactnativedemo artifact_env: E2E_REACT_NATIVE_ANDROID_APP_PATH ready_marker: catalog-tab -os_tracks: +os_version_tags: - latest suites: - id: launch-smoke diff --git a/e2e/lib/e2e_matrix.rb b/e2e/lib/e2e_matrix.rb index cd80fba2..5aa91765 100644 --- a/e2e/lib/e2e_matrix.rb +++ b/e2e/lib/e2e_matrix.rb @@ -2,6 +2,19 @@ require "yaml" +# Expands the compact E2E test matrix in config/matrix.yml into one fully-resolved +# run per (application x os_version_tag x suite) combination. +# +# Unlike GitHub Actions, Bitrise has no built-in matrix/strategy support, so we +# do the fan-out ourselves: `expand` produces the JSON list of runs that the +# pipeline consumes, and Bitrise parallelizes over it via `run_at`/`count`. +# +# Today the numbers make the config and the expansion look equivalent - +# 2 applications + 1 os_version_tag + 1 suite, and 2 * 1 * 1 = 2 runs - so it can look +# like a plain YAML-to-JSON copy. The point is the multiplication, not the copy: +# adding a single os_version_tag (e.g. a minimum-supported OS) transparently duplicates +# every application x suite across that OS, turning an additive config change into +# a multiplicative set of runs without hand-writing each one. class E2EMatrix attr_reader :config_path @@ -16,9 +29,9 @@ def initialize(config_path, config) def expand applications.flat_map do |application| - os_tracks.flat_map do |os_track| + os_version_tags.flat_map do |os_version_tag| suites.map do |suite| - build_run(application, os_track, suite) + build_run(application, os_version_tag, suite) end end end @@ -33,53 +46,53 @@ def validation_errors errors = [] errors << "version must be 1" unless @config.fetch("version", nil) == 1 validate_collection(errors, "applications", applications) - validate_collection(errors, "os_tracks", os_tracks) + validate_collection(errors, "os_version_tags", os_version_tags) validate_collection(errors, "suites", suites) validate_applications(errors) - validate_os_tracks(errors) + validate_os_version_tags(errors) validate_suites(errors) errors end private - def build_run(application, os_track, suite) + def build_run(application, os_version_tag, suite) platform = application.fetch("platform") - os_track_id = os_track_id(os_track) + os_version_tag_id = os_version_tag_id(os_version_tag) suite_id = suite.fetch("id") application_id = application.fetch("id") { - "id" => "#{application_id}-#{os_track_id}-#{suite_id}", + "id" => "#{application_id}-#{os_version_tag_id}-#{suite_id}", "application_id" => application_id, "target" => application.fetch("target"), "platform" => platform, - "os_track" => os_track_id, - "device_selector" => device_selector(platform, os_track), + "os_version_tag" => os_version_tag_id, + "device_selector" => device_selector(platform, os_version_tag), "app_id" => application.fetch("app_id"), "artifact_env" => application.fetch("artifact_env"), "execute" => suite.fetch("execute"), "ready_marker" => application.fetch("ready_marker"), - "status_context" => "checkout-kit/e2e/#{application_id}/#{os_track_id}/#{suite_id}" + "status_context" => "checkout-kit/e2e/#{application_id}/#{os_version_tag_id}/#{suite_id}" } end - def device_selector(platform, os_track) - return os_track.fetch("device_selector") if os_track.is_a?(Hash) && os_track.key?("device_selector") + def device_selector(platform, os_version_tag) + return os_version_tag.fetch("device_selector") if os_version_tag.is_a?(Hash) && os_version_tag.key?("device_selector") - "#{platform}:phone:#{os_track_id(os_track)}" + "#{platform}:phone:#{os_version_tag_id(os_version_tag)}" end - def os_track_id(os_track) - os_track.is_a?(Hash) ? os_track.fetch("id") : os_track + def os_version_tag_id(os_version_tag) + os_version_tag.is_a?(Hash) ? os_version_tag.fetch("id") : os_version_tag end def applications @config.fetch("applications", []) || [] end - def os_tracks - @config.fetch("os_tracks", []) || [] + def os_version_tags + @config.fetch("os_version_tags", []) || [] end def suites @@ -104,19 +117,19 @@ def validate_applications(errors) end end - def validate_os_tracks(errors) - return unless os_tracks.is_a?(Array) + def validate_os_version_tags(errors) + return unless os_version_tags.is_a?(Array) - ids = os_tracks.map { |os_track| safe_os_track_id(os_track) } - errors << "os_track ids must be unique" unless ids.compact.uniq.length == ids.compact.length - os_tracks.each do |os_track| - id = safe_os_track_id(os_track) - errors << "os_track missing id" if id.to_s.empty? + ids = os_version_tags.map { |os_version_tag| safe_os_version_tag_id(os_version_tag) } + errors << "os_version_tag ids must be unique" unless ids.compact.uniq.length == ids.compact.length + os_version_tags.each do |os_version_tag| + id = safe_os_version_tag_id(os_version_tag) + errors << "os_version_tag missing id" if id.to_s.empty? end end - def safe_os_track_id(os_track) - os_track_id(os_track) + def safe_os_version_tag_id(os_version_tag) + os_version_tag_id(os_version_tag) rescue KeyError nil end From 6ab205001b1404c10aaee824606495a95e939623 Mon Sep 17 00:00:00 2001 From: Kieran Osgood Date: Wed, 8 Jul 2026 15:52:10 +0100 Subject: [PATCH 3/3] Address E2E matrix launch review feedback --- e2e/README.md | 6 +++--- e2e/config/matrix.yml | 4 ++-- e2e/lib/e2e_matrix.rb | 7 ++++++- e2e/scripts/e2e_matrix | 2 +- 4 files changed, 12 insertions(+), 7 deletions(-) diff --git a/e2e/README.md b/e2e/README.md index 425c608f..eb942c00 100644 --- a/e2e/README.md +++ b/e2e/README.md @@ -43,7 +43,7 @@ React Native iOS: ```bash E2E_APP_ID=com.shopify.checkoutkit.reactnativedemo \ -E2E_READY_MARKER=catalog-tab \ +E2E_READY_MARKER=checkout-kit-sample-ready \ maestro --platform ios test e2e/tests/shared/launch-smoke.yaml ``` @@ -51,7 +51,7 @@ React Native Android: ```bash E2E_APP_ID=com.shopify.checkoutkit.reactnativedemo \ -E2E_READY_MARKER=catalog-tab \ +E2E_READY_MARKER=checkout-kit-sample-ready \ maestro --platform android test e2e/tests/shared/launch-smoke.yaml ``` @@ -110,7 +110,7 @@ ruby e2e/scripts/e2e_matrix expand --index 0 Shared flows rely on stable cross-app identifiers. The launch smoke requires each target app to expose this ready marker: -- `catalog-tab` +- `checkout-kit-sample-ready` Future shared flows should add identifiers here before they are used across React Native, Swift, and Android sample apps. diff --git a/e2e/config/matrix.yml b/e2e/config/matrix.yml index bddfb8cb..53608764 100644 --- a/e2e/config/matrix.yml +++ b/e2e/config/matrix.yml @@ -5,13 +5,13 @@ applications: platform: ios app_id: com.shopify.checkoutkit.reactnativedemo artifact_env: E2E_REACT_NATIVE_IOS_APP_PATH - ready_marker: catalog-tab + ready_marker: checkout-kit-sample-ready - id: react-native-android target: react-native platform: android app_id: com.shopify.checkoutkit.reactnativedemo artifact_env: E2E_REACT_NATIVE_ANDROID_APP_PATH - ready_marker: catalog-tab + ready_marker: checkout-kit-sample-ready os_version_tags: - latest suites: diff --git a/e2e/lib/e2e_matrix.rb b/e2e/lib/e2e_matrix.rb index 5aa91765..3e780dab 100644 --- a/e2e/lib/e2e_matrix.rb +++ b/e2e/lib/e2e_matrix.rb @@ -39,6 +39,10 @@ def expand def run_at(index) runs = expand + unless index.is_a?(Integer) && index >= 0 && index < runs.length + raise IndexError, "index #{index} outside of run range 0...#{runs.length}" + end + runs.fetch(index) end @@ -139,7 +143,8 @@ def validate_suites(errors) validate_unique_ids(errors, "suite", suites) suites.each do |suite| - id = suite.fetch("id", "") + id = suite.fetch("id", "") + errors << "suite missing id" if id.to_s.empty? execute = suite.fetch("execute", "") errors << "suite #{id} missing execute" if execute.empty? next if execute.empty? diff --git a/e2e/scripts/e2e_matrix b/e2e/scripts/e2e_matrix index 5f521888..de97ae08 100755 --- a/e2e/scripts/e2e_matrix +++ b/e2e/scripts/e2e_matrix @@ -6,7 +6,7 @@ require "optparse" require_relative "../lib/e2e_matrix" options = { - config: "e2e/config/matrix.yml" + config: File.expand_path("../config/matrix.yml", __dir__) } parser = OptionParser.new do |opts|