From abb39189679ab5ec5227d0c0b038af6aaf7bd451 Mon Sep 17 00:00:00 2001 From: Kieran Osgood Date: Mon, 29 Jun 2026 16:52:53 +0100 Subject: [PATCH] Document GitHub E2E reporter class Assisted-By: devx/9ae89f69-61a5-46e3-8faf-006d105b0cdd --- e2e/BITRISE.md | 10 ++ e2e/bitrise.yml | 18 ++- e2e/lib/e2e_github_reporter.rb | 210 +++++++++++++++++++++++++++ e2e/lib/json_http_client.rb | 46 ++++++ e2e/scripts/execute_browserstack_run | 46 ++---- e2e/scripts/report_e2e_results | 53 +++++++ 6 files changed, 349 insertions(+), 34 deletions(-) create mode 100644 e2e/lib/e2e_github_reporter.rb create mode 100644 e2e/lib/json_http_client.rb create mode 100755 e2e/scripts/report_e2e_results diff --git a/e2e/BITRISE.md b/e2e/BITRISE.md index ea9eb254a..a6f717ac6 100644 --- a/e2e/BITRISE.md +++ b/e2e/BITRISE.md @@ -73,6 +73,8 @@ The `e2e-execute-browserstack-run` workflow authenticates with BrowserStack usin | `BROWSERSTACK_USERNAME` | BrowserStack API username. | | `BROWSERSTACK_ACCESS_KEY` | BrowserStack API access key. | +BrowserStack artifact links in GitHub reports require access to BrowserStack App Automate. Sign in to [BrowserStack App Automate](https://app-automate.browserstack.com/dashboard/v2/builds) before opening build, video, screenshot, or log links. + ## Code signing React Native iOS IPA generation uses Bitrise's certificate and profile installer before running `xcodebuild archive` and `xcodebuild -exportArchive`. @@ -98,6 +100,14 @@ The launch smoke suite sends only non-sensitive Maestro environment values to Br Do not pass storefront tokens or customer data through BrowserStack Maestro environment variables without explicit review, because those values are visible in BrowserStack dashboards. +## GitHub reporting + +The `e2e-report` workflow creates commit statuses, Check Runs, and sticky PR comments using the short-lived token generated by the Bitrise GitHub App. + +The Bitrise project has **Project settings > Repository > Extend GitHub App permissions to builds** enabled. Bitrise exposes the build-scoped GitHub App token as `GIT_HTTP_PASSWORD`; the report workflow maps it to `GITHUB_TOKEN` before running `e2e/scripts/report_e2e_results`. + +Green runs update statuses and Check Runs without creating new PR comments. Failing runs update a sticky PR failure comment with direct BrowserStack evidence links. + ## Caching React Native Android E2E builds use the released native Maven artifact versions declared by the React Native sample and module configuration. Do not pass the React Native `--local` flag or set local native SDK override environment variables for these builds. diff --git a/e2e/bitrise.yml b/e2e/bitrise.yml index 2f4fa903e..059de0d7c 100644 --- a/e2e/bitrise.yml +++ b/e2e/bitrise.yml @@ -228,11 +228,21 @@ workflows: - git-clone@8: {} - pull-intermediate-files@1: inputs: - - artifact_sources: |- - e2e-execute-browserstack-run + - artifact_sources: e2e-execute-browserstack-run.* - script@1: - title: Placeholder E2E report + title: Report E2E results to GitHub + timeout: 900 + no_output_timeout: 450 inputs: - content: |- set -euo pipefail - echo "Phase 2 placeholder for aggregate E2E reporting" + source e2e/scripts/bitrise_ci_helpers + e2e_log "Checking GitHub reporting configuration" + if [ -z "${GITHUB_TOKEN:-}" ]; then + : "${GIT_HTTP_PASSWORD:?GITHUB_TOKEN or Bitrise GitHub App GIT_HTTP_PASSWORD is required. Enable Project settings > Repository > Extend GitHub App permissions to builds.}" + export GITHUB_TOKEN="$GIT_HTTP_PASSWORD" + fi + : "${BITRISE_GIT_COMMIT:?BITRISE_GIT_COMMIT is required}" + results_root="${E2E_BROWSERSTACK_RESULTS_DIR:-$BITRISE_DEPLOY_DIR/e2e/results}" + e2e_log "Reporting E2E results to GitHub" + e2e/scripts/report_e2e_results --results-root "$results_root" diff --git a/e2e/lib/e2e_github_reporter.rb b/e2e/lib/e2e_github_reporter.rb new file mode 100644 index 000000000..1e3af973f --- /dev/null +++ b/e2e/lib/e2e_github_reporter.rb @@ -0,0 +1,210 @@ +# frozen_string_literal: true + +require "json" +require_relative "json_http_client" + +# Publishes normalized E2E run results back to GitHub as commit statuses, +# check runs, and a sticky pull request comment for failures. +class E2EGitHubReporter + COMMENT_MARKER = "" + + def initialize(results, repository:, sha:, pr_number:, token: nil, strict: false) + @results = results + @repository = repository + @sha = sha + @pr_number = pr_number + @token = token + @strict = strict + end + + def publish! + commit_status_payloads.each { |payload| client.post_json("/repos/#{@repository}/statuses/#{@sha}", payload) } + client.post_json("/repos/#{@repository}/check-runs", check_run_payload) + sync_failure_comment + end + + def markdown_summary + lines = [] + lines << "## Checkout Kit E2E results" + lines << "" + lines << "| Status | Suite | Target | Platform | OS version tag | Device |" + lines << "|---|---|---|---|---|---|" + @results.each do |result| + lines << "| #{status_icon(result)} | `#{result.fetch("execute")}` | #{result.fetch("target")} | #{result.fetch("platform")} | #{result.fetch("os_version_tag")} | #{device_cell(result)} |" + end + failure_lines = failed_results.flat_map { |result| failure_details(result) } + unless failure_lines.empty? + lines << "" + lines << "## Failures" + lines << "" + if soft_fail? + lines << "> [!CAUTION]" + lines << "> These are currently non-blocking, but failures may indicate an issue you need to resolve before merging." + lines << "> If you believe there is a flaky assertion, please raise a ticket in the #checkout-kit-devs channel so it can be addressed" + lines << "" + end + lines << "> BrowserStack artifacts require BrowserStack access. Sign in to [BrowserStack App Automate](https://app-automate.browserstack.com/dashboard/v2/builds) before opening artifact links." + lines.concat(failure_lines) + end + lines.join("\n") + end + + def commit_status_payloads + @results.map do |result| + { + state: result.fetch("passed") ? "success" : "failure", + context: result.fetch("status_context"), + description: status_description(result), + target_url: browserstack_build_url(result) + } + end + end + + def failure_comment_body + return nil if failed_results.empty? + + [COMMENT_MARKER, markdown_summary].join("\n") + end + + private + + def check_run_payload + conclusion = failed_results.empty? ? "success" : "failure" + { + name: "Checkout Kit E2E", + head_sha: @sha, + status: "completed", + conclusion: conclusion, + output: { + title: "Checkout Kit E2E #{conclusion}", + summary: markdown_summary + } + } + end + + def sync_failure_comment + body = failure_comment_body + existing = existing_failure_comment + if body + if existing + client.patch_json("/repos/#{@repository}/issues/comments/#{existing.fetch("id")}", {body: body}) + else + client.post_json("/repos/#{@repository}/issues/#{@pr_number}/comments", {body: body}) + end + elsif existing + client.patch_json("/repos/#{@repository}/issues/comments/#{existing.fetch("id")}", {body: "#{COMMENT_MARKER}\n✅ Checkout Kit E2E failures resolved."}) + end + end + + def existing_failure_comment + issue_comments.find do |comment| + comment.fetch("body", "").include?(COMMENT_MARKER) + end + end + + def failed_results + @results.reject { |result| result.fetch("passed") } + end + + def soft_fail? + !@strict + end + + def failure_details(result) + lines = [] + lines << "" + lines << "### #{failure_heading(result)}" + lines << "" + lines << "| Test | Status | Artifacts |" + lines << "|---|---|---|" + tests = result.fetch("failed_tests", []) + if tests.empty? + lines << "| — | #{status_icon(result)} | #{artifact_links(nil, result)} |" + else + tests.each do |testcase| + lines << "| `#{testcase.fetch("name", "unknown")}` | ❌ | #{artifact_links(testcase, result)} |" + end + end + lines + end + + def failure_heading(result) + suite = File.basename(result.fetch("execute"), ".*") + "#{os_label(result.fetch("platform"))} — #{suite}" + end + + def artifact_links(testcase, result) + links = ["[BrowserStack](#{browserstack_build_url(result)})"] + if testcase + link_fields.each do |label, key| + url = artifact_url(testcase[key]) + links << "[#{label}](#{url})" if url + end + end + links.join(" · ") + end + + def artifact_url(value) + case value + when String then value.empty? ? nil : value + when Array then value.find { |entry| entry.is_a?(String) && !entry.empty? } + end + end + + def link_fields + { + "Video" => "video", + "Screenshot" => "screenshots", + "Maestro commands" => "maestro_commands", + "Maestro log" => "maestro_log", + "Device log" => "device_log", + "Network log" => "network_log" + } + end + + def device_cell(result) + name = result.fetch("resolved_device") + version = result["resolved_os_version"] + version && !version.empty? ? "#{name}
#{os_label(result.fetch("platform"))} #{version}" : name + end + + def os_label(platform) + platform == "ios" ? "iOS" : "Android" + end + + def status_icon(result) + result.fetch("passed") ? "✅" : "❌" + end + + def status_description(result) + description = result.fetch("passed") ? "passed" : "#{result.fetch("status")} on #{result.fetch("resolved_device")}" + description[0, 140] + end + + def browserstack_build_url(result) + "https://app-automate.browserstack.com/dashboard/v2/builds/#{result.fetch("build_id")}" + end + + def issue_comments + comments = [] + page = 1 + loop do + batch = client.get("/repos/#{@repository}/issues/#{@pr_number}/comments?per_page=100&page=#{page}") + break unless batch.is_a?(Array) && !batch.empty? + + comments.concat(batch) + break if batch.length < 100 + + page += 1 + end + comments + end + + def client + @client ||= JsonHttpClient.new(host: "api.github.com", error_label: "GitHub", default_headers: {"Accept" => "application/vnd.github+json"}) do |request| + raise "GitHub token is required" unless @token + + request["Authorization"] = "Bearer #{@token}" + end + end +end diff --git a/e2e/lib/json_http_client.rb b/e2e/lib/json_http_client.rb new file mode 100644 index 000000000..092e223be --- /dev/null +++ b/e2e/lib/json_http_client.rb @@ -0,0 +1,46 @@ +# frozen_string_literal: true + +require "json" +require "net/http" + +# Small JSON-over-HTTPS client shared by the BrowserStack executor and the +# GitHub reporter. Each caller supplies its host, an error label, default +# headers, and an authenticator block that stamps credentials onto the request. +class JsonHttpClient + def initialize(host:, error_label:, default_headers: {}, &authenticator) + @host = host + @error_label = error_label + @default_headers = default_headers + @authenticator = authenticator + end + + def get(path) + execute(Net::HTTP::Get.new(path)) + end + + def post_json(path, body) + execute(with_json_body(Net::HTTP::Post.new(path), body)) + end + + def patch_json(path, body) + execute(with_json_body(Net::HTTP::Patch.new(path), body)) + end + + def execute(request) + @default_headers.each { |name, value| request[name] = value } + @authenticator&.call(request) + response = Net::HTTP.start(@host, 443, use_ssl: true) { |http| http.request(request) } + body = response.body.to_s.empty? ? {} : JSON.parse(response.body) + return body if response.is_a?(Net::HTTPSuccess) + + raise "#{@error_label} request failed #{response.code}: #{body}" + end + + private + + def with_json_body(request, body) + request["Content-Type"] = "application/json" + request.body = JSON.generate(body) + request + end +end diff --git a/e2e/scripts/execute_browserstack_run b/e2e/scripts/execute_browserstack_run index f55cef07b..63e91a273 100755 --- a/e2e/scripts/execute_browserstack_run +++ b/e2e/scripts/execute_browserstack_run @@ -7,8 +7,8 @@ require "json" require "net/http" require "optparse" require "securerandom" -require "uri" require_relative "../lib/browserstack_device_resolver" +require_relative "../lib/json_http_client" # Executes one row from the BrowserStack run plan, from artifact upload through # build polling and normalized result persistence. @@ -20,6 +20,9 @@ class BrowserStackRunExecutor @options = options @username = ENV.fetch("BROWSERSTACK_USERNAME") @access_key = ENV.fetch("BROWSERSTACK_ACCESS_KEY") + @client = JsonHttpClient.new(host: API_HOST, error_label: "BrowserStack") do |request| + request.basic_auth(@username, @access_key) + end end def run @@ -42,16 +45,17 @@ class BrowserStackRunExecutor def resolve_device(run) override = ENV["E2E_DEVICE_OVERRIDE"] || ENV["E2E_#{run.fetch("platform").upcase}_DEVICE_OVERRIDE"] if override && !override.empty? + name, separator, version = override.rpartition("-") return { "device_selector" => run.fetch("device_selector"), "browserstack_device" => override, - "resolved_device" => override.split("-").first, - "resolved_os_version" => override.split("-").last + "resolved_device" => separator.empty? ? override : name, + "resolved_os_version" => separator.empty? ? nil : version } end - devices = get_json("/app-automate/devices.json") - limits = get_json("/app-automate/device_tier_limits.json") + devices = @client.get("/app-automate/devices.json") + limits = @client.get("/app-automate/device_tier_limits.json") BrowserStackDeviceResolver.new(devices, limits).resolve(run.fetch("device_selector")) end @@ -75,7 +79,7 @@ class BrowserStackRunExecutor E2E_READY_MARKER: run.fetch("ready_marker") } } - response = post_json("/app-automate/maestro/v2/#{run.fetch("platform")}/build", body) + response = @client.post_json("/app-automate/maestro/v2/#{run.fetch("platform")}/build", body) write_json("build-start.json", response) response end @@ -83,7 +87,7 @@ class BrowserStackRunExecutor def poll_build(build_id) deadline = Time.now + ENV.fetch("E2E_BROWSERSTACK_TIMEOUT_SECONDS", "1800").to_i loop do - response = get_json("/app-automate/maestro/v2/builds/#{build_id}") + response = @client.get("/app-automate/maestro/v2/builds/#{build_id}") write_json("build-status.json", response) return response if TERMINAL_STATUSES.include?(response.fetch("status").to_s.downcase) raise "BrowserStack build timed out: #{build_id}" if Time.now >= deadline @@ -96,7 +100,7 @@ class BrowserStackRunExecutor build_id = build_status.fetch("id") build_status.fetch("devices", []).flat_map do |device| device.fetch("sessions", []).map do |session| - get_json("/app-automate/maestro/v2/builds/#{build_id}/sessions/#{session.fetch("id")}") + @client.get("/app-automate/maestro/v2/builds/#{build_id}/sessions/#{session.fetch("id")}") end end end @@ -117,7 +121,9 @@ class BrowserStackRunExecutor "os_version_tag" => run.fetch("os_version_tag"), "execute" => run.fetch("execute"), "device_selector" => device.fetch("device_selector"), - "resolved_device" => device.fetch("browserstack_device"), + "resolved_device" => device.fetch("resolved_device"), + "resolved_os_version" => device.fetch("resolved_os_version"), + "browserstack_device" => device.fetch("browserstack_device"), "app_url" => app.fetch("app_url"), "test_suite_url" => suite.fetch("test_suite_url"), "build_id" => build.fetch("build_id"), @@ -141,27 +147,7 @@ class BrowserStackRunExecutor request = Net::HTTP::Post.new(path) request["Content-Type"] = "multipart/form-data; boundary=#{boundary}" request.body = body - execute_request(request) - end - - def get_json(path) - execute_request(Net::HTTP::Get.new(path)) - end - - def post_json(path, body) - request = Net::HTTP::Post.new(path) - request["Content-Type"] = "application/json" - request.body = JSON.generate(body) - execute_request(request) - end - - def execute_request(request) - request.basic_auth(@username, @access_key) - response = Net::HTTP.start(API_HOST, 443, use_ssl: true) { |http| http.request(request) } - parsed = JSON.parse(response.body) - return parsed if response.is_a?(Net::HTTPSuccess) - - raise "BrowserStack request failed #{response.code}: #{parsed}" + @client.execute(request) end def custom_id(run, suffix) diff --git a/e2e/scripts/report_e2e_results b/e2e/scripts/report_e2e_results new file mode 100755 index 000000000..9b359ebb7 --- /dev/null +++ b/e2e/scripts/report_e2e_results @@ -0,0 +1,53 @@ +#!/usr/bin/env ruby +# frozen_string_literal: true + +require "json" +require "optparse" +require_relative "../lib/e2e_github_reporter" + +options = { + results_root: ENV["E2E_BROWSERSTACK_RESULTS_DIR"] || File.join(Dir.pwd, "e2e-results"), + repository: ENV["GITHUB_REPOSITORY"] || "Shopify/checkout-kit", + sha: ENV["BITRISE_GIT_COMMIT"], + pr_number: ENV["BITRISE_PULL_REQUEST"], + token: ENV["GITHUB_TOKEN"], + strict: ENV.fetch("E2E_STRICT", "false") == "true" +} + +OptionParser.new do |opts| + opts.on("--results-root PATH") { |path| options[:results_root] = path } + opts.on("--repository REPOSITORY") { |repository| options[:repository] = repository } + opts.on("--sha SHA") { |sha| options[:sha] = sha } + opts.on("--pr NUMBER") { |number| options[:pr_number] = number } +end.parse! + +pr_number = options[:pr_number].to_s.strip +if pr_number.empty? || pr_number == "false" || pr_number == "0" + warn "No pull request associated with this build; skipping GitHub E2E report" + exit 0 +end + +%i[sha token].each do |key| + next unless options[key].nil? || options[key].to_s.strip.empty? + + abort "#{key} is required" +end +options[:pr_number] = Integer(options[:pr_number]) + +result_paths = Dir.glob(File.join(options.fetch(:results_root), "**", "result.json")) +if result_paths.empty? + warn "No E2E result files found under #{options.fetch(:results_root)}" + exit 1 +end + +results = result_paths.sort.map { |path| JSON.parse(File.read(path)) } +reporter = E2EGitHubReporter.new( + results, + repository: options.fetch(:repository), + sha: options.fetch(:sha), + pr_number: options.fetch(:pr_number), + token: options.fetch(:token), + strict: options.fetch(:strict) +) +reporter.publish! +puts reporter.markdown_summary